diff --git a/integration-tests/ci-visibility/vitest-tests/early-flake-detection.mjs b/integration-tests/ci-visibility/vitest-tests/early-flake-detection.mjs new file mode 100644 index 00000000000..a85036dac8e --- /dev/null +++ b/integration-tests/ci-visibility/vitest-tests/early-flake-detection.mjs @@ -0,0 +1,33 @@ +import { describe, test, expect } from 'vitest' +import { sum } from './sum' + +let numAttempt = 0 +let numOtherAttempt = 0 + +describe('early flake detection', () => { + test('can retry tests that eventually pass', { repeats: process.env.SHOULD_REPEAT && 2 }, () => { + expect(sum(1, 2)).to.equal(numAttempt++ > 1 ? 3 : 4) + }) + + test('can retry tests that always pass', { repeats: process.env.SHOULD_REPEAT && 2 }, () => { + if (process.env.ALWAYS_FAIL) { + expect(sum(1, 2)).to.equal(4) + } else { + expect(sum(1, 2)).to.equal(3) + } + }) + + test('does not retry if it is not new', () => { + expect(sum(1, 2)).to.equal(3) + }) + + test.skip('does not retry if the test is skipped', () => { + expect(sum(1, 2)).to.equal(3) + }) + + if (process.env.SHOULD_ADD_EVENTUALLY_FAIL) { + test('can retry tests that eventually fail', () => { + expect(sum(1, 2)).to.equal(numOtherAttempt++ < 3 ? 3 : 4) + }) + } +}) diff --git a/integration-tests/vitest/vitest.spec.js b/integration-tests/vitest/vitest.spec.js index f2fb9a6ef12..de38feee9da 100644 --- a/integration-tests/vitest/vitest.spec.js +++ b/integration-tests/vitest/vitest.spec.js @@ -19,10 +19,17 @@ const { TEST_COMMAND, TEST_LEVEL_EVENT_TYPES, TEST_SOURCE_FILE, - TEST_SOURCE_START + TEST_SOURCE_START, + TEST_IS_NEW, + TEST_NAME, + TEST_EARLY_FLAKE_ENABLED, + TEST_EARLY_FLAKE_ABORT_REASON, + TEST_SUITE } = require('../../packages/dd-trace/src/plugins/util/test') const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env') +const NUM_RETRIES_EFD = 3 + const versions = ['1.6.0', 'latest'] const linePctMatchRegex = /Lines\s+:\s+([\d.]+)%/ @@ -227,7 +234,7 @@ versions.forEach((version) => { }).then(() => done()).catch(done) childProcess = exec( - './node_modules/.bin/vitest run', // TODO: change tests we run + './node_modules/.bin/vitest run', { cwd, env: { @@ -265,7 +272,7 @@ versions.forEach((version) => { }).then(() => done()).catch(done) childProcess = exec( - './node_modules/.bin/vitest run', // TODO: change tests we run + './node_modules/.bin/vitest run', { cwd, env: { @@ -306,7 +313,7 @@ versions.forEach((version) => { }).then(() => done()).catch(done) childProcess = exec( - './node_modules/.bin/vitest run', // TODO: change tests we run + './node_modules/.bin/vitest run', { cwd, env: { @@ -352,7 +359,7 @@ versions.forEach((version) => { }) }) - // only works for >=2.0.0 + // total code coverage only works for >=2.0.0 if (version === 'latest') { const coverageProviders = ['v8', 'istanbul'] @@ -405,5 +412,489 @@ versions.forEach((version) => { }) }) } + // maybe only latest version? + context('early flake detection', () => { + it('retries new tests', (done) => { + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + } + } + }) + + receiver.setKnownTests({ + vitest: { + 'ci-visibility/vitest-tests/early-flake-detection.mjs': [ + // 'early flake detection can retry tests that eventually pass', // will be considered new + // 'early flake detection can retry tests that always pass', // will be considered new + // 'early flake detection can retry tests that eventually fail', // will be considered new + // 'early flake detection does not retry if the test is skipped', // skipped so not retried + 'early flake detection does not retry if it is not new' + ] + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const tests = events.filter(event => event.type === 'test').map(test => test.content) + + assert.equal(tests.length, 14) + + assert.includeMembers(tests.map(test => test.meta[TEST_NAME]), [ + 'early flake detection can retry tests that eventually pass', + 'early flake detection can retry tests that eventually pass', + 'early flake detection can retry tests that eventually pass', + 'early flake detection can retry tests that eventually pass', + 'early flake detection can retry tests that eventually fail', + 'early flake detection can retry tests that eventually fail', + 'early flake detection can retry tests that eventually fail', + 'early flake detection can retry tests that eventually fail', + 'early flake detection can retry tests that always pass', + 'early flake detection can retry tests that always pass', + 'early flake detection can retry tests that always pass', + 'early flake detection can retry tests that always pass', + 'early flake detection does not retry if it is not new', + 'early flake detection does not retry if the test is skipped' + ]) + const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true') + assert.equal(newTests.length, 12) // 4 executions of the three new tests + + const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + assert.equal(retriedTests.length, 9) // 3 retries of the three new tests + + // exit code should be 0 and test session should be reported as passed, + // even though there are some failing executions + const failedTests = tests.filter(test => test.meta[TEST_STATUS] === 'fail') + assert.equal(failedTests.length, 3) + const testSessionEvent = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSessionEvent.meta, TEST_STATUS, 'pass') + assert.propertyVal(testSessionEvent.meta, TEST_EARLY_FLAKE_ENABLED, 'true') + }) + + childProcess = exec( + './node_modules/.bin/vitest run', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TEST_DIR: 'ci-visibility/vitest-tests/early-flake-detection*', + NODE_OPTIONS: '--import dd-trace/register.js -r dd-trace/ci/init', + SHOULD_ADD_EVENTUALLY_FAIL: '1' + }, + stdio: 'pipe' + } + ) + + childProcess.on('exit', (exitCode) => { + eventsPromise.then(() => { + assert.equal(exitCode, 0) + done() + }).catch(done) + }) + }) + + it('fails if all the attempts fail', (done) => { + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + } + } + }) + + receiver.setKnownTests({ + vitest: { + 'ci-visibility/vitest-tests/early-flake-detection.mjs': [ + // 'early flake detection can retry tests that eventually pass', // will be considered new + // 'early flake detection can retry tests that always pass', // will be considered new + // 'early flake detection does not retry if the test is skipped', // skipped so not retried + 'early flake detection does not retry if it is not new' + ] + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const tests = events.filter(event => event.type === 'test').map(test => test.content) + + assert.equal(tests.length, 10) + + assert.includeMembers(tests.map(test => test.meta[TEST_NAME]), [ + 'early flake detection can retry tests that eventually pass', + 'early flake detection can retry tests that eventually pass', + 'early flake detection can retry tests that eventually pass', + 'early flake detection can retry tests that eventually pass', + 'early flake detection can retry tests that always pass', + 'early flake detection can retry tests that always pass', + 'early flake detection can retry tests that always pass', + 'early flake detection can retry tests that always pass', + 'early flake detection does not retry if it is not new', + 'early flake detection does not retry if the test is skipped' + ]) + const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true') + assert.equal(newTests.length, 8) // 4 executions of the two new tests + + const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + assert.equal(retriedTests.length, 6) // 3 retries of the two new tests + + // the multiple attempts did not result in a single pass, + // so the test session should be reported as failed + const failedTests = tests.filter(test => test.meta[TEST_STATUS] === 'fail') + assert.equal(failedTests.length, 6) + const testSessionEvent = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSessionEvent.meta, TEST_STATUS, 'fail') + assert.propertyVal(testSessionEvent.meta, TEST_EARLY_FLAKE_ENABLED, 'true') + }) + + childProcess = exec( + './node_modules/.bin/vitest run', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TEST_DIR: 'ci-visibility/vitest-tests/early-flake-detection*', + NODE_OPTIONS: '--import dd-trace/register.js -r dd-trace/ci/init', + ALWAYS_FAIL: 'true' + }, + stdio: 'pipe' + } + ) + + childProcess.on('exit', (exitCode) => { + eventsPromise.then(() => { + assert.equal(exitCode, 1) + done() + }).catch(done) + }) + }) + + it('bails out of EFD if the percentage of new tests is too high', (done) => { + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + }, + faulty_session_threshold: 0 + } + }) + + receiver.setKnownTests({ + vitest: {} + }) // tests from ci-visibility/vitest-tests/early-flake-detection.mjs will be new + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ABORT_REASON, 'faulty') + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + assert.equal(tests.length, 4) + + const newTests = tests.filter( + test => test.meta[TEST_IS_NEW] === 'true' + ) + // no new tests + assert.equal(newTests.length, 0) + }) + + childProcess = exec( + './node_modules/.bin/vitest run', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TEST_DIR: 'ci-visibility/vitest-tests/early-flake-detection*', + NODE_OPTIONS: '--import dd-trace/register.js -r dd-trace/ci/init', + DD_TRACE_DEBUG: '1', + DD_TRACE_LOG_LEVEL: 'error' + }, + stdio: 'pipe' + } + ) + + childProcess.on('exit', (exitCode) => { + eventsPromise.then(() => { + assert.equal(exitCode, 1) + done() + }).catch(done) + }) + }) + + it('is disabled if DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED is false', (done) => { + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + } + } + }) + + receiver.setKnownTests({ + vitest: { + 'ci-visibility/vitest-tests/early-flake-detection.mjs': [ + // 'early flake detection can retry tests that eventually pass', // will be considered new + // 'early flake detection can retry tests that always pass', // will be considered new + // 'early flake detection does not retry if the test is skipped', // skipped so not retried + 'early flake detection does not retry if it is not new' + ] + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const tests = events.filter(event => event.type === 'test').map(test => test.content) + + assert.equal(tests.length, 4) + + assert.includeMembers(tests.map(test => test.meta[TEST_NAME]), [ + 'early flake detection can retry tests that eventually pass', + 'early flake detection can retry tests that always pass', + 'early flake detection does not retry if it is not new', + 'early flake detection does not retry if the test is skipped' + ]) + const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true') + assert.equal(newTests.length, 0) + + const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + assert.equal(retriedTests.length, 0) + + const failedTests = tests.filter(test => test.meta[TEST_STATUS] === 'fail') + assert.equal(failedTests.length, 1) + const testSessionEvent = events.find(event => event.type === 'test_session_end').content + assert.equal(testSessionEvent.meta[TEST_STATUS], 'fail') + }) + + childProcess = exec( + './node_modules/.bin/vitest run', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TEST_DIR: 'ci-visibility/vitest-tests/early-flake-detection*', + NODE_OPTIONS: '--import dd-trace/register.js -r dd-trace/ci/init', + DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED: 'false' + }, + stdio: 'pipe' + } + ) + + childProcess.on('exit', (exitCode) => { + eventsPromise.then(() => { + assert.equal(exitCode, 1) + done() + }).catch(done) + }) + }) + + it('does not run EFD if the known tests request fails', (done) => { + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + } + } + }) + + receiver.setKnownTestsResponseCode(500) + receiver.setKnownTests({}) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const tests = events.filter(event => event.type === 'test').map(test => test.content) + + assert.equal(tests.length, 4) + + assert.includeMembers(tests.map(test => test.meta[TEST_NAME]), [ + 'early flake detection can retry tests that eventually pass', + 'early flake detection can retry tests that always pass', + 'early flake detection does not retry if it is not new', + 'early flake detection does not retry if the test is skipped' + ]) + const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true') + assert.equal(newTests.length, 0) + + const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + assert.equal(retriedTests.length, 0) + + const failedTests = tests.filter(test => test.meta[TEST_STATUS] === 'fail') + assert.equal(failedTests.length, 1) + const testSessionEvent = events.find(event => event.type === 'test_session_end').content + assert.equal(testSessionEvent.meta[TEST_STATUS], 'fail') + }) + + childProcess = exec( + './node_modules/.bin/vitest run', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TEST_DIR: 'ci-visibility/vitest-tests/early-flake-detection*', + NODE_OPTIONS: '--import dd-trace/register.js -r dd-trace/ci/init' + }, + stdio: 'pipe' + } + ) + + childProcess.on('exit', (exitCode) => { + eventsPromise.then(() => { + assert.equal(exitCode, 1) + done() + }).catch(done) + }) + }) + + it('works when the cwd is not the repository root', (done) => { + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + } + } + }) + + receiver.setKnownTests({ + vitest: { + 'ci-visibility/subproject/vitest-test.mjs': [ + 'context can report passed test' // no test will be considered new + ] + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const tests = events.filter(event => event.type === 'test').map(test => test.content) + + // no retries + assert.equal(tests.length, 1) + + assert.propertyVal(tests[0].meta, TEST_SUITE, 'ci-visibility/subproject/vitest-test.mjs') + // it's not considered new + assert.notProperty(tests[0].meta, TEST_IS_NEW) + }) + + childProcess = exec( + '../../node_modules/.bin/vitest run', + { + cwd: `${cwd}/ci-visibility/subproject`, + env: { + ...getCiVisAgentlessConfig(receiver.port), + NODE_OPTIONS: '--import dd-trace/register.js -r dd-trace/ci/init', // ESM requires more flags + TEST_DIR: './vitest-test.mjs' + }, + stdio: 'inherit' + } + ) + + childProcess.on('exit', (exitCode) => { + eventsPromise.then(() => { + assert.equal(exitCode, 0) + done() + }).catch(done) + }) + }) + + it('works with repeats config when EFD is disabled', (done) => { + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: false + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const tests = events.filter(event => event.type === 'test').map(test => test.content) + + assert.equal(tests.length, 8) + + assert.includeMembers(tests.map(test => test.meta[TEST_NAME]), [ + 'early flake detection can retry tests that eventually pass', + 'early flake detection can retry tests that eventually pass', + 'early flake detection can retry tests that eventually pass', // repeated twice + 'early flake detection can retry tests that always pass', + 'early flake detection can retry tests that always pass', + 'early flake detection can retry tests that always pass', // repeated twice + 'early flake detection does not retry if it is not new', + 'early flake detection does not retry if the test is skipped' + ]) + const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true') + assert.equal(newTests.length, 0) // no new test detected + + const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + assert.equal(retriedTests.length, 4) // 2 repetitions on 2 tests + + // vitest reports the test as failed if any of the repetitions fail, so we'll follow that + // TODO: we might want to improve htis + const failedTests = tests.filter(test => test.meta[TEST_STATUS] === 'fail') + assert.equal(failedTests.length, 3) + + const testSessionEvent = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSessionEvent.meta, TEST_STATUS, 'fail') + assert.notProperty(testSessionEvent.meta, TEST_EARLY_FLAKE_ENABLED) + }) + + childProcess = exec( + './node_modules/.bin/vitest run', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TEST_DIR: 'ci-visibility/vitest-tests/early-flake-detection*', + NODE_OPTIONS: '--import dd-trace/register.js -r dd-trace/ci/init', + SHOULD_REPEAT: '1' + }, + stdio: 'pipe' + } + ) + + childProcess.on('exit', (exitCode) => { + eventsPromise.then(() => { + assert.equal(exitCode, 1) + done() + }).catch(done) + }) + }) + }) }) }) diff --git a/packages/datadog-instrumentations/src/vitest.js b/packages/datadog-instrumentations/src/vitest.js index a90daf7bfec..f0117e0e8c0 100644 --- a/packages/datadog-instrumentations/src/vitest.js +++ b/packages/datadog-instrumentations/src/vitest.js @@ -1,5 +1,6 @@ const { addHook, channel, AsyncResource } = require('./helpers/instrument') const shimmer = require('../../datadog-shimmer') +const log = require('../../dd-trace/src/log') // test hooks const testStartCh = channel('ci:vitest:test:start') @@ -7,6 +8,7 @@ const testFinishTimeCh = channel('ci:vitest:test:finish-time') const testPassCh = channel('ci:vitest:test:pass') const testErrorCh = channel('ci:vitest:test:error') const testSkipCh = channel('ci:vitest:test:skip') +const isNewTestCh = channel('ci:vitest:test:is-new') // test suite hooks const testSuiteStartCh = channel('ci:vitest:test-suite:start') @@ -17,9 +19,13 @@ const testSuiteErrorCh = channel('ci:vitest:test-suite:error') const testSessionStartCh = channel('ci:vitest:session:start') const testSessionFinishCh = channel('ci:vitest:session:finish') const libraryConfigurationCh = channel('ci:vitest:library-configuration') +const knownTestsCh = channel('ci:vitest:known-tests') +const isEarlyFlakeDetectionFaultyCh = channel('ci:vitest:is-early-flake-detection-faulty') const taskToAsync = new WeakMap() - +const taskToStatuses = new WeakMap() +const newTasks = new WeakSet() +const switchedStatuses = new WeakSet() const sessionAsyncResource = new AsyncResource('bound-anonymous-fn') function isReporterPackage (vitestPackage) { @@ -108,20 +114,61 @@ function getSortWrapper (sort) { // will not work. This will be a known limitation. let isFlakyTestRetriesEnabled = false let flakyTestRetriesCount = 0 + let isEarlyFlakeDetectionEnabled = false + let earlyFlakeDetectionNumRetries = 0 + let isEarlyFlakeDetectionFaulty = false + let knownTests = {} try { const { err, libraryConfig } = await getChannelPromise(libraryConfigurationCh) if (!err) { isFlakyTestRetriesEnabled = libraryConfig.isFlakyTestRetriesEnabled flakyTestRetriesCount = libraryConfig.flakyTestRetriesCount + isEarlyFlakeDetectionEnabled = libraryConfig.isEarlyFlakeDetectionEnabled + earlyFlakeDetectionNumRetries = libraryConfig.earlyFlakeDetectionNumRetries } } catch (e) { isFlakyTestRetriesEnabled = false + isEarlyFlakeDetectionEnabled = false } + if (isFlakyTestRetriesEnabled && !this.ctx.config.retry && flakyTestRetriesCount > 0) { this.ctx.config.retry = flakyTestRetriesCount } + if (isEarlyFlakeDetectionEnabled) { + const knownTestsResponse = await getChannelPromise(knownTestsCh) + if (!knownTestsResponse.err) { + knownTests = knownTestsResponse.knownTests + const testFilepaths = await this.ctx.getTestFilepaths() + + isEarlyFlakeDetectionFaultyCh.publish({ + knownTests: knownTests.vitest || {}, + testFilepaths, + onDone: (isFaulty) => { + isEarlyFlakeDetectionFaulty = isFaulty + } + }) + if (isEarlyFlakeDetectionFaulty) { + isEarlyFlakeDetectionEnabled = false + log.warn('Early flake detection is disabled because the number of new tests is too high.') + } else { + // TODO: use this to pass session and module IDs to the worker, instead of polluting process.env + // Note: setting this.ctx.config.provide directly does not work because it's cached + try { + const workspaceProject = this.ctx.getCoreWorkspaceProject() + workspaceProject._provided._ddKnownTests = knownTests.vitest + workspaceProject._provided._ddIsEarlyFlakeDetectionEnabled = isEarlyFlakeDetectionEnabled + workspaceProject._provided._ddEarlyFlakeDetectionNumRetries = earlyFlakeDetectionNumRetries + } catch (e) { + log.warn('Could not send known tests to workers so Early Flake Detection will not work.') + } + } + } else { + isEarlyFlakeDetectionEnabled = false + } + } + let testCodeCoverageLinesTotal if (this.ctx.coverageProvider?.generateCoverage) { @@ -154,6 +201,8 @@ function getSortWrapper (sort) { status: getSessionStatus(this.state), testCodeCoverageLinesTotal, error, + isEarlyFlakeDetectionEnabled, + isEarlyFlakeDetectionFaulty, onFinish }) }) @@ -188,12 +237,83 @@ addHook({ file: 'dist/runners.js' }, (vitestPackage) => { const { VitestTestRunner } = vitestPackage + + // `onBeforeRunTask` is run before any repetition or attempt is run + shimmer.wrap(VitestTestRunner.prototype, 'onBeforeRunTask', onBeforeRunTask => async function (task) { + const testName = getTestName(task) + try { + const { + _ddKnownTests: knownTests, + _ddIsEarlyFlakeDetectionEnabled: isEarlyFlakeDetectionEnabled, + _ddEarlyFlakeDetectionNumRetries: numRepeats + } = globalThis.__vitest_worker__.providedContext + + if (isEarlyFlakeDetectionEnabled) { + isNewTestCh.publish({ + knownTests, + testSuiteAbsolutePath: task.file.filepath, + testName, + onDone: (isNew) => { + if (isNew) { + task.repeats = numRepeats + newTasks.add(task) + taskToStatuses.set(task, []) + } + } + }) + } + } catch (e) { + log.error('Vitest workers could not parse known tests, so Early Flake Detection will not work.') + } + + return onBeforeRunTask.apply(this, arguments) + }) + + // `onAfterRunTask` is run after all repetitions or attempts are run + shimmer.wrap(VitestTestRunner.prototype, 'onAfterRunTask', onAfterRunTask => async function (task) { + const { + _ddIsEarlyFlakeDetectionEnabled: isEarlyFlakeDetectionEnabled + } = globalThis.__vitest_worker__.providedContext + + if (isEarlyFlakeDetectionEnabled && taskToStatuses.has(task)) { + const statuses = taskToStatuses.get(task) + // If the test has passed at least once, we consider it passed + if (statuses.includes('pass')) { + if (task.result.state === 'fail') { + switchedStatuses.add(task) + } + task.result.state = 'pass' + } + } + + return onAfterRunTask.apply(this, arguments) + }) + // test start (only tests that are not marked as skip or todo) + // `onBeforeTryTask` is run for every repetition and attempt of the test shimmer.wrap(VitestTestRunner.prototype, 'onBeforeTryTask', onBeforeTryTask => async function (task, retryInfo) { if (!testStartCh.hasSubscribers) { return onBeforeTryTask.apply(this, arguments) } - const { retry: numAttempt } = retryInfo + const testName = getTestName(task) + let isNew = false + let isEarlyFlakeDetectionEnabled = false + + try { + const { + _ddIsEarlyFlakeDetectionEnabled + } = globalThis.__vitest_worker__.providedContext + + isEarlyFlakeDetectionEnabled = _ddIsEarlyFlakeDetectionEnabled + + if (isEarlyFlakeDetectionEnabled) { + isNew = newTasks.has(task) + } + } catch (e) { + log.error('Vitest workers could not parse known tests, so Early Flake Detection will not work.') + } + const { retry: numAttempt, repeats: numRepetition } = retryInfo + // We finish the previous test here because we know it has failed already if (numAttempt > 0) { const asyncResource = taskToAsync.get(task) @@ -205,14 +325,58 @@ addHook({ } } + const lastExecutionStatus = task.result.state + + // These clauses handle task.repeats, whether EFD is enabled or not + // The only thing that EFD does is to forcefully pass the test if it has passed at least once + if (numRepetition > 0 && numRepetition < task.repeats) { // it may or may have not failed + // Here we finish the earlier iteration, + // as long as it's not the _last_ iteration (which will be finished normally) + + // TODO: check test duration (not to repeat if it's too slow) + const asyncResource = taskToAsync.get(task) + if (asyncResource) { + if (lastExecutionStatus === 'fail') { + const testError = task.result?.errors?.[0] + asyncResource.runInAsyncScope(() => { + testErrorCh.publish({ error: testError }) + }) + } else { + asyncResource.runInAsyncScope(() => { + testPassCh.publish({ task }) + }) + } + if (isEarlyFlakeDetectionEnabled) { + const statuses = taskToStatuses.get(task) + statuses.push(lastExecutionStatus) + // If we don't "reset" the result.state to "pass", once a repetition fails, + // vitest will always consider the test as failed, so we can't read the actual status + task.result.state = 'pass' + } + } + } else if (numRepetition === task.repeats) { + const asyncResource = taskToAsync.get(task) + if (lastExecutionStatus === 'fail') { + const testError = task.result?.errors?.[0] + asyncResource.runInAsyncScope(() => { + testErrorCh.publish({ error: testError }) + }) + } else { + asyncResource.runInAsyncScope(() => { + testPassCh.publish({ task }) + }) + } + } + const asyncResource = new AsyncResource('bound-anonymous-fn') taskToAsync.set(task, asyncResource) asyncResource.runInAsyncScope(() => { testStartCh.publish({ - testName: getTestName(task), + testName, testSuiteAbsolutePath: task.file.filepath, - isRetry: numAttempt > 0 + isRetry: numAttempt > 0 || numRepetition > 0, + isNew }) }) return onBeforeTryTask.apply(this, arguments) @@ -230,7 +394,7 @@ addHook({ const asyncResource = taskToAsync.get(task) if (asyncResource) { - // We don't finish here because the test might fail in a later hook + // We don't finish here because the test might fail in a later hook (afterEach) asyncResource.runInAsyncScope(() => { testFinishTimeCh.publish({ status, task }) }) @@ -332,19 +496,21 @@ addHook({ testTasks.forEach(task => { const testAsyncResource = taskToAsync.get(task) const { result } = task + // We have to trick vitest into thinking that the test has passed + // but we want to report it as failed if it did fail + const isSwitchedStatus = switchedStatuses.has(task) if (result) { const { state, duration, errors } = result if (state === 'skip') { // programmatic skip testSkipCh.publish({ testName: getTestName(task), testSuiteAbsolutePath: task.file.filepath }) - } else if (state === 'pass') { + } else if (state === 'pass' && !isSwitchedStatus) { if (testAsyncResource) { testAsyncResource.runInAsyncScope(() => { testPassCh.publish({ task }) }) } - } else if (state === 'fail') { - // If it's failing, we have no accurate finish time, so we have to use `duration` + } else if (state === 'fail' || isSwitchedStatus) { let testError if (errors?.length) { diff --git a/packages/datadog-plugin-vitest/src/index.js b/packages/datadog-plugin-vitest/src/index.js index 149f3e5882f..34617bdb1ac 100644 --- a/packages/datadog-plugin-vitest/src/index.js +++ b/packages/datadog-plugin-vitest/src/index.js @@ -7,13 +7,17 @@ const { getTestSuitePath, getTestSuiteCommonTags, getTestSessionName, + getIsFaultyEarlyFlakeDetection, TEST_SOURCE_FILE, TEST_IS_RETRY, TEST_CODE_COVERAGE_LINES_PCT, TEST_CODE_OWNERS, TEST_LEVEL_EVENT_TYPES, TEST_SESSION_NAME, - TEST_SOURCE_START + TEST_SOURCE_START, + TEST_IS_NEW, + TEST_EARLY_FLAKE_ENABLED, + TEST_EARLY_FLAKE_ABORT_REASON } = require('../../dd-trace/src/plugins/util/test') const { COMPONENT } = require('../../dd-trace/src/constants') const { @@ -37,7 +41,26 @@ class VitestPlugin extends CiPlugin { this.taskToFinishTime = new WeakMap() - this.addSub('ci:vitest:test:start', ({ testName, testSuiteAbsolutePath, isRetry }) => { + this.addSub('ci:vitest:test:is-new', ({ knownTests, testSuiteAbsolutePath, testName, onDone }) => { + const testSuite = getTestSuitePath(testSuiteAbsolutePath, this.repositoryRoot) + const testsForThisTestSuite = knownTests[testSuite] || [] + onDone(!testsForThisTestSuite.includes(testName)) + }) + + this.addSub('ci:vitest:is-early-flake-detection-faulty', ({ + knownTests, + testFilepaths, + onDone + }) => { + const isFaulty = getIsFaultyEarlyFlakeDetection( + testFilepaths.map(testFilepath => getTestSuitePath(testFilepath, this.repositoryRoot)), + knownTests, + this.libraryConfig.earlyFlakeDetectionFaultyThreshold + ) + onDone(isFaulty) + }) + + this.addSub('ci:vitest:test:start', ({ testName, testSuiteAbsolutePath, isRetry, isNew }) => { const testSuite = getTestSuitePath(testSuiteAbsolutePath, this.repositoryRoot) const store = storage.getStore() @@ -47,6 +70,9 @@ class VitestPlugin extends CiPlugin { if (isRetry) { extraTags[TEST_IS_RETRY] = 'true' } + if (isNew) { + extraTags[TEST_IS_NEW] = 'true' + } const span = this.startTestSpan( testName, @@ -195,7 +221,14 @@ class VitestPlugin extends CiPlugin { } }) - this.addSub('ci:vitest:session:finish', ({ status, onFinish, error, testCodeCoverageLinesTotal }) => { + this.addSub('ci:vitest:session:finish', ({ + status, + error, + testCodeCoverageLinesTotal, + isEarlyFlakeDetectionEnabled, + isEarlyFlakeDetectionFaulty, + onFinish + }) => { this.testSessionSpan.setTag(TEST_STATUS, status) this.testModuleSpan.setTag(TEST_STATUS, status) if (error) { @@ -206,6 +239,12 @@ class VitestPlugin extends CiPlugin { this.testModuleSpan.setTag(TEST_CODE_COVERAGE_LINES_PCT, testCodeCoverageLinesTotal) this.testSessionSpan.setTag(TEST_CODE_COVERAGE_LINES_PCT, testCodeCoverageLinesTotal) } + if (isEarlyFlakeDetectionEnabled) { + this.testSessionSpan.setTag(TEST_EARLY_FLAKE_ENABLED, 'true') + } + if (isEarlyFlakeDetectionFaulty) { + this.testSessionSpan.setTag(TEST_EARLY_FLAKE_ABORT_REASON, 'faulty') + } this.testModuleSpan.finish() this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'module') this.testSessionSpan.finish()