From 27ed0b2e714d4b7438748d1563b9b9ed4a7f06a1 Mon Sep 17 00:00:00 2001 From: Juan Fernandez Date: Mon, 23 Sep 2024 17:01:44 +0200 Subject: [PATCH] more tests --- .../vitest-tests/early-flake-detection.mjs | 4 + integration-tests/vitest/vitest.spec.js | 74 ++++++++++++++++++- 2 files changed, 74 insertions(+), 4 deletions(-) diff --git a/integration-tests/ci-visibility/vitest-tests/early-flake-detection.mjs b/integration-tests/ci-visibility/vitest-tests/early-flake-detection.mjs index e4b3bf8f78d..1dcfbc3b73a 100644 --- a/integration-tests/ci-visibility/vitest-tests/early-flake-detection.mjs +++ b/integration-tests/ci-visibility/vitest-tests/early-flake-detection.mjs @@ -15,4 +15,8 @@ describe('early flake detection', () => { test('does not retry if it is not new', () => { expect(sum(1, 2)).to.equal(3) }) + + test.skip('does not retry if the test is skipped', () => { + expect(sum(1, 2)).to.equal(3) + }) }) diff --git a/integration-tests/vitest/vitest.spec.js b/integration-tests/vitest/vitest.spec.js index 53d8cc647c2..a9e5c99c155 100644 --- a/integration-tests/vitest/vitest.spec.js +++ b/integration-tests/vitest/vitest.spec.js @@ -429,6 +429,7 @@ versions.forEach((version) => { 'ci-visibility/vitest-tests/early-flake-detection.mjs': [ // 'early flake detection can retry tests that eventually pass', // will be considered new // 'early flake detection can retry tests that never pass', // will be considered new + // 'early flake detection does not retry if the test is skipped', // skipped so not retried 'early flake detection does not retry if it is not new' ] } @@ -440,7 +441,7 @@ versions.forEach((version) => { const tests = events.filter(event => event.type === 'test').map(test => test.content) - assert.equal(tests.length, 9) + assert.equal(tests.length, 10) assert.includeMembers(tests.map(test => test.meta[TEST_NAME]), [ 'early flake detection can retry tests that eventually pass', @@ -451,7 +452,8 @@ versions.forEach((version) => { 'early flake detection can retry tests that always pass', 'early flake detection can retry tests that always pass', 'early flake detection can retry tests that always pass', - 'early flake detection does not retry if it is not new' + 'early flake detection does not retry if it is not new', + 'early flake detection does not retry if the test is skipped' ]) const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true') assert.equal(newTests.length, 8) // 4 executions of the two new tests @@ -517,12 +519,13 @@ versions.forEach((version) => { const tests = events.filter(event => event.type === 'test').map(test => test.content) - assert.equal(tests.length, 3) + assert.equal(tests.length, 4) assert.includeMembers(tests.map(test => test.meta[TEST_NAME]), [ 'early flake detection can retry tests that eventually pass', 'early flake detection can retry tests that always pass', - 'early flake detection does not retry if it is not new' + 'early flake detection does not retry if it is not new', + 'early flake detection does not retry if the test is skipped' ]) const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true') assert.equal(newTests.length, 0) @@ -557,6 +560,69 @@ versions.forEach((version) => { }).catch(done) }) }) + + it('does not run EFD if the known tests request fails', (done) => { + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + } + } + }) + + receiver.setKnownTestsResponseCode(500) + receiver.setKnownTests({}) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const tests = events.filter(event => event.type === 'test').map(test => test.content) + + assert.equal(tests.length, 4) + + assert.includeMembers(tests.map(test => test.meta[TEST_NAME]), [ + 'early flake detection can retry tests that eventually pass', + 'early flake detection can retry tests that always pass', + 'early flake detection does not retry if it is not new', + 'early flake detection does not retry if the test is skipped' + ]) + const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true') + assert.equal(newTests.length, 0) + + const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + assert.equal(retriedTests.length, 0) + + const failedTests = tests.filter(test => test.meta[TEST_STATUS] === 'fail') + assert.equal(failedTests.length, 1) + const testSessionEvent = events.find(event => event.type === 'test_session_end').content + assert.equal(testSessionEvent.meta[TEST_STATUS], 'fail') + }) + + childProcess = exec( + './node_modules/.bin/vitest run', // TODO: change tests we run + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TEST_DIR: 'ci-visibility/vitest-tests/early-flake-detection*', + NODE_OPTIONS: '--import dd-trace/register.js -r dd-trace/ci/init' + }, + stdio: 'pipe' + } + ) + + childProcess.on('exit', (exitCode) => { + eventsPromise.then(() => { + assert.equal(exitCode, 1) + done() + }).catch(done) + }) + }) }) }) })