more tests

DataDog · Sep 23, 2024 · 27ed0b2 · 27ed0b2
1 parent feac24d
commit 27ed0b2
Show file tree

Hide file tree

Showing 2 changed files with 74 additions and 4 deletions.
diff --git a/integration-tests/ci-visibility/vitest-tests/early-flake-detection.mjs b/integration-tests/ci-visibility/vitest-tests/early-flake-detection.mjs
@@ -15,4 +15,8 @@ describe('early flake detection', () => {
   test('does not retry if it is not new', () => {
     expect(sum(1, 2)).to.equal(3)
   })
+
+  test.skip('does not retry if the test is skipped', () => {
+    expect(sum(1, 2)).to.equal(3)
+  })
 })
diff --git a/integration-tests/vitest/vitest.spec.js b/integration-tests/vitest/vitest.spec.js
@@ -429,6 +429,7 @@ versions.forEach((version) => {
             'ci-visibility/vitest-tests/early-flake-detection.mjs': [
               // 'early flake detection can retry tests that eventually pass', // will be considered new
               // 'early flake detection can retry tests that never pass', // will be considered new
+              // 'early flake detection does not retry if the test is skipped', // skipped so not retried
               'early flake detection does not retry if it is not new'
             ]
           }
@@ -440,7 +441,7 @@ versions.forEach((version) => {
 
             const tests = events.filter(event => event.type === 'test').map(test => test.content)
 
-            assert.equal(tests.length, 9)
+            assert.equal(tests.length, 10)
 
             assert.includeMembers(tests.map(test => test.meta[TEST_NAME]), [
               'early flake detection can retry tests that eventually pass',
@@ -451,7 +452,8 @@ versions.forEach((version) => {
               'early flake detection can retry tests that always pass',
               'early flake detection can retry tests that always pass',
               'early flake detection can retry tests that always pass',
-              'early flake detection does not retry if it is not new'
+              'early flake detection does not retry if it is not new',
+              'early flake detection does not retry if the test is skipped'
             ])
             const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true')
             assert.equal(newTests.length, 8) // 4 executions of the two new tests
@@ -517,12 +519,13 @@ versions.forEach((version) => {
 
             const tests = events.filter(event => event.type === 'test').map(test => test.content)
 
-            assert.equal(tests.length, 3)
+            assert.equal(tests.length, 4)
 
             assert.includeMembers(tests.map(test => test.meta[TEST_NAME]), [
               'early flake detection can retry tests that eventually pass',
               'early flake detection can retry tests that always pass',
-              'early flake detection does not retry if it is not new'
+              'early flake detection does not retry if it is not new',
+              'early flake detection does not retry if the test is skipped'
             ])
             const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true')
             assert.equal(newTests.length, 0)
@@ -557,6 +560,69 @@ versions.forEach((version) => {
           }).catch(done)
         })
       })
+
+      it('does not run EFD if the known tests request fails', (done) => {
+        receiver.setSettings({
+          itr_enabled: false,
+          code_coverage: false,
+          tests_skipping: false,
+          early_flake_detection: {
+            enabled: true,
+            slow_test_retries: {
+              '5s': NUM_RETRIES_EFD
+            }
+          }
+        })
+
+        receiver.setKnownTestsResponseCode(500)
+        receiver.setKnownTests({})
+
+        const eventsPromise = receiver
+          .gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', payloads => {
+            const events = payloads.flatMap(({ payload }) => payload.events)
+
+            const tests = events.filter(event => event.type === 'test').map(test => test.content)
+
+            assert.equal(tests.length, 4)
+
+            assert.includeMembers(tests.map(test => test.meta[TEST_NAME]), [
+              'early flake detection can retry tests that eventually pass',
+              'early flake detection can retry tests that always pass',
+              'early flake detection does not retry if it is not new',
+              'early flake detection does not retry if the test is skipped'
+            ])
+            const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true')
+            assert.equal(newTests.length, 0)
+
+            const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true')
+            assert.equal(retriedTests.length, 0)
+
+            const failedTests = tests.filter(test => test.meta[TEST_STATUS] === 'fail')
+            assert.equal(failedTests.length, 1)
+            const testSessionEvent = events.find(event => event.type === 'test_session_end').content
+            assert.equal(testSessionEvent.meta[TEST_STATUS], 'fail')
+          })
+
+        childProcess = exec(
+          './node_modules/.bin/vitest run', // TODO: change tests we run
+          {
+            cwd,
+            env: {
+              ...getCiVisAgentlessConfig(receiver.port),
+              TEST_DIR: 'ci-visibility/vitest-tests/early-flake-detection*',
+              NODE_OPTIONS: '--import dd-trace/register.js -r dd-trace/ci/init'
+            },
+            stdio: 'pipe'
+          }
+        )
+
+        childProcess.on('exit', (exitCode) => {
+          eventsPromise.then(() => {
+            assert.equal(exitCode, 1)
+            done()
+          }).catch(done)
+        })
+      })
     })
   })
 })