Run with Updated Version of Flakeguard (#15385)

* Run with updated version of flakeguard * Condense flakeguard runs * Update version again * Updates flakeguard version and inputs * Fix flakeguard references
smartcontractkit · Nov 22, 2024 · ac7a739 · ac7a739
1 parent 00777b8
commit ac7a739
Show file tree

Hide file tree

Showing 4 changed files with 28 additions and 47 deletions.
diff --git a/.github/workflows/ci-core.yml b/.github/workflows/ci-core.yml
@@ -463,15 +463,15 @@ jobs:
           SONAR_SCANNER_OPTS: "-Xms6g -Xmx8g"
 
   trigger-flaky-test-detection-for-root-project:
-    name: Find New Flaky Tests In Chainlink Project
-    uses: ./.github/workflows/find-new-flaky-tests.yml
+    name: Flakeguard Root Project
+    uses: ./.github/workflows/flakeguard.yml
     if: ${{ github.event_name == 'pull_request' }}
     with:
       repoUrl: 'https://github.com/smartcontractkit/chainlink'
       projectPath: '.'
       baseRef: ${{ github.base_ref }}
       headRef: ${{ github.head_ref }}
-      runThreshold: '0.99'
+      maxPassRatio: '1.0'
       findByTestFilesDiff: true
       findByAffectedPackages: false
       slackNotificationAfterTestsChannelId: 'C07TRF65CNS' #flaky-test-detector-notifications
@@ -480,16 +480,16 @@ jobs:
       SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}
 
   trigger-flaky-test-detection-for-deployment-project:
-    name: Find New Flaky Tests In Deployment Project
-    uses: ./.github/workflows/find-new-flaky-tests.yml
+    name: Flakeguard Deployment Project
+    uses: ./.github/workflows/flakeguard.yml
     needs: [filter]
     if: ${{ github.event_name == 'pull_request' && needs.filter.outputs.deployment-changes == 'true'}}
     with:
       repoUrl: 'https://github.com/smartcontractkit/chainlink'
       projectPath: 'deployment'
       baseRef: ${{ github.base_ref }}
       headRef: ${{ github.head_ref }}
-      runThreshold: '0.99'
+      maxPassRatio: '1.0'
       findByTestFilesDiff: true
       findByAffectedPackages: false
       slackNotificationAfterTestsChannelId: 'C07TRF65CNS' #flaky-test-detector-notifications

diff --git a/...ub/workflows/run-find-new-flaky-tests.yml → .github/workflows/flakeguard-on-demand.yml b/...ub/workflows/run-find-new-flaky-tests.yml → .github/workflows/flakeguard-on-demand.yml
@@ -1,6 +1,9 @@
-name: Find Flaky Tests
+name: Flakeguard On Demand
 
 on:
+  schedule:
+    # Run every night at 3:00 AM UTC
+    - cron: '0 3 * * *'
   workflow_dispatch:
     inputs:
       repoUrl:
@@ -26,12 +29,12 @@ on:
         required: false
         type: boolean
         description: 'Run all tests in the project.'
-        default: false      
-      runThreshold:
+        default: true      
+      maxPassRatio:
         required: false
         type: string
-        description: 'The threshold for the number of times a test can fail before being considered flaky.'
-        default: '0.8'
+        description: 'The maximum (non-inclusive) pass ratio threshold for a test to be considered a failure. Any tests below this pass rate will be considered flaky.'
+        default: '1.0'
       findByTestFilesDiff:
         required: false
         type: boolean
@@ -41,31 +44,32 @@ on:
         required: false
         type: boolean
         description: 'Find new or updated test packages by comparing affected packages.'
-        default: true
+        default: false
       slack_notification_after_tests_channel_id:
         description: "Slack channel ID to send the notification to for failed tests."
         required: false
         type: string
       extraArgs:
         required: false
         type: string
-        default: '{}'
-        description: 'JSON of extra arguments for the workflow.'        
+        default: '{ "skipped_tests": "TestChainComponents", "test_repeat_count": "5", "all_tests_runner": "ubuntu22.04-32cores-128GB", "all_tests_runner_count": "3", "min_pass_ratio": "0", "run_with_race": "false" }'
+        description: 'JSON of extra arguments for the workflow.'    
 
 jobs:
   trigger-flaky-test-detection:
     name: Find Flaky Tests
-    uses: ./.github/workflows/find-new-flaky-tests.yml
+    uses: ./.github/workflows/flakeguard.yml
     with:
       repoUrl: ${{ inputs.repoUrl }}
       baseRef: ${{ inputs.baseRef }}
       projectPath: ${{ inputs.projectPath }}
       headRef: ${{ inputs.headRef }}
-      runThreshold: ${{ inputs.runThreshold }}
+      maxPassRatio: ${{ inputs.maxPassRatio }}
       runAllTests: ${{ inputs.runAllTests }}
       findByTestFilesDiff: ${{ inputs.findByTestFilesDiff }}
       findByAffectedPackages: ${{ inputs.findByAffectedPackages }}
       slackNotificationAfterTestsChannelId: ${{ inputs.slack_notification_after_tests_channel_id }}
       extraArgs: ${{ inputs.extraArgs }}
     secrets:
-      SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}
+      SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}
+
diff --git a/.github/workflows/find-new-flaky-tests.yml → .github/workflows/flakeguard.yml b/.github/workflows/find-new-flaky-tests.yml → .github/workflows/flakeguard.yml
@@ -1,4 +1,4 @@
-name: Find Flaky Tests
+name: Flakeguard
 
 on:
   workflow_call:
@@ -25,11 +25,11 @@ on:
         type: boolean
         description: 'Run all tests in the project.'
         default: false    
-      runThreshold:
+      maxPassRatio:
         required: false
         type: string
-        description: 'The threshold for the number of times a test can fail before being considered flaky.'
-        default: '0.9'
+        description: 'The maximum (non-inclusive) pass ratio threshold for a test to be considered a failure. Any tests below this pass rate will be considered flaky.'
+        default: '1.0'
       findByTestFilesDiff:
         required: false
         type: boolean
@@ -64,7 +64,6 @@ env:
   DEFAULT_RUNNER: 'ubuntu-latest' # The default runner to use for running tests.
   UPLOAD_ALL_TEST_RESULTS: ${{ fromJson(inputs.extraArgs)['upload_all_test_results'] || 'false' }} # Whether to upload all test results as artifacts.
   PRINT_FAILED_TESTS: ${{ fromJson(inputs.extraArgs)['print_failed_tests'] || 'false' }} # Whether to print failed tests in the GitHub console.
-  MIN_PASS_RATIO: ${{ fromJson(inputs.extraArgs)['min_pass_ratio'] || '0.001' }} # The minimum pass ratio for a test to be considered as flaky. Used to distinguish between tests that are truly flaky (with inconsistent results) and those that are consistently failing. Set to 0 if you want to consider all failed tests as flaky.
 
 jobs:
   get-tests:
@@ -99,7 +98,7 @@ jobs:
 
       - name: Install flakeguard
         shell: bash
-        run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@8b02ed1703ef40755a4c46ff454cf4ff2e89275d
+        run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@fa6c4ebc22c1d3c5e91dd39efeb76c4a88fc3174
 
       - name: Find new or updated test packages
         if: ${{ inputs.runAllTests == false }}
@@ -258,11 +257,11 @@ jobs:
 
       - name: Install flakeguard
         shell: bash
-        run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@8b02ed1703ef40755a4c46ff454cf4ff2e89275d
+        run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@be06798af83ef6d9f7cf04e8b10a8484520c5061
 
       - name: Run tests with flakeguard
         shell: bash
-        run: flakeguard run --project-path=${{ inputs.projectPath }} --test-packages=${{ matrix.testPackages }} --run-count=${{ env.TEST_REPEAT_COUNT }} --min-pass-ratio=${{ env.MIN_PASS_RATIO }} --threshold=${{ inputs.runThreshold }} --race=${{ env.RUN_WITH_RACE }} --skip-tests=${{ env.SKIPPED_TESTS }} --print-failed-tests=${{ env.PRINT_FAILED_TESTS }} --output-json=test-result.json
+        run: flakeguard run --project-path=${{ inputs.projectPath }} --test-packages=${{ matrix.testPackages }} --run-count=${{ env.TEST_REPEAT_COUNT }} --max-pass-ratio=${{ inputs.maxPassRatio }} --race=${{ env.RUN_WITH_RACE }} --skip-tests=${{ env.SKIPPED_TESTS }} --print-failed-tests=${{ env.PRINT_FAILED_TESTS }} --output-json=test-result.json
         env:
           CL_DATABASE_URL: ${{ env.DB_URL }}
 

diff --git a/.github/workflows/run-nightly-flaky-test-detector.yml b/.github/workflows/run-nightly-flaky-test-detector.yml