diff --git a/.github/workflows/ci-core.yml b/.github/workflows/ci-core.yml index 5c931ed9870..7f5f0c091f8 100644 --- a/.github/workflows/ci-core.yml +++ b/.github/workflows/ci-core.yml @@ -463,15 +463,15 @@ jobs: SONAR_SCANNER_OPTS: "-Xms6g -Xmx8g" trigger-flaky-test-detection-for-root-project: - name: Find New Flaky Tests In Chainlink Project - uses: ./.github/workflows/find-new-flaky-tests.yml + name: Flakeguard Root Project + uses: ./.github/workflows/flakeguard.yml if: ${{ github.event_name == 'pull_request' }} with: repoUrl: 'https://github.com/smartcontractkit/chainlink' projectPath: '.' baseRef: ${{ github.base_ref }} headRef: ${{ github.head_ref }} - runThreshold: '0.99' + maxPassRatio: '1.0' findByTestFilesDiff: true findByAffectedPackages: false slackNotificationAfterTestsChannelId: 'C07TRF65CNS' #flaky-test-detector-notifications @@ -480,8 +480,8 @@ jobs: SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }} trigger-flaky-test-detection-for-deployment-project: - name: Find New Flaky Tests In Deployment Project - uses: ./.github/workflows/find-new-flaky-tests.yml + name: Flakeguard Deployment Project + uses: ./.github/workflows/flakeguard.yml needs: [filter] if: ${{ github.event_name == 'pull_request' && needs.filter.outputs.deployment-changes == 'true'}} with: @@ -489,7 +489,7 @@ jobs: projectPath: 'deployment' baseRef: ${{ github.base_ref }} headRef: ${{ github.head_ref }} - runThreshold: '0.99' + maxPassRatio: '1.0' findByTestFilesDiff: true findByAffectedPackages: false slackNotificationAfterTestsChannelId: 'C07TRF65CNS' #flaky-test-detector-notifications diff --git a/.github/workflows/run-find-new-flaky-tests.yml b/.github/workflows/flakeguard-on-demand.yml similarity index 74% rename from .github/workflows/run-find-new-flaky-tests.yml rename to .github/workflows/flakeguard-on-demand.yml index d1318719349..0ac8444a542 100644 --- a/.github/workflows/run-find-new-flaky-tests.yml +++ b/.github/workflows/flakeguard-on-demand.yml @@ -1,6 +1,9 @@ -name: Find Flaky Tests +name: Flakeguard On Demand on: + schedule: + # Run every night at 3:00 AM UTC + - cron: '0 3 * * *' workflow_dispatch: inputs: repoUrl: @@ -26,12 +29,12 @@ on: required: false type: boolean description: 'Run all tests in the project.' - default: false - runThreshold: + default: true + maxPassRatio: required: false type: string - description: 'The threshold for the number of times a test can fail before being considered flaky.' - default: '0.8' + description: 'The maximum (non-inclusive) pass ratio threshold for a test to be considered a failure. Any tests below this pass rate will be considered flaky.' + default: '1.0' findByTestFilesDiff: required: false type: boolean @@ -41,7 +44,7 @@ on: required: false type: boolean description: 'Find new or updated test packages by comparing affected packages.' - default: true + default: false slack_notification_after_tests_channel_id: description: "Slack channel ID to send the notification to for failed tests." required: false @@ -49,23 +52,24 @@ on: extraArgs: required: false type: string - default: '{}' - description: 'JSON of extra arguments for the workflow.' + default: '{ "skipped_tests": "TestChainComponents", "test_repeat_count": "5", "all_tests_runner": "ubuntu22.04-32cores-128GB", "all_tests_runner_count": "3", "min_pass_ratio": "0", "run_with_race": "false" }' + description: 'JSON of extra arguments for the workflow.' jobs: trigger-flaky-test-detection: name: Find Flaky Tests - uses: ./.github/workflows/find-new-flaky-tests.yml + uses: ./.github/workflows/flakeguard.yml with: repoUrl: ${{ inputs.repoUrl }} baseRef: ${{ inputs.baseRef }} projectPath: ${{ inputs.projectPath }} headRef: ${{ inputs.headRef }} - runThreshold: ${{ inputs.runThreshold }} + maxPassRatio: ${{ inputs.maxPassRatio }} runAllTests: ${{ inputs.runAllTests }} findByTestFilesDiff: ${{ inputs.findByTestFilesDiff }} findByAffectedPackages: ${{ inputs.findByAffectedPackages }} slackNotificationAfterTestsChannelId: ${{ inputs.slack_notification_after_tests_channel_id }} extraArgs: ${{ inputs.extraArgs }} secrets: - SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }} \ No newline at end of file + SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }} + \ No newline at end of file diff --git a/.github/workflows/find-new-flaky-tests.yml b/.github/workflows/flakeguard.yml similarity index 96% rename from .github/workflows/find-new-flaky-tests.yml rename to .github/workflows/flakeguard.yml index a685f4f5e70..0fd9c0c41fb 100644 --- a/.github/workflows/find-new-flaky-tests.yml +++ b/.github/workflows/flakeguard.yml @@ -1,4 +1,4 @@ -name: Find Flaky Tests +name: Flakeguard on: workflow_call: @@ -25,11 +25,11 @@ on: type: boolean description: 'Run all tests in the project.' default: false - runThreshold: + maxPassRatio: required: false type: string - description: 'The threshold for the number of times a test can fail before being considered flaky.' - default: '0.9' + description: 'The maximum (non-inclusive) pass ratio threshold for a test to be considered a failure. Any tests below this pass rate will be considered flaky.' + default: '1.0' findByTestFilesDiff: required: false type: boolean @@ -64,7 +64,6 @@ env: DEFAULT_RUNNER: 'ubuntu-latest' # The default runner to use for running tests. UPLOAD_ALL_TEST_RESULTS: ${{ fromJson(inputs.extraArgs)['upload_all_test_results'] || 'false' }} # Whether to upload all test results as artifacts. PRINT_FAILED_TESTS: ${{ fromJson(inputs.extraArgs)['print_failed_tests'] || 'false' }} # Whether to print failed tests in the GitHub console. - MIN_PASS_RATIO: ${{ fromJson(inputs.extraArgs)['min_pass_ratio'] || '0.001' }} # The minimum pass ratio for a test to be considered as flaky. Used to distinguish between tests that are truly flaky (with inconsistent results) and those that are consistently failing. Set to 0 if you want to consider all failed tests as flaky. jobs: get-tests: @@ -99,7 +98,7 @@ jobs: - name: Install flakeguard shell: bash - run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@8b02ed1703ef40755a4c46ff454cf4ff2e89275d + run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@fa6c4ebc22c1d3c5e91dd39efeb76c4a88fc3174 - name: Find new or updated test packages if: ${{ inputs.runAllTests == false }} @@ -258,11 +257,11 @@ jobs: - name: Install flakeguard shell: bash - run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@8b02ed1703ef40755a4c46ff454cf4ff2e89275d + run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@be06798af83ef6d9f7cf04e8b10a8484520c5061 - name: Run tests with flakeguard shell: bash - run: flakeguard run --project-path=${{ inputs.projectPath }} --test-packages=${{ matrix.testPackages }} --run-count=${{ env.TEST_REPEAT_COUNT }} --min-pass-ratio=${{ env.MIN_PASS_RATIO }} --threshold=${{ inputs.runThreshold }} --race=${{ env.RUN_WITH_RACE }} --skip-tests=${{ env.SKIPPED_TESTS }} --print-failed-tests=${{ env.PRINT_FAILED_TESTS }} --output-json=test-result.json + run: flakeguard run --project-path=${{ inputs.projectPath }} --test-packages=${{ matrix.testPackages }} --run-count=${{ env.TEST_REPEAT_COUNT }} --max-pass-ratio=${{ inputs.maxPassRatio }} --race=${{ env.RUN_WITH_RACE }} --skip-tests=${{ env.SKIPPED_TESTS }} --print-failed-tests=${{ env.PRINT_FAILED_TESTS }} --output-json=test-result.json env: CL_DATABASE_URL: ${{ env.DB_URL }} diff --git a/.github/workflows/run-nightly-flaky-test-detector.yml b/.github/workflows/run-nightly-flaky-test-detector.yml deleted file mode 100644 index 1c5dc72d4a3..00000000000 --- a/.github/workflows/run-nightly-flaky-test-detector.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: Run Nightly Flaky Test Detector - -on: - schedule: - # Run every night at 3:00 AM UTC - - cron: '0 3 * * *' - workflow_dispatch: # Allows manual trigger for debugging - -jobs: - trigger-flaky-test-detection: - name: Find Flaky Tests - uses: ./.github/workflows/find-new-flaky-tests.yml - with: - repoUrl: 'https://github.com/smartcontractkit/chainlink' - baseRef: 'origin/develop' - projectPath: '.' - runThreshold: '1' - runAllTests: true - extraArgs: '{ "skipped_tests": "TestChainComponents", "test_repeat_count": "5", "all_tests_runner": "ubuntu22.04-32cores-128GB", "all_tests_runner_count": "3", "min_pass_ratio": "0", "run_with_race": "false" }' - secrets: - SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }} - \ No newline at end of file