test(systest): improve systest resilience (#6171) #20087
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| env: | |
| go-version: "1.22" | |
| GCLOUD_KEY: ${{ secrets.GCLOUD_KEY }} | |
| PROJECT_NAME: ${{ secrets.PROJECT_NAME }} | |
| CLUSTER_NAME: ${{ secrets.CLUSTER_NAME }} | |
| CLUSTER_ZONE: ${{ secrets.CLUSTER_ZONE }} | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| ES_USER: ${{ secrets.ES_USER }} | |
| ES_PASS: ${{ secrets.ES_PASS }} | |
| MAIN_ES_IP: ${{ secrets.MAIN_ES_IP }} | |
| TD_QUEUE_NAME: ${{ secrets.TD_QUEUE_NAME }} | |
| TD_QUEUE_ZONE: ${{ secrets.TD_QUEUE_ZONE }} | |
| DUMP_QUEUE_NAME: ${{ secrets.DUMP_QUEUE_NAME }} | |
| DUMP_QUEUE_ZONE: ${{ secrets.DUMP_QUEUE_ZONE }} | |
| # If needed the variable GOPRIVATE can be set in the repository settings :: Secrets and Variables > action > Variables : GOPRIVATE | |
| # Trigger the workflow on all pull requests, and on push to specific branches | |
| on: | |
| # Allow manually triggering this workflow | |
| workflow_dispatch: | |
| # run for all pull requests and pushes to certain branches | |
| pull_request: | |
| push: | |
| branches: | |
| - staging | |
| - trying | |
| concurrency: | |
| group: ${{ github.base_ref == 'staging' && 'smci-staging' || format('smci-{0}-{1}', github.workflow, github.ref) }} | |
| cancel-in-progress: ${{ github.base_ref == 'staging' && false || true }} | |
| jobs: | |
| ## stage 0: check which files were changed | |
| filter-changes: | |
| runs-on: ubuntu-22.04 | |
| outputs: | |
| nondocchanges: ${{ steps.filter.outputs.nondoc }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| ssh-key: ${{ secrets.GH_ACTION_PRIVATE_KEY }} | |
| - uses: dorny/paths-filter@v3 | |
| id: filter | |
| with: | |
| # All patterns have to match for it to be considered a non-doc change. | |
| # If only markdown files or CODEOWNERS are changed non-doc will be considered true and not trigger | |
| # building the code and executing the tests. | |
| predicate-quantifier: 'every' | |
| filters: | | |
| nondoc: | |
| - '!**/*.md' | |
| - '!.github/CODEOWNERS' | |
| ## stage 1: run unit tests and app tests as a prerequisite | |
| ## these run on all pushes to all pull requests, all branches | |
| ## note that secrets may not be accessible in this phase | |
| quicktests: | |
| runs-on: ubuntu-22.04 | |
| needs: filter-changes | |
| if: ${{ needs.filter-changes.outputs.nondocchanges == 'true' }} | |
| # should not take more than 2-3 mins | |
| timeout-minutes: 5 | |
| steps: | |
| - name: Add OpenCL support | |
| run: sudo apt-get update -q && sudo apt-get install -qy ocl-icd-opencl-dev libpocl2 | |
| - name: checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| ssh-key: ${{ secrets.GH_ACTION_PRIVATE_KEY }} | |
| - uses: extractions/netrc@v2 | |
| with: | |
| machine: github.com | |
| username: ${{ secrets.GH_ACTION_TOKEN_USER }} | |
| password: ${{ secrets.GH_ACTION_TOKEN }} | |
| if: vars.GOPRIVATE | |
| - name: set up go | |
| uses: actions/setup-go@v5 | |
| with: | |
| check-latest: true | |
| go-version: ${{ env.go-version }} | |
| - name: fmt, tidy, generate | |
| run: | | |
| make install | |
| make test-fmt | |
| make test-tidy | |
| make test-generate | |
| lint: | |
| runs-on: ubuntu-22.04 | |
| needs: filter-changes | |
| if: ${{ needs.filter-changes.outputs.nondocchanges == 'true' }} | |
| # should not take more than 15-16 mins | |
| timeout-minutes: 18 | |
| steps: | |
| - name: Add OpenCL support | |
| run: sudo apt-get update -q && sudo apt-get install -qy ocl-icd-opencl-dev libpocl2 | |
| - name: checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| ssh-key: ${{ secrets.GH_ACTION_PRIVATE_KEY }} | |
| - uses: extractions/netrc@v2 | |
| with: | |
| machine: github.com | |
| username: ${{ secrets.GH_ACTION_TOKEN_USER }} | |
| password: ${{ secrets.GH_ACTION_TOKEN }} | |
| if: vars.GOPRIVATE | |
| - name: set up go | |
| uses: actions/setup-go@v5 | |
| with: | |
| check-latest: true | |
| go-version: ${{ env.go-version }} | |
| - name: setup env | |
| run: make install | |
| - name: staticcheck | |
| run: make staticcheck | |
| - name: lint | |
| run: make lint-github-action | |
| build-tools: | |
| runs-on: ${{ matrix.os }} | |
| needs: filter-changes | |
| if: ${{ needs.filter-changes.outputs.nondocchanges == 'true' }} | |
| strategy: | |
| fail-fast: true | |
| matrix: | |
| os: | |
| - ubuntu-22.04 | |
| - ubuntu-latest-arm-8-cores | |
| - macos-13 | |
| - [self-hosted, macOS, ARM64, go-spacemesh] | |
| - windows-2022 | |
| steps: | |
| - name: Add OpenCL support - Ubuntu | |
| if: ${{ matrix.os == 'ubuntu-22.04' || matrix.os == 'ubuntu-latest-arm-8-cores' }} | |
| run: sudo apt-get update -q && sudo apt-get install -qy ocl-icd-opencl-dev libpocl2 | |
| - name: disable Windows Defender - Windows | |
| if: ${{ matrix.os == 'windows-2022' }} | |
| run: | | |
| Set-MpPreference -DisableRealtimeMonitoring $true | |
| - name: Set new git config - Windows | |
| if: ${{ matrix.os == 'windows-2022' }} | |
| run: | | |
| git config --global pack.window 1 | |
| git config --global core.compression 0 | |
| git config --global http.postBuffer 524288000 | |
| - name: checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| ssh-key: ${{ secrets.GH_ACTION_PRIVATE_KEY }} | |
| - uses: extractions/netrc@v2 | |
| with: | |
| machine: github.com | |
| username: ${{ secrets.GH_ACTION_TOKEN_USER }} | |
| password: ${{ secrets.GH_ACTION_TOKEN }} | |
| if: vars.GOPRIVATE | |
| - name: set up go | |
| uses: actions/setup-go@v5 | |
| with: | |
| check-latest: true | |
| go-version: ${{ env.go-version }} | |
| - name: setup env | |
| run: make install | |
| - name: build merge-nodes | |
| run: make merge-nodes | |
| - name: build gen-p2p-identity | |
| run: make gen-p2p-identity | |
| - name: build bootstrapper | |
| run: make bootstrapper | |
| build: | |
| runs-on: ${{ matrix.os }} | |
| needs: filter-changes | |
| if: ${{ needs.filter-changes.outputs.nondocchanges == 'true' }} | |
| strategy: | |
| fail-fast: true | |
| matrix: | |
| os: | |
| - ubuntu-22.04 | |
| - ubuntu-latest-arm-8-cores | |
| - macos-13 | |
| - [self-hosted, macOS, ARM64, go-spacemesh] | |
| - windows-2022 | |
| steps: | |
| - name: Add OpenCL support - Ubuntu | |
| if: ${{ matrix.os == 'ubuntu-22.04' || matrix.os == 'ubuntu-latest-arm-8-cores' }} | |
| run: sudo apt-get update -q && sudo apt-get install -qy ocl-icd-opencl-dev libpocl2 | |
| - name: disable Windows Defender - Windows | |
| if: ${{ matrix.os == 'windows-2022' }} | |
| run: | | |
| Set-MpPreference -DisableRealtimeMonitoring $true | |
| - name: Set new git config - Windows | |
| if: ${{ matrix.os == 'windows-2022' }} | |
| run: | | |
| git config --global pack.window 1 | |
| git config --global core.compression 0 | |
| git config --global http.postBuffer 524288000 | |
| - name: checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| ssh-key: ${{ secrets.GH_ACTION_PRIVATE_KEY }} | |
| - uses: extractions/netrc@v2 | |
| with: | |
| machine: github.com | |
| username: ${{ secrets.GH_ACTION_TOKEN_USER }} | |
| password: ${{ secrets.GH_ACTION_TOKEN }} | |
| if: vars.GOPRIVATE | |
| - name: set up go | |
| uses: actions/setup-go@v5 | |
| with: | |
| check-latest: true | |
| go-version: ${{ env.go-version }} | |
| cache: ${{ runner.arch != 'arm64' }} | |
| - name: setup env | |
| run: make install | |
| - name: build | |
| timeout-minutes: 5 | |
| run: make build | |
| unittests: | |
| runs-on: ${{ matrix.os }} | |
| needs: filter-changes | |
| if: ${{ needs.filter-changes.outputs.nondocchanges == 'true' }} | |
| strategy: | |
| fail-fast: true | |
| matrix: | |
| os: | |
| - ubuntu-22.04 | |
| - ubuntu-latest-arm-8-cores | |
| # FIXME: reenable the macos runner | |
| # - macos-13 | |
| - [self-hosted, macOS, ARM64, go-spacemesh] | |
| - windows-2022 | |
| steps: | |
| # as we use some request to localhost, sometimes it gives us flaky tests. try to disable tcp offloading for fix it | |
| # https://github.com/actions/virtual-environments/issues/1187 | |
| - name: disable TCP/UDP offload - Ubuntu | |
| if: ${{ matrix.os == 'ubuntu-22.04' }} | |
| run: | | |
| sudo ethtool -K eth0 tx off | |
| sudo ethtool -K eth0 rx off | |
| - name: disable TCP/UDP offload - MacOS | |
| if: ${{ matrix.os == 'macos-13' || matrix.os == '[self-hosted, macOS, ARM64, go-spacemesh]' }} | |
| run: | | |
| sudo sysctl -w net.link.generic.system.hwcksum_tx=0 | |
| sudo sysctl -w net.link.generic.system.hwcksum_rx=0 | |
| - name: disable TCP/UDP offload - Windows | |
| if: ${{ matrix.os == 'windows-2022' }} | |
| run: | | |
| Disable-NetAdapterChecksumOffload -Name * -TcpIPv4 -UdpIPv4 -TcpIPv6 -UdpIPv6 | |
| - name: disable Windows Defender - Windows | |
| if: ${{ matrix.os == 'windows-2022' }} | |
| run: | | |
| Set-MpPreference -DisableRealtimeMonitoring $true | |
| - name: Set new git config - Windows | |
| if: ${{ matrix.os == 'windows-2022' }} | |
| run: | | |
| git config --global pack.window 1 | |
| git config --global core.compression 0 | |
| git config --global http.postBuffer 524288000 | |
| - name: checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| lfs: true | |
| ssh-key: ${{ secrets.GH_ACTION_PRIVATE_KEY }} | |
| - uses: extractions/netrc@v2 | |
| with: | |
| machine: github.com | |
| username: ${{ secrets.GH_ACTION_TOKEN_USER }} | |
| password: ${{ secrets.GH_ACTION_TOKEN }} | |
| if: vars.GOPRIVATE | |
| - name: set up go | |
| uses: actions/setup-go@v5 | |
| with: | |
| check-latest: true | |
| go-version: ${{ env.go-version }} | |
| cache: ${{ runner.arch != 'arm64' }} | |
| - name: Add OpenCL support - Ubuntu | |
| if: ${{ matrix.os == 'ubuntu-22.04' || matrix.os == 'ubuntu-latest-arm-8-cores' }} | |
| run: sudo apt-get update -q && sudo apt-get install -qy ocl-icd-opencl-dev libpocl2 | |
| - name: Add OpenCL support - Windows | |
| if: ${{ matrix.os == 'windows-2022' }} | |
| run: choco install opencl-intel-cpu-runtime | |
| - name: setup env | |
| run: make install | |
| - name: Clear test cache | |
| run: make clear-test-cache | |
| - name: unit tests | |
| timeout-minutes: 35 | |
| env: | |
| GOTESTSUM_FORMAT: standard-quiet | |
| GOTESTSUM_JUNITFILE: unit-tests.xml | |
| run: make test | |
| - name: Publish Test Report | |
| uses: mikepenz/action-junit-report@v4 | |
| # always run even if the previous step fails | |
| if: always() | |
| with: | |
| report_paths: "**/unit-tests.xml" | |
| annotate_only: true | |
| ci-status: | |
| # run regardless of status of previous jobs but skip if the required secret is not accessible | |
| if: always() | |
| needs: | |
| - filter-changes | |
| - quicktests | |
| - lint | |
| - build-tools | |
| - build | |
| - unittests | |
| runs-on: ubuntu-22.04 | |
| env: | |
| # short-circuit success if no non-doc files were modified | |
| # this is the easiest way to access success/failure state of previous jobs in this workflow | |
| status: ${{ (needs.filter-changes.outputs.nondocchanges == 'false' || (needs.quicktests.result == 'success' && needs.build.result == 'success' && needs.lint.result == 'success' && needs.unittests.result == 'success')) && 'success' || 'failure' }} | |
| steps: | |
| - uses: act10ns/slack@v2 | |
| name: Slack notification | |
| # skip if the secret is not accessible | |
| if: env.SLACK_WEBHOOK_URL | |
| with: | |
| status: ${{ env.status }} | |
| - name: Mark the job as succeeded | |
| if: env.status == 'success' | |
| run: exit 0 | |
| - name: Mark the job as failed | |
| if: env.status != 'success' | |
| run: exit 1 |