fixing flaky replication tests #1667
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test Spec | |
| on: | |
| push: | |
| paths: | |
| - .github/workflows/test-spec.yml | |
| - package*.json | |
| - spec/** | |
| - tests/** | |
| - tools/src/tester/** | |
| - tsconfig.json | |
| pull_request: | |
| paths: | |
| - .github/workflows/test-spec.yml | |
| - package*.json | |
| - spec/** | |
| - tests/** | |
| - tools/src/tester/** | |
| - tsconfig.json | |
| jobs: | |
| test-opensearch-spec: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| entry: | |
| - version: 1.3.17 | |
| admin_password: admin | |
| - version: 2.0.0 | |
| admin_password: admin | |
| - version: 2.18.0 | |
| - version: 2.18.0 | |
| tests: plugins/index_state_management | |
| - version: 2.18.0 | |
| tests: plugins/ml | |
| - version: 2.18.0 | |
| tests: routing | |
| - version: 2.18.0 | |
| tests: snapshot | |
| - version: 2.18.0 | |
| tests: dangling | |
| url: http://localhost:9200 | |
| - version: 2.18.0 | |
| tests: plugins/replication | |
| url: http://localhost:9200 | |
| - version: 2.18.0 | |
| tests: plugins/streaming | |
| - version: 2.18.0 | |
| tests: plugins/notifications | |
| - version: 2.18.0 | |
| tests: plugins/query_insights | |
| - version: 2.18.0 | |
| tests: plugins/workload-management | |
| - version: 2.18.0 | |
| tests: plugins/analysis | |
| - version: 2.19.0 | |
| hub: opensearchstaging | |
| ref: '@sha256:4da23e0137b2b67206d23b36fcf0914cc39b3bf19310c782f536e4934b86f6cc' | |
| - version: 3.0.0 | |
| hub: opensearchstaging | |
| ref: '@sha256:727643acdfebed77bfdb26362dbcff536b7ea02a0cc4ae2da2521729171333de' | |
| name: test-opensearch-spec (version=${{ matrix.entry.version }}, hub=${{ matrix.entry.hub || 'opensearchproject' }}, tests=${{ matrix.entry.tests || 'default' }}) | |
| runs-on: ubuntu-latest | |
| env: | |
| OPENSEARCH_DOCKER_HUB_PROJECT: ${{ matrix.entry.hub || 'opensearchproject' }} | |
| OPENSEARCH_DOCKER_REF: ${{ matrix.entry.ref }} | |
| OPENSEARCH_VERSION: ${{ matrix.entry.version }} | |
| OPENSEARCH_PASSWORD: ${{ matrix.entry.admin_password || 'myStrongPassword123!' }} | |
| OPENSEARCH_JAVA_OPTS: ${{ matrix.entry.opts }} -Xms8g -Xmx8g | |
| steps: | |
| - name: Checkout Repo | |
| uses: actions/checkout@v4 | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version: '20' | |
| - name: Install Dependencies | |
| run: npm ci | |
| - name: Run OpenSearch Cluster | |
| working-directory: tests/${{ matrix.entry.tests || 'default' }} | |
| run: docker compose up -d | |
| - name: Generate Test Files Hash | |
| id: tests | |
| run: echo "hash=${{ matrix.entry.version }}-${{ hashFiles(format('tests/{0}', matrix.entry.tests || 'default')) }}" >> $GITHUB_OUTPUT | |
| - name: Run Tests | |
| run: | | |
| npm run test:spec -- \ | |
| --opensearch-insecure \ | |
| --opensearch-version=${{ matrix.entry.version }} \ | |
| --coverage coverage/test-spec-coverage-${{ steps.tests.outputs.hash }}.json \ | |
| --opensearch-url=${{ matrix.entry.url || 'https://localhost:9200'}} \ | |
| --tests=tests/${{ matrix.entry.tests || 'default' }} | |
| - name: Get Container Logs | |
| if: failure() | |
| run: | | |
| for container_id in $(docker ps -aqf "ancestor=opensearchproject/opensearch:${{ matrix.entry.version }}"); do \ | |
| echo Dumping logs from $container_id ... && \ | |
| docker logs $container_id \ | |
| ; done | |
| - name: Upload Test Coverage Results | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: coverage-${{ matrix.entry.version }}-${{ steps.tests.outputs.hash }} | |
| path: coverage/test-spec-coverage-${{ steps.tests.outputs.hash }}.json | |
| merge-coverage: | |
| runs-on: ubuntu-latest | |
| needs: test-opensearch-spec | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download Spec Coverage Data | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: coverage | |
| - name: Display Missing Test Paths | |
| run: | | |
| jq -sc ' | |
| (map(.operations) | add | unique) as $all | | |
| (map(.evaluated_operations) | add | unique) as $evaluated | | |
| $all-$evaluated | | |
| sort_by(.path) | | |
| .[] | | |
| "\(.method) \(.path)" | |
| ' $(find ./ -name "test-spec-coverage-*.json") | |
| - name: Combine Test Coverage Data | |
| shell: bash -eo pipefail {0} | |
| run: | | |
| jq -sc ' | |
| (map(.operations) | add | unique | length) as $total_operations_count | | |
| (map(.evaluated_operations) | add | unique | length) as $evaluated_operations_count | | |
| { | |
| total_operations_count: $total_operations_count, | |
| evaluated_operations_count: $evaluated_operations_count, | |
| evaluated_paths_pct: $evaluated_operations_count | (10000 * . / $total_operations_count | round / 100) | |
| } | |
| ' $(find ./coverage -name "test-spec-coverage-*.json") > ./coverage/coverage.json | |
| cat ./coverage/coverage.json | |
| - name: Construct Comment Data Payload | |
| if: github.event_name == 'pull_request' | |
| shell: bash -eo pipefail {0} | |
| run: | | |
| jq \ | |
| --arg pr_number ${PR_NUMBER} \ | |
| --slurpfile test_coverage ./coverage/coverage.json \ | |
| --null-input ' | |
| { | |
| "pr_number": ($pr_number), | |
| "comment_identifier": "## Spec Test Coverage Analysis", | |
| "template_name": "pr-test-coverage-analysis", | |
| "template_data": { | |
| "test_coverage": ($test_coverage[0]) | |
| } | |
| } | |
| ' | tee pr-comment.json | |
| env: | |
| PR_NUMBER: ${{ github.event.pull_request.number }} | |
| - name: Upload PR Comment Payload | |
| uses: actions/upload-artifact@v4 | |
| if: github.event_name == 'pull_request' | |
| with: | |
| name: pr-comment | |
| path: pr-comment.json |