feat(job-runner): enable snuba jobs command to run from JSON manifest… #18159
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: ci | |
| on: | |
| push: | |
| branches: | |
| - master | |
| pull_request: | |
| jobs: | |
| files-changed: | |
| name: detect what files changed | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 3 | |
| # Map a step output to a job output | |
| outputs: | |
| api_changes: ${{ steps.changes.outputs.api_changes }} | |
| steps: | |
| - uses: actions/checkout@7884fcad6b5d53d10323aee724dc68d8b9096a2e # v2 | |
| - name: Check for backend file changes | |
| uses: getsentry/paths-filter@66f7f1844185eb7fb6738ea4ea59d74bb99199e5 # v2 | |
| id: changes | |
| with: | |
| token: ${{ github.token }} | |
| filters: .github/file-filters.yml | |
| linting: | |
| name: "pre-commit hooks" # (includes Python formatting + linting) | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| name: Checkout code | |
| - name: Internal github app token | |
| id: token | |
| uses: getsentry/action-github-app-token@97c9e23528286821f97fba885c1b1123284b29cc # v2.0.0 | |
| continue-on-error: true | |
| with: | |
| app_id: ${{ vars.SENTRY_INTERNAL_APP_ID }} | |
| private_key: ${{ secrets.SENTRY_INTERNAL_APP_PRIVATE_KEY }} | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.11' | |
| - uses: actions/cache@v3 | |
| with: | |
| path: ~/.cache/pre-commit | |
| key: cache-epoch-1|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml') }} | |
| - name: Setup pre-commit | |
| run: make setup-git | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| make install-python-dependencies | |
| - uses: getsentry/paths-filter@v2 | |
| id: files | |
| with: | |
| # Enable listing of files matching each filter. | |
| # Paths to files will be available in `${FILTER_NAME}_files` output variable. | |
| # Paths will be escaped and space-delimited. | |
| # Output is usable as command line argument list in linux shell | |
| list-files: shell | |
| # It doesn't make sense to lint deleted files. | |
| # Therefore we specify we are only interested in added or modified files. | |
| filters: | | |
| all: | |
| - added|modified: '**/*' | |
| - name: Run pre-commit checks | |
| # Run pre-commit to lint and format check files that were changed (but not deleted) compared to master. | |
| # XXX: there is a very small chance that it'll expand to exceed Linux's limits | |
| # `getconf ARG_MAX` - max # bytes of args + environ for exec() | |
| # we skip the `no-commit-to-branch` because in CI we are in fact on master already | |
| # and we have merged to it | |
| run: | | |
| SKIP=no-commit-to-branch pre-commit run --files ${{ steps.files.outputs.all_files }} | |
| # If working tree is dirty, commit and update if we have a token | |
| - name: Apply any pre-commit fixed files | |
| if: steps.token.outcome == 'success' && github.ref != 'refs/heads/master' && always() | |
| uses: getsentry/action-github-commit@v1.0.0 | |
| with: | |
| github-token: ${{ steps.token.outputs.token }} | |
| rust-linting: | |
| name: "Linting - Rust" | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| name: Checkout code | |
| - name: Run linter | |
| run: | | |
| make lint-rust format-rust-ci | |
| typing: | |
| name: "mypy typing" | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| name: Checkout code | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.11' | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| make install-python-dependencies | |
| - name: Run mypy | |
| run: | | |
| make backend-typing | |
| config-validation: | |
| name: "Dataset Config Validation" | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 2 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| name: Checkout code | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.11' | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| make install-python-dependencies | |
| - name: Validate configs | |
| run: | | |
| make validate-configs | |
| snuba-image: | |
| name: Build snuba CI image | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| outputs: | |
| branch: ${{ steps.branch.outputs.branch }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v3 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v2 | |
| - name: Get branch name | |
| id: branch | |
| # strip `refs/heads/` from $GITHUB_REF and replace `/` with `-` so that | |
| # it can be used as a docker tag | |
| run: echo "branch=$(echo ${GITHUB_REF#refs/heads/} | tr / -)" >> "$GITHUB_OUTPUT" | |
| # We are only using ghcr here for CI as `setup-gcloud` is a bit slow | |
| # Should revisit this when we move off of google cloud build (we may want to move these to GCR) | |
| - name: Registry login | |
| run: | | |
| echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io -u $GITHUB_ACTOR --password-stdin | |
| - name: Build snuba docker image for CI | |
| uses: docker/build-push-action@v4 | |
| if: github.repository_owner == 'getsentry' | |
| with: | |
| context: . | |
| # push: true | |
| load: true | |
| target: testing | |
| tags: | | |
| ghcr.io/getsentry/snuba-ci:${{ github.sha }} | |
| ghcr.io/getsentry/snuba-ci:${{ steps.branch.outputs.branch }} | |
| ghcr.io/getsentry/snuba-ci:latest | |
| cache-from: | | |
| type=registry,ref=ghcr.io/getsentry/snuba-ci:${{ github.sha }} | |
| type=registry,ref=ghcr.io/getsentry/snuba-ci:${{ steps.branch.outputs.branch }} | |
| type=registry,ref=ghcr.io/getsentry/snuba-ci:latest | |
| # cache-to is re-implemented using the next step, to run conditionally for contributors | |
| - name: Publish images for cache | |
| if: steps.branch.outputs.branch == 'master' || github.event.pull_request.head.repo.full_name == github.repository | |
| # outside contributors won't be able to push to the docker registry | |
| # ignore the failures in this step | |
| continue-on-error: ${{ github.event_name == 'pull_request' }} | |
| run: | | |
| # Useful to speed up CI | |
| docker push ghcr.io/getsentry/snuba-ci:${{ steps.branch.outputs.branch }} | |
| docker push ghcr.io/getsentry/snuba-ci:${{ github.sha }} | |
| if [ "${{ steps.branch.outputs.branch }}" == 'master' ]; then | |
| # The latest tag should only be published on `master` | |
| docker push ghcr.io/getsentry/snuba-ci:latest | |
| fi | |
| tests: | |
| needs: [linting, snuba-image] | |
| name: Tests and code coverage | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| strategy: | |
| matrix: | |
| snuba_settings: | |
| [ | |
| "test", | |
| "test_rust", | |
| "test_distributed", | |
| "test_distributed_migrations", | |
| ] | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v2 | |
| - name: Registry login | |
| run: | | |
| echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io -u $GITHUB_ACTOR --password-stdin | |
| - name: Docker set up | |
| run: | | |
| docker network create --attachable cloudbuild | |
| - name: Docker Snuba Rust tests | |
| run: | | |
| SNUBA_IMAGE=ghcr.io/getsentry/snuba-ci:${{ github.sha }} SNUBA_SETTINGS=test docker compose -f docker-compose.gcb.yml run --rm snuba-test-rust | |
| if: ${{ matrix.snuba_settings == 'test_rust' }} | |
| - name: Docker Snuba tests | |
| run: | | |
| SNUBA_IMAGE=ghcr.io/getsentry/snuba-ci:${{ github.sha }} SNUBA_SETTINGS=${{ matrix.snuba_settings }} docker compose -f docker-compose.gcb.yml run --rm snuba-test | |
| if: ${{ matrix.snuba_settings == 'test' || matrix.snuba_settings == 'test_distributed' }} | |
| - name: Docker Snuba Multi-Node Tests | |
| run: | | |
| SNUBA_IMAGE=ghcr.io/getsentry/snuba-ci:${{ github.sha }} SNUBA_SETTINGS=test_distributed_migrations docker compose --profile multi_node -f docker-compose.gcb.yml up -d | |
| SNUBA_IMAGE=ghcr.io/getsentry/snuba-ci:${{ github.sha }} SNUBA_SETTINGS=test_distributed_migrations TEST_LOCATION=test_distributed_migrations docker compose --profile multi_node -f docker-compose.gcb.yml run --rm snuba-test | |
| if: ${{ matrix.snuba_settings == 'test_distributed_migrations' }} | |
| - name: Docker Snuba Init Tests | |
| run: | | |
| SNUBA_IMAGE=ghcr.io/getsentry/snuba-ci:${{ github.sha }} SNUBA_SETTINGS=test_initialization TEST_LOCATION=test_initialization docker compose -f docker-compose.gcb.yml run --rm snuba-test | |
| if: ${{ matrix.snuba_settings == 'test' }} | |
| - name: Upload test results to Codecov | |
| if: ${{ !cancelled() }} | |
| uses: codecov/test-results-action@v1 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| - name: Upload to codecov | |
| run: | | |
| curl -Os https://uploader.codecov.io/latest/linux/codecov && chmod +x codecov && ./codecov -t ${CODECOV_TOKEN} | |
| admin-tests: | |
| needs: [linting] | |
| name: Front end tests for snuba admin | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v2 | |
| name: Checkout code | |
| - uses: volta-cli/action@v4 | |
| - name: Set up and run tests through yarn | |
| run: cd snuba/admin && yarn install && yarn run test --coverage | |
| - name: Upload to codecov | |
| run: | | |
| curl -Os https://uploader.codecov.io/latest/linux/codecov && chmod +x codecov && ./codecov -t ${CODECOV_TOKEN} | |
| sentry: | |
| needs: [snuba-image, files-changed] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 | |
| strategy: | |
| matrix: | |
| instance: [0, 1, 2, 3] | |
| env: | |
| MIGRATIONS_TEST_MIGRATE: 1 | |
| # XXX: `MATRIX_INSTANCE_TOTAL` must be hardcoded to the length of `strategy.matrix.instance`. | |
| MATRIX_INSTANCE_TOTAL: 4 | |
| TEST_GROUP_STRATEGY: ROUND_ROBIN | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v2 | |
| - name: Registry login | |
| run: | | |
| echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io -u $GITHUB_ACTOR --password-stdin | |
| - name: Checkout sentry | |
| uses: actions/checkout@v2 | |
| with: | |
| repository: getsentry/sentry | |
| path: sentry | |
| - name: Setup steps | |
| id: setup | |
| run: | | |
| pip install --upgrade pip wheel | |
| # We cannot execute actions that are not placed under .github of the main repo | |
| mkdir -p .github/actions | |
| cp -r sentry/.github/actions/* .github/actions | |
| - name: Setup Sentry | |
| id: setup-sentry | |
| uses: ./.github/actions/setup-sentry | |
| with: | |
| workdir: sentry | |
| snuba: false | |
| kafka: true | |
| clickhouse: true | |
| - name: Start snuba | |
| run: | | |
| docker run -d --rm \ | |
| -p 127.0.0.1:1218:1218 \ | |
| -e PYTHONUNBUFFERED=1 \ | |
| -e SNUBA_SETTINGS=docker \ | |
| -e DEBUG=1 \ | |
| -e DEFAULT_BROKERS=sentry_kafka:9092 \ | |
| -e CLICKHOUSE_HOST=sentry_clickhouse \ | |
| -e CLICKHOUSE_PORT=9000 \ | |
| -e CLICKHOUSE_HTTP_PORT=8123 \ | |
| -e REDIS_HOST=sentry_redis \ | |
| -e REDIS_PORT=6379 \ | |
| -e REDIS_DB=1 \ | |
| --name sentry_snuba \ | |
| --network sentry \ | |
| ghcr.io/getsentry/snuba-ci:${{ github.sha }} | |
| docker exec sentry_snuba snuba migrations migrate --force | |
| - name: Run snuba tests | |
| if: needs.files-changed.outputs.api_changes == 'false' | |
| working-directory: sentry | |
| run: | | |
| pytest -k 'not __In' tests \ | |
| -m snuba_ci \ | |
| -vv --cov . --cov-report="xml:.artifacts/snuba.coverage.xml" | |
| - name: Run full tests | |
| # TODO: Adjust which changed files we care about | |
| if: needs.files-changed.outputs.api_changes == 'true' | |
| working-directory: sentry | |
| run: | | |
| pytest -k 'not __In' tests/snuba \ | |
| tests/sentry/eventstream/kafka \ | |
| tests/sentry/post_process_forwarder \ | |
| tests/sentry/snuba \ | |
| tests/sentry/eventstore/snuba \ | |
| tests/sentry/search/events \ | |
| tests/sentry/event_manager \ | |
| tests/sentry/api/endpoints/test_organization_profiling_functions.py \ | |
| tests/snuba/api/endpoints/test_organization_events_stats_mep.py \ | |
| tests/sentry/sentry_metrics/querying \ | |
| tests/snuba/test_snql_snuba.py \ | |
| tests/snuba/test_metrics_layer.py \ | |
| -vv --cov . --cov-report="xml:.artifacts/snuba.coverage.xml" | |
| - name: Run CI module tests | |
| if: needs.files-changed.outputs.api_changes == 'true' | |
| working-directory: sentry | |
| run: pytest -k 'not __In' tests -vv -m snuba_ci | |
| clickhouse-versions: | |
| needs: [linting, snuba-image] | |
| name: Tests on multiple clickhouse versions | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| version: | |
| [ | |
| "23.8.11.29.altinitystable", | |
| ] | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v2 | |
| - name: Registry login | |
| run: | | |
| echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io -u $GITHUB_ACTOR --password-stdin | |
| - name: Docker set up | |
| run: | | |
| docker network create --attachable cloudbuild | |
| - name: Docker Snuba Test other ClickHouse versions | |
| run: | | |
| export CLICKHOUSE_IMAGE=ghcr.io/getsentry/image-mirror-altinity-clickhouse-server:${{matrix.version}} | |
| SNUBA_IMAGE=ghcr.io/getsentry/snuba-ci:${{ github.sha }} SNUBA_SETTINGS=test docker compose -f docker-compose.gcb.yml run --rm snuba-test | |
| - name: Upload to codecov | |
| run: | | |
| curl -Os https://uploader.codecov.io/latest/linux/codecov && chmod +x codecov && ./codecov -t ${CODECOV_TOKEN} | |
| self-hosted-end-to-end: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| steps: | |
| - name: Checkout Snuba | |
| uses: actions/checkout@v3 | |
| - name: Run Sentry self-hosted e2e CI | |
| uses: getsentry/action-self-hosted-e2e-tests@main | |
| with: | |
| project_name: snuba | |
| docker_repo: getsentry/snuba | |
| image_url: us-central1-docker.pkg.dev/sentryio/snuba/image:${{ github.event.pull_request.head.sha || github.sha }} | |
| docker_password: ${{ secrets.DOCKER_HUB_RW_TOKEN }} | |
| publish-to-dockerhub: | |
| name: Publish Snuba to DockerHub | |
| runs-on: ubuntu-20.04 | |
| if: ${{ (github.ref_name == 'master') }} | |
| steps: | |
| - uses: actions/checkout@v3 # v3.1.0 | |
| - name: Pull the test image | |
| id: image_pull | |
| env: | |
| IMAGE_URL: us-central1-docker.pkg.dev/sentryio/snuba/image:${{ github.sha }} | |
| shell: bash | |
| run: | | |
| echo "We poll for the Docker image that the GCB/GHA build produces until it succeeds or this job times out." | |
| echo "Polling for $IMAGE_URL" | |
| timeout 20m bash -c 'until docker pull "$IMAGE_URL" 2>/dev/null; do sleep 10; done' | |
| - name: Get short SHA for docker tag | |
| id: short_sha | |
| shell: bash | |
| run: | | |
| SHORT_SHA=$(git rev-parse --short "$GITHUB_SHA") | |
| if [[ -z "$SHORT_SHA" ]]; then | |
| echo "Short SHA empty? Re-running rev-parse." | |
| git rev-parse --short "$GITHUB_SHA" | |
| else | |
| echo "sha=$SHORT_SHA" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Push built docker image | |
| shell: bash | |
| env: | |
| SHORT_SHA: ${{ steps.short_sha.outputs.sha }} | |
| IMAGE_URL: us-central1-docker.pkg.dev/sentryio/snuba/image:${{ github.sha }} | |
| run: | | |
| # only login if the password is set | |
| if [[ "${{ secrets.DOCKER_HUB_RW_TOKEN }}" ]]; then echo "${{ secrets.DOCKER_HUB_RW_TOKEN }}" | docker login --username=sentrybuilder --password-stdin; fi | |
| # We push 3 tags to Dockerhub: | |
| # first, the full sha of the commit | |
| docker tag ${IMAGE_URL} getsentry/snuba:${GITHUB_SHA} | |
| docker push getsentry/snuba:${GITHUB_SHA} | |
| # second, the short sha of the commit | |
| docker tag ${IMAGE_URL} getsentry/snuba:${SHORT_SHA} | |
| docker push getsentry/snuba:${SHORT_SHA} | |
| # finally, nightly | |
| docker tag ${IMAGE_URL} getsentry/snuba:nightly | |
| docker push getsentry/snuba:nightly |