ci.yml

name: ci
on:
  push:
    branches:
      - master
  pull_request:

jobs:
  linting:
    name: "pre-commit hooks" # (includes Python formatting + linting)
    runs-on: ubuntu-latest
    timeout-minutes: 5
    steps:
      - uses: actions/checkout@v3
        name: Checkout code
      - name: Internal github app token
        id: token
        uses: getsentry/action-github-app-token@97c9e23528286821f97fba885c1b1123284b29cc # v2.0.0
        continue-on-error: true
        with:
          app_id: ${{ vars.SENTRY_INTERNAL_APP_ID }}
          private_key: ${{ secrets.SENTRY_INTERNAL_APP_PRIVATE_KEY }}
      - uses: actions/setup-python@v4
        with:
          python-version: 3.8
      - uses: actions/cache@v3
        with:
          path: ~/.cache/pre-commit
          key: cache-epoch-1|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml') }}
      - name: Setup pre-commit
        run: make setup-git
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          make install-python-dependencies
      - uses: getsentry/paths-filter@v2
        id: files
        with:
          # Enable listing of files matching each filter.
          # Paths to files will be available in `${FILTER_NAME}_files` output variable.
          # Paths will be escaped and space-delimited.
          # Output is usable as command line argument list in linux shell
          list-files: shell

          # It doesn't make sense to lint deleted files.
          # Therefore we specify we are only interested in added or modified files.
          filters: |
            all:
              - added|modified: '**/*'
      - name: Run pre-commit checks
        # Run pre-commit to lint and format check files that were changed (but not deleted) compared to master.
        # XXX: there is a very small chance that it'll expand to exceed Linux's limits
        #      `getconf ARG_MAX` - max # bytes of args + environ for exec()
        # we skip the `no-commit-to-branch` because in CI we are in fact on master already
        # and we have merged to it
        run: |
          SKIP=no-commit-to-branch pre-commit run --files ${{ steps.files.outputs.all_files }}

      # If working tree is dirty, commit and update if we have a token
      - name: Apply any pre-commit fixed files
        if: steps.token.outcome == 'success' && github.ref != 'refs/heads/master' && always()
        uses: getsentry/action-github-commit@v1.0.0
        with:
          github-token: ${{ steps.token.outputs.token }}

  typing:
    name: "mypy typing"
    runs-on: ubuntu-latest
    timeout-minutes: 5
    steps:
      - uses: actions/checkout@v3
        name: Checkout code
      - uses: actions/setup-python@v4
        with:
          python-version: 3.8
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          make install-python-dependencies
      - name: Run mypy
        run: |
          make backend-typing

  config-validation:
    name: "Dataset Config Validation"
    runs-on: ubuntu-latest
    timeout-minutes: 2
    steps:
      - uses: actions/checkout@v3
        name: Checkout code
      - uses: actions/setup-python@v4
        with:
          python-version: 3.8
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          make install-python-dependencies
      - name: Validate configs
        run: |
          make validate-configs

  snuba-image:
    name: Build snuba CI image
    runs-on: ubuntu-latest
    timeout-minutes: 20
    outputs:
      branch: ${{ steps.branch.outputs.branch }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v3

      - name: Get branch name
        id: branch
        # strip `refs/heads/` from $GITHUB_REF and replace `/` with `-` so that
        # it can be used as a docker tag
        run: echo "branch=$(echo ${GITHUB_REF#refs/heads/} | tr / -)" >> "$GITHUB_OUTPUT"

      # We are only using ghcr here for CI as `setup-gcloud` is a bit slow
      # Should revisit this when we move off of google cloud build (we may want to move these to GCR)
      - name: Registry login
        run: |
          echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io -u $GITHUB_ACTOR --password-stdin

      # These are pulled in order to be able to use docker layer caching
      - name: Pull snuba CI images
        if: github.repository_owner == 'getsentry'
        run: |
          docker pull ghcr.io/getsentry/snuba-ci:${{ github.sha }} || \
            docker pull ghcr.io/getsentry/snuba-ci:${{ steps.branch.outputs.branch }} || \
            docker pull ghcr.io/getsentry/snuba-ci:latest || true

      - name: Build snuba docker image for CI
        if: github.repository_owner == 'getsentry'
        run: |
          docker build . \
            --build-arg BUILDKIT_INLINE_CACHE=1 \
            --build-arg SHOULD_BUILD_RUST=false \
            -t ghcr.io/getsentry/snuba-ci:latest \
            -t ghcr.io/getsentry/snuba-ci:${{ github.sha }} \
            -t ghcr.io/getsentry/snuba-ci:${{ steps.branch.outputs.branch }} \
            --cache-from ghcr.io/getsentry/snuba-ci:latest \
            --cache-from ghcr.io/getsentry/snuba-ci:${{ steps.branch.outputs.branch }} \
            --cache-from ghcr.io/getsentry/snuba-ci:${{ github.sha }} \
            --target testing

      - name: Publish images for cache
        if: steps.branch.outputs.branch == 'master' || github.event.pull_request.head.repo.full_name == github.repository
        # outside contributors won't be able to push to the docker registry
        # ignore the failures in this step
        continue-on-error: ${{ github.event_name == 'pull_request' }}
        run: |
          # Useful to speed up CI
          docker push ghcr.io/getsentry/snuba-ci:${{ steps.branch.outputs.branch }}
          docker push ghcr.io/getsentry/snuba-ci:${{ github.sha }}
          if [ "${{ steps.branch.outputs.branch }}" == 'master' ]; then
            # The latest tag should only be published on `master`
            docker push ghcr.io/getsentry/snuba-ci:latest
          fi

  tests:
    needs: [linting, snuba-image]
    name: Tests and code coverage
    runs-on: ubuntu-latest
    timeout-minutes: 30
    strategy:
      matrix:
        snuba_settings:
          ["test", "test_distributed", "test_distributed_migrations"]
    steps:
      - uses: actions/checkout@v2
        name: Checkout code

      - name: Pull snuba CI images
        run: |
          docker pull ghcr.io/getsentry/snuba-ci:${{ github.sha }} || \
            docker pull ghcr.io/getsentry/snuba-ci:${{ needs.snuba-image.outputs.branch }} || \
            docker pull ghcr.io/getsentry/snuba-ci:latest || true

      - name: Build snuba docker image for CI
        run: |
          docker build . \
            -t snuba-test \
            --build-arg SHOULD_BUILD_RUST=false \
            --cache-from ghcr.io/getsentry/snuba-ci:${{ github.sha }} \
            --cache-from ghcr.io/getsentry/snuba-ci:${{ needs.snuba-image.outputs.branch }} \
            --cache-from ghcr.io/getsentry/snuba-ci:latest \
            --target testing

      - name: Docker set up
        run: |
          docker network create --attachable cloudbuild

      - name: Docker Snuba tests
        run: |
          SNUBA_IMAGE=snuba-test SNUBA_SETTINGS=${{ matrix.snuba_settings }} docker-compose -f docker-compose.gcb.yml run --rm snuba-test
        if: ${{ matrix.snuba_settings != 'test_distributed_migrations' }}

      - name: Docker Snuba Multi-Node Tests
        run: |
          SNUBA_SETTINGS=test_distributed_migrations docker-compose --profile multi_node -f docker-compose.gcb.yml up -d
          SNUBA_IMAGE=snuba-test SNUBA_SETTINGS=test_distributed_migrations TEST_LOCATION=test_distributed_migrations docker-compose --profile multi_node -f docker-compose.gcb.yml run --rm snuba-test
        if: ${{ matrix.snuba_settings == 'test_distributed_migrations' }}

      - name: Docker Snuba Init Tests
        run: |
          SNUBA_IMAGE=snuba-test SNUBA_SETTINGS=test_initialization TEST_LOCATION=test_initialization docker-compose -f docker-compose.gcb.yml run --rm snuba-test
        if: ${{ matrix.snuba_settings == 'test' }}

      - name: Upload to codecov
        run: |
          curl -Os https://uploader.codecov.io/latest/linux/codecov && chmod +x codecov && ./codecov -t ${CODECOV_TOKEN}

  admin-tests:
    needs: [linting]
    name: Front end tests for snuba admin
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
        name: Checkout code
      - uses: volta-cli/action@v4
      - name: Set up and run tests through yarn
        run: cd snuba/admin && yarn install && yarn run test --coverage
      - name: Upload to codecov
        run: |
          curl -Os https://uploader.codecov.io/latest/linux/codecov && chmod +x codecov && ./codecov -t ${CODECOV_TOKEN}

  sentry:
    needs: [snuba-image]
    runs-on: ubuntu-latest
    timeout-minutes: 90

    strategy:
      matrix:
        instance: [0, 1]

    env:
      # XXX: MATRIX_INSTANCE_TOTAL must be hardcoded to the length of strategy.matrix.instance.
      MATRIX_INSTANCE_TOTAL: 2
      MIGRATIONS_TEST_MIGRATE: 1

    steps:
      # Checkout codebase
      - name: Checkout snuba
        uses: actions/checkout@v2

      - name: Pull snuba CI images
        run: |
          docker pull ghcr.io/getsentry/snuba-ci:${{ github.sha }} || \
            docker pull ghcr.io/getsentry/snuba-ci:${{ needs.snuba-image.outputs.branch }} || \
            docker pull ghcr.io/getsentry/snuba-ci:latest || true

      - name: Build snuba docker image for CI
        run: |
          docker build . \
            -t snuba-test \
            --build-arg SHOULD_BUILD_RUST=false \
            --cache-from ghcr.io/getsentry/snuba-ci:latest \
            --cache-from ghcr.io/getsentry/snuba-ci:${{ needs.snuba-image.outputs.branch }} \
            --cache-from ghcr.io/getsentry/snuba-ci:${{ github.sha }} \
            --target testing

      # Checkout Sentry and run integration tests against latest snuba
      # Make sure this is after `docker build`, otherwise we'll break docker cache
      - name: Checkout sentry
        uses: actions/checkout@v2
        with:
          repository: getsentry/sentry
          path: sentry

      - name: Setup steps
        id: setup
        run: |
          pip install --upgrade pip wheel
          # We cannot execute actions that are not placed under .github of the main repo
          mkdir -p .github/actions
          cp -r sentry/.github/actions/* .github/actions

      - name: Setup Sentry
        id: setup-sentry
        uses: ./.github/actions/setup-sentry
        with:
          workdir: sentry
          cache-files-hash: ${{ hashFiles('sentry/requirements**.txt') }}
          python-version: 3.8
          snuba: false
          kafka: true
          clickhouse: true

      - name: Start snuba
        run: |
          docker run -d --rm \
            -p 127.0.0.1:1218:1218 \
            -e PYTHONUNBUFFERED=1 \
            -e SNUBA_SETTINGS=docker \
            -e DEBUG=1 \
            -e DEFAULT_BROKERS=sentry_kafka:9092 \
            -e CLICKHOUSE_HOST=sentry_clickhouse \
            -e CLICKHOUSE_PORT=9000 \
            -e CLICKHOUSE_HTTP_PORT=8123 \
            -e REDIS_HOST=sentry_redis \
            -e REDIS_PORT=6379 \
            -e REDIS_DB=1 \
            --name sentry_snuba \
            --network sentry \
            snuba-test
          docker exec sentry_snuba snuba migrations migrate --force

      - name: Run snuba tests
        working-directory: sentry
        run: |
          make test-snuba
  self-hosted-end-to-end:
    needs: [snuba-image]
    runs-on: ubuntu-latest
    # temporary, remove once we are confident the action is working
    continue-on-error: true

    steps:
      - name: Checkout Snuba
        uses: actions/checkout@v3
      - name: Run Sentry self-hosted e2e CI
        uses: getsentry/action-self-hosted-e2e-tests@f45ef07793b2cc805a9a9401819f486da449a90a
        with:
          project_name: snuba
          docker_repo: getsentry/snuba
          image_url: us.gcr.io/sentryio/snuba:${{ github.event.pull_request.head.sha || github.sha }}
          docker_password: ${{ secrets.DOCKER_HUB_RW_TOKEN }}