feat(admin-auth): Add TTL cache for admin roles #11093
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: ci | |
| on: | |
| push: | |
| branches: | |
| - master | |
| pull_request: | |
| jobs: | |
| linting: | |
| name: "pre-commit hooks" # (includes Python formatting + linting) | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| name: Checkout code | |
| - name: Internal github app token | |
| id: token | |
| uses: getsentry/action-github-app-token@97c9e23528286821f97fba885c1b1123284b29cc # v2.0.0 | |
| continue-on-error: true | |
| with: | |
| app_id: ${{ vars.SENTRY_INTERNAL_APP_ID }} | |
| private_key: ${{ secrets.SENTRY_INTERNAL_APP_PRIVATE_KEY }} | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: 3.8 | |
| - uses: actions/cache@v3 | |
| with: | |
| path: ~/.cache/pre-commit | |
| key: cache-epoch-1|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml') }} | |
| - name: Setup pre-commit | |
| run: make setup-git | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| make install-python-dependencies | |
| - uses: getsentry/paths-filter@v2 | |
| id: files | |
| with: | |
| # Enable listing of files matching each filter. | |
| # Paths to files will be available in `${FILTER_NAME}_files` output variable. | |
| # Paths will be escaped and space-delimited. | |
| # Output is usable as command line argument list in linux shell | |
| list-files: shell | |
| # It doesn't make sense to lint deleted files. | |
| # Therefore we specify we are only interested in added or modified files. | |
| filters: | | |
| all: | |
| - added|modified: '**/*' | |
| - name: Run pre-commit checks | |
| # Run pre-commit to lint and format check files that were changed (but not deleted) compared to master. | |
| # XXX: there is a very small chance that it'll expand to exceed Linux's limits | |
| # `getconf ARG_MAX` - max # bytes of args + environ for exec() | |
| # we skip the `no-commit-to-branch` because in CI we are in fact on master already | |
| # and we have merged to it | |
| run: | | |
| SKIP=no-commit-to-branch pre-commit run --files ${{ steps.files.outputs.all_files }} | |
| # If working tree is dirty, commit and update if we have a token | |
| - name: Apply any pre-commit fixed files | |
| if: steps.token.outcome == 'success' && github.ref != 'refs/heads/master' && always() | |
| uses: getsentry/action-github-commit@v1.0.0 | |
| with: | |
| github-token: ${{ steps.token.outputs.token }} | |
| typing: | |
| name: "mypy typing" | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| name: Checkout code | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: 3.8 | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| make install-python-dependencies | |
| - name: Run mypy | |
| run: | | |
| make backend-typing | |
| config-validation: | |
| name: "Dataset Config Validation" | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 2 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| name: Checkout code | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: 3.8 | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| make install-python-dependencies | |
| - name: Validate configs | |
| run: | | |
| make validate-configs | |
| snuba-image: | |
| name: Build snuba CI image | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| outputs: | |
| branch: ${{ steps.branch.outputs.branch }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v3 | |
| - name: Get branch name | |
| id: branch | |
| # strip `refs/heads/` from $GITHUB_REF and replace `/` with `-` so that | |
| # it can be used as a docker tag | |
| run: echo "branch=$(echo ${GITHUB_REF#refs/heads/} | tr / -)" >> "$GITHUB_OUTPUT" | |
| # We are only using ghcr here for CI as `setup-gcloud` is a bit slow | |
| # Should revisit this when we move off of google cloud build (we may want to move these to GCR) | |
| - name: Registry login | |
| run: | | |
| echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io -u $GITHUB_ACTOR --password-stdin | |
| # These are pulled in order to be able to use docker layer caching | |
| - name: Pull snuba CI images | |
| if: github.repository_owner == 'getsentry' | |
| run: | | |
| docker pull ghcr.io/getsentry/snuba-ci:${{ github.sha }} || \ | |
| docker pull ghcr.io/getsentry/snuba-ci:${{ steps.branch.outputs.branch }} || \ | |
| docker pull ghcr.io/getsentry/snuba-ci:latest || true | |
| - name: Build snuba docker image for CI | |
| if: github.repository_owner == 'getsentry' | |
| run: | | |
| docker build . \ | |
| --build-arg BUILDKIT_INLINE_CACHE=1 \ | |
| --build-arg SHOULD_BUILD_RUST=false \ | |
| -t ghcr.io/getsentry/snuba-ci:latest \ | |
| -t ghcr.io/getsentry/snuba-ci:${{ github.sha }} \ | |
| -t ghcr.io/getsentry/snuba-ci:${{ steps.branch.outputs.branch }} \ | |
| --cache-from ghcr.io/getsentry/snuba-ci:latest \ | |
| --cache-from ghcr.io/getsentry/snuba-ci:${{ steps.branch.outputs.branch }} \ | |
| --cache-from ghcr.io/getsentry/snuba-ci:${{ github.sha }} \ | |
| --target testing | |
| - name: Publish images for cache | |
| if: steps.branch.outputs.branch == 'master' || github.event.pull_request.head.repo.full_name == github.repository | |
| # outside contributors won't be able to push to the docker registry | |
| # ignore the failures in this step | |
| continue-on-error: ${{ github.event_name == 'pull_request' }} | |
| run: | | |
| # Useful to speed up CI | |
| docker push ghcr.io/getsentry/snuba-ci:${{ steps.branch.outputs.branch }} | |
| docker push ghcr.io/getsentry/snuba-ci:${{ github.sha }} | |
| if [ "${{ steps.branch.outputs.branch }}" == 'master' ]; then | |
| # The latest tag should only be published on `master` | |
| docker push ghcr.io/getsentry/snuba-ci:latest | |
| fi | |
| tests: | |
| needs: [linting, snuba-image] | |
| name: Tests and code coverage | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 | |
| strategy: | |
| matrix: | |
| snuba_settings: | |
| ["test", "test_distributed", "test_distributed_migrations"] | |
| steps: | |
| - uses: actions/checkout@v2 | |
| name: Checkout code | |
| - name: Pull snuba CI images | |
| run: | | |
| docker pull ghcr.io/getsentry/snuba-ci:${{ github.sha }} || \ | |
| docker pull ghcr.io/getsentry/snuba-ci:${{ needs.snuba-image.outputs.branch }} || \ | |
| docker pull ghcr.io/getsentry/snuba-ci:latest || true | |
| - name: Build snuba docker image for CI | |
| run: | | |
| docker build . \ | |
| -t snuba-test \ | |
| --build-arg SHOULD_BUILD_RUST=false \ | |
| --cache-from ghcr.io/getsentry/snuba-ci:${{ github.sha }} \ | |
| --cache-from ghcr.io/getsentry/snuba-ci:${{ needs.snuba-image.outputs.branch }} \ | |
| --cache-from ghcr.io/getsentry/snuba-ci:latest \ | |
| --target testing | |
| - name: Docker set up | |
| run: | | |
| docker network create --attachable cloudbuild | |
| - name: Docker Snuba tests | |
| run: | | |
| SNUBA_IMAGE=snuba-test SNUBA_SETTINGS=${{ matrix.snuba_settings }} docker-compose -f docker-compose.gcb.yml run --rm snuba-test | |
| if: ${{ matrix.snuba_settings != 'test_distributed_migrations' }} | |
| - name: Docker Snuba Multi-Node Tests | |
| run: | | |
| SNUBA_SETTINGS=test_distributed_migrations docker-compose --profile multi_node -f docker-compose.gcb.yml up -d | |
| SNUBA_IMAGE=snuba-test SNUBA_SETTINGS=test_distributed_migrations TEST_LOCATION=test_distributed_migrations docker-compose --profile multi_node -f docker-compose.gcb.yml run --rm snuba-test | |
| if: ${{ matrix.snuba_settings == 'test_distributed_migrations' }} | |
| - name: Docker Snuba Init Tests | |
| run: | | |
| SNUBA_IMAGE=snuba-test SNUBA_SETTINGS=test_initialization TEST_LOCATION=test_initialization docker-compose -f docker-compose.gcb.yml run --rm snuba-test | |
| if: ${{ matrix.snuba_settings == 'test' }} | |
| - name: Upload to codecov | |
| run: | | |
| curl -Os https://uploader.codecov.io/latest/linux/codecov && chmod +x codecov && ./codecov -t ${CODECOV_TOKEN} | |
| admin-tests: | |
| needs: [linting] | |
| name: Front end tests for snuba admin | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v2 | |
| name: Checkout code | |
| - name: Set up and run tests through yarn | |
| run: cd snuba/admin && yarn install && yarn run test --coverage | |
| - name: Upload to codecov | |
| run: | | |
| curl -Os https://uploader.codecov.io/latest/linux/codecov && chmod +x codecov && ./codecov -t ${CODECOV_TOKEN} | |
| sentry: | |
| needs: [snuba-image] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 90 | |
| strategy: | |
| matrix: | |
| instance: [0, 1] | |
| env: | |
| # XXX: MATRIX_INSTANCE_TOTAL must be hardcoded to the length of strategy.matrix.instance. | |
| MATRIX_INSTANCE_TOTAL: 2 | |
| MIGRATIONS_TEST_MIGRATE: 1 | |
| steps: | |
| # Checkout codebase | |
| - name: Checkout snuba | |
| uses: actions/checkout@v2 | |
| - name: Pull snuba CI images | |
| run: | | |
| docker pull ghcr.io/getsentry/snuba-ci:${{ github.sha }} || \ | |
| docker pull ghcr.io/getsentry/snuba-ci:${{ needs.snuba-image.outputs.branch }} || \ | |
| docker pull ghcr.io/getsentry/snuba-ci:latest || true | |
| - name: Build snuba docker image for CI | |
| run: | | |
| docker build . \ | |
| -t snuba-test \ | |
| --build-arg SHOULD_BUILD_RUST=false \ | |
| --cache-from ghcr.io/getsentry/snuba-ci:latest \ | |
| --cache-from ghcr.io/getsentry/snuba-ci:${{ needs.snuba-image.outputs.branch }} \ | |
| --cache-from ghcr.io/getsentry/snuba-ci:${{ github.sha }} \ | |
| --target testing | |
| # Checkout Sentry and run integration tests against latest snuba | |
| # Make sure this is after `docker build`, otherwise we'll break docker cache | |
| - name: Checkout sentry | |
| uses: actions/checkout@v2 | |
| with: | |
| repository: getsentry/sentry | |
| path: sentry | |
| - name: Setup steps | |
| id: setup | |
| run: | | |
| pip install --upgrade pip wheel | |
| # We cannot execute actions that are not placed under .github of the main repo | |
| mkdir -p .github/actions | |
| cp -r sentry/.github/actions/* .github/actions | |
| - name: Setup Sentry | |
| id: setup-sentry | |
| uses: ./.github/actions/setup-sentry | |
| with: | |
| workdir: sentry | |
| cache-files-hash: ${{ hashFiles('sentry/requirements**.txt') }} | |
| python-version: 3.8 | |
| snuba: false | |
| kafka: true | |
| clickhouse: true | |
| - name: Start snuba | |
| run: | | |
| docker run -d --rm \ | |
| -p 127.0.0.1:1218:1218 \ | |
| -e PYTHONUNBUFFERED=1 \ | |
| -e SNUBA_SETTINGS=docker \ | |
| -e DEBUG=1 \ | |
| -e DEFAULT_BROKERS=sentry_kafka:9092 \ | |
| -e CLICKHOUSE_HOST=sentry_clickhouse \ | |
| -e CLICKHOUSE_PORT=9000 \ | |
| -e CLICKHOUSE_HTTP_PORT=8123 \ | |
| -e REDIS_HOST=sentry_redis \ | |
| -e REDIS_PORT=6379 \ | |
| -e REDIS_DB=1 \ | |
| --name sentry_snuba \ | |
| --network sentry \ | |
| snuba-test | |
| docker exec sentry_snuba snuba migrations migrate --force | |
| - name: Run snuba tests | |
| working-directory: sentry | |
| run: | | |
| make test-snuba | |
| self-hosted-end-to-end: | |
| needs: [snuba-image] | |
| runs-on: ubuntu-latest | |
| # temporary, remove once we are confident the action is working | |
| continue-on-error: true | |
| steps: | |
| - name: Checkout Snuba | |
| uses: actions/checkout@v3 | |
| - name: Run Sentry self-hosted e2e CI | |
| uses: getsentry/action-self-hosted-e2e-tests@f45ef07793b2cc805a9a9401819f486da449a90a | |
| with: | |
| project_name: snuba | |
| docker_repo: getsentry/snuba | |
| image_url: us.gcr.io/sentryio/snuba:${{ github.event.pull_request.head.sha || github.sha }} | |
| docker_password: ${{ secrets.DOCKER_HUB_RW_TOKEN }} |