Merge remote-tracking branch 'origin/master' into enoch/add-tracing-s…

…yntax-highlighting

.github/workflows/ci.yml

@@ -15,7 +15,7 @@ jobs:
  name: Checkout code
  - name: Internal github app token
  id: token
- uses: getsentry/action-github-app-token@97c9e23528286821f97fba885c1b1123284b29cc  # v2.0.0
+ uses: getsentry/action-github-app-token@97c9e23528286821f97fba885c1b1123284b29cc # v2.0.0
  continue-on-error: true
  with:
  app_id: ${{ vars.SENTRY_INTERNAL_APP_ID }}
@@ -161,7 +161,7 @@ jobs:
  needs: [linting, snuba-image]
  name: Tests and code coverage
  runs-on: ubuntu-latest
- timeout-minutes: 60
+ timeout-minutes: 30
  strategy:
  matrix:
  snuba_settings:
@@ -206,7 +206,6 @@ jobs:
  SNUBA_IMAGE=snuba-test SNUBA_SETTINGS=test_initialization TEST_LOCATION=test_initialization docker-compose -f docker-compose.gcb.yml run --rm snuba-test
  if: ${{ matrix.snuba_settings == 'test' }}
 
-
  - name: Upload to codecov
  run: |
  curl -Os https://uploader.codecov.io/latest/linux/codecov && chmod +x codecov && ./codecov -t ${CODECOV_TOKEN}

.github/workflows/lint-pipelines.yml

@@ -18,3 +18,16 @@ jobs:
  - uses: getsentry/action-setup-gocd-cli@2f7943ce1a380dea121fd6338a60dc9aabf8e7f1 # v1.0.1
  - name: Lint Pipelines with gocd-cli
  run: ./.github/workflows/lint-pipelines.sh
+
+ render:
+ name: Render GoCD Pipelines with Jsonnet
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3
+ - uses: getsentry/action-gocd-jsonnet@v0
+ with:
+ jb-install: true
+ check-for-changes: true
+ convert-to-yaml: true
+ jsonnet-dir: gocd/templates
+ generated-dir: gocd/generated-pipelines

.gitignore

@@ -13,3 +13,4 @@ node_modules
 .vscode/*.log
 snuba/admin/dist/bundle.js*
 tmp/
+gocd/templates/vendor/

Makefile

@@ -84,3 +84,13 @@ lint-rust:
  cd rust_snuba && cargo clippy -- -D warnings
 
 .PHONY: lint-rust
+
+gocd:
+ rm -rf ./gocd/generated-pipelines
+ mkdir -p ./gocd/generated-pipelines
+ cd ./gocd/templates && jb install
+ find . -type f \( -name '*.libsonnet' -o -name '*.jsonnet' \) -print0 | xargs -n 1 -0 jsonnetfmt -i
+ find . -type f \( -name '*.libsonnet' -o -name '*.jsonnet' \) -print0 | xargs -n 1 -0 jsonnet-lint -J ./gocd/templates/vendor
+ cd ./gocd/templates && jsonnet -J vendor -m ../generated-pipelines ./snuba.jsonnet
+ cd ./gocd/generated-pipelines && find . -type f \( -name '*.yaml' \) -print0 | xargs -n 1 -0 yq -p json -o yaml -i
+.PHONY: gocd

docs-requirements.txt

@@ -1,5 +1,5 @@
 jsonschema2md==0.4.0
 fastjsonschema==2.16.2
-sentry-sdk==1.18.0
+sentry-sdk==1.26.0
 myst-parser==0.18.0
 sphinx==5.1.1

gocd/README.md

@@ -0,0 +1,59 @@
+# Snuba Pipelines
+
+Relay is in the process of moving to a set of rendered jsonnet pipelines.
+
+## Jsonnet
+
+You can render the jsonnet pipelines by running:
+
+```
+make gocd
+```
+
+This will clean, fmt, lint and generate the GoCD pipelines to
+`./gocd/generated-pipelines`.
+
+
+The snuba pipelines are using the https://github.com/getsentry/gocd-jsonnet
+libraries to generate the pipeline for each region.
+
+## Files
+
+Below is a description of the directories in the `gocd/` directory.
+
+### `gocd/templates/`
+
+These are a set of jsonnet and libsonnet files which are used
+to generate the GoCD pipelines. This avoids duplication across
+our pipelines as we deploy to multiple regions.
+
+The `gocd/templates/snuba.jsonnet` file is the entry point for the
+generated pipelines.
+
+`gocd/templates/pipelines/snuba.libsonnet` define the pipeline behaviors.
+This library defines the GoCD pipeline, following the same naming
+as the [GoCD yaml pipelines](https://github.com/tomzo/gocd-yaml-config-plugin#readme).
+
+`gocd/templates/bash/*.sh` are shell scripts that are inlined in the
+resulting pipelines. This seperation means syntax highlighting and
+extra tooling works for bash scripts.
+
+`gocd/templates/jsonnetfile.json` and `gocd/templates/jsonnetfile.lock.json`
+are used by [jsonnet-bundler](https://github.com/jsonnet-bundler/jsonnet-bundler#readme), a package manager for jsonnet.
+
+You can update jsonnet dependencies by runnning `jb update`.
+
+### `gocd/generated-pipelines/`
+
+The current setup of GoCD at sentry is only able to look for
+yaml pipelines in a repo, so the genered pipelines have the be
+commited.
+
+The dev-infra team is working on a GoCD plugin that will accept
+the jsonnet directly, removing the need for commiting the
+generated-pipelines.
+
+### `gocd/pipelines/`
+
+These are the original pipelines and will be used until we move
+to the jsonnet pipelines.

gocd/generated-pipelines/snuba-next-monitor.yaml

@@ -0,0 +1,266 @@
+format_version: 10
+pipelines:
+ deploy-snuba-next-monitor:
+ display_order: 2
+ environment_variables:
+ GITHUB_TOKEN: '{{SECRET:[devinfra-github][token]}}'
+ GOCD_ACCESS_TOKEN: '{{SECRET:[devinfra][gocd_access_token]}}'
+ SENTRY_REGION: monitor
+ group: snuba-next
+ lock_behavior: unlockWhenFinished
+ materials:
+ deploy-snuba-next-us-pipeline-complete:
+ pipeline: deploy-snuba-next-us
+ stage: pipeline-complete
+ snuba_repo:
+ branch: master
+ destination: snuba
+ git: git@github.com:getsentry/snuba.git
+ shallow_clone: false
+ stages:
+ - ready:
+ jobs:
+ ready:
+ tasks:
+ - exec:
+ command: true
+ - wait:
+ approval:
+ type: manual
+ jobs:
+ wait:
+ tasks:
+ - exec:
+ command: true
+ - checks:
+ jobs:
+ checks:
+ elastic_profile_id: snuba
+ tasks:
+ - script: |
+ ##!/bin/bash
+
+ /devinfra/scripts/checks/githubactions/checkruns.py \
+ getsentry/snuba \
+ ${GO_REVISION_SNUBA_REPO} \
+ "Tests and code coverage (test)" \
+ "Tests and code coverage (test_distributed)" \
+ "Tests and code coverage (test_distributed_migrations)" \
+ "Dataset Config Validation" \
+ "sentry (0)" \
+ "sentry (1)" \
+ "self-hosted-end-to-end"
+ - script: |
+ ##!/bin/bash
+
+ /devinfra/scripts/checks/googlecloud/checkcloudbuild.py \
+ ${GO_REVISION_SNUBA_REPO} \
+ sentryio \
+ "us.gcr.io/sentryio/snuba"
+ - script: |
+ ##!/bin/bash
+
+ deploy_sha=`snuba/scripts/fetch_service_refs.py --pipeline "deploy-snuba"`
+ snuba/scripts/check-migrations.py --to $deploy_sha --workdir snuba
+ timeout: 1800
+ - deploy-canary:
+ fetch_materials: true
+ jobs:
+ create-sentry-release:
+ elastic_profile_id: snuba
+ environment_variables:
+ SENTRY_AUTH_TOKEN: '{{SECRET:[devinfra-sentryio][token]}}'
+ SENTRY_ORG: sentry
+ SENTRY_PROJECT: snuba
+ tasks:
+ - script: |
+ ##!/bin/bash
+
+ sentry-cli releases new "${GO_REVISION_SNUBA_REPO}"
+ sentry-cli releases set-commits "${GO_REVISION_SNUBA_REPO}" --commit "getsentry/snuba@${GO_REVISION_SNUBA_REPO}"
+ sentry-cli releases deploys "${GO_REVISION_SNUBA_REPO}" new -e canary
+ timeout: 300
+ deploy-canary:
+ elastic_profile_id: snuba
+ environment_variables:
+ LABEL_SELECTOR: service=snuba,is_canary=true
+ tasks:
+ - script: |
+ ##!/bin/bash
+
+ eval $(/devinfra/scripts/regions/project_env_vars.py --region="${SENTRY_REGION}")
+
+ /devinfra/scripts/k8s/k8stunnel \
+ && /devinfra/scripts/k8s/k8s-deploy.py \
+ --context="gke_${GCP_PROJECT}_${GKE_REGION}-${GKE_CLUSTER_ZONE}_${GKE_CLUSTER}" \
+ --label-selector="${LABEL_SELECTOR}" \
+ --image="us.gcr.io/sentryio/snuba:${GO_REVISION_SNUBA_REPO}" \
+ --container-name="api" \
+ --container-name="consumer" \
+ --container-name="errors-consumer" \
+ --container-name="errors-replacer" \
+ --container-name="events-subscriptions-executor" \
+ --container-name="events-subscriptions-scheduler" \
+ --container-name="generic-metrics-counters-consumer" \
+ --container-name="generic-metrics-counters-subscriptions-executor" \
+ --container-name="generic-metrics-counters-subscriptions-scheduler" \
+ --container-name="generic-metrics-distributions-consumer" \
+ --container-name="generic-metrics-distributions-subscriptions-executor" \
+ --container-name="generic-metrics-distributions-subscriptions-scheduler" \
+ --container-name="generic-metrics-sets-consumer" \
+ --container-name="generic-metrics-sets-subscriptions-executor" \
+ --container-name="generic-metrics-sets-subscriptions-scheduler" \
+ --container-name="loadbalancer-outcomes-consumer" \
+ --container-name="loadtest-errors-consumer" \
+ --container-name="loadtest-loadbalancer-outcomes-consumer" \
+ --container-name="loadtest-outcomes-consumer" \
+ --container-name="loadtest-transactions-consumer" \
+ --container-name="metrics-consumer" \
+ --container-name="metrics-counters-subscriptions-scheduler" \
+ --container-name="metrics-sets-subscriptions-scheduler" \
+ --container-name="metrics-subscriptions-executor" \
+ --container-name="outcomes-billing-consumer" \
+ --container-name="outcomes-consumer" \
+ --container-name="profiles-consumer" \
+ --container-name="profiling-functions-consumer" \
+ --container-name="querylog-consumer" \
+ --container-name="replacer" \
+ --container-name="replays-consumer" \
+ --container-name="search-issues-consumer" \
+ --container-name="snuba-admin" \
+ --container-name="transactions-consumer-new" \
+ --container-name="transactions-subscriptions-executor" \
+ --container-name="transactions-subscriptions-scheduler" \
+ --container-name="rust-querylog-consumer" \
+ --container-name="spans-consumer" \
+ --container-name="dlq-consumer" \
+ && /devinfra/scripts/k8s/k8s-deploy.py \
+ --label-selector="${LABEL_SELECTOR}" \
+ --image="us.gcr.io/sentryio/snuba:${GO_REVISION_SNUBA_REPO}" \
+ --type="cronjob" \
+ --container-name="cleanup" \
+ --container-name="optimize"
+ timeout: 1200
+ - deploy-primary:
+ fetch_materials: true
+ jobs:
+ create-sentry-release:
+ elastic_profile_id: snuba
+ environment_variables:
+ SENTRY_AUTH_TOKEN: '{{SECRET:[devinfra-sentryio][token]}}'
+ SENTRY_ORG: sentry
+ SENTRY_PROJECT: snuba
+ tasks:
+ - script: |
+ ##!/bin/bash
+
+ sentry-cli releases deploys "${GO_REVISION_SNUBA_REPO}" new -e production
+ sentry-cli releases finalize "${GO_REVISION_SNUBA_REPO}"
+ timeout: 300
+ deploy-canary:
+ elastic_profile_id: snuba
+ environment_variables:
+ LABEL_SELECTOR: service=snuba
+ tasks:
+ - script: |
+ ##!/bin/bash
+
+ eval $(/devinfra/scripts/regions/project_env_vars.py --region="${SENTRY_REGION}")
+
+ /devinfra/scripts/k8s/k8stunnel \
+ && /devinfra/scripts/k8s/k8s-deploy.py \
+ --context="gke_${GCP_PROJECT}_${GKE_REGION}-${GKE_CLUSTER_ZONE}_${GKE_CLUSTER}" \
+ --label-selector="${LABEL_SELECTOR}" \
+ --image="us.gcr.io/sentryio/snuba:${GO_REVISION_SNUBA_REPO}" \
+ --container-name="api" \
+ --container-name="consumer" \
+ --container-name="errors-consumer" \
+ --container-name="errors-replacer" \
+ --container-name="events-subscriptions-executor" \
+ --container-name="events-subscriptions-scheduler" \
+ --container-name="generic-metrics-counters-consumer" \
+ --container-name="generic-metrics-counters-subscriptions-executor" \
+ --container-name="generic-metrics-counters-subscriptions-scheduler" \
+ --container-name="generic-metrics-distributions-consumer" \
+ --container-name="generic-metrics-distributions-subscriptions-executor" \
+ --container-name="generic-metrics-distributions-subscriptions-scheduler" \
+ --container-name="generic-metrics-sets-consumer" \
+ --container-name="generic-metrics-sets-subscriptions-executor" \
+ --container-name="generic-metrics-sets-subscriptions-scheduler" \
+ --container-name="loadbalancer-outcomes-consumer" \
+ --container-name="loadtest-errors-consumer" \
+ --container-name="loadtest-loadbalancer-outcomes-consumer" \
+ --container-name="loadtest-outcomes-consumer" \
+ --container-name="loadtest-transactions-consumer" \
+ --container-name="metrics-consumer" \
+ --container-name="metrics-counters-subscriptions-scheduler" \
+ --container-name="metrics-sets-subscriptions-scheduler" \
+ --container-name="metrics-subscriptions-executor" \
+ --container-name="outcomes-billing-consumer" \
+ --container-name="outcomes-consumer" \
+ --container-name="profiles-consumer" \
+ --container-name="profiling-functions-consumer" \
+ --container-name="querylog-consumer" \
+ --container-name="replacer" \
+ --container-name="replays-consumer" \
+ --container-name="search-issues-consumer" \
+ --container-name="snuba-admin" \
+ --container-name="transactions-consumer-new" \
+ --container-name="transactions-subscriptions-executor" \
+ --container-name="transactions-subscriptions-scheduler" \
+ --container-name="rust-querylog-consumer" \
+ --container-name="spans-consumer" \
+ --container-name="dlq-consumer" \
+ && /devinfra/scripts/k8s/k8s-deploy.py \
+ --label-selector="${LABEL_SELECTOR}" \
+ --image="us.gcr.io/sentryio/snuba:${GO_REVISION_SNUBA_REPO}" \
+ --type="cronjob" \
+ --container-name="cleanup" \
+ --container-name="optimize"
+ timeout: 1200
+ - migrate:
+ fetch_materials: true
+ jobs:
+ migrate:
+ elastic_profile_id: snuba
+ tasks:
+ - script: |
+ ##!/bin/bash
+
+ eval $(/devinfra/scripts/regions/project_env_vars.py --region="${SENTRY_REGION}")
+
+ /devinfra/scripts/k8s/k8stunnel \
+ && /devinfra/scripts/k8s/k8s-spawn-job.py \
+ --context="gke_${GCP_PROJECT}_${GKE_REGION}-${GKE_CLUSTER_ZONE}_${GKE_CLUSTER}" \
+ --label-selector="service=snuba-admin" \
+ --container-name="snuba-admin" \
+ "snuba-migrate" "us.gcr.io/sentryio/snuba:${GO_REVISION_SNUBA_REPO}" \
+ -- snuba migrations migrate -r complete -r partial
+ - plugin:
+ configuration:
+ id: script-executor
+ version: 1
+ options:
+ script: |
+ ##!/bin/bash
+
+ eval $(/devinfra/scripts/regions/project_env_vars.py --region="${SENTRY_REGION}")
+
+ /devinfra/scripts/k8s/k8stunnel \
+ && /devinfra/scripts/k8s/k8s-spawn-job.py \
+ --context="gke_${GCP_PROJECT}_${GKE_REGION}-${GKE_CLUSTER_ZONE}_${GKE_CLUSTER}" \
+ --label-selector="service=snuba-admin" \
+ --container-name="snuba-admin" \
+ "snuba-migrate-reverse" "us.gcr.io/sentryio/snuba:${GO_REVISION_SNUBA_REPO}" \
+ -- snuba migrations reverse-in-progress
+ run_if: failed
+ timeout: 1200
+ - pipeline-complete:
+ approval:
+ allow_only_on_success: true
+ type: success
+ jobs:
+ pipeline-complete:
+ tasks:
+ - exec:
+ command: true