Introduce pipedream

getsentry · Jun 27, 2023 · ffce590 · ffce590
1 parent d5c494a
commit ffce590
Show file tree

Hide file tree

Showing 18 changed files with 871 additions and 0 deletions.
diff --git a/.gitignore b/.gitignore
@@ -13,3 +13,4 @@ node_modules
 .vscode/*.log
 snuba/admin/dist/bundle.js*
 tmp/
+gocd/templates/vendor/
diff --git a/Makefile b/Makefile
@@ -84,3 +84,13 @@ lint-rust:
 	cd rust_snuba && cargo clippy -- -D warnings
 
 .PHONY: lint-rust
+
+gocd:
+	rm -rf ./gocd/generated-pipelines
+	mkdir -p ./gocd/generated-pipelines
+	cd ./gocd/templates && jb install
+	find . -type f \( -name '*.libsonnet' -o -name '*.jsonnet' \) -print0 | xargs -n 1 -0 jsonnetfmt -i
+	find . -type f \( -name '*.libsonnet' -o -name '*.jsonnet' \) -print0 | xargs -n 1 -0 jsonnet-lint -J ./gocd/templates/vendor
+	cd ./gocd/templates && jsonnet -J vendor -m ../generated-pipelines ./snuba.jsonnet
+	cd ./gocd/generated-pipelines && find . -type f \( -name '*.yaml' \) -print0 | xargs -n 1 -0 yq -p json -o yaml -i
+.PHONY: gocd
diff --git a/gocd/README.md b/gocd/README.md
@@ -0,0 +1,59 @@
+# Snuba Pipelines
+
+Relay is in the process of moving to a set of rendered jsonnet pipelines.
+
+## Jsonnet
+
+You can render the jsonnet pipelines by running:
+
+```
+make gocd
+```
+
+This will clean, fmt, lint and generate the GoCD pipelines to
+`./gocd/generated-pipelines`.
+
+
+The snuba pipelines are using the https://github.com/getsentry/gocd-jsonnet
+libraries to generate the pipeline for each region.
+
+## Files
+
+Below is a description of the directories in the `gocd/` directory.
+
+### `gocd/templates/`
+
+These are a set of jsonnet and libsonnet files which are used
+to generate the GoCD pipelines. This avoids duplication across
+our pipelines as we deploy to multiple regions.
+
+The `gocd/templates/snuba.jsonnet` file is the entry point for the
+generated pipelines.
+
+`gocd/templates/pipelines/snuba.libsonnet` define the pipeline behaviors.
+This library defines the GoCD pipeline, following the same naming
+as the [GoCD yaml pipelines](https://github.com/tomzo/gocd-yaml-config-plugin#readme).
+
+`gocd/templates/bash/*.sh` are shell scripts that are inlined in the
+resulting pipelines. This seperation means syntax highlighting and
+extra tooling works for bash scripts.
+
+`gocd/templates/jsonnetfile.json` and `gocd/templates/jsonnetfile.lock.json`
+are used by [jsonnet-bundler](https://github.com/jsonnet-bundler/jsonnet-bundler#readme), a package manager for jsonnet.
+
+You can update jsonnet dependencies by runnning `jb update`.
+
+### `gocd/generated-pipelines/`
+
+The current setup of GoCD at sentry is only able to look for
+yaml pipelines in a repo, so the genered pipelines have the be
+commited.
+
+The dev-infra team is working on a GoCD plugin that will accept
+the jsonnet directly, removing the need for commiting the
+generated-pipelines.
+
+### `gocd/pipelines/`
+
+These are the original pipelines and will be used until we move
+to the jsonnet pipelines.
diff --git a/gocd/generated-pipelines/snuba-next-monitor.yaml b/gocd/generated-pipelines/snuba-next-monitor.yaml
@@ -0,0 +1,251 @@
+format_version: 10
+pipelines:
+  deploy-snuba-next-monitor:
+    environment_variables:
+      GITHUB_TOKEN: '{{SECRET:[devinfra-github][token]}}'
+      GOCD_ACCESS_TOKEN: '{{SECRET:[devinfra][gocd_access_token]}}'
+      SENTRY_REGION: monitor
+    group: snuba-next-regions
+    lock_behavior: unlockWhenFinished
+    materials:
+      deploy-snuba-next-us-pipeline-complete:
+        pipeline: deploy-snuba-next-us
+        stage: pipeline-complete
+      snuba_repo:
+        branch: master
+        destination: snuba
+        git: git@github.com:getsentry/snuba.git
+        shallow_clone: false
+    stages:
+      - checks:
+          jobs:
+            checks:
+              elastic_profile_id: snuba
+              tasks:
+                - script: |
+                    ##!/bin/bash
+
+                    /devinfra/scripts/checks/githubactions/checkruns.py \
+                      getsentry/snuba \
+                      ${GO_REVISION_SNUBA_REPO} \
+                      "Tests and code coverage (test)" \
+                      "Tests and code coverage (test_distributed)" \
+                      "Tests and code coverage (test_distributed_migrations)" \
+                      "Dataset Config Validation" \
+                      "sentry (0)" \
+                      "sentry (1)" \
+                      "self-hosted-end-to-end"
+                - script: |
+                    ##!/bin/bash
+
+                    /devinfra/scripts/checks/googlecloud/checkcloudbuild.py \
+                      ${GO_REVISION_SNUBA_REPO} \
+                      sentryio \
+                      "us.gcr.io/sentryio/snuba"
+                - script: |
+                    ##!/bin/bash
+
+                    deploy_sha=`snuba/scripts/fetch_service_refs.py --pipeline "deploy-snuba"`
+                    snuba/scripts/check-migrations.py --to $deploy_sha --workdir snuba
+              timeout: 1800
+      - deploy-canary:
+          fetch_materials: true
+          jobs:
+            create-sentry-release:
+              elastic_profile_id: snuba
+              environment_variables:
+                SENTRY_AUTH_TOKEN: '{{SECRET:[devinfra-sentryio][token]}}'
+                SENTRY_ORG: sentry
+                SENTRY_PROJECT: snuba
+              tasks:
+                - script: |
+                    ##!/bin/bash
+
+                    sentry-cli releases new "${GO_REVISION_SNUBA_REPO}"
+                    sentry-cli releases set-commits "${GO_REVISION_SNUBA_REPO}" --commit "getsentry/snuba@${GO_REVISION_SNUBA_REPO}"
+                    sentry-cli releases deploys "${GO_REVISION_SNUBA_REPO}" new -e canary
+              timeout: 300
+            deploy-canary:
+              elastic_profile_id: snuba
+              environment_variables:
+                LABEL_SELECTOR: service=snuba,is_canary=true
+              tasks:
+                - script: |
+                    ##!/bin/bash
+
+                    eval $(/devinfra/scripts/regions/project_env_vars.py --region="${SENTRY_REGION}")
+
+                    /devinfra/scripts/k8s/k8stunnel \
+                    && /devinfra/scripts/k8s/k8s-deploy.py \
+                      --context="gke_${GCP_PROJECT}_${GKE_REGION}-${GKE_CLUSTER_ZONE}_${GKE_CLUSTER}" \
+                      --label-selector="${LABEL_SELECTOR}" \
+                      --image="us.gcr.io/sentryio/snuba:${GO_REVISION_SNUBA_REPO}" \
+                      --container-name="api" \
+                      --container-name="consumer" \
+                      --container-name="errors-consumer" \
+                      --container-name="errors-replacer" \
+                      --container-name="events-subscriptions-executor" \
+                      --container-name="events-subscriptions-scheduler" \
+                      --container-name="generic-metrics-counters-consumer" \
+                      --container-name="generic-metrics-counters-subscriptions-executor" \
+                      --container-name="generic-metrics-counters-subscriptions-scheduler" \
+                      --container-name="generic-metrics-distributions-consumer" \
+                      --container-name="generic-metrics-distributions-subscriptions-executor" \
+                      --container-name="generic-metrics-distributions-subscriptions-scheduler" \
+                      --container-name="generic-metrics-sets-consumer" \
+                      --container-name="generic-metrics-sets-subscriptions-executor" \
+                      --container-name="generic-metrics-sets-subscriptions-scheduler" \
+                      --container-name="loadbalancer-outcomes-consumer" \
+                      --container-name="loadtest-errors-consumer" \
+                      --container-name="loadtest-loadbalancer-outcomes-consumer" \
+                      --container-name="loadtest-outcomes-consumer" \
+                      --container-name="loadtest-transactions-consumer" \
+                      --container-name="metrics-consumer" \
+                      --container-name="metrics-counters-subscriptions-scheduler" \
+                      --container-name="metrics-sets-subscriptions-scheduler" \
+                      --container-name="metrics-subscriptions-executor" \
+                      --container-name="outcomes-billing-consumer" \
+                      --container-name="outcomes-consumer" \
+                      --container-name="profiles-consumer" \
+                      --container-name="profiling-functions-consumer" \
+                      --container-name="querylog-consumer" \
+                      --container-name="replacer" \
+                      --container-name="replays-consumer" \
+                      --container-name="search-issues-consumer" \
+                      --container-name="snuba-admin" \
+                      --container-name="transactions-consumer-new" \
+                      --container-name="transactions-subscriptions-executor" \
+                      --container-name="transactions-subscriptions-scheduler" \
+                      --container-name="rust-querylog-consumer" \
+                      --container-name="spans-consumer" \
+                      --container-name="dlq-consumer" \
+                    && /devinfra/scripts/k8s/k8s-deploy.py \
+                      --label-selector="${LABEL_SELECTOR}" \
+                      --image="us.gcr.io/sentryio/snuba:${GO_REVISION_SNUBA_REPO}" \
+                      --type="cronjob" \
+                      --container-name="cleanup" \
+                      --container-name="optimize"
+              timeout: 1200
+      - deploy-primary:
+          fetch_materials: true
+          jobs:
+            create-sentry-release:
+              elastic_profile_id: snuba
+              environment_variables:
+                SENTRY_AUTH_TOKEN: '{{SECRET:[devinfra-sentryio][token]}}'
+                SENTRY_ORG: sentry
+                SENTRY_PROJECT: snuba
+              tasks:
+                - script: |
+                    ##!/bin/bash
+
+                    sentry-cli releases deploys "${GO_REVISION_SNUBA_REPO}" new -e production
+                    sentry-cli releases finalize "${GO_REVISION_SNUBA_REPO}"
+              timeout: 300
+            deploy-canary:
+              elastic_profile_id: snuba
+              environment_variables:
+                LABEL_SELECTOR: service=snuba
+              tasks:
+                - script: |
+                    ##!/bin/bash
+
+                    eval $(/devinfra/scripts/regions/project_env_vars.py --region="${SENTRY_REGION}")
+
+                    /devinfra/scripts/k8s/k8stunnel \
+                    && /devinfra/scripts/k8s/k8s-deploy.py \
+                      --context="gke_${GCP_PROJECT}_${GKE_REGION}-${GKE_CLUSTER_ZONE}_${GKE_CLUSTER}" \
+                      --label-selector="${LABEL_SELECTOR}" \
+                      --image="us.gcr.io/sentryio/snuba:${GO_REVISION_SNUBA_REPO}" \
+                      --container-name="api" \
+                      --container-name="consumer" \
+                      --container-name="errors-consumer" \
+                      --container-name="errors-replacer" \
+                      --container-name="events-subscriptions-executor" \
+                      --container-name="events-subscriptions-scheduler" \
+                      --container-name="generic-metrics-counters-consumer" \
+                      --container-name="generic-metrics-counters-subscriptions-executor" \
+                      --container-name="generic-metrics-counters-subscriptions-scheduler" \
+                      --container-name="generic-metrics-distributions-consumer" \
+                      --container-name="generic-metrics-distributions-subscriptions-executor" \
+                      --container-name="generic-metrics-distributions-subscriptions-scheduler" \
+                      --container-name="generic-metrics-sets-consumer" \
+                      --container-name="generic-metrics-sets-subscriptions-executor" \
+                      --container-name="generic-metrics-sets-subscriptions-scheduler" \
+                      --container-name="loadbalancer-outcomes-consumer" \
+                      --container-name="loadtest-errors-consumer" \
+                      --container-name="loadtest-loadbalancer-outcomes-consumer" \
+                      --container-name="loadtest-outcomes-consumer" \
+                      --container-name="loadtest-transactions-consumer" \
+                      --container-name="metrics-consumer" \
+                      --container-name="metrics-counters-subscriptions-scheduler" \
+                      --container-name="metrics-sets-subscriptions-scheduler" \
+                      --container-name="metrics-subscriptions-executor" \
+                      --container-name="outcomes-billing-consumer" \
+                      --container-name="outcomes-consumer" \
+                      --container-name="profiles-consumer" \
+                      --container-name="profiling-functions-consumer" \
+                      --container-name="querylog-consumer" \
+                      --container-name="replacer" \
+                      --container-name="replays-consumer" \
+                      --container-name="search-issues-consumer" \
+                      --container-name="snuba-admin" \
+                      --container-name="transactions-consumer-new" \
+                      --container-name="transactions-subscriptions-executor" \
+                      --container-name="transactions-subscriptions-scheduler" \
+                      --container-name="rust-querylog-consumer" \
+                      --container-name="spans-consumer" \
+                      --container-name="dlq-consumer" \
+                    && /devinfra/scripts/k8s/k8s-deploy.py \
+                      --label-selector="${LABEL_SELECTOR}" \
+                      --image="us.gcr.io/sentryio/snuba:${GO_REVISION_SNUBA_REPO}" \
+                      --type="cronjob" \
+                      --container-name="cleanup" \
+                      --container-name="optimize"
+              timeout: 1200
+      - migrate:
+          fetch_materials: true
+          jobs:
+            migrate:
+              elastic_profile_id: snuba
+              tasks:
+                - script: |
+                    ##!/bin/bash
+
+                    eval $(/devinfra/scripts/regions/project_env_vars.py --region="${SENTRY_REGION}")
+
+                    /devinfra/scripts/k8s/k8stunnel \
+                    && /devinfra/scripts/k8s/k8s-spawn-job.py \
+                      --context="gke_${GCP_PROJECT}_${GKE_REGION}-${GKE_CLUSTER_ZONE}_${GKE_CLUSTER}" \
+                      --label-selector="service=snuba-admin" \
+                      --container-name="snuba-admin" \
+                      "snuba-migrate" "us.gcr.io/sentryio/snuba:${GO_REVISION_SNUBA_REPO}" \
+                      -- snuba migrations migrate -r complete -r partial
+                - plugin:
+                    configuration:
+                      id: script-executor
+                      version: 1
+                    options:
+                      script: |
+                        ##!/bin/bash
+
+                        eval $(/devinfra/scripts/regions/project_env_vars.py --region="${SENTRY_REGION}")
+
+                        /devinfra/scripts/k8s/k8stunnel \
+                        && /devinfra/scripts/k8s/k8s-spawn-job.py \
+                          --context="gke_${GCP_PROJECT}_${GKE_REGION}-${GKE_CLUSTER_ZONE}_${GKE_CLUSTER}" \
+                          --label-selector="service=snuba-admin" \
+                          --container-name="snuba-admin" \
+                          "snuba-migrate-reverse" "us.gcr.io/sentryio/snuba:${GO_REVISION_SNUBA_REPO}" \
+                          -- snuba migrations reverse-in-progress
+                    run_if: failed
+              timeout: 1200
+      - pipeline-complete:
+          approval:
+            allow_only_on_success: true
+            type: success
+          jobs:
+            pipeline-complete:
+              tasks:
+                - exec:
+                    command: true