Introduce relay pipedream (#2248)

Co-authored-by: Iker Barriocanal <32816711+iker-barriocanal@users.noreply.github.com>

.github/workflows/lint-pipelines.yml

@@ -22,3 +22,16 @@ jobs:
  echo "GoCD YAML Linting"
  find "gocd" -name "*.yaml" -type f -print0 | \
  xargs -0 -I'{}' bash -c 'printf "\n🔎 Linting {}\n\t" && gocd-cli configrepo syntax --yaml --raw "{}"'
+
+ render:
+ name: Render GoCD Pipelines with Jsonnet
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3
+ - uses: getsentry/action-gocd-jsonnet@v0
+ with:
+ jb-install: true
+ check-for-changes: true
+ convert-to-yaml: true
+ jsonnet-dir: gocd/templates
+ generated-dir: gocd/generated-pipelines

.gitignore

@@ -17,3 +17,6 @@ target
 # NPM utilities for building docs
 /node_modules/
 package-lock.json
+
+# Jsonnet-bundler
+gocd/templates/vendor/

Makefile

@@ -176,3 +176,13 @@ clean-target-dir:
 help: ## this help
  @ awk 'BEGIN {FS = ":.*##"; printf "Usage: make \033[36m<target>\033[0m\n\nTargets:\n"} /^[a-zA-Z_-]+:.*?##/ { printf " \033[36m%-10s\033[0m\t%s\n", $$1, $$2 }' $(MAKEFILE_LIST) | column -s$$'\t' -t
 .PHONY: help
+
+gocd: ## Build GoCD pipelines
+ @ rm -rf ./gocd/generated-pipelines
+ @ mkdir -p ./gocd/generated-pipelines
+ @ cd ./gocd/templates && jb install
+ @ find . -type f \( -name '*.libsonnet' -o -name '*.jsonnet' \) -print0 | xargs -n 1 -0 jsonnetfmt -i
+ @ find . -type f \( -name '*.libsonnet' -o -name '*.jsonnet' \) -print0 | xargs -n 1 -0 jsonnet-lint -J ./gocd/templates/vendor
+ @ cd ./gocd/templates && jsonnet -J vendor -m ../generated-pipelines ./relay.jsonnet
+ @ cd ./gocd/generated-pipelines && find . -type f \( -name '*.yaml' \) -print0 | xargs -n 1 -0 yq -p json -o yaml -i
+.PHONY: gocd

gocd/README.md

@@ -0,0 +1,59 @@
+# Relay Pipelines
+
+Relay is in the process of moving to a set of rendered jsonnet pipelines.
+
+## Jsonnet
+
+You can render the jsonnet pipelines by running:
+
+```
+make gocd
+```
+
+This will clean, fmt, lint and generate the GoCD pipelines to
+`./gocd/generated-pipelines`.
+
+
+The Relay pipelines are using the https://github.com/getsentry/gocd-jsonnet
+libraries to generate the pipeline for each region.
+
+## Files
+
+Below is a description of the directories in the `gocd/` directory.
+
+### `gocd/templates/`
+
+These are a set of jsonnet and libsonnet files which are used
+to generate the relay pipelines. This avoids duplication across
+our GoCD pipeline files as we deploy to multiple regions.
+
+The `gocd/templates/relay.jsonnet` file is the entry point for the
+relay pipelines.
+
+`gocd/templates/libs/*.libsonnet` define the pipeline behaviors for
+deploy relay and relay-pops. These libraries are used to create a
+GoCD pipeline, following the same naming as the
+[GoCD yaml pipelines](https://github.com/tomzo/gocd-yaml-config-plugin#readme).
+
+`gocd/templates/bash/*.sh` are shell scripts that are inlined in the
+result pipelines. This seperation means syntax highlighting and
+extra tooling works for relay's bash scripts.
+
+`gocd/templates/jsonnetfile.json` and `gocd/templates/jsonnetfile.lock.json`
+are used by [jsonnet-bundler](https://github.com/jsonnet-bundler/jsonnet-bundler#readme), a package manager for jsonnet.
+
+You can update jsonnet dependencies by runnning `jb update`.
+
+### `gocd/generated-pipelines/`
+
+The current setup of GoCD at sentry is only able to look for
+yaml pipelines in a repo, so relay has to commit the generated
+pipelines.
+
+The dev-infra team is working on a GoCD plugin that will accept
+the jsonnet directly, removing the need for the generated-pipelines.
+
+### `gocd/pipelines/`
+
+These are the original relay pipelines and will be used until we move
+to the jsonnet pipelines.

gocd/generated-pipelines/relay-next-monitor.yaml

@@ -0,0 +1,109 @@
+format_version: 10
+pipelines:
+ deploy-relay-next-monitor:
+ display_order: 2
+ environment_variables:
+ SENTRY_REGION: monitor
+ group: relay-next
+ lock_behavior: unlockWhenFinished
+ materials:
+ deploy-relay-next-us-pipeline-complete:
+ pipeline: deploy-relay-next-us
+ stage: pipeline-complete
+ relay_repo:
+ branch: master
+ destination: relay
+ git: git@github.com:getsentry/relay.git
+ shallow_clone: true
+ stages:
+ - checks:
+ fetch_materials: true
+ jobs:
+ checks:
+ elastic_profile_id: relay
+ environment_variables:
+ GITHUB_TOKEN: '{{SECRET:[devinfra-github][token]}}'
+ tasks:
+ - script: |
+ ##!/bin/bash
+
+ /devinfra/scripts/checks/githubactions/checkruns.py \
+ getsentry/relay \
+ "${GO_REVISION_RELAY_REPO}" \
+ "Integration Tests" \
+ "Test (macos-latest)" \
+ "Test (windows-latest)" \
+ "Test All Features (ubuntu-latest)" \
+ "Push GCR Docker Image"
+ timeout: 1800
+ - deploy-production:
+ fetch_materials: true
+ jobs:
+ create_sentry_release:
+ elastic_profile_id: relay
+ environment_variables:
+ SENTRY_AUTH_TOKEN: '{{SECRET:[devinfra-temp][relay_sentry_auth_token]}}'
+ SENTRY_ORG: sentry
+ SENTRY_PROJECT: relay
+ SENTRY_URL: https://sentry.my.sentry.io/
+ tasks:
+ - script: |
+ ##!/bin/bash
+
+ ./relay/scripts/create-sentry-release "${GO_REVISION_RELAY_REPO}"
+ timeout: 1200
+ deploy:
+ elastic_profile_id: relay
+ tasks:
+ - script: |
+ ##!/bin/bash
+
+ eval $(/devinfra/scripts/regions/project_env_vars.py --region="${SENTRY_REGION}")
+
+ /devinfra/scripts/k8s/k8stunnel
+
+ /devinfra/scripts/k8s/k8s-deploy.py \
+ --label-selector="service=relay,deploy_if_production=true" \
+ --image="us.gcr.io/sentryio/relay:${GO_REVISION_RELAY_REPO}" \
+ --container-name="relay"
+ timeout: 1200
+ - progress-to-pops:
+ approval:
+ allow_only_on_success: true
+ type: manual
+ jobs:
+ progress-to-pops:
+ elastic_profile_id: relay
+ tasks:
+ - exec:
+ command: true
+ timeout: 1200
+ - deploy-pops:
+ fetch_materials: true
+ jobs:
+ deploy-pops-monitor:
+ elastic_profile_id: relay-pop
+ environment_variables:
+ SENTRY_REGION: monitor
+ tasks:
+ - script: |
+ ##!/bin/bash
+
+ eval $(/devinfra/scripts/regions/project_env_vars.py --region="${SENTRY_REGION}")
+
+ /devinfra/scripts/k8s/k8stunnel
+
+ /devinfra/scripts/k8s/k8s-deploy.py \
+ --label-selector="service=relay-pop" \
+ --image="us.gcr.io/sentryio/relay:${GO_REVISION_RELAY_REPO}" \
+ --container-name="relay"
+ timeout: 1200
+ - pipeline-complete:
+ approval:
+ allow_only_on_success: true
+ type: success
+ jobs:
+ pipeline-complete:
+ tasks:
+ - exec:
+ command: true

gocd/generated-pipelines/relay-next-us.yaml

@@ -0,0 +1,173 @@
+format_version: 10
+pipelines:
+ deploy-relay-next-us:
+ display_order: 1
+ environment_variables:
+ SENTRY_REGION: us
+ group: relay-next
+ lock_behavior: unlockWhenFinished
+ materials:
+ deploy-relay-next-pipeline-complete:
+ pipeline: deploy-relay-next
+ stage: pipeline-complete
+ relay_repo:
+ branch: master
+ destination: relay
+ git: git@github.com:getsentry/relay.git
+ shallow_clone: true
+ stages:
+ - checks:
+ fetch_materials: true
+ jobs:
+ checks:
+ elastic_profile_id: relay
+ environment_variables:
+ GITHUB_TOKEN: '{{SECRET:[devinfra-github][token]}}'
+ tasks:
+ - script: |
+ ##!/bin/bash
+
+ /devinfra/scripts/checks/githubactions/checkruns.py \
+ getsentry/relay \
+ "${GO_REVISION_RELAY_REPO}" \
+ "Integration Tests" \
+ "Test (macos-latest)" \
+ "Test (windows-latest)" \
+ "Test All Features (ubuntu-latest)" \
+ "Push GCR Docker Image"
+ timeout: 1800
+ - deploy-production:
+ fetch_materials: true
+ jobs:
+ create_sentry_release:
+ elastic_profile_id: relay
+ environment_variables:
+ SENTRY_AUTH_TOKEN: '{{SECRET:[devinfra-temp][relay_sentry_auth_token]}}'
+ SENTRY_ORG: sentry
+ SENTRY_PROJECT: relay
+ SENTRY_URL: https://sentry.my.sentry.io/
+ tasks:
+ - script: |
+ ##!/bin/bash
+
+ ./relay/scripts/create-sentry-release "${GO_REVISION_RELAY_REPO}"
+ timeout: 1200
+ deploy:
+ elastic_profile_id: relay
+ tasks:
+ - script: |
+ ##!/bin/bash
+
+ eval $(/devinfra/scripts/regions/project_env_vars.py --region="${SENTRY_REGION}")
+
+ /devinfra/scripts/k8s/k8stunnel
+
+ /devinfra/scripts/k8s/k8s-deploy.py \
+ --label-selector="service=relay,deploy_if_production=true" \
+ --image="us.gcr.io/sentryio/relay:${GO_REVISION_RELAY_REPO}" \
+ --container-name="relay"
+ timeout: 1200
+ - progress-to-pops:
+ approval:
+ allow_only_on_success: true
+ type: manual
+ jobs:
+ progress-to-pops:
+ elastic_profile_id: relay
+ tasks:
+ - exec:
+ command: true
+ timeout: 1200
+ - deploy-pops:
+ fetch_materials: true
+ jobs:
+ create_sentry_release:
+ elastic_profile_id: relay
+ environment_variables:
+ SENTRY_AUTH_TOKEN: '{{SECRET:[devinfra-temp][relay_sentry_auth_token]}}'
+ SENTRY_ORG: sentry
+ SENTRY_PROJECT: pop-relay
+ SENTRY_URL: https://sentry.my.sentry.io/
+ tasks:
+ - script: |
+ ##!/bin/bash
+
+ ./relay/scripts/create-sentry-release "${GO_REVISION_RELAY_REPO}"
+ timeout: 1200
+ deploy-pops-us-pop-1:
+ elastic_profile_id: relay-pop
+ environment_variables:
+ SENTRY_REGION: us-pop-1
+ tasks:
+ - script: |
+ ##!/bin/bash
+
+ eval $(/devinfra/scripts/regions/project_env_vars.py --region="${SENTRY_REGION}")
+
+ /devinfra/scripts/k8s/k8stunnel
+
+ /devinfra/scripts/k8s/k8s-deploy.py \
+ --label-selector="service=relay-pop" \
+ --image="us.gcr.io/sentryio/relay:${GO_REVISION_RELAY_REPO}" \
+ --container-name="relay"
+ timeout: 1200
+ deploy-pops-us-pop-2:
+ elastic_profile_id: relay-pop
+ environment_variables:
+ SENTRY_REGION: us-pop-2
+ tasks:
+ - script: |
+ ##!/bin/bash
+
+ eval $(/devinfra/scripts/regions/project_env_vars.py --region="${SENTRY_REGION}")
+
+ /devinfra/scripts/k8s/k8stunnel
+
+ /devinfra/scripts/k8s/k8s-deploy.py \
+ --label-selector="service=relay-pop" \
+ --image="us.gcr.io/sentryio/relay:${GO_REVISION_RELAY_REPO}" \
+ --container-name="relay"
+ timeout: 1200
+ deploy-pops-us-pop-3:
+ elastic_profile_id: relay-pop
+ environment_variables:
+ SENTRY_REGION: us-pop-3
+ tasks:
+ - script: |
+ ##!/bin/bash
+
+ eval $(/devinfra/scripts/regions/project_env_vars.py --region="${SENTRY_REGION}")
+
+ /devinfra/scripts/k8s/k8stunnel
+
+ /devinfra/scripts/k8s/k8s-deploy.py \
+ --label-selector="service=relay-pop" \
+ --image="us.gcr.io/sentryio/relay:${GO_REVISION_RELAY_REPO}" \
+ --container-name="relay"
+ timeout: 1200
+ deploy-pops-us-pop-4:
+ elastic_profile_id: relay-pop
+ environment_variables:
+ SENTRY_REGION: us-pop-4
+ tasks:
+ - script: |
+ ##!/bin/bash
+
+ eval $(/devinfra/scripts/regions/project_env_vars.py --region="${SENTRY_REGION}")
+
+ /devinfra/scripts/k8s/k8stunnel
+
+ /devinfra/scripts/k8s/k8s-deploy.py \
+ --label-selector="service=relay-pop" \
+ --image="us.gcr.io/sentryio/relay:${GO_REVISION_RELAY_REPO}" \
+ --container-name="relay"
+ timeout: 1200
+ - pipeline-complete:
+ approval:
+ allow_only_on_success: true
+ type: success
+ jobs:
+ pipeline-complete:
+ tasks:
+ - exec:
+ command: true