From 41abecbfbb375d7d09faa9b8bfea10ed40350acc Mon Sep 17 00:00:00 2001 From: Ian Woodard <17186604+IanWoodard@users.noreply.github.com> Date: Wed, 28 Feb 2024 12:15:35 -0800 Subject: [PATCH] Rearchitect GoCD pipelines (#3131) In this PR, I've restructured Relay's GoCD pipelines to separate Processing And PoPs into distinct pipelines. As we previously discussed, this change aims to improve efficiency and robustness of Relay's deployment process. Key enhancements include: * Addition of canary deployments for both Processing and PoPs, providing an early warning system for issues in our deployment process * Introduction of soak-time stage with Sentry and DataDog checks, allowing us to detect and address issues early on and stop problematic deploys from rolling out further. * Inclusion of dedicated rollback pipelines for both Processing and PoPs, increasing the speed at which we can mitigate issues. #skip-changelog --- Makefile | 2 + gocd/templates/bash/check-datadog-status.sh | 5 + gocd/templates/bash/check-sentry-errors.sh | 25 ++ .../templates/bash/check-sentry-new-errors.sh | 23 ++ gocd/templates/bash/deploy-pop-canary.sh | 10 + .../bash/deploy-processing-canary.sh | 10 + gocd/templates/bash/deploy-processing.sh | 10 + gocd/templates/bash/pause-current-pipeline.sh | 4 + gocd/templates/bash/wait-canary.sh | 4 + gocd/templates/bash/wait-soak.sh | 4 + ...ops.libsonnet => relay-pops-old.libsonnet} | 0 gocd/templates/libs/utils.libsonnet | 33 +++ gocd/templates/pipelines/pops.libsonnet | 232 ++++++++++++++++++ gocd/templates/pipelines/processing.libsonnet | 159 ++++++++++++ .../{libs => pipelines}/relay.libsonnet | 2 +- gocd/templates/pops.jsonnet | 25 ++ gocd/templates/processing.jsonnet | 25 ++ gocd/templates/relay.jsonnet | 2 +- scripts/create-sentry-release | 6 +- 19 files changed, 577 insertions(+), 4 deletions(-) create mode 100644 gocd/templates/bash/check-datadog-status.sh create mode 100644 gocd/templates/bash/check-sentry-errors.sh create mode 100644 gocd/templates/bash/check-sentry-new-errors.sh create mode 100644 gocd/templates/bash/deploy-pop-canary.sh create mode 100644 gocd/templates/bash/deploy-processing-canary.sh create mode 100644 gocd/templates/bash/deploy-processing.sh create mode 100644 gocd/templates/bash/pause-current-pipeline.sh create mode 100644 gocd/templates/bash/wait-canary.sh create mode 100644 gocd/templates/bash/wait-soak.sh rename gocd/templates/libs/{relay-pops.libsonnet => relay-pops-old.libsonnet} (100%) create mode 100644 gocd/templates/libs/utils.libsonnet create mode 100644 gocd/templates/pipelines/pops.libsonnet create mode 100644 gocd/templates/pipelines/processing.libsonnet rename gocd/templates/{libs => pipelines}/relay.libsonnet (97%) create mode 100644 gocd/templates/pops.jsonnet create mode 100644 gocd/templates/processing.jsonnet diff --git a/Makefile b/Makefile index 72b242a1bf..863396fed3 100644 --- a/Makefile +++ b/Makefile @@ -185,6 +185,8 @@ gocd: ## Build GoCD pipelines @ find . -type f \( -name '*.libsonnet' -o -name '*.jsonnet' \) -print0 | xargs -n 1 -0 jsonnetfmt -i @ find . -type f \( -name '*.libsonnet' -o -name '*.jsonnet' \) -print0 | xargs -n 1 -0 jsonnet-lint -J ./gocd/templates/vendor @ cd ./gocd/templates && jsonnet --ext-code output-files=true -J vendor -m ../generated-pipelines ./relay.jsonnet + @ cd ./gocd/templates && jsonnet --ext-code output-files=true -J vendor -m ../generated-pipelines ./pops.jsonnet + @ cd ./gocd/templates && jsonnet --ext-code output-files=true -J vendor -m ../generated-pipelines ./relay.jsonnet @ cd ./gocd/generated-pipelines && find . -type f \( -name '*.yaml' \) -print0 | xargs -n 1 -0 yq -p json -o yaml -i .PHONY: gocd diff --git a/gocd/templates/bash/check-datadog-status.sh b/gocd/templates/bash/check-datadog-status.sh new file mode 100644 index 0000000000..cd7d6c3716 --- /dev/null +++ b/gocd/templates/bash/check-datadog-status.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +/devinfra/scripts/checks/datadog/monitor_status.py \ + ${DATADOG_MONITOR_IDS} \ + --skip-check=false diff --git a/gocd/templates/bash/check-sentry-errors.sh b/gocd/templates/bash/check-sentry-errors.sh new file mode 100644 index 0000000000..1d7d82e08b --- /dev/null +++ b/gocd/templates/bash/check-sentry-errors.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# shellcheck disable=SC2206 +project_ids=(${SENTRY_PROJECT_IDS}) +# shellcheck disable=SC2206 +project_slugs=(${SENTRY_PROJECTS}) + + +if [ ${##project_ids[@]} -ne ${##project_slugs[@]} ]; then + echo "Error: SENTRY_PROJECT_IDS and SENTRY_PROJECTS must have the same number of elements" + exit 1 +fi + +for i in "${!project_ids[@]}"; do + /devinfra/scripts/checks/sentry/release_error_events.py \ + --project-id="${project_ids[i]}" \ + --project-slug="${project_slugs[i]}" \ + --release="relay@${GO_REVISION_GETSENTRY_REPO}" \ + --duration=5 \ + --error-events-limit="${ERROR_LIMIT}" \ + --dry-run="${DRY_RUN}" \ + --single-tenant="${SENTRY_SINGLE_TENANT}" \ + --skip-check="${SKIP_CANARY_CHECKS}" \ + --sentry-base="${SENTRY_BASE}" +done diff --git a/gocd/templates/bash/check-sentry-new-errors.sh b/gocd/templates/bash/check-sentry-new-errors.sh new file mode 100644 index 0000000000..bd1decd44a --- /dev/null +++ b/gocd/templates/bash/check-sentry-new-errors.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# shellcheck disable=SC2206 +project_ids=(${SENTRY_PROJECT_IDS}) +# shellcheck disable=SC2206 +project_slugs=(${SENTRY_PROJECTS}) + +if [ ${##project_ids[@]} -ne ${##project_slugs[@]} ]; then + echo "Error: SENTRY_PROJECT_IDS and SENTRY_PROJECTS must have the same number of elements" + exit 1 +fi + +for i in "${!project_ids[@]}"; do + /devinfra/scripts/checks/sentry/release_new_issues.py \ + --project-id="${project_ids[i]}" \ + --project-slug="${project_slugs[i]}" \ + --release="relay@${GO_REVISION_GETSENTRY_REPO}" \ + --new-issues-limit=0 \ + --dry-run="${DRY_RUN}" \ + --single-tenant="${SENTRY_SINGLE_TENANT}" \ + --skip-check="${SKIP_CANARY_CHECKS}" \ + --sentry-base="${SENTRY_BASE}" +done diff --git a/gocd/templates/bash/deploy-pop-canary.sh b/gocd/templates/bash/deploy-pop-canary.sh new file mode 100644 index 0000000000..3411fb7995 --- /dev/null +++ b/gocd/templates/bash/deploy-pop-canary.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +eval $(/devinfra/scripts/regions/project_env_vars.py --region="${SENTRY_REGION}") + +/devinfra/scripts/k8s/k8stunnel + +/devinfra/scripts/k8s/k8s-deploy.py \ + --label-selector="service=relay-pop,env=canary" \ + --image="us.gcr.io/sentryio/relay-pop:${GO_REVISION_RELAY_REPO}" \ + --container-name="relay" diff --git a/gocd/templates/bash/deploy-processing-canary.sh b/gocd/templates/bash/deploy-processing-canary.sh new file mode 100644 index 0000000000..00b524b4da --- /dev/null +++ b/gocd/templates/bash/deploy-processing-canary.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +eval $(/devinfra/scripts/regions/project_env_vars.py --region="${SENTRY_REGION}") + +/devinfra/scripts/k8s/k8stunnel + +/devinfra/scripts/k8s/k8s-deploy.py \ + --label-selector="service=relay,env=canary" \ + --image="us.gcr.io/sentryio/relay:${GO_REVISION_RELAY_REPO}" \ + --container-name="relay" diff --git a/gocd/templates/bash/deploy-processing.sh b/gocd/templates/bash/deploy-processing.sh new file mode 100644 index 0000000000..4b3145f50d --- /dev/null +++ b/gocd/templates/bash/deploy-processing.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +eval $(/devinfra/scripts/regions/project_env_vars.py --region="${SENTRY_REGION}") + +/devinfra/scripts/k8s/k8stunnel + +/devinfra/scripts/k8s/k8s-deploy.py \ + --label-selector="service=relay" \ + --image="us.gcr.io/sentryio/relay:${GO_REVISION_RELAY_REPO}" \ + --container-name="relay" diff --git a/gocd/templates/bash/pause-current-pipeline.sh b/gocd/templates/bash/pause-current-pipeline.sh new file mode 100644 index 0000000000..0e807df418 --- /dev/null +++ b/gocd/templates/bash/pause-current-pipeline.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +gocd-pause-current-pipeline \ + --pause-message="${PAUSE_MESSAGE}" diff --git a/gocd/templates/bash/wait-canary.sh b/gocd/templates/bash/wait-canary.sh new file mode 100644 index 0000000000..77ff6c512f --- /dev/null +++ b/gocd/templates/bash/wait-canary.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +# Wait for 5 minutes +sleep 300 diff --git a/gocd/templates/bash/wait-soak.sh b/gocd/templates/bash/wait-soak.sh new file mode 100644 index 0000000000..77ff6c512f --- /dev/null +++ b/gocd/templates/bash/wait-soak.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +# Wait for 5 minutes +sleep 300 diff --git a/gocd/templates/libs/relay-pops.libsonnet b/gocd/templates/libs/relay-pops-old.libsonnet similarity index 100% rename from gocd/templates/libs/relay-pops.libsonnet rename to gocd/templates/libs/relay-pops-old.libsonnet diff --git a/gocd/templates/libs/utils.libsonnet b/gocd/templates/libs/utils.libsonnet new file mode 100644 index 0000000000..888186f576 --- /dev/null +++ b/gocd/templates/libs/utils.libsonnet @@ -0,0 +1,33 @@ +local gocdtasks = import 'github.com/getsentry/gocd-jsonnet/libs/gocd-tasks.libsonnet'; + +{ + pause_on_failure(): { + plugin: { + options: gocdtasks.script(importstr '../bash/pause-current-pipeline.sh'), + run_if: 'failed', + configuration: { + id: 'script-executor', + version: 1, + }, + }, + }, + github_checks(): [ + { + checks: { + fetch_materials: true, + jobs: { + checks: { + environment_variables: { + GITHUB_TOKEN: '{{SECRET:[devinfra-github][token]}}', + }, + timeout: 1800, + elastic_profile_id: 'relay', + tasks: [ + gocdtasks.script(importstr '../bash/github-check-runs.sh'), + ], + }, + }, + }, + }, + ], +} diff --git a/gocd/templates/pipelines/pops.libsonnet b/gocd/templates/pipelines/pops.libsonnet new file mode 100644 index 0000000000..80bcf2787f --- /dev/null +++ b/gocd/templates/pipelines/pops.libsonnet @@ -0,0 +1,232 @@ +local utils = import '../libs/utils.libsonnet'; +local gocdtasks = import 'github.com/getsentry/gocd-jsonnet/libs/gocd-tasks.libsonnet'; + +local canary_region_pops = { + de: [], + // TODO: Check that these are right + us: ['us-pop-1', 'us-pop-regional-1'], +}; + +local region_pops = { + de: [ + 'de-pop-1', + 'de-pop-2', + ], + us: [ + 'us-pop-1', + 'us-pop-2', + 'us-pop-3', + 'us-pop-4', + 'us-pop-regional-1', + 'us-pop-regional-2', + 'us-pop-regional-3', + 'us-pop-regional-4', + ], +}; + +// The purpose of this stage is to let the deployment soak for a while and +// detect any issues that might have been introduced. +local soak_time(region) = + if region == 's4s' || region == 'us' then + [ + { + 'soak-time': { + jobs: { + soak: { + environment_variables: { + SENTRY_REGION: region, + GOCD_ACCESS_TOKEN: '{{SECRET:[devinfra][gocd_access_token]}}', + SENTRY_AUTH_TOKEN: '{{SECRET:[devinfra-sentryio][token]}}', + DATADOG_API_KEY: '{{SECRET:[devinfra][sentry_datadog_api_key]}}', + DATADOG_APP_KEY: '{{SECRET:[devinfra][sentry_datadog_app_key]}}', + // Datadog monitor IDs for the soak time + DATADOG_MONITOR_IDS: '137575470 22592147 27804625 22634395 22635255', + SENTRY_PROJECTS: if region == 's4s' then 'sentry-for-sentry' else 'pop-relay relay', + SENTRY_PROJECT_IDS: if region == 's4s' then '1513938' else '9 4', + SENTRY_SINGLE_TENANT: if region == 's4s' then 'true' else 'false', + SENTRY_BASE: if region == 's4s' then 'https://sentry.io/api/0' else 'https://sentry.my.sentry.io/api/0', + // TODO: Set a proper error limit + ERROR_LIMIT: 500, + PAUSE_MESSAGE: 'Detecting issues in the deployment. Pausing pipeline.', + // TODO: Switch dry run to false once we're confident in the soak time + DRY_RUN: 'true', + }, + elastic_profile_id: 'relay-pop', + tasks: [ + gocdtasks.script(importstr '../bash/wait-soak.sh'), + gocdtasks.script(importstr '../bash/check-sentry-errors.sh'), + gocdtasks.script(importstr '../bash/check-sentry-new-errors.sh'), + gocdtasks.script(importstr '../bash/check-datadog-status.sh'), + utils.pause_on_failure(), + ], + }, + }, + }, + }, + ] + else + []; + +// Create a gocd job that will run the deploy-pop-canary script, +// wait for a few minutes, and check the status of the canary deployment. +local deploy_pop_canary_job(region) = + { + timeout: 1200, + elastic_profile_id: 'relay-pop', + environment_variables: { + SENTRY_REGION: region, + GOCD_ACCESS_TOKEN: '{{SECRET:[devinfra][gocd_access_token]}}', + SENTRY_AUTH_TOKEN: '{{SECRET:[devinfra-sentryio][token]}}', + DATADOG_API_KEY: '{{SECRET:[devinfra][sentry_datadog_api_key]}}', + DATADOG_APP_KEY: '{{SECRET:[devinfra][sentry_datadog_app_key]}}', + // Datadog monitor IDs for the canary deployment + DATADOG_MONITOR_IDS: '137575470 22592147 27804625 22634395 22635255', + SENTRY_PROJECTS: 'pop-relay relay', + SENTRY_PROJECT_IDS: '9 4', + SENTRY_SINGLE_TENANT: 'false', + SENTRY_BASE: 'https://sentry.my.sentry.io/api/0', + // TODO: Set a proper error limit + ERROR_LIMIT: 500, + PAUSE_MESSAGE: 'Pausing pipeline due to canary failure.', + // TODO: Switch dry run to false once we're confident in the soak time + DRY_RUN: 'true', + }, + tasks: [ + gocdtasks.script(importstr '../bash/deploy-pop-canary.sh'), + gocdtasks.script(importstr '../bash/wait-canary.sh'), + gocdtasks.script(importstr '../bash/check-sentry-errors.sh'), + gocdtasks.script(importstr '../bash/check-sentry-new-errors.sh'), + gocdtasks.script(importstr '../bash/check-datadog-status.sh'), + utils.pause_on_failure(), + ], + }; + +// Create a gocd job that will run the deploy-pop script +local deploy_pop_job(region) = + { + timeout: 1200, + elastic_profile_id: 'relay-pop', + environment_variables: { + SENTRY_REGION: region, + }, + tasks: [ + gocdtasks.script(importstr '../bash/deploy-pop.sh'), + ], + }; + +// Iterate over a list of regions and create a job for each +local deploy_jobs(regions, deploy_job, partition='-') = + { + ['deploy-primary' + partition + region]: deploy_job(region) + for region in regions + }; + +// The purpose of this stage is to deploy a canary to all canary PoPs for a given region +// and wait for a few minutes to see if there are any issues. +local deploy_canary_pops_stage(region) = + { + 'deploy-canary': { + fetch_materials: true, + jobs: { + create_sentry_release: { + timeout: 1200, + elastic_profile_id: 'relay', + environment_variables: { + SENTRY_ORG: 'sentry', + SENTRY_PROJECT: 'pop-relay', + SENTRY_URL: 'https://sentry.my.sentry.io/', + // Temporary; self-service encrypted secrets aren't implemented yet. + // This should really be rotated to an internal integration token. + SENTRY_AUTH_TOKEN: '{{SECRET:[devinfra-temp][relay_sentry_auth_token]}}', + SENTRY_ENVIRONMENT: 'canary', + }, + tasks: [ + gocdtasks.script(importstr '../bash/create-sentry-relay-release.sh'), + ], + }, + }, + }, + } { + 'deploy-canary'+: { + fetch_materials: true, + jobs+: deploy_jobs( + [region] + canary_region_pops[region], + deploy_pop_canary_job, + '-canary-', + ), + }, + }; + +// The purpose of this stage is to deploy to all PoPs for a given region as well +// as create a sentry release. +local deploy_pops_stage(region) = + { + 'deploy-primary': { + fetch_materials: true, + jobs: { + // PoPs have their own Sentry project, which requires separate symbol upload via + // create-sentry-release. They could be moved into the same project with a different + // environment to avoid this. + create_sentry_release: { + timeout: 1200, + elastic_profile_id: 'relay', + environment_variables: { + SENTRY_ORG: if region == 's4s' then 'sentry-st' else 'sentry', + SENTRY_PROJECT: if region == 's4s' then 'sentry-for-sentry' else 'pop-relay', + SENTRY_URL: if region == 's4s' then 'https://sentry-st.sentry.io/' else 'https://sentry.my.sentry.io/', + // Temporary; self-service encrypted secrets aren't implemented yet. + // This should really be rotated to an internal integration token. + SENTRY_AUTH_TOKEN: if region == 's4s' then '{{SECRET:[devinfra-temp][relay_sentry_st_auth_token]}}' else '{{SECRET:[devinfra-temp][relay_sentry_auth_token]}}', + }, + tasks: [ + gocdtasks.script(importstr '../bash/create-sentry-relay-pop-release.sh'), + ], + }, + }, + }, + } { + 'deploy-primary'+: { + jobs+: deploy_jobs( + [region] + region_pops[region], + deploy_pop_job, + ), + }, + }; + +// The purpose of this stage is to deploy to a single PoP for a given region. +local deploy_generic_pops_stage(region) = + { + 'deploy-primary': { + fetch_materials: true, + jobs: { + ['deploy-primary-' + region]: deploy_pop_job(region), + }, + }, + }; + +// The US region deploys create a sentry release and deploys to a number +// of clusters, other regions only deploy to a single cluster. +local deployment_stages(region) = + if region == 'us' || region == 'de' then + // The canary stage is only for the US and DE regions + [deploy_canary_pops_stage(region), deploy_pops_stage(region)] + else + [deploy_generic_pops_stage(region)]; + + +function(region) { + environment_variables: { + SENTRY_REGION: region, + }, + group: 'relay-pops-next', + lock_behavior: 'unlockWhenFinished', + materials: { + relay_repo: { + git: 'git@github.com:getsentry/relay.git', + shallow_clone: true, + branch: 'master', + destination: 'relay', + }, + }, + stages: utils.github_checks() + deployment_stages(region) + soak_time(region), +} diff --git a/gocd/templates/pipelines/processing.libsonnet b/gocd/templates/pipelines/processing.libsonnet new file mode 100644 index 0000000000..3524d4c3b7 --- /dev/null +++ b/gocd/templates/pipelines/processing.libsonnet @@ -0,0 +1,159 @@ +local utils = import '../libs/utils.libsonnet'; +local gocdtasks = import 'github.com/getsentry/gocd-jsonnet/libs/gocd-tasks.libsonnet'; + +// The purpose of this stage is to let the deployment soak for a while and +// detect any issues that might have been introduced. +local soak_time(region) = + if region == 's4s' || region == 'us' then + [ + { + 'soak-time': { + jobs: { + soak: { + environment_variables: { + SENTRY_REGION: region, + GOCD_ACCESS_TOKEN: '{{SECRET:[devinfra][gocd_access_token]}}', + SENTRY_AUTH_TOKEN: '{{SECRET:[devinfra-sentryio][token]}}', + DATADOG_API_KEY: '{{SECRET:[devinfra][sentry_datadog_api_key]}}', + DATADOG_APP_KEY: '{{SECRET:[devinfra][sentry_datadog_app_key]}}', + // Datadog monitor IDs for the soak time + // TODO: Add the monitor IDs + DATADOG_MONITOR_IDS: '137566884 14146876 137619914 14030245', + SENTRY_PROJECTS: if region == 's4s' then 'sentry-for-sentry' else 'relay pop-relay', + SENTRY_PROJECT_IDS: if region == 's4s' then '1513938' else '4 9', + SENTRY_SINGLE_TENANT: if region == 's4s' then 'true' else 'false', + SENTRY_BASE: if region == 's4s' then 'https://sentry.io/api/0' else 'https://sentry.my.sentry.io/api/0', + // TODO: Set a proper error limit + ERROR_LIMIT: 500, + PAUSE_MESSAGE: 'Detecting issues in the deployment. Pausing pipeline.', + // TODO: Switch dry run to false once we're confident in the soak time + DRY_RUN: 'true', + }, + elastic_profile_id: 'relay', + tasks: [ + gocdtasks.script(importstr '../bash/wait-soak.sh'), + gocdtasks.script(importstr '../bash/check-sentry-errors.sh'), + gocdtasks.script(importstr '../bash/check-sentry-new-errors.sh'), + gocdtasks.script(importstr '../bash/check-datadog-status.sh'), + utils.pause_on_failure(), + ], + }, + }, + }, + }, + ] + else + []; + +// The purpose of this stage is to deploy a canary for a given region and wait for a few minutes +// to see if there are any issues. +local deploy_canary(region) = + if region == 'us' then + [ + { + 'deploy-canary': { + fetch_materials: true, + jobs: { + create_sentry_release: { + environment_variables: { + SENTRY_ORG: 'sentry', + SENTRY_PROJECT: 'relay', + SENTRY_URL: 'https://sentry.my.sentry.io/', + // Temporary; self-service encrypted secrets aren't implemented yet. + // This should really be rotated to an internal integration token. + SENTRY_AUTH_TOKEN: '{{SECRET:[devinfra-temp][relay_sentry_auth_token]}}', + SENTRY_ENVIRONMENT: 'canary', + }, + timeout: 1200, + elastic_profile_id: 'relay', + tasks: [ + gocdtasks.script(importstr '../bash/create-sentry-relay-release.sh'), + ], + }, + deploy: { + timeout: 1200, + elastic_profile_id: 'relay', + environment_variables: { + SENTRY_REGION: region, + GOCD_ACCESS_TOKEN: '{{SECRET:[devinfra][gocd_access_token]}}', + SENTRY_AUTH_TOKEN: '{{SECRET:[devinfra-sentryio][token]}}', + DATADOG_API_KEY: '{{SECRET:[devinfra][sentry_datadog_api_key]}}', + DATADOG_APP_KEY: '{{SECRET:[devinfra][sentry_datadog_app_key]}}', + // Datadog monitor IDs for the canary deployment + DATADOG_MONITOR_IDS: '137566884 14146876 137619914 14030245', + SENTRY_PROJECTS: 'relay pop-relay', + SENTRY_PROJECT_IDS: '4 9', + SENTRY_SINGLE_TENANT: 'false', + SENTRY_BASE: 'https://sentry.my.sentry.io/api/0', + // TODO: Set a proper error limit + ERROR_LIMIT: 500, + PAUSE_MESSAGE: 'Pausing pipeline due to canary failure.', + // TODO: Switch dry run to false once we're confident in the canary + DRY_RUN: 'true', + }, + tasks: [ + gocdtasks.script(importstr '../bash/deploy-processing-canary.sh'), + gocdtasks.script(importstr '../bash/wait-canary.sh'), + gocdtasks.script(importstr '../bash/check-sentry-errors.sh'), + gocdtasks.script(importstr '../bash/check-sentry-new-errors.sh'), + gocdtasks.script(importstr '../bash/check-datadog-status.sh'), + utils.pause_on_failure(), + ], + }, + }, + }, + }, + ] + else + []; + +// The purpose of this stage is to deploy to production +local deploy_primary(region) = [ + { + 'deploy-primary': { + fetch_materials: true, + jobs: { + create_sentry_release: { + environment_variables: { + SENTRY_ORG: if region == 's4s' then 'sentry-st' else 'sentry', + SENTRY_PROJECT: if region == 's4s' then 'sentry-for-sentry' else 'relay', + SENTRY_URL: if region == 's4s' then 'https://sentry-st.sentry.io/' else 'https://sentry.my.sentry.io/', + // Temporary; self-service encrypted secrets aren't implemented yet. + // This should really be rotated to an internal integration token. + SENTRY_AUTH_TOKEN: if region == 's4s' then '{{SECRET:[devinfra-temp][relay_sentry_st_auth_token]}}' else '{{SECRET:[devinfra-temp][relay_sentry_auth_token]}}', + }, + timeout: 1200, + elastic_profile_id: 'relay', + tasks: [ + gocdtasks.script(importstr '../bash/create-sentry-relay-release.sh'), + ], + }, + deploy: { + timeout: 1200, + elastic_profile_id: 'relay', + tasks: [ + gocdtasks.script(importstr '../bash/deploy-processing.sh'), + ], + }, + }, + }, + }, +]; + + +function(region) { + environment_variables: { + SENTRY_REGION: region, + }, + group: 'relay-next', + lock_behavior: 'unlockWhenFinished', + materials: { + relay_repo: { + git: 'git@github.com:getsentry/relay.git', + shallow_clone: true, + branch: 'master', + destination: 'relay', + }, + }, + stages: utils.github_checks() + deploy_canary(region) + deploy_primary(region) + soak_time(region), +} diff --git a/gocd/templates/libs/relay.libsonnet b/gocd/templates/pipelines/relay.libsonnet similarity index 97% rename from gocd/templates/libs/relay.libsonnet rename to gocd/templates/pipelines/relay.libsonnet index 0633f22f0c..46b0911319 100644 --- a/gocd/templates/libs/relay.libsonnet +++ b/gocd/templates/pipelines/relay.libsonnet @@ -1,4 +1,4 @@ -local pops = import './relay-pops.libsonnet'; +local pops = import '../libs/relay-pops-old.libsonnet'; local gocdtasks = import 'github.com/getsentry/gocd-jsonnet/libs/gocd-tasks.libsonnet'; function(region) { diff --git a/gocd/templates/pops.jsonnet b/gocd/templates/pops.jsonnet new file mode 100644 index 0000000000..9c060f2dd8 --- /dev/null +++ b/gocd/templates/pops.jsonnet @@ -0,0 +1,25 @@ +// Learn more about GoCD Pipedream here: +// https://www.notion.so/sentry/Pipedreams-in-GoCD-with-Jsonnet-430f46b87fa14650a80adf6708b088d9 + +local pops = import './pipelines/pops.libsonnet'; +local pipedream = import 'github.com/getsentry/gocd-jsonnet/libs/pipedream.libsonnet'; + +local pipedream_config = { + name: 'relay-pops', + auto_deploy: false, + materials: { + relay_repo: { + git: 'git@github.com:getsentry/relay.git', + shallow_clone: true, + branch: 'master', + destination: 'relay', + }, + }, + rollback: { + material_name: 'relay_repo', + stage: 'deploy-primary', + elastic_profile_id: 'relay-pop', + }, +}; + +pipedream.render(pipedream_config, pops) diff --git a/gocd/templates/processing.jsonnet b/gocd/templates/processing.jsonnet new file mode 100644 index 0000000000..27c5f29b25 --- /dev/null +++ b/gocd/templates/processing.jsonnet @@ -0,0 +1,25 @@ +// Learn more about GoCD Pipedream here: +// https://www.notion.so/sentry/Pipedreams-in-GoCD-with-Jsonnet-430f46b87fa14650a80adf6708b088d9 + +local processing = import './pipelines/processing.libsonnet'; +local pipedream = import 'github.com/getsentry/gocd-jsonnet/libs/pipedream.libsonnet'; + +local pipedream_config = { + name: 'relay-processing', + auto_deploy: false, + materials: { + relay_repo: { + git: 'git@github.com:getsentry/relay.git', + shallow_clone: true, + branch: 'master', + destination: 'relay', + }, + }, + rollback: { + material_name: 'relay_repo', + stage: 'deploy-primary', + elastic_profile_id: 'relay', + }, +}; + +pipedream.render(pipedream_config, processing) diff --git a/gocd/templates/relay.jsonnet b/gocd/templates/relay.jsonnet index 6f2b169b01..e36410bd81 100644 --- a/gocd/templates/relay.jsonnet +++ b/gocd/templates/relay.jsonnet @@ -1,7 +1,7 @@ // Learn more about GoCD Pipedream here: // https://www.notion.so/sentry/Pipedreams-in-GoCD-with-Jsonnet-430f46b87fa14650a80adf6708b088d9 -local relay = import './libs/relay.libsonnet'; +local relay = import './pipelines/relay.libsonnet'; local pipedream = import 'github.com/getsentry/gocd-jsonnet/libs/pipedream.libsonnet'; local pipedream_config = { diff --git a/scripts/create-sentry-release b/scripts/create-sentry-release index eb98805094..1b4c6b2f53 100755 --- a/scripts/create-sentry-release +++ b/scripts/create-sentry-release @@ -8,11 +8,13 @@ # SENTRY_AUTH_TOKEN: Sentry auth token (https://sentry.io/settings/account/api/auth-tokens/) # SENTRY_ORG: Organization slug # SENTRY_PROJECT: Project slug +# Optional environment variables: +# SENTRY_ENVIRONMENT: Environment name (default: production) set -eu REVISION="${1:-}" NAME="${2:-relay}" -GITHUB_PROJECT="getsentry/relay" +ENVIRONMENT="${SENTRY_ENVIRONMENT:-production}" if [ -z "${REVISION}" ]; then echo 'No revision specified' && exit 1 @@ -46,6 +48,6 @@ echo "Creating a new release and deploy for ${RELEASE} in Sentry..." sentry-cli releases new "${RELEASE}" # NOTE: Disabled until the repository is properly configured in Sentry # sentry-cli releases set-commits "${RELEASE}" --commit "${GITHUB_PROJECT}@${REVISION}" -sentry-cli releases deploys "${RELEASE}" new -e production +sentry-cli releases deploys "${RELEASE}" new -e "${ENVIRONMENT}" sentry-cli releases finalize "${RELEASE}" echo 'Deploy created.'