From de431301a6eaf4dc61a12dc34cbb44d74059970c Mon Sep 17 00:00:00 2001 From: "Matt Gaunt-Seo @ Sentry.io" <112419115+mattgauntseo-sentry@users.noreply.github.com> Date: Mon, 17 Jul 2023 10:40:37 -0700 Subject: [PATCH] Update jsonnet libs and add rollback pipeline (#2301) --- Makefile | 2 +- .../relay-next-monitor.yaml | 4 +- gocd/generated-pipelines/relay-next-us.yaml | 6 +- .../rollback-relay-next.yaml | 57 +++++++++++++++++++ gocd/templates/jsonnetfile.json | 4 +- gocd/templates/jsonnetfile.lock.json | 6 +- gocd/templates/libs/relay-pops.libsonnet | 2 +- gocd/templates/libs/relay.libsonnet | 2 +- gocd/templates/relay.jsonnet | 6 +- 9 files changed, 75 insertions(+), 14 deletions(-) create mode 100644 gocd/generated-pipelines/rollback-relay-next.yaml diff --git a/Makefile b/Makefile index 47a8d1fe2a..e7d55285ed 100644 --- a/Makefile +++ b/Makefile @@ -180,7 +180,7 @@ help: ## this help gocd: ## Build GoCD pipelines @ rm -rf ./gocd/generated-pipelines @ mkdir -p ./gocd/generated-pipelines - @ cd ./gocd/templates && jb install + @ cd ./gocd/templates && jb install && jb update @ find . -type f \( -name '*.libsonnet' -o -name '*.jsonnet' \) -print0 | xargs -n 1 -0 jsonnetfmt -i @ find . -type f \( -name '*.libsonnet' -o -name '*.jsonnet' \) -print0 | xargs -n 1 -0 jsonnet-lint -J ./gocd/templates/vendor @ cd ./gocd/templates && jsonnet -J vendor -m ../generated-pipelines ./relay.jsonnet diff --git a/gocd/generated-pipelines/relay-next-monitor.yaml b/gocd/generated-pipelines/relay-next-monitor.yaml index 622acd1799..03762140e7 100644 --- a/gocd/generated-pipelines/relay-next-monitor.yaml +++ b/gocd/generated-pipelines/relay-next-monitor.yaml @@ -7,8 +7,8 @@ pipelines: group: relay-next lock_behavior: unlockWhenFinished materials: - deploy-relay-next-us-pipeline-complete: - pipeline: deploy-relay-next-us + deploy-relay-next-pipeline-complete: + pipeline: deploy-relay-next stage: pipeline-complete relay_repo: branch: master diff --git a/gocd/generated-pipelines/relay-next-us.yaml b/gocd/generated-pipelines/relay-next-us.yaml index 29ea1cd999..a6ce7e9f77 100644 --- a/gocd/generated-pipelines/relay-next-us.yaml +++ b/gocd/generated-pipelines/relay-next-us.yaml @@ -1,14 +1,14 @@ format_version: 10 pipelines: deploy-relay-next-us: - display_order: 1 + display_order: 3 environment_variables: SENTRY_REGION: us group: relay-next lock_behavior: unlockWhenFinished materials: - deploy-relay-next-pipeline-complete: - pipeline: deploy-relay-next + deploy-relay-next-monitor-pipeline-complete: + pipeline: deploy-relay-next-monitor stage: pipeline-complete relay_repo: branch: master diff --git a/gocd/generated-pipelines/rollback-relay-next.yaml b/gocd/generated-pipelines/rollback-relay-next.yaml new file mode 100644 index 0000000000..adc57723b6 --- /dev/null +++ b/gocd/generated-pipelines/rollback-relay-next.yaml @@ -0,0 +1,57 @@ +format_version: 10 +pipelines: + rollback-relay-next: + display_order: 1 + environment_variables: + ALL_PIPELINE_FLAGS: --pipeline="deploy-relay-next-monitor" --pipeline="deploy-relay-next-us" --pipeline="deploy-relay-next" + GOCD_ACCESS_TOKEN: '{{SECRET:[devinfra][gocd_access_token]}}' + REGION_PIPELINE_FLAGS: --pipeline="deploy-relay-next-monitor" --pipeline="deploy-relay-next-us" + ROLLBACK_MATERIAL_NAME: relay_repo + ROLLBACK_STAGE: deploy-production + group: relay-next + lock_behavior: unlockWhenFinished + materials: + deploy-relay-next-us-pipeline-complete: + pipeline: deploy-relay-next-us + stage: pipeline-complete + stages: + - pause_pipelines: + approval: + type: manual + jobs: + rollback: + tasks: + - script: | + ##!/bin/bash + + ## Note: $ALL_PIPELINE_FLAGS has no quoting, for word expansion + ## shellcheck disable=SC2086 + if [[ "${ALL_PIPELINE_FLAGS:-}" ]]; then + set -- $ALL_PIPELINE_FLAGS + fi + + ## Pause all pipelines in the pipedream + gocd-pause-and-cancel-pipelines \ + --pause-message="This pipeline is being rolled back, please check with team before un-pausing." \ + "$@" + start_rollback: + jobs: + rollback: + tasks: + - script: | + ##!/bin/bash + + ## Note: $REGION_PIPELINE_FLAGS has no quoting, for word expansion + ## shellcheck disable=SC2086 + if [[ "${REGION_PIPELINE_FLAGS:-}" ]]; then + set -- $REGION_PIPELINE_FLAGS + fi + + ## Get sha from the given pipeline run to deploy to all pipedream pipelines. + sha=$(gocd-sha-for-pipeline --material-name="${ROLLBACK_MATERIAL_NAME}") + + gocd-emergency-deploy \ + --commit-sha="${sha}" \ + --deploy-stage="${ROLLBACK_STAGE}" \ + --pause-message="This pipeline was rolled back, please check with team before un-pausing." \ + "$@" diff --git a/gocd/templates/jsonnetfile.json b/gocd/templates/jsonnetfile.json index c05e780858..81740f8301 100644 --- a/gocd/templates/jsonnetfile.json +++ b/gocd/templates/jsonnetfile.json @@ -5,10 +5,10 @@ "source": { "git": { "remote": "https://github.com/getsentry/gocd-jsonnet.git", - "subdir": "v1.0.0" + "subdir": "libs" } }, - "version": "main" + "version": "v1.1.4" } ], "legacyImports": true diff --git a/gocd/templates/jsonnetfile.lock.json b/gocd/templates/jsonnetfile.lock.json index b5361cb8d5..eab3524854 100644 --- a/gocd/templates/jsonnetfile.lock.json +++ b/gocd/templates/jsonnetfile.lock.json @@ -5,11 +5,11 @@ "source": { "git": { "remote": "https://github.com/getsentry/gocd-jsonnet.git", - "subdir": "v1.0.0" + "subdir": "libs" } }, - "version": "7636a8579792e6e1e66b0b567583e5e05764c39f", - "sum": "eOpSoGZ9y3+6O3qllOXVBdiHfQ2xwcnAJWqlE4k3Rj8=" + "version": "fc28e0fb504269698df9b19109543057eb5549cd", + "sum": "/R7fdDl7Sg9wqOEoDVkp49qiuMVtzjvAcqVGX4HbGG4=" } ], "legacyImports": false diff --git a/gocd/templates/libs/relay-pops.libsonnet b/gocd/templates/libs/relay-pops.libsonnet index 2d1076c62e..479bcecf13 100644 --- a/gocd/templates/libs/relay-pops.libsonnet +++ b/gocd/templates/libs/relay-pops.libsonnet @@ -1,5 +1,5 @@ local STAGE_NAME = 'deploy-pops'; -local gocdtasks = import 'github.com/getsentry/gocd-jsonnet/v1.0.0/gocd-tasks.libsonnet'; +local gocdtasks = import 'github.com/getsentry/gocd-jsonnet/libs/gocd-tasks.libsonnet'; // Create a gocd job that will run the deploy-pop script local deploy_pop_job(region) = diff --git a/gocd/templates/libs/relay.libsonnet b/gocd/templates/libs/relay.libsonnet index 48fa2a59c8..7ed89a0b56 100644 --- a/gocd/templates/libs/relay.libsonnet +++ b/gocd/templates/libs/relay.libsonnet @@ -1,5 +1,5 @@ local pops = import './relay-pops.libsonnet'; -local gocdtasks = import 'github.com/getsentry/gocd-jsonnet/v1.0.0/gocd-tasks.libsonnet'; +local gocdtasks = import 'github.com/getsentry/gocd-jsonnet/libs/gocd-tasks.libsonnet'; function(region) { environment_variables: { diff --git a/gocd/templates/relay.jsonnet b/gocd/templates/relay.jsonnet index 0291ac8021..ca7412a511 100644 --- a/gocd/templates/relay.jsonnet +++ b/gocd/templates/relay.jsonnet @@ -1,5 +1,5 @@ local relay = import './libs/relay.libsonnet'; -local pipedream = import 'github.com/getsentry/gocd-jsonnet/v1.0.0/pipedream.libsonnet'; +local pipedream = import 'github.com/getsentry/gocd-jsonnet/libs/pipedream.libsonnet'; local pipedream_config = { name: 'relay-next', @@ -12,6 +12,10 @@ local pipedream_config = { destination: 'relay', }, }, + rollback: { + material_name: 'relay_repo', + stage: 'deploy-production', + }, }; pipedream.render(pipedream_config, relay)