From 48ac90c78391e45fe1ae8a6efe182720d946e14b Mon Sep 17 00:00:00 2001 From: "Matt Gaunt-Seo @ Sentry.io" <112419115+mattgauntseo-sentry@users.noreply.github.com> Date: Wed, 5 Jul 2023 11:00:46 -0700 Subject: [PATCH] Add script for customer deployments (#4438) --- .../snuba-next-monitor.yaml | 46 +++++++++++++------ gocd/generated-pipelines/snuba-next-us.yaml | 31 +++++++------ gocd/templates/bash/migrate-reverse.sh | 15 +++--- gocd/templates/bash/migrate-st.sh | 26 +++++++++++ gocd/templates/bash/migrate.sh | 14 +++--- gocd/templates/pipelines/snuba.libsonnet | 8 +++- 6 files changed, 99 insertions(+), 41 deletions(-) create mode 100644 gocd/templates/bash/migrate-st.sh diff --git a/gocd/generated-pipelines/snuba-next-monitor.yaml b/gocd/generated-pipelines/snuba-next-monitor.yaml index 2434398287..bf140b0dcf 100644 --- a/gocd/generated-pipelines/snuba-next-monitor.yaml +++ b/gocd/generated-pipelines/snuba-next-monitor.yaml @@ -223,19 +223,36 @@ pipelines: jobs: migrate: elastic_profile_id: snuba + environment_variables: + SNUBA_SERVICE_NAME: snuba tasks: - script: | ##!/bin/bash + ## At the time of writing (2023-06-28) the single tenant deployments + ## have been using a different migration process compared to the + ## US deployment of snuba. + ## This script should be merged with migrate.sh if we can figure + ## out a common migration script for all regions. + eval $(/devinfra/scripts/regions/project_env_vars.py --region="${SENTRY_REGION}") + /devinfra/scripts/k8s/k8stunnel - /devinfra/scripts/k8s/k8stunnel \ - && /devinfra/scripts/k8s/k8s-spawn-job.py \ - --context="gke_${GCP_PROJECT}_${GKE_REGION}-${GKE_CLUSTER_ZONE}_${GKE_CLUSTER}" \ - --label-selector="service=snuba-admin" \ - --container-name="snuba-admin" \ - "snuba-migrate" "us.gcr.io/sentryio/snuba:${GO_REVISION_SNUBA_REPO}" \ - -- snuba migrations migrate -r complete -r partial + /devinfra/scripts/k8s/k8s-spawn-job.py \ + --label-selector="service=${SNUBA_SERVICE_NAME}" \ + --container-name="${SNUBA_SERVICE_NAME}" \ + "snuba-bootstrap" \ + "us.gcr.io/sentryio/snuba:${GO_REVISION_SNUBA_REPO}" \ + -- \ + snuba bootstrap --force --no-migrate + + /devinfra/scripts/k8s/k8s-spawn-job.py \ + --label-selector="service=${SNUBA_SERVICE_NAME}" \ + --container-name="${SNUBA_SERVICE_NAME}" \ + "snuba-migrate" \ + "us.gcr.io/sentryio/snuba:${GO_REVISION_SNUBA_REPO}" \ + -- \ + snuba migrations migrate --force - plugin: configuration: id: script-executor @@ -245,14 +262,15 @@ pipelines: ##!/bin/bash eval $(/devinfra/scripts/regions/project_env_vars.py --region="${SENTRY_REGION}") + /devinfra/scripts/k8s/k8stunnel - /devinfra/scripts/k8s/k8stunnel \ - && /devinfra/scripts/k8s/k8s-spawn-job.py \ - --context="gke_${GCP_PROJECT}_${GKE_REGION}-${GKE_CLUSTER_ZONE}_${GKE_CLUSTER}" \ - --label-selector="service=snuba-admin" \ - --container-name="snuba-admin" \ - "snuba-migrate-reverse" "us.gcr.io/sentryio/snuba:${GO_REVISION_SNUBA_REPO}" \ - -- snuba migrations reverse-in-progress + /devinfra/scripts/k8s/k8s-spawn-job.py \ + --label-selector="service=${SNUBA_SERVICE_NAME}" \ + --container-name="${SNUBA_SERVICE_NAME}" \ + "snuba-migrate-reverse" \ + "us.gcr.io/sentryio/snuba:${GO_REVISION_SNUBA_REPO}" \ + -- \ + snuba migrations reverse-in-progress run_if: failed timeout: 1200 - pipeline-complete: diff --git a/gocd/generated-pipelines/snuba-next-us.yaml b/gocd/generated-pipelines/snuba-next-us.yaml index 94d5d794b7..cb8be8ea53 100644 --- a/gocd/generated-pipelines/snuba-next-us.yaml +++ b/gocd/generated-pipelines/snuba-next-us.yaml @@ -223,19 +223,23 @@ pipelines: jobs: migrate: elastic_profile_id: snuba + environment_variables: + SNUBA_SERVICE_NAME: snuba-admin tasks: - script: | ##!/bin/bash eval $(/devinfra/scripts/regions/project_env_vars.py --region="${SENTRY_REGION}") + /devinfra/scripts/k8s/k8stunnel - /devinfra/scripts/k8s/k8stunnel \ - && /devinfra/scripts/k8s/k8s-spawn-job.py \ + /devinfra/scripts/k8s/k8s-spawn-job.py \ --context="gke_${GCP_PROJECT}_${GKE_REGION}-${GKE_CLUSTER_ZONE}_${GKE_CLUSTER}" \ - --label-selector="service=snuba-admin" \ - --container-name="snuba-admin" \ - "snuba-migrate" "us.gcr.io/sentryio/snuba:${GO_REVISION_SNUBA_REPO}" \ - -- snuba migrations migrate -r complete -r partial + --label-selector="service=${SNUBA_SERVICE_NAME}" \ + --container-name="${SNUBA_SERVICE_NAME}" \ + "snuba-migrate" \ + "us.gcr.io/sentryio/snuba:${GO_REVISION_SNUBA_REPO}" \ + -- \ + snuba migrations migrate -r complete -r partial - plugin: configuration: id: script-executor @@ -245,14 +249,15 @@ pipelines: ##!/bin/bash eval $(/devinfra/scripts/regions/project_env_vars.py --region="${SENTRY_REGION}") + /devinfra/scripts/k8s/k8stunnel - /devinfra/scripts/k8s/k8stunnel \ - && /devinfra/scripts/k8s/k8s-spawn-job.py \ - --context="gke_${GCP_PROJECT}_${GKE_REGION}-${GKE_CLUSTER_ZONE}_${GKE_CLUSTER}" \ - --label-selector="service=snuba-admin" \ - --container-name="snuba-admin" \ - "snuba-migrate-reverse" "us.gcr.io/sentryio/snuba:${GO_REVISION_SNUBA_REPO}" \ - -- snuba migrations reverse-in-progress + /devinfra/scripts/k8s/k8s-spawn-job.py \ + --label-selector="service=${SNUBA_SERVICE_NAME}" \ + --container-name="${SNUBA_SERVICE_NAME}" \ + "snuba-migrate-reverse" \ + "us.gcr.io/sentryio/snuba:${GO_REVISION_SNUBA_REPO}" \ + -- \ + snuba migrations reverse-in-progress run_if: failed timeout: 1200 - pipeline-complete: diff --git a/gocd/templates/bash/migrate-reverse.sh b/gocd/templates/bash/migrate-reverse.sh index a3832e1475..83d68fee81 100644 --- a/gocd/templates/bash/migrate-reverse.sh +++ b/gocd/templates/bash/migrate-reverse.sh @@ -1,11 +1,12 @@ #!/bin/bash eval $(/devinfra/scripts/regions/project_env_vars.py --region="${SENTRY_REGION}") +/devinfra/scripts/k8s/k8stunnel -/devinfra/scripts/k8s/k8stunnel \ -&& /devinfra/scripts/k8s/k8s-spawn-job.py \ - --context="gke_${GCP_PROJECT}_${GKE_REGION}-${GKE_CLUSTER_ZONE}_${GKE_CLUSTER}" \ - --label-selector="service=snuba-admin" \ - --container-name="snuba-admin" \ - "snuba-migrate-reverse" "us.gcr.io/sentryio/snuba:${GO_REVISION_SNUBA_REPO}" \ - -- snuba migrations reverse-in-progress +/devinfra/scripts/k8s/k8s-spawn-job.py \ + --label-selector="service=${SNUBA_SERVICE_NAME}" \ + --container-name="${SNUBA_SERVICE_NAME}" \ + "snuba-migrate-reverse" \ + "us.gcr.io/sentryio/snuba:${GO_REVISION_SNUBA_REPO}" \ + -- \ + snuba migrations reverse-in-progress diff --git a/gocd/templates/bash/migrate-st.sh b/gocd/templates/bash/migrate-st.sh new file mode 100644 index 0000000000..a76a5cdb63 --- /dev/null +++ b/gocd/templates/bash/migrate-st.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +# At the time of writing (2023-06-28) the single tenant deployments +# have been using a different migration process compared to the +# US deployment of snuba. +# This script should be merged with migrate.sh if we can figure +# out a common migration script for all regions. + +eval $(/devinfra/scripts/regions/project_env_vars.py --region="${SENTRY_REGION}") +/devinfra/scripts/k8s/k8stunnel + +/devinfra/scripts/k8s/k8s-spawn-job.py \ + --label-selector="service=${SNUBA_SERVICE_NAME}" \ + --container-name="${SNUBA_SERVICE_NAME}" \ + "snuba-bootstrap" \ + "us.gcr.io/sentryio/snuba:${GO_REVISION_SNUBA_REPO}" \ + -- \ + snuba bootstrap --force --no-migrate + +/devinfra/scripts/k8s/k8s-spawn-job.py \ + --label-selector="service=${SNUBA_SERVICE_NAME}" \ + --container-name="${SNUBA_SERVICE_NAME}" \ + "snuba-migrate" \ + "us.gcr.io/sentryio/snuba:${GO_REVISION_SNUBA_REPO}" \ + -- \ + snuba migrations migrate --force diff --git a/gocd/templates/bash/migrate.sh b/gocd/templates/bash/migrate.sh index a9e58b3eca..a1966d28e8 100644 --- a/gocd/templates/bash/migrate.sh +++ b/gocd/templates/bash/migrate.sh @@ -1,11 +1,13 @@ #!/bin/bash eval $(/devinfra/scripts/regions/project_env_vars.py --region="${SENTRY_REGION}") +/devinfra/scripts/k8s/k8stunnel -/devinfra/scripts/k8s/k8stunnel \ -&& /devinfra/scripts/k8s/k8s-spawn-job.py \ +/devinfra/scripts/k8s/k8s-spawn-job.py \ --context="gke_${GCP_PROJECT}_${GKE_REGION}-${GKE_CLUSTER_ZONE}_${GKE_CLUSTER}" \ - --label-selector="service=snuba-admin" \ - --container-name="snuba-admin" \ - "snuba-migrate" "us.gcr.io/sentryio/snuba:${GO_REVISION_SNUBA_REPO}" \ - -- snuba migrations migrate -r complete -r partial + --label-selector="service=${SNUBA_SERVICE_NAME}" \ + --container-name="${SNUBA_SERVICE_NAME}" \ + "snuba-migrate" \ + "us.gcr.io/sentryio/snuba:${GO_REVISION_SNUBA_REPO}" \ + -- \ + snuba migrations migrate -r complete -r partial diff --git a/gocd/templates/pipelines/snuba.libsonnet b/gocd/templates/pipelines/snuba.libsonnet index 8f67bea696..e929b61c3c 100644 --- a/gocd/templates/pipelines/snuba.libsonnet +++ b/gocd/templates/pipelines/snuba.libsonnet @@ -103,8 +103,14 @@ function(region) { migrate: { timeout: 1200, elastic_profile_id: 'snuba', + environment_variables: { + SNUBA_SERVICE_NAME: if region == 'monitor' || std.startsWith(region, 'customer-') then 'snuba' else 'snuba-admin', + }, tasks: [ - gocdtasks.script(importstr '../bash/migrate.sh'), + if region == 'monitor' || std.startsWith(region, 'customer-') then + gocdtasks.script(importstr '../bash/migrate-st.sh') + else + gocdtasks.script(importstr '../bash/migrate.sh'), { plugin: { options: gocdtasks.script(importstr '../bash/migrate-reverse.sh'),