From 79da5b181f68b5a9947e023cbbf8ef5df8153592 Mon Sep 17 00:00:00 2001 From: Ian Woodard <17186604+IanWoodard@users.noreply.github.com> Date: Wed, 8 May 2024 15:05:10 -0700 Subject: [PATCH 1/7] WIP move to pipedream --- .github/workflows/validate-pipelines.yml | 6 ++ .gitignore | 3 + Makefile | 15 ++++ gocd/pipelines/chartcuterie.yaml | 68 ------------------ gocd/templates/bash/check-cloudbuild.sh | 6 ++ gocd/templates/bash/check-github-runs.sh | 6 ++ gocd/templates/bash/deploy.sh | 9 +++ gocd/templates/bash/wait-canary.sh | 3 + gocd/templates/chartcuterie.jsonnet | 24 +++++++ gocd/templates/jsonnetfile.json | 15 ++++ gocd/templates/jsonnetfile.lock.json | 16 +++++ .../pipelines/chartcuterie.libsonnet | 71 +++++++++++++++++++ 12 files changed, 174 insertions(+), 68 deletions(-) create mode 100644 Makefile delete mode 100644 gocd/pipelines/chartcuterie.yaml create mode 100644 gocd/templates/bash/check-cloudbuild.sh create mode 100644 gocd/templates/bash/check-github-runs.sh create mode 100644 gocd/templates/bash/deploy.sh create mode 100644 gocd/templates/bash/wait-canary.sh create mode 100644 gocd/templates/chartcuterie.jsonnet create mode 100644 gocd/templates/jsonnetfile.json create mode 100644 gocd/templates/jsonnetfile.lock.json create mode 100644 gocd/templates/pipelines/chartcuterie.libsonnet diff --git a/.github/workflows/validate-pipelines.yml b/.github/workflows/validate-pipelines.yml index d092ce0..7809d5f 100644 --- a/.github/workflows/validate-pipelines.yml +++ b/.github/workflows/validate-pipelines.yml @@ -25,6 +25,7 @@ jobs: filters: | gocd: - 'gocd/**' + - '.github/workflows/validate-pipelines.yml' validate: if: needs.files-changed.outputs.gocd == 'true' @@ -47,6 +48,11 @@ jobs: token_format: 'id_token' id_token_audience: '610575311308-9bsjtgqg4jm01mt058rncpopujgk3627.apps.googleusercontent.com' id_token_include_email: true + - uses: getsentry/action-gocd-jsonnet@v1 + with: + jb-install: true + jsonnet-dir: gocd/templates + generated-dir: gocd/generated-pipelines - uses: getsentry/action-validate-gocd-pipelines@v1 with: configrepo: chartcuterie__master diff --git a/.gitignore b/.gitignore index 491fc35..ef0c9de 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,5 @@ node_modules lib + +/gocd/generated-pipelines +/gocd/templates/vendor diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..d982510 --- /dev/null +++ b/Makefile @@ -0,0 +1,15 @@ +gocd: ## Build GoCD pipelines + rm -rf ./gocd/generated-pipelines + mkdir -p ./gocd/generated-pipelines + cd ./gocd/templates && jb install && jb update + + # Format + find . -type f \( -name '*.libsonnet' -o -name '*.jsonnet' \) -print0 | xargs -n 1 -0 jsonnetfmt -i + # Lint + find . -type f \( -name '*.libsonnet' -o -name '*.jsonnet' \) -print0 | xargs -n 1 -0 jsonnet-lint -J ./gocd/templates/vendor + # Build + cd ./gocd/templates && find . -type f \( -name '*.jsonnet' \) -print0 | xargs -n 1 -0 jsonnet --ext-code output-files=true -J vendor -m ../generated-pipelines + + # Convert JSON to yaml + cd ./gocd/generated-pipelines && find . -type f \( -name '*.yaml' \) -print0 | xargs -n 1 -0 yq -p json -o yaml -i +.PHONY: gocd diff --git a/gocd/pipelines/chartcuterie.yaml b/gocd/pipelines/chartcuterie.yaml deleted file mode 100644 index 782fade..0000000 --- a/gocd/pipelines/chartcuterie.yaml +++ /dev/null @@ -1,68 +0,0 @@ -# More information on gocd-flavor YAML can be found here: -# - https://github.com/tomzo/gocd-yaml-config-plugin#pipeline -# - https://www.notion.so/sentry/GoCD-New-Service-Quickstart-6d8db7a6964049b3b0e78b8a4b52e25d -format_version: 10 -pipelines: - deploy-chartcuterie: - environment_variables: - GCP_PROJECT: internal-sentry - GKE_CLUSTER: zdpwkxst - GKE_REGION: us-central1 - GKE_CLUSTER_ZONE: b - GKE_BASTION_ZONE: b - # Required for checkruns. - GITHUB_TOKEN: "{{SECRET:[devinfra-github][token]}}" - group: chartcuterie - lock_behavior: unlockWhenFinished - materials: - chartcuterie_repo: - git: git@github.com:getsentry/chartcuterie.git - shallow_clone: true - branch: master - destination: chartcuterie - stages: - - checks: - jobs: - checks: - timeout: 1200 - elastic_profile_id: chartcuterie - tasks: - - script: | - /devinfra/scripts/checks/githubactions/checkruns.py \ - getsentry/chartcuterie \ - ${GO_REVISION_CHARTCUTERIE_REPO} \ - build - - script: | - /devinfra/scripts/checks/googlecloud/checkcloudbuild.py \ - ${GO_REVISION_CHARTCUTERIE_REPO} \ - sentryio \ - "us.gcr.io/sentryio/chartcuterie" - - deploy-canary: - jobs: - deploy: - timeout: 600 - elastic_profile_id: chartcuterie - tasks: - - script: | - /devinfra/scripts/k8s/k8stunnel \ - && /devinfra/scripts/k8s/k8s-deploy.py \ - --context="gke_${GCP_PROJECT}_${GKE_REGION}-${GKE_CLUSTER_ZONE}_${GKE_CLUSTER}" \ - --label-selector="service=chartcuterie,env=canary" \ - --image="us.gcr.io/sentryio/chartcuterie:${GO_REVISION_CHARTCUTERIE_REPO}" \ - --container-name="chartcuterie" - - script: | - /devinfra/scripts/canary/canarychecks.py \ - --wait-minutes=5 - - deploy-primary: - jobs: - deploy: - timeout: 600 - elastic_profile_id: chartcuterie - tasks: - - script: | - /devinfra/scripts/k8s/k8stunnel \ - && /devinfra/scripts/k8s/k8s-deploy.py \ - --context="gke_${GCP_PROJECT}_${GKE_REGION}-${GKE_CLUSTER_ZONE}_${GKE_CLUSTER}" \ - --label-selector="service=chartcuterie" \ - --image="us.gcr.io/sentryio/chartcuterie:${GO_REVISION_CHARTCUTERIE_REPO}" \ - --container-name="chartcuterie" diff --git a/gocd/templates/bash/check-cloudbuild.sh b/gocd/templates/bash/check-cloudbuild.sh new file mode 100644 index 0000000..b651195 --- /dev/null +++ b/gocd/templates/bash/check-cloudbuild.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +/devinfra/scripts/checks/googlecloud/checkcloudbuild.py \ + "${GO_REVISION_CHARTCUTERIE_REPO}" \ + sentryio \ + "us.gcr.io/sentryio/chartcuterie" diff --git a/gocd/templates/bash/check-github-runs.sh b/gocd/templates/bash/check-github-runs.sh new file mode 100644 index 0000000..f88447a --- /dev/null +++ b/gocd/templates/bash/check-github-runs.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +/devinfra/scripts/checks/githubactions/checkruns.py \ + getsentry/chartcuterie \ + "${GO_REVISION_CHARTCUTERIE_REPO}" \ + build diff --git a/gocd/templates/bash/deploy.sh b/gocd/templates/bash/deploy.sh new file mode 100644 index 0000000..93fd25f --- /dev/null +++ b/gocd/templates/bash/deploy.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +eval $(/devinfra/scripts/regions/project_env_vars.py --region="${SENTRY_REGION}") + +/devinfra/scripts/k8s/k8stunnel \ + && /devinfra/scripts/k8s/k8s-deploy.py \ + --label-selector="${LABEL_SELECTOR}" \ + --image="us.gcr.io/sentryio/chartcuterie:${GO_REVISION_CHARTCUTERIE_REPO}" \ + --container-name="chartcuterie" diff --git a/gocd/templates/bash/wait-canary.sh b/gocd/templates/bash/wait-canary.sh new file mode 100644 index 0000000..6abe472 --- /dev/null +++ b/gocd/templates/bash/wait-canary.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +sleep 300 diff --git a/gocd/templates/chartcuterie.jsonnet b/gocd/templates/chartcuterie.jsonnet new file mode 100644 index 0000000..8268bb1 --- /dev/null +++ b/gocd/templates/chartcuterie.jsonnet @@ -0,0 +1,24 @@ +local chartcuterie = import './pipelines/chartcuterie.libsonnet'; +local pipedream = import 'github.com/getsentry/gocd-jsonnet/libs/pipedream.libsonnet'; + +// Pipedream can be configured using this object, you can learn more about the +// configuration options here: https://github.com/getsentry/gocd-jsonnet#readme +local pipedream_config = { + name: 'chartcuterie', + auto_deploy: false, + materials: { + chartcuterie_repo: { + git: 'git@github.com:getsentry/chartcuterie.git', + shallow_clone: true, + branch: 'master', + destination: 'chartcuterie', + }, + }, + rollback: { + material_name: 'chartcuterie_repo', + stage: 'deploy-primary', + elastic_profile_id: 'chartcuterie', + }, +}; + +pipedream.render(pipedream_config, chartcuterie) diff --git a/gocd/templates/jsonnetfile.json b/gocd/templates/jsonnetfile.json new file mode 100644 index 0000000..3ce163f --- /dev/null +++ b/gocd/templates/jsonnetfile.json @@ -0,0 +1,15 @@ +{ + "version": 1, + "dependencies": [ + { + "source": { + "git": { + "remote": "https://github.com/getsentry/gocd-jsonnet.git", + "subdir": "libs" + } + }, + "version": "v2.8" + } + ], + "legacyImports": true +} diff --git a/gocd/templates/jsonnetfile.lock.json b/gocd/templates/jsonnetfile.lock.json new file mode 100644 index 0000000..6987c37 --- /dev/null +++ b/gocd/templates/jsonnetfile.lock.json @@ -0,0 +1,16 @@ +{ + "version": 1, + "dependencies": [ + { + "source": { + "git": { + "remote": "https://github.com/getsentry/gocd-jsonnet.git", + "subdir": "libs" + } + }, + "version": "172de74e6347127957707d990f20f3e99c6c1c46", + "sum": "pBB9Jio0EnAgB7811tnly/S1B5fz6BeYoi4hQ7KgsHM=" + } + ], + "legacyImports": false +} diff --git a/gocd/templates/pipelines/chartcuterie.libsonnet b/gocd/templates/pipelines/chartcuterie.libsonnet new file mode 100644 index 0000000..b784b05 --- /dev/null +++ b/gocd/templates/pipelines/chartcuterie.libsonnet @@ -0,0 +1,71 @@ +local gocdtasks = import 'github.com/getsentry/gocd-jsonnet/libs/gocd-tasks.libsonnet'; + +function(region) { + environment_variables: { + // SENTRY_REGION is used by the dev-infra scripts to connect to GKE + SENTRY_REGION: region, + }, + materials: { + chartcuterie_repo: { + git: 'git@github.com:getsentry/chartcuterie.git', + shallow_clone: true, + branch: 'master', + destination: 'chartcuterie', + }, + }, + lock_behavior: 'unlockWhenFinished', + stages: [ + { + checks: { + fetch_materials: true, + jobs: { + checks: { + timeout: 1200, + environment_variables: { + GITHUB_TOKEN: '{{SECRET:[devinfra-github][token]}}', + }, + tasks: [ + gocdtasks.script(importstr '../bash/check-github-runs.sh'), + gocdtasks.script(importstr '../bash/check-cloudbuild.sh'), + ], + }, + }, + }, + }, + { + 'deploy-canary': { + fetch_materials: true, + jobs: { + deploy: { + timeout: 600, + elastic_profile_id: 'chartcuterie', + environment_variables: { + LABEL_SELECTOR: 'service=chartcuterie,env=canary', + }, + tasks: [ + gocdtasks.script(importstr '../bash/deploy.sh'), + gocdtasks.script(importstr '../bash/wait-canary.sh'), + ], + }, + }, + }, + }, + { + 'deploy-primary': { + fetch_materials: true, + jobs: { + deploy: { + timeout: 600, + elastic_profile_id: 'chartcuterie', + environment_variables: { + LABEL_SELECTOR: 'service=chartcuterie', + }, + tasks: [ + gocdtasks.script(importstr '../bash/deploy.sh'), + ], + }, + }, + }, + }, + ], +} From fd705338cb200a07c514542b215a7832eff95d17 Mon Sep 17 00:00:00 2001 From: Ian Woodard <17186604+IanWoodard@users.noreply.github.com> Date: Tue, 28 May 2024 14:44:35 -0700 Subject: [PATCH 2/7] Bumping gocd-jsonnet version --- gocd/templates/chartcuterie.jsonnet | 3 ++- gocd/templates/jsonnetfile.json | 2 +- gocd/templates/jsonnetfile.lock.json | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/gocd/templates/chartcuterie.jsonnet b/gocd/templates/chartcuterie.jsonnet index 8268bb1..63caa5b 100644 --- a/gocd/templates/chartcuterie.jsonnet +++ b/gocd/templates/chartcuterie.jsonnet @@ -5,7 +5,7 @@ local pipedream = import 'github.com/getsentry/gocd-jsonnet/libs/pipedream.libso // configuration options here: https://github.com/getsentry/gocd-jsonnet#readme local pipedream_config = { name: 'chartcuterie', - auto_deploy: false, + auto_deploy: true, materials: { chartcuterie_repo: { git: 'git@github.com:getsentry/chartcuterie.git', @@ -19,6 +19,7 @@ local pipedream_config = { stage: 'deploy-primary', elastic_profile_id: 'chartcuterie', }, + exclude_regions: ['s4s', 'customer-1', 'customer-2', 'customer-3', 'customer-3', 'customer-4', 'customer-6', 'customer-7'], }; pipedream.render(pipedream_config, chartcuterie) diff --git a/gocd/templates/jsonnetfile.json b/gocd/templates/jsonnetfile.json index 3ce163f..050d936 100644 --- a/gocd/templates/jsonnetfile.json +++ b/gocd/templates/jsonnetfile.json @@ -8,7 +8,7 @@ "subdir": "libs" } }, - "version": "v2.8" + "version": "v2.10.0" } ], "legacyImports": true diff --git a/gocd/templates/jsonnetfile.lock.json b/gocd/templates/jsonnetfile.lock.json index 6987c37..94bb9ef 100644 --- a/gocd/templates/jsonnetfile.lock.json +++ b/gocd/templates/jsonnetfile.lock.json @@ -8,8 +8,8 @@ "subdir": "libs" } }, - "version": "172de74e6347127957707d990f20f3e99c6c1c46", - "sum": "pBB9Jio0EnAgB7811tnly/S1B5fz6BeYoi4hQ7KgsHM=" + "version": "74ae5728e2d7ed39fdd43cf3b2d28dde7e4567a1", + "sum": "AKMGYALLyaVVVjTNnZy64PoCDA8QjxTbHBe5dCnE4tE=" } ], "legacyImports": false From 7b1f88f0bd4fe00d87027edcfe33c6aa5e5bbfde Mon Sep 17 00:00:00 2001 From: Ian Woodard <17186604+IanWoodard@users.noreply.github.com> Date: Tue, 28 May 2024 14:53:17 -0700 Subject: [PATCH 3/7] Updating gocd scripts --- gocd/templates/bash/check-cloudbuild.sh | 2 +- gocd/templates/bash/deploy.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/gocd/templates/bash/check-cloudbuild.sh b/gocd/templates/bash/check-cloudbuild.sh index b651195..effb7a3 100644 --- a/gocd/templates/bash/check-cloudbuild.sh +++ b/gocd/templates/bash/check-cloudbuild.sh @@ -3,4 +3,4 @@ /devinfra/scripts/checks/googlecloud/checkcloudbuild.py \ "${GO_REVISION_CHARTCUTERIE_REPO}" \ sentryio \ - "us.gcr.io/sentryio/chartcuterie" + "us-central1-docker.pkg.dev/sentryio/chartcuterie/image" diff --git a/gocd/templates/bash/deploy.sh b/gocd/templates/bash/deploy.sh index 93fd25f..ad337e3 100644 --- a/gocd/templates/bash/deploy.sh +++ b/gocd/templates/bash/deploy.sh @@ -5,5 +5,5 @@ eval $(/devinfra/scripts/regions/project_env_vars.py --region="${SENTRY_REGION}" /devinfra/scripts/k8s/k8stunnel \ && /devinfra/scripts/k8s/k8s-deploy.py \ --label-selector="${LABEL_SELECTOR}" \ - --image="us.gcr.io/sentryio/chartcuterie:${GO_REVISION_CHARTCUTERIE_REPO}" \ + --image="us-central1-docker.pkg.dev/sentryio/chartcuterie/image:${GO_REVISION_CHARTCUTERIE_REPO}" \ --container-name="chartcuterie" From f0b888ade2dca5479db9642f6eea329697020c77 Mon Sep 17 00:00:00 2001 From: Raj Joshi Date: Tue, 28 May 2024 15:13:30 -0700 Subject: [PATCH 4/7] removed excluded regions --- gocd/templates/chartcuterie.jsonnet | 1 - 1 file changed, 1 deletion(-) diff --git a/gocd/templates/chartcuterie.jsonnet b/gocd/templates/chartcuterie.jsonnet index 63caa5b..4ee5660 100644 --- a/gocd/templates/chartcuterie.jsonnet +++ b/gocd/templates/chartcuterie.jsonnet @@ -19,7 +19,6 @@ local pipedream_config = { stage: 'deploy-primary', elastic_profile_id: 'chartcuterie', }, - exclude_regions: ['s4s', 'customer-1', 'customer-2', 'customer-3', 'customer-3', 'customer-4', 'customer-6', 'customer-7'], }; pipedream.render(pipedream_config, chartcuterie) From 4f2ded42d4d99d2d72f5507ccec93a42ecb6861a Mon Sep 17 00:00:00 2001 From: Raj Joshi Date: Tue, 28 May 2024 15:42:45 -0700 Subject: [PATCH 5/7] updated bash scripts to use named arguments --- gocd/templates/bash/check-cloudbuild.sh | 6 +++--- gocd/templates/bash/check-github-runs.sh | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/gocd/templates/bash/check-cloudbuild.sh b/gocd/templates/bash/check-cloudbuild.sh index effb7a3..1218687 100644 --- a/gocd/templates/bash/check-cloudbuild.sh +++ b/gocd/templates/bash/check-cloudbuild.sh @@ -1,6 +1,6 @@ #!/bin/bash /devinfra/scripts/checks/googlecloud/checkcloudbuild.py \ - "${GO_REVISION_CHARTCUTERIE_REPO}" \ - sentryio \ - "us-central1-docker.pkg.dev/sentryio/chartcuterie/image" + --sha="${GO_REVISION_CHARTCUTERIE_REPO}" \ + --project="sentryio" \ + --image_name="us-central1-docker.pkg.dev/sentryio/chartcuterie/image" diff --git a/gocd/templates/bash/check-github-runs.sh b/gocd/templates/bash/check-github-runs.sh index f88447a..bd93c57 100644 --- a/gocd/templates/bash/check-github-runs.sh +++ b/gocd/templates/bash/check-github-runs.sh @@ -1,6 +1,6 @@ #!/bin/bash /devinfra/scripts/checks/githubactions/checkruns.py \ - getsentry/chartcuterie \ - "${GO_REVISION_CHARTCUTERIE_REPO}" \ - build + --repo="getsentry/chartcuterie" \ + --sha="${GO_REVISION_CHARTCUTERIE_REPO}" \ + --check_names="build" From 96d90fd759d2c6a63163dc95ac6cac8be20ca75a Mon Sep 17 00:00:00 2001 From: Raj Joshi Date: Thu, 30 May 2024 13:23:16 -0700 Subject: [PATCH 6/7] exclude non de/us regions --- gocd/templates/chartcuterie.jsonnet | 1 + 1 file changed, 1 insertion(+) diff --git a/gocd/templates/chartcuterie.jsonnet b/gocd/templates/chartcuterie.jsonnet index 4ee5660..9a07a9c 100644 --- a/gocd/templates/chartcuterie.jsonnet +++ b/gocd/templates/chartcuterie.jsonnet @@ -19,6 +19,7 @@ local pipedream_config = { stage: 'deploy-primary', elastic_profile_id: 'chartcuterie', }, + exclude_regions: ['customer-1', 'customer-2', 'customer-3', 'customer-4', 'customer-5', 'customer-6', 'customer-7', 's4s'], }; pipedream.render(pipedream_config, chartcuterie) From d5d0752df654ca0f92c29c280653d922cefefe73 Mon Sep 17 00:00:00 2001 From: Ian Woodard <17186604+IanWoodard@users.noreply.github.com> Date: Thu, 30 May 2024 15:28:01 -0700 Subject: [PATCH 7/7] Cleaning up excluded regions --- gocd/templates/chartcuterie.jsonnet | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gocd/templates/chartcuterie.jsonnet b/gocd/templates/chartcuterie.jsonnet index 9a07a9c..5c43176 100644 --- a/gocd/templates/chartcuterie.jsonnet +++ b/gocd/templates/chartcuterie.jsonnet @@ -19,7 +19,7 @@ local pipedream_config = { stage: 'deploy-primary', elastic_profile_id: 'chartcuterie', }, - exclude_regions: ['customer-1', 'customer-2', 'customer-3', 'customer-4', 'customer-5', 'customer-6', 'customer-7', 's4s'], + exclude_regions: ['s4s', 'customer-1', 'customer-2', 'customer-3', 'customer-4', 'customer-6', 'customer-7'], }; pipedream.render(pipedream_config, chartcuterie)