Merge pull request #1747 from Logflare/chore/workflow-dispatch-cloudb…

…uild chore: allow workflow_dispatch for cloudbuild trigger, move gcloud scripts to makefile
Logflare · Oct 12, 2023 · 55862e3 · 55862e3
2 parents 2052b98 + e1622b3
commit 55862e3
Show file tree

Hide file tree

Showing 4 changed files with 70 additions and 29 deletions.
diff --git a/.github/workflows/trigger-cloudbuild.yml b/.github/workflows/trigger-cloudbuild.yml
@@ -1,14 +1,15 @@
 name: Cloudbuild Build & Deploy
 
 on:
+  workflow_dispatch:
   workflow_call:
 
 permissions:
   contents: read
 
 jobs:
   build_image:
-    name: Build Prod Versioned GCP Image
+    name: Build & Deploy GCP Image
     strategy:
       matrix:
         env: [prod, staging]
@@ -47,27 +48,34 @@ jobs:
         with:
           version: '418.0.0'
           project_id: ${{ env.project_id }}
-      - name: Bake image for non-versioned (staging only)
+      - name: Build & deploy for staging (main branch)
         if: github.event_name != 'release' && matrix.env == 'staging'
-        run: |
-          gcloud builds submit projects/${{ env.project_id }}/locations/${{ env.region }}/connections/${{ env.host_connection}}/repositories/${{ env.repository_name }} --config=cloudbuild/staging/build-image.yaml --substitutions=_IMAGE_TAG=dev-${{ env.short_sha }}
-      - name: Bake image for versioned (prod and staging)
-        if: github.event_name == 'release'
-        run: |
-          gcloud builds submit projects/${{ env.project_id }}/locations/${{ env.region }}/connections/${{ env.host_connection}}/repositories/${{ env.repository_name }} --config=cloudbuild/${{ matrix.env }}/build-image.yaml --substitutions=_IMAGE_TAG=${{ env.version }}
+        run: make deploy.staging.main
+
+      # - name: Bake image for versioned (prod and staging)
+      #   if: github.event_name == 'release'
+      #   run: |
+      #     gcloud builds submit \
+      #         projects/${{ env.project_id }}/locations/${{ env.region }}/connections/${{ env.host_connection}}/repositories/${{ env.repository_name }} \
+      #         --config=cloudbuild/${{ matrix.env }}/build-image.yaml \
+      #         --substitutions=_IMAGE_TAG=${{ env.version }} \
+      #         --region=${{ env.region }}
 
 
   # only do auto-deploy for staging non-versioned
-  deploy:
-    runs-on: ubuntu-latest
-    needs:
-      - build_image
-    if: github.ref == 'refs/heads/main' && github.event_name == 'pull_request'
-    name: Deploy to Staging
-    steps:
-      - name: Env vars
-        run: |
-          echo "short_sha=$(git rev-parse --short HEAD)" >> "$GITHUB_ENV"
-      - name: Deploy
-        run: |
-          gcloud builds submit --no-source --config=cloudbuild/staging/deploy.yaml --substitutions=_IMAGE_TAG=${{ env.short_sha }}
+  # deploy:
+  #   runs-on: ubuntu-latest
+  #   needs:
+  #     - build_image
+  #   if: github.ref == 'refs/heads/main' && github.event_name == 'pull_request'
+  #   name: Deploy to Staging
+  #   steps:
+  #     - name: Env vars
+  #       run: |
+  #         echo "short_sha=$(git rev-parse --short HEAD)" >> "$GITHUB_ENV"
+  #     - name: Deploy
+  #       run: |
+  #         gcloud builds submit \
+  #             --no-source \
+  #             --config=cloudbuild/staging/deploy.yaml \
+  #             --substitutions=_IMAGE_TAG=dev-${{ env.short_sha }}
diff --git a/Makefile b/Makefile
@@ -2,10 +2,11 @@ GCLOUD_LOCATION ?= us-central1
 GCLOUD_KEYRING ?= logflare-keyring-us-central1
 GCLOUD_KEY ?= logflare-secrets-key
 GCLOUD_PROJECT ?= logflare-staging
-
 ERL_COOKIE ?= monster
 
 ENV ?= dev
+SHA_IMAGE_TAG ?= dev-$(shell git rev-parse --short HEAD)
+VERSION ?= $(shell cat ./VERSION)
 
 help:
 	@cat DEVELOPMENT.md
@@ -98,3 +99,34 @@ grpc.protoc:
 	trap 'rm -rf "$$dir"' EXIT; \
 	git clone https://github.com/open-telemetry/opentelemetry-proto.git $$dir; \
 	protoc -I=$$dir --elixir_out=plugins=grpc:$(PWD)/lib/logflare_grpc $$(find $$dir -iname '*.proto')
+
+
+# manual deployment scripts
+
+deploy.staging.main:
+	gcloud builds submit \
+		projects/logflare-staging/locations/us-central1/connections/github-logflare/repositories/Logflare-logflare \
+		--revision=main  \
+		--config=cloudbuild/staging/build-image.yaml \
+		--substitutions=_IMAGE_TAG=$(SHA_IMAGE_TAG) \
+		--region=us-central1
+
+	gcloud builds submit \
+		--no-source \
+		--config=./cloudbuild/staging/deploy.yaml \
+		--substitutions=_IMAGE_TAG=$(SHA_IMAGE_TAG)
+
+deploy.staging.versioned:
+	gcloud builds submit \
+		projects/logflare-staging/locations/us-central1/connections/github-logflare/repositories/Logflare-logflare \
+		--revision=main  \
+		--config=cloudbuild/staging/build-image.yaml \
+		--substitutions=_IMAGE_TAG=$(VERSION) \
+		--region=us-central1
+
+	gcloud builds submit \
+		--no-source \
+		--config=./cloudbuild/staging/deploy.yaml \
+		--substitutions=_IMAGE_TAG=$(VERSION),_INSTANCE_TYPE=c2d-standard-8,_CLUSTER=versioned
+
+.PHONY: deploy.staging.main
diff --git a/cloudbuild/staging/build-image.yaml b/cloudbuild/staging/build-image.yaml
@@ -3,7 +3,7 @@ steps:
     args:
       - kms
       - decrypt
-      - --ciphertext-file=cloudbuild/gcloud_staging.json.enc
+      - --ciphertext-file=./cloudbuild/gcloud_staging.json.enc
       - --plaintext-file=gcloud.json
       - --location=us-central1
       - --keyring=logflare-keyring-us-central1
@@ -51,7 +51,7 @@ steps:
         "--build-arg",
         "TAG_VERSION=${_IMAGE_TAG}",
         "-f",
-        "cloudbuild/secret_setup.Dockerfile",
+        "./cloudbuild/secret_setup.Dockerfile",
         "-t",
         "${_CONTAINER_IMAGE}",
         "-t",
@@ -63,11 +63,11 @@ substitutions:
     # override this manually with cli as we aren't using cloud trigger anymore
     _ENV: staging
     _IMAGE_TAG: $SHORT_SHA
-    _CONTAINER_IMAGE: gcr.io/$PROJECT_ID/logflare_app:${_IMAGE_TAG}
+    _CONTAINER_IMAGE: gcr.io/${PROJECT_ID}/logflare_app:${_IMAGE_TAG}
 
 images:
-  - "gcr.io/$PROJECT_ID/logflare_app:${_IMAGE_TAG}"
-  - "gcr.io/$PROJECT_ID/logflare_app:latest"
+  - "gcr.io/${PROJECT_ID}/logflare_app:${_IMAGE_TAG}"
+  - "gcr.io/${PROJECT_ID}/logflare_app:latest"
 
 timeout: 1800s
 

diff --git a/cloudbuild/staging/deploy.yaml b/cloudbuild/staging/deploy.yaml
@@ -7,7 +7,7 @@ steps:
       - instance-templates
       - create-with-container
       - $_TEMPLATE_NAME
-      - --machine-type=e2-standard-2
+      - --machine-type=${_INSTANCE_TYPE}
       - --project=logflare-staging
       - --network-interface=network=default,network-tier=PREMIUM
       - --maintenance-policy=MIGRATE
@@ -47,7 +47,8 @@ steps:
 
 substitutions:
     _COOKIE: default
-    _CLUSTER: main # default value
+    _CLUSTER: main
+    _INSTANCE_TYPE: e2-standard-2
     _INSTANCE_GROUP: instance-group-staging-${_CLUSTER}
     _IMAGE_TAG: $SHORT_SHA
     _TEMPLATE_NAME: logflare-staging-${_CLUSTER}-cluster-${_IMAGE_TAG}