feat: k8s continuous deployment

.github/workflows/ci.yml

@@ -59,17 +59,17 @@ jobs:
         name: Set main branch tag
         if: ${{ env.BRANCH == 'main' }}
         run: |
-          echo "ENV_TAG=prod" >> $GITHUB_ENV
+          echo "DEPLOY_ENV=prod" >> $GITHUB_ENV
       -
         name: Set rc branch tag
         if: ${{ env.BRANCH == 'release-candidate' }}
         run: |
-          echo "ENV_TAG=tnet" >> $GITHUB_ENV
+          echo "DEPLOY_ENV=tnet" >> $GITHUB_ENV
       -
         name: Set develop branch tag
-        if: ${{ env.BRANCH == 'develop' || env.ENV_TAG == '' }}
+        if: ${{ env.BRANCH == 'develop' || env.DEPLOY_ENV == '' }}
         run: |
-          echo "ENV_TAG=dev" >> $GITHUB_ENV
+          echo "DEPLOY_ENV=dev" >> $GITHUB_ENV
       -
         name: Set publish flag
         if: ${{ env.BRANCH == 'main' || env.BRANCH == 'release-candidate' || env.BRANCH == 'develop' }}
@@ -81,7 +81,20 @@ jobs:
       -
         name: Push Docker image
         if: ${{ env.PUBLISH == 'true' }}
-        run: dagger do push -w "actions:push:\"${{ env.AWS_REGION }}\":\"${{ env.ENV_TAG }}\":\"${{ env.BRANCH }}\":\"${{ env.SHA }}\":\"${{ env.SHA_TAG }}\":\"${{ env.VERSION }}\":_" -p ${{ env.DAGGER_PLAN }}
+        run: dagger do push -w "actions:push:\"${{ env.AWS_REGION }}\":\"${{ env.DEPLOY_ENV }}\":\"${{ env.BRANCH }}\":\"${{ env.SHA }}\":\"${{ env.SHA_TAG }}\":\"${{ env.VERSION }}\":_" -p ${{ env.DAGGER_PLAN }}
+      -
+        name: Schedule k8s deployment
+        run: |
+          # Schedule deployment
+          make DEPLOY_ENV="$DEPLOY_ENV" DEPLOY_TAG=${{ env.SHA }} schedule-k8s-deployment
+          # Schedule post-deployment tests
+          make DEPLOY_ENV="$DEPLOY_ENV" TEST_SELECTOR="correctness/fast" schedule-tests
+
+          # If deploying to QA, also deploy to Dev and run post-deployment tests.
+          if [[ "$DEPLOY_ENV" == "qa" ]]; then
+            make DEPLOY_ENV="dev" DEPLOY_TAG=${{ env.SHA }} schedule-k8s-deployment
+            make DEPLOY_ENV="dev" TEST_SELECTOR="correctness/fast" schedule-tests
+          fi
       -
         name: Set commit status "success"
         run: dagger do success -p ${{ env.STATUS_PLAN }}

Makefile

@@ -0,0 +1,19 @@
+# Makefile provides an API for CI related tasks
+# Using the makefile is not required however CI
+# uses the specific targets within the file.
+# Therefore may be useful in ensuring a change
+# is ready to pass CI checks.
+
+# ECS environment to deploy image to
+DEPLOY_ENV ?= dev
+
+# Docker image tag to deploy
+DEPLOY_TAG ?= latest
+
+.PHONY: schedule-k8s-deployment
+schedule-k8s-deployment:
+	./ci-scripts/schedule_k8s_deploy.sh "${DEPLOY_ENV}" "${DEPLOY_TAG}"
+
+.PHONY: schedule-tests
+schedule-tests:
+	./ci-scripts/schedule_tests.sh "${DEPLOY_ENV}" "${TEST_SELECTOR}"

ci-scripts/schedule_k8s_deploy.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+
+id=$(uuidgen)
+job_id=$(uuidgen)
+now=$(date +%s%N)
+ttl=$(date +%s -d "14 days")
+image=ceramicnetwork/js-ceramic:${2-dev}
+network=${1-dev}
+environment=ceramic-v4-${network}
+
+docker run --rm -i \
+  -e "AWS_REGION=$AWS_REGION" \
+  -e "AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID" \
+  -e "AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY" \
+  -v ~/.aws:/root/.aws \
+  -v "$PWD":/aws \
+  amazon/aws-cli dynamodb put-item --table-name "ceramic-$network-ops" --item \
+  "{                                                                \
+    \"id\":     {\"S\": \"$id\"},                                   \
+    \"job\":    {\"S\": \"$job_id\"},                               \
+    \"ts\":     {\"N\": \"$now\"},                                  \
+    \"ttl\":    {\"N\": \"$ttl\"},                                  \
+    \"stage\":  {\"S\": \"queued\"},                                \
+    \"type\":   {\"S\": \"workflow\"},                              \
+    \"params\": {                                                   \
+      \"M\": {                                                      \
+        \"name\":     {\"S\": \"Deploy k8s $network JS-CERAMIC\"},  \
+        \"org\":      {\"S\": \"3box\"},                            \
+        \"repo\":     {\"S\": \"ceramic-infra\"},                   \
+        \"ref\":      {\"S\": \"main\"},                            \
+        \"workflow\": {\"S\": \"update_image.yml\"},                \
+        \"labels\":   {\"L\": [{\"S\": \"deploy\"}]},               \
+        \"inputs\":   {                                             \
+          \"M\": {                                                  \
+            \"ceramic_image\":     {\"S\": \"$image\"},             \
+            \"environment\":       {\"S\": \"$environment\"}        \
+          }                                                         \
+        }                                                           \
+      }                                                             \
+    }                                                               \
+  }"

ci-scripts/schedule_tests.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+id=$(uuidgen)
+job_id=$(uuidgen)
+# Schedule tests for 15 minutes in the future to allow the network to stabilize. This assumes that the deployment is
+# successful with the right image being deployed, which might not always be the case if the deployment fails for some
+# reason. In the future, this can be done better via the CD manager, which will check for the network being ready with
+# the right image before scheduling tests.
+now=$(date +%s%N -d "15 minutes")
+ttl=$(date +%s -d "14 days")
+network=${1-dev}
+test_selector=${2-.}
+
+docker run --rm -i \
+  -e "AWS_REGION=$AWS_REGION" \
+  -e "AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID" \
+  -e "AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY" \
+  -v ~/.aws:/root/.aws \
+  -v "$PWD":/aws \
+  amazon/aws-cli dynamodb put-item --table-name "ceramic-$network-ops" --item \
+  "{                                                        \
+    \"id\":     {\"S\": \"$id\"},                           \
+    \"job\":    {\"S\": \"$job_id\"},                       \
+    \"ts\":     {\"N\": \"$now\"},                          \
+    \"ttl\":    {\"N\": \"$ttl\"},                          \
+    \"stage\":  {\"S\": \"queued\"},                        \
+    \"type\":   {\"S\": \"workflow\"},                      \
+    \"params\": {                                           \
+      \"M\": {                                              \
+        \"name\":     {\"S\": \"Post-Deployment Tests\"},   \
+        \"org\":      {\"S\": \"3box\"},                    \
+        \"repo\":     {\"S\": \"ceramic-tests\"},           \
+        \"ref\":      {\"S\": \"main\"},                    \
+        \"workflow\": {\"S\": \"run-durable.yml\"},         \
+        \"labels\":   {\"L\": [{\"S\": \"test\"}]},         \
+        \"inputs\":   {                                     \
+          \"M\": {                                          \
+            \"test_selector\": {\"S\": \"$test_selector\"}  \
+          }                                                 \
+        }                                                   \
+      }                                                     \
+    }                                                       \
+  }"