Deploy from Local (#1826)

* Scripts moved into an external file so we can deploy from local if Github Actions are down

* Set -ex command added

* Fixing edge cases where deployments fail if there are no tags to cleanup

* Increased timeout time for logs & added changes to workers per core & memory limit on deploy

.github/workflows/production.yml

@@ -30,29 +30,8 @@ jobs:
         run: |
           echo Login
           oc login "${{ secrets.OPENSHIFT_CLUSTER }}" --token="${{ secrets.OC4_PROD_TOKEN }}"
-          echo Promote
-          bash openshift/scripts/oc_promote.sh ${SUFFIX} apply
-          bash openshift/scripts/oc_promote_ubuntu.sh ${SUFFIX} apply
-          echo Provision database
-          CPU_REQUEST=75m CPU_LIMIT=2000m MEMORY_REQUEST=2Gi MEMORY_LIMIT=16Gi PVC_SIZE=45Gi PROJ_TARGET=e1e498-prod bash openshift/scripts/oc_provision_db.sh prod apply
-          echo Deploy API
-          WORKERS_PER_CORE=0.25 CPU_REQUEST=100m CPU_LIMIT=500m MEMORY_REQUEST=3Gi MEMORY_LIMIT=6Gi REPLICAS=3 PROJ_TARGET=e1e498-prod VANITY_DOMAIN=psu.nrs.gov.bc.ca SECOND_LEVEL_DOMAIN=apps.silver.devops.gov.bc.ca bash ./openshift/scripts/oc_deploy.sh prod apply
-          echo Env Canada Subscriber
-          PROJ_TARGET=e1e498-prod bash openshift/scripts/oc_provision_ec_gdps_cronjob.sh prod apply
-          PROJ_TARGET=e1e498-prod bash openshift/scripts/oc_provision_ec_hrdps_cronjob.sh prod apply
-          PROJ_TARGET=e1e498-prod bash openshift/scripts/oc_provision_ec_rdps_cronjob.sh prod apply
-          echo C-Haines
-          PROJ_TARGET=e1e498-prod bash openshift/scripts/oc_provision_c_haines_cronjob.sh prod apply
-          echo BC FireWeather cronjobs
-          echo "Run forecast at 8h30 PDT and 16h30 PDT (so before and after noon)"
-          PROJ_TARGET=e1e498-prod SCHEDULE="30 * * * *" bash openshift/scripts/oc_provision_wfwx_noon_forecasts_cronjob.sh prod apply
-          PROJ_TARGET=e1e498-prod SCHEDULE="15 * * * *" bash openshift/scripts/oc_provision_wfwx_hourly_actuals_cronjob.sh prod apply
-          echo Configure backups
-          PROJ_TARGET=e1e498-prod CPU_REQUEST=1000m CPU_LIMIT=2000m bash openshift/scripts/oc_provision_backup_s3_postgres_cronjob.sh prod apply
-          PROJ_TARGET=e1e498-prod CPU_REQUEST=50m CPU_LIMIT=500m BACKUP_VOLUME_SIZE=3Gi bash openshift/scripts/oc_provision_backup_mariadb.sh prod apply
-          PROJ_TARGET=e1e498-prod CPU_REQUEST=50m CPU_LIMIT=500m bash openshift/scripts/oc_provision_backup_mariadb_cronjob.sh prod apply
-          echo Configure
-          PROJ_TARGET=e1e498-prod CERTBOT_STAGING=false DRYRUN=false bash openshift/scripts/oc_provision_certbot_cronjob.sh prod apply
+          PROJ_TARGET=e1e498-prod bash openshift/scripts/oc_deploy_to_production.sh ${SUFFIX} apply
+
   set_deployment_status_success:
     name: Set production deployment successful
     needs: [deploy_to_production]

openshift/scripts/oc_deploy.sh

@@ -63,7 +63,7 @@ OC_CANCEL_ALL_PREV_DEPLOY="oc -n ${PROJ_TARGET} rollout cancel dc/${NAME_OBJ} ||
 # Deploy and follow the progress
 #
 OC_DEPLOY="oc -n ${PROJ_TARGET} rollout latest dc/${NAME_OBJ}"
-OC_LOG="oc -n ${PROJ_TARGET} logs -f --pod-running-timeout=1m dc/${NAME_OBJ}"
+OC_LOG="oc -n ${PROJ_TARGET} logs -f --pod-running-timeout=2m dc/${NAME_OBJ}"
 if [ ! "${APPLY}" ]; then
   OC_CANCEL_ALL_PREV_DEPLOY=""
   OC_DEPLOY="${OC_DEPLOY} --dry-run || true" # in case there is no previous rollout

openshift/scripts/oc_deploy_to_production.sh

@@ -0,0 +1,51 @@
+#!/bin/sh -l
+#
+source "$(dirname ${0})/common/common"
+
+#
+# OpenShift Deploy Helper
+#   
+#   Pulls in logic for SUFFIX variable when declaring PR name
+#   Intended for use with a pull request-based pipeline.
+#   Suffixes incl.: pr-###.
+#
+# Usage:
+#                         
+#    [PROJ_TARGET] ${THIS_FILE} [SUFFIX] [RUN-TYPE]
+#
+# Examples:
+#
+#   Provide a PR number.
+#
+#   PROJ_TARGET=e1e498-prod ${THIS_FILE} pr-0 apply
+#   ${THIS_FILE} pr-0 dry-run
+#
+RUN_TYPE=$2
+# Target project override for Dev or Prod deployments
+#
+PROJ_TARGET="${PROJ_TARGET:-${PROJ_DEV}}"
+
+set -ex
+echo Promote
+bash $(dirname ${0})/oc_promote.sh ${SUFFIX} ${RUN_TYPE}
+bash $(dirname ${0})/oc_promote_ubuntu.sh ${SUFFIX} ${RUN_TYPE}
+echo Provision database
+CPU_REQUEST=75m CPU_LIMIT=2000m MEMORY_REQUEST=2Gi MEMORY_LIMIT=16Gi PVC_SIZE=45Gi PROJ_TARGET=${PROJ_TARGET} bash $(dirname ${0})/oc_provision_db.sh prod ${RUN_TYPE}
+echo Deploy API
+WORKERS_PER_CORE=0.25 CPU_REQUEST=100m CPU_LIMIT=500m MEMORY_REQUEST=3Gi MEMORY_LIMIT=6Gi REPLICAS=3 PROJ_TARGET=${PROJ_TARGET} VANITY_DOMAIN=psu.nrs.gov.bc.ca SECOND_LEVEL_DOMAIN=apps.silver.devops.gov.bc.ca bash $(dirname ${0})/oc_deploy.sh prod ${RUN_TYPE}
+echo Env Canada Subscriber
+PROJ_TARGET=${PROJ_TARGET} bash $(dirname ${0})/oc_provision_ec_gdps_cronjob.sh prod ${RUN_TYPE}
+PROJ_TARGET=${PROJ_TARGET} bash $(dirname ${0})/oc_provision_ec_hrdps_cronjob.sh prod ${RUN_TYPE} 
+PROJ_TARGET=${PROJ_TARGET} bash $(dirname ${0})/oc_provision_ec_rdps_cronjob.sh prod ${RUN_TYPE}
+echo C-Haines
+PROJ_TARGET=${PROJ_TARGET} bash $(dirname ${0})/oc_provision_c_haines_cronjob.sh prod ${RUN_TYPE}
+echo BC FireWeather cronjobs
+echo "Run forecast at 8h30 PDT and 16h30 PDT (so before and after noon)"
+PROJ_TARGET=${PROJ_TARGET} SCHEDULE="30 * * * *" bash $(dirname ${0})/oc_provision_wfwx_noon_forecasts_cronjob.sh prod ${RUN_TYPE}
+PROJ_TARGET=${PROJ_TARGET} SCHEDULE="15 * * * *" bash $(dirname ${0})/oc_provision_wfwx_hourly_actuals_cronjob.sh prod ${RUN_TYPE}
+echo Configure backups
+PROJ_TARGET=${PROJ_TARGET} CPU_REQUEST=1000m CPU_LIMIT=2000m bash $(dirname ${0})/oc_provision_backup_s3_postgres_cronjob.sh prod ${RUN_TYPE}
+PROJ_TARGET=${PROJ_TARGET} CPU_REQUEST=50m CPU_LIMIT=500m BACKUP_VOLUME_SIZE=3Gi bash $(dirname ${0})/oc_provision_backup_mariadb.sh prod ${RUN_TYPE}
+PROJ_TARGET=${PROJ_TARGET} CPU_REQUEST=50m CPU_LIMIT=500m bash $(dirname ${0})/oc_provision_backup_mariadb_cronjob.sh prod ${RUN_TYPE}
+echo Configure
+PROJ_TARGET=${PROJ_TARGET} CERTBOT_STAGING=false DRYRUN=false bash $(dirname ${0})/oc_provision_certbot_cronjob.sh prod ${RUN_TYPE}

openshift/scripts/oc_promote.sh

@@ -45,12 +45,17 @@ done
 if [ "${APPLY}" ]; then
 	eval "${OC_IMG_IMPORT}"
 	eval "${OC_IMG_RETAG}"
-
-	for PRUNE in "${OC_IMG_PRUNE[@]}"; do
-		eval "${PRUNE}"
-	done
+	if ! [ -z ${OC_IMG_PRUNE+x} ]; then
+		for PRUNE in "${OC_IMG_PRUNE[@]}"; do
+			eval "${PRUNE}"
+		done
+	fi
 fi
 
 # Provide oc command instruction
 #
-display_helper "${OC_IMG_IMPORT}" "${OC_IMG_RETAG}" "${OC_IMG_PRUNE[@]}"
+if ! [ -z ${OC_IMG_PRUNE+x} ]; then
+	display_helper "${OC_IMG_IMPORT}" "${OC_IMG_RETAG}" "${OC_IMG_PRUNE[@]}"
+else
+	display_helper "${OC_IMG_IMPORT}" "${OC_IMG_RETAG}"
+fi

openshift/scripts/oc_promote_ubuntu.sh

@@ -45,12 +45,17 @@ done
 if [ "${APPLY}" ]; then
 	eval "${OC_IMG_IMPORT}"
 	eval "${OC_IMG_RETAG}"
-
-	for PRUNE in "${OC_IMG_PRUNE[@]}"; do
-		eval "${PRUNE}"
-	done
+	if ! [ -z ${OC_IMG_PRUNE+x} ]; then
+		for PRUNE in "${OC_IMG_PRUNE[@]}"; do
+			eval "${PRUNE}"
+		done
+	fi
 fi
 
 # Provide oc command instruction
 #
-display_helper "${OC_IMG_IMPORT}" "${OC_IMG_RETAG}" "${OC_IMG_PRUNE[@]}"
+if ! [ -z ${OC_IMG_PRUNE+x} ]; then
+	display_helper "${OC_IMG_IMPORT}" "${OC_IMG_RETAG}" "${OC_IMG_PRUNE[@]}"
+else
+	display_helper "${OC_IMG_IMPORT}" "${OC_IMG_RETAG}"
+fi