From 155c987c625a4a55e42188c9777f7a9bba492a53 Mon Sep 17 00:00:00 2001 From: JulianForeman <71847719+JulianForeman@users.noreply.github.com> Date: Thu, 31 Mar 2022 15:58:47 -0700 Subject: [PATCH] Deploy from Local (#1826) * Scripts moved into an external file so we can deploy from local if Github Actions are down * Set -ex command added * Fixing edge cases where deployments fail if there are no tags to cleanup * Increased timeout time for logs & added changes to workers per core & memory limit on deploy --- .github/workflows/production.yml | 25 +--------- openshift/scripts/oc_deploy.sh | 2 +- openshift/scripts/oc_deploy_to_production.sh | 51 ++++++++++++++++++++ openshift/scripts/oc_promote.sh | 15 ++++-- openshift/scripts/oc_promote_ubuntu.sh | 15 ++++-- 5 files changed, 74 insertions(+), 34 deletions(-) create mode 100755 openshift/scripts/oc_deploy_to_production.sh diff --git a/.github/workflows/production.yml b/.github/workflows/production.yml index f3113ef36..6d6eec541 100644 --- a/.github/workflows/production.yml +++ b/.github/workflows/production.yml @@ -30,29 +30,8 @@ jobs: run: | echo Login oc login "${{ secrets.OPENSHIFT_CLUSTER }}" --token="${{ secrets.OC4_PROD_TOKEN }}" - echo Promote - bash openshift/scripts/oc_promote.sh ${SUFFIX} apply - bash openshift/scripts/oc_promote_ubuntu.sh ${SUFFIX} apply - echo Provision database - CPU_REQUEST=75m CPU_LIMIT=2000m MEMORY_REQUEST=2Gi MEMORY_LIMIT=16Gi PVC_SIZE=45Gi PROJ_TARGET=e1e498-prod bash openshift/scripts/oc_provision_db.sh prod apply - echo Deploy API - WORKERS_PER_CORE=0.25 CPU_REQUEST=100m CPU_LIMIT=500m MEMORY_REQUEST=3Gi MEMORY_LIMIT=6Gi REPLICAS=3 PROJ_TARGET=e1e498-prod VANITY_DOMAIN=psu.nrs.gov.bc.ca SECOND_LEVEL_DOMAIN=apps.silver.devops.gov.bc.ca bash ./openshift/scripts/oc_deploy.sh prod apply - echo Env Canada Subscriber - PROJ_TARGET=e1e498-prod bash openshift/scripts/oc_provision_ec_gdps_cronjob.sh prod apply - PROJ_TARGET=e1e498-prod bash openshift/scripts/oc_provision_ec_hrdps_cronjob.sh prod apply - PROJ_TARGET=e1e498-prod bash openshift/scripts/oc_provision_ec_rdps_cronjob.sh prod apply - echo C-Haines - PROJ_TARGET=e1e498-prod bash openshift/scripts/oc_provision_c_haines_cronjob.sh prod apply - echo BC FireWeather cronjobs - echo "Run forecast at 8h30 PDT and 16h30 PDT (so before and after noon)" - PROJ_TARGET=e1e498-prod SCHEDULE="30 * * * *" bash openshift/scripts/oc_provision_wfwx_noon_forecasts_cronjob.sh prod apply - PROJ_TARGET=e1e498-prod SCHEDULE="15 * * * *" bash openshift/scripts/oc_provision_wfwx_hourly_actuals_cronjob.sh prod apply - echo Configure backups - PROJ_TARGET=e1e498-prod CPU_REQUEST=1000m CPU_LIMIT=2000m bash openshift/scripts/oc_provision_backup_s3_postgres_cronjob.sh prod apply - PROJ_TARGET=e1e498-prod CPU_REQUEST=50m CPU_LIMIT=500m BACKUP_VOLUME_SIZE=3Gi bash openshift/scripts/oc_provision_backup_mariadb.sh prod apply - PROJ_TARGET=e1e498-prod CPU_REQUEST=50m CPU_LIMIT=500m bash openshift/scripts/oc_provision_backup_mariadb_cronjob.sh prod apply - echo Configure - PROJ_TARGET=e1e498-prod CERTBOT_STAGING=false DRYRUN=false bash openshift/scripts/oc_provision_certbot_cronjob.sh prod apply + PROJ_TARGET=e1e498-prod bash openshift/scripts/oc_deploy_to_production.sh ${SUFFIX} apply + set_deployment_status_success: name: Set production deployment successful needs: [deploy_to_production] diff --git a/openshift/scripts/oc_deploy.sh b/openshift/scripts/oc_deploy.sh index 95339ea3d..70df08f96 100755 --- a/openshift/scripts/oc_deploy.sh +++ b/openshift/scripts/oc_deploy.sh @@ -63,7 +63,7 @@ OC_CANCEL_ALL_PREV_DEPLOY="oc -n ${PROJ_TARGET} rollout cancel dc/${NAME_OBJ} || # Deploy and follow the progress # OC_DEPLOY="oc -n ${PROJ_TARGET} rollout latest dc/${NAME_OBJ}" -OC_LOG="oc -n ${PROJ_TARGET} logs -f --pod-running-timeout=1m dc/${NAME_OBJ}" +OC_LOG="oc -n ${PROJ_TARGET} logs -f --pod-running-timeout=2m dc/${NAME_OBJ}" if [ ! "${APPLY}" ]; then OC_CANCEL_ALL_PREV_DEPLOY="" OC_DEPLOY="${OC_DEPLOY} --dry-run || true" # in case there is no previous rollout diff --git a/openshift/scripts/oc_deploy_to_production.sh b/openshift/scripts/oc_deploy_to_production.sh new file mode 100755 index 000000000..21cf578a5 --- /dev/null +++ b/openshift/scripts/oc_deploy_to_production.sh @@ -0,0 +1,51 @@ +#!/bin/sh -l +# +source "$(dirname ${0})/common/common" + +# +# OpenShift Deploy Helper +# +# Pulls in logic for SUFFIX variable when declaring PR name +# Intended for use with a pull request-based pipeline. +# Suffixes incl.: pr-###. +# +# Usage: +# +# [PROJ_TARGET] ${THIS_FILE} [SUFFIX] [RUN-TYPE] +# +# Examples: +# +# Provide a PR number. +# +# PROJ_TARGET=e1e498-prod ${THIS_FILE} pr-0 apply +# ${THIS_FILE} pr-0 dry-run +# +RUN_TYPE=$2 +# Target project override for Dev or Prod deployments +# +PROJ_TARGET="${PROJ_TARGET:-${PROJ_DEV}}" + +set -ex +echo Promote +bash $(dirname ${0})/oc_promote.sh ${SUFFIX} ${RUN_TYPE} +bash $(dirname ${0})/oc_promote_ubuntu.sh ${SUFFIX} ${RUN_TYPE} +echo Provision database +CPU_REQUEST=75m CPU_LIMIT=2000m MEMORY_REQUEST=2Gi MEMORY_LIMIT=16Gi PVC_SIZE=45Gi PROJ_TARGET=${PROJ_TARGET} bash $(dirname ${0})/oc_provision_db.sh prod ${RUN_TYPE} +echo Deploy API +WORKERS_PER_CORE=0.25 CPU_REQUEST=100m CPU_LIMIT=500m MEMORY_REQUEST=3Gi MEMORY_LIMIT=6Gi REPLICAS=3 PROJ_TARGET=${PROJ_TARGET} VANITY_DOMAIN=psu.nrs.gov.bc.ca SECOND_LEVEL_DOMAIN=apps.silver.devops.gov.bc.ca bash $(dirname ${0})/oc_deploy.sh prod ${RUN_TYPE} +echo Env Canada Subscriber +PROJ_TARGET=${PROJ_TARGET} bash $(dirname ${0})/oc_provision_ec_gdps_cronjob.sh prod ${RUN_TYPE} +PROJ_TARGET=${PROJ_TARGET} bash $(dirname ${0})/oc_provision_ec_hrdps_cronjob.sh prod ${RUN_TYPE} +PROJ_TARGET=${PROJ_TARGET} bash $(dirname ${0})/oc_provision_ec_rdps_cronjob.sh prod ${RUN_TYPE} +echo C-Haines +PROJ_TARGET=${PROJ_TARGET} bash $(dirname ${0})/oc_provision_c_haines_cronjob.sh prod ${RUN_TYPE} +echo BC FireWeather cronjobs +echo "Run forecast at 8h30 PDT and 16h30 PDT (so before and after noon)" +PROJ_TARGET=${PROJ_TARGET} SCHEDULE="30 * * * *" bash $(dirname ${0})/oc_provision_wfwx_noon_forecasts_cronjob.sh prod ${RUN_TYPE} +PROJ_TARGET=${PROJ_TARGET} SCHEDULE="15 * * * *" bash $(dirname ${0})/oc_provision_wfwx_hourly_actuals_cronjob.sh prod ${RUN_TYPE} +echo Configure backups +PROJ_TARGET=${PROJ_TARGET} CPU_REQUEST=1000m CPU_LIMIT=2000m bash $(dirname ${0})/oc_provision_backup_s3_postgres_cronjob.sh prod ${RUN_TYPE} +PROJ_TARGET=${PROJ_TARGET} CPU_REQUEST=50m CPU_LIMIT=500m BACKUP_VOLUME_SIZE=3Gi bash $(dirname ${0})/oc_provision_backup_mariadb.sh prod ${RUN_TYPE} +PROJ_TARGET=${PROJ_TARGET} CPU_REQUEST=50m CPU_LIMIT=500m bash $(dirname ${0})/oc_provision_backup_mariadb_cronjob.sh prod ${RUN_TYPE} +echo Configure +PROJ_TARGET=${PROJ_TARGET} CERTBOT_STAGING=false DRYRUN=false bash $(dirname ${0})/oc_provision_certbot_cronjob.sh prod ${RUN_TYPE} diff --git a/openshift/scripts/oc_promote.sh b/openshift/scripts/oc_promote.sh index 6cda83ec2..055ef6b8e 100755 --- a/openshift/scripts/oc_promote.sh +++ b/openshift/scripts/oc_promote.sh @@ -45,12 +45,17 @@ done if [ "${APPLY}" ]; then eval "${OC_IMG_IMPORT}" eval "${OC_IMG_RETAG}" - - for PRUNE in "${OC_IMG_PRUNE[@]}"; do - eval "${PRUNE}" - done + if ! [ -z ${OC_IMG_PRUNE+x} ]; then + for PRUNE in "${OC_IMG_PRUNE[@]}"; do + eval "${PRUNE}" + done + fi fi # Provide oc command instruction # -display_helper "${OC_IMG_IMPORT}" "${OC_IMG_RETAG}" "${OC_IMG_PRUNE[@]}" +if ! [ -z ${OC_IMG_PRUNE+x} ]; then + display_helper "${OC_IMG_IMPORT}" "${OC_IMG_RETAG}" "${OC_IMG_PRUNE[@]}" +else + display_helper "${OC_IMG_IMPORT}" "${OC_IMG_RETAG}" +fi diff --git a/openshift/scripts/oc_promote_ubuntu.sh b/openshift/scripts/oc_promote_ubuntu.sh index d2276acb8..b629213ee 100755 --- a/openshift/scripts/oc_promote_ubuntu.sh +++ b/openshift/scripts/oc_promote_ubuntu.sh @@ -45,12 +45,17 @@ done if [ "${APPLY}" ]; then eval "${OC_IMG_IMPORT}" eval "${OC_IMG_RETAG}" - - for PRUNE in "${OC_IMG_PRUNE[@]}"; do - eval "${PRUNE}" - done + if ! [ -z ${OC_IMG_PRUNE+x} ]; then + for PRUNE in "${OC_IMG_PRUNE[@]}"; do + eval "${PRUNE}" + done + fi fi # Provide oc command instruction # -display_helper "${OC_IMG_IMPORT}" "${OC_IMG_RETAG}" "${OC_IMG_PRUNE[@]}" +if ! [ -z ${OC_IMG_PRUNE+x} ]; then + display_helper "${OC_IMG_IMPORT}" "${OC_IMG_RETAG}" "${OC_IMG_PRUNE[@]}" +else + display_helper "${OC_IMG_IMPORT}" "${OC_IMG_RETAG}" +fi \ No newline at end of file