CrunchyDB s3 standby cluster (bcgov#3407)

Adds configuration and instructions on spinning up a standby cluster that bootstraps itself from the configured `pgbackrest` repo, stored in s3.

closes bcgov#3305

docs/DB.md

@@ -69,3 +69,17 @@ Install postgis extension:
 ```psql
 create extension postgis;
 ```
+
+### CrunchyDB Openshift Cluster
+
+The database is deployed using the CrunchyDB postgres operator: https://github.com/CrunchyData/postgres-operator
+
+#### Standby Cluster Restore
+
+To spin up a standby cluster that bootstraps itself from our configured pgbackrest repo run:
+
+`PROJ_TARGET=<your-namespace> BUCKET=<your-bucket> bash openshift/scripts/oc_provision_crunchy_standby.sh <your-suffix> apply`
+
+Further details here: https://access.crunchydata.com/documentation/postgres-operator/latest/tutorials/backups-disaster-recovery/disaster-recovery#repo-based-standby
+
+In the case of needing promote the standby cluster to the primary cluster, follow instructions here: https://access.crunchydata.com/documentation/postgres-operator/latest/tutorials/backups-disaster-recovery/disaster-recovery#promoting-a-standby-cluster

openshift/scripts/oc_provision_crunchy_standby.sh

@@ -0,0 +1,71 @@
+#!/bin/sh -l
+#
+source "$(dirname ${0})/common/common"
+
+#%
+#% OpenShift Deploy Helper
+#%
+#%   Intended to deploy a once-off standby cluster which replicates from a pgbackrest repo
+#%   Suffixes incl.: pr-###.
+#%
+#% Usage:
+#%
+#%    PROJ_TARGET={namespace-target} BUCKET={your-s3-bucket} ${THIS_FILE} [SUFFIX] [apply]
+#%
+#% Examples:
+#%
+#%   Provide a PR number. Defaults to a dry-run.
+#%   PROJ_TARGET={namespace-target} BUCKET={your-s3-bucket} ${THIS_FILE} pr-0
+#%
+#%   Apply when satisfied.
+#%   PROJ_TARGET={namespace-target} BUCKET={your-s3-bucket} ${THIS_FILE} pr-0 apply
+#%
+
+
+# Target project override for Dev or Prod deployments
+#
+PROJ_TARGET="${PROJ_TARGET:-${PROJ_DEV}}"
+
+# Prepare names for crunchy ephemeral instance for this PR.
+IMAGE_STREAM_NAMESPACE=${IMAGE_STREAM_NAMESPACE:-${PROJ_TOOLS}}
+EPHEMERAL_STORAGE=${EPHEMERAL_STORAGE:-'False'}
+
+# Process template
+OC_PROCESS="oc -n ${PROJ_TARGET} process -f ${TEMPLATE_PATH}/crunchy_standby.yaml \
+-p SUFFIX=${SUFFIX} \
+-p TARGET_NAMESPACE=${PROJ_TARGET} \
+-p BUCKET=${BUCKET} \
+-p DATA_SIZE=45Gi \
+-p WAL_SIZE=15Gi \
+ ${IMAGE_NAME:+ " -p IMAGE_NAME=${IMAGE_NAME}"} \
+ ${IMAGE_TAG:+ " -p IMAGE_TAG=${IMAGE_TAG}"} \
+ ${IMAGE_REGISTRY:+ " -p IMAGE_REGISTRY=${IMAGE_REGISTRY}"} \
+ -p CPU_REQUEST=75m \
+ -p CPU_LIMIT=2000m \
+ -p MEMORY_REQUEST=2Gi \
+ -p MEMORY_LIMIT=16Gi"
+
+
+# In order to avoid running out of storage quota in our development environment, use
+# ephemeral storage by removing the pvc request from the template.
+if [ "$EPHEMERAL_STORAGE" = "True" ]
+then
+    # Pipe the template to jq, and delete the pvc and volume claim items from the template.
+    OC_PROCESS="${OC_PROCESS} | jq 'del(.items[2].spec.template.spec.volumes[0].persistentVolumeClaim) \
+| del(.items[2].spec.volumeClaimTemplates)'"
+fi
+
+# Apply template (apply or use --dry-run)
+#
+OC_APPLY="oc -n ${PROJ_TARGET} apply -f -"
+[ "${APPLY}" ] || OC_APPLY="${OC_APPLY} --dry-run=client"
+
+# Execute commands
+#
+
+eval "${OC_PROCESS}"
+eval "${OC_PROCESS} | ${OC_APPLY}"
+
+# Provide oc command instruction
+#
+display_helper "${OC_PROCESS} | ${OC_APPLY}"

openshift/templates/crunchy_standby.yaml

@@ -0,0 +1,123 @@
+apiVersion: template.openshift.io/v1
+kind: Template
+metadata:
+  name: wps-crunchydb-standby
+  annotations:
+    "openshift.io/display-name": wps-crunchydb-standby
+labels:
+  app.kubernetes.io/part-of: wps-crunchydb-standby
+  app: wps-crunchydb-standby
+parameters:
+  - description: Namespace in which database resides
+    displayName: Target Namespace
+    name: TARGET_NAMESPACE
+    required: true
+  - name: BUCKET
+    description: S3 bucket name
+    required: true
+  - name: DATA_SIZE
+    description: Data PVC size
+    required: true
+  - name: WAL_SIZE
+    description: WAL PVC size
+    required: true
+  - name: CPU_REQUEST
+    description: |
+      Starting amount of CPU the container can use.
+    displayName: CPU REQUEST
+    value: "50m"
+  - name: CPU_LIMIT
+    description: Maximum amount of CPU the container can use.
+    displayName: CPU Limit
+    value: "250m"
+  - name: MEMORY_REQUEST
+    description: Starting amount of memory the container can use.
+    displayName: Memory Request
+    value: 256Mi
+  - name: MEMORY_LIMIT
+    description: Maximum amount of memory the container can use.
+    displayName: Memory Limit
+    value: 1Gi
+  - name: SUFFIX
+    description: Deployment suffix, e.g. pr-###
+    required: true
+  - name: IMAGE_REGISTRY
+    value: image-registry.apps.silver.devops.gov.bc.ca
+  - name: IMAGE_NAMESPACE
+    value: e1e498-tools
+  - name: GLOBAL_NAME
+    description: Name of global Module
+    value: wps-global
+objects:
+  - apiVersion: v1
+    data:
+      init.sql: |-
+        \c wps\\
+        CREATE EXTENSION postgis;
+    kind: ConfigMap
+    metadata:
+      name: wps-init-sql
+  - apiVersion: postgres-operator.crunchydata.com/v1beta1
+    kind: PostgresCluster
+    metadata:
+      name: wps-crunchydb-standby
+    spec:
+      image: artifacts.developer.gov.bc.ca/bcgov-docker-local/crunchy-postgres-gis:ubi8-14.7-3.2-0
+      metadata:
+        name: wps-crunchydb-standby
+        labels:
+          app: wps-crunchydb-standby
+      databaseInitSQL:
+        key: init.sql
+        name: wps-init-sql
+      users:
+        - name: wps
+          databases:
+            - postgres
+            - wps
+          options: "SUPERUSER"
+      postgresVersion: 14
+      instances:
+        - name: crunchy
+          replicas: 1
+          resources:
+            requests:
+              cpu: ${CPU_REQUEST}
+              memory: ${MEMORY_REQUEST}
+            limits:
+              cpu: ${CPU_LIMIT}
+              memory: ${MEMORY_LIMIT}
+          dataVolumeClaimSpec:
+            accessModes:
+              - "ReadWriteOnce"
+            resources:
+              requests:
+                storage: ${DATA_SIZE}
+            storageClassName: netapp-block-standard
+          walVolumeClaimSpec:
+            accessModes:
+              - "ReadWriteOnce"
+            resources:
+              requests:
+                storage: ${WAL_SIZE}
+            storageClassName: netapp-block-standard
+      backups:
+        pgbackrest:
+          image: artifacts.developer.gov.bc.ca/bcgov-docker-local/crunchy-pgbackrest:ubi8-2.41-4
+          configuration:
+            - secret:
+                name: crunchy-pgbackrest
+                items:
+                  - key: conf
+                    path: s3.conf
+          global:
+            repo1-path: /pgbackrest/${SUFFIX}/repo1
+          repos:
+            - name: repo1
+              s3:
+                bucket: ${BUCKET}
+                endpoint: nrs.objectstore.gov.bc.ca
+                region: "ca-central-1"
+      standby:
+        enabled: true
+        repoName: repo1