Merge branch 'main' into renovate/poetry-all-minor-patch

.github/workflows/integration.yml

@@ -247,7 +247,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python-version: [3.6.15]
+        python-version: [3.11.2]
     steps:
       - name: Checkout repo
         uses: actions/checkout@v4
@@ -265,7 +265,7 @@ jobs:
           cache-name: cache-poetry-installer
         with:
           path: "~/poetry_installer"
-          key: "poetry-installer-1.1.11"
+          key: "poetry-installer-1.7.1"
       - name: Download poetry installer
         if: steps.cache-poetry-installer.outputs.cache-hit != 'true'
         run: |
@@ -275,10 +275,9 @@ jobs:
       - name: Install poetry (api)
         run: |
           cd ~/poetry_installer
-          python install-poetry.py --version 1.1.11
+          python install-poetry.py --version 1.7.1
           poetry config virtualenvs.create true
           poetry config virtualenvs.in-project false
-          poetry config experimental.new-installer false
       # poetry cache folder: /home/runner/.cache/pypoetry
       - name: Cache poetry
         uses: actions/cache@v4

openshift/s3-backup/README.md

@@ -15,7 +15,13 @@ oc -n e1e498-tools process -f build.yaml | oc -n e1e498-tools apply -f -
 ### Apply template using a specified branch and version
 
 ```bash
-oc -n e1e498-tools -p VERSION=some-date -p GIT_BRANCH=my-branch process -f build.yaml | oc -n e1e498-tools apply -f -
+oc -n e1e498-tools process -p VERSION=some-date -p GIT_BRANCH=my-branch process -f build.yaml | oc -n e1e498-tools apply -f -
+```
+
+### Kick off the build
+
+```bash
+oc -n e1e498-tools start-build s3-backup --follow
 ```
 
 ### Re-tag for production

openshift/s3-backup/docker/Dockerfile

@@ -1,10 +1,12 @@
-FROM centos/postgresql-12-centos7
+FROM postgres:14
 
 # Set the locale (important for poetry)
 ENV LC_ALL en_US.UTF-8
 ENV LANG en_US.UTF-8
 ENV LANGUAGE en_US.UTF-8
 
+RUN apt-get update && apt-get -y install unzip
+
 # Download the Amazon CLI installer.
 ADD "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" /tmp/awscliv2.zip
 
@@ -13,22 +15,17 @@ USER root
 RUN unzip /tmp/awscliv2.zip -d /tmp/ &&\
     /tmp/aws/install
 
-RUN yum install -y python3 python3-setuptools
+RUN apt-get -y install python3 python3-setuptools python3-pip
 
-# Install poetry.
-# This is not the recommended way to install poetry, but it works.
-RUN pip3 install --upgrade pip setuptools wheel
-RUN pip3 install --upgrade poetry
+# We don't care that we break system packages, this is only used for backup related tasks
+RUN pip3 install poetry --break-system-packages
 
-# Getting poetry to play nice with centos in openshift is a nightmare, so we just
+# Getting poetry to play nice in openshift is a nightmare, so we just
 # take the easy route and create a requirements.txt file.
 COPY pyproject.toml poetry.lock /tmp/
 RUN cd /tmp && \
     python3 -m poetry export -f requirements.txt --output requirements.txt && \
-    pip3 install -r requirements.txt
-
-# Switch back to default user - 26 is the postgres user.
-USER 26
+    pip3 install -r requirements.txt --break-system-packages
 
 # Copy scripts.
 COPY backup_to_s3.sh .

openshift/s3-backup/docker/pyproject.toml

@@ -6,7 +6,7 @@ authors = ["Sybrand Strauss <sybrand.strauss@gov.bc.ca>"]
 license = "Apache License, Version 2.0, January 2004"
 
 [tool.poetry.dependencies]
-python = ">=3.6.8,<=3.6.15"
+python = ">=3.10.4"
 aiobotocore = "^2.0.0"
 python-decouple = "^3.5"
 

openshift/scripts/oc_provision_backup_s3_postgres_cronjob.sh

@@ -29,11 +29,10 @@ PROJ_TARGET="${PROJ_TARGET:-${PROJ_DEV}}"
 # Prepare variables for backups
 JOB_NAME="backup-postgres-${APP_NAME}-${SUFFIX}"
 IMAGE_NAMESPACE=${PROJ_TOOLS}
-CLUSTER_NAME="patroni-${APP_NAME}-${SUFFIX}"
+CLUSTER_NAME="wps-crunchydb-${SUFFIX}"
 
 OC_PROCESS="oc -n ${PROJ_TARGET} process -f ${TEMPLATE_PATH}/backup-s3-postgres-cronjob.yaml \
-    -p DATABASE_SERVICE_NAME=patroni-${APP_NAME}-${SUFFIX}-leader \
-    -p DATABASE_DEPLOYMENT_NAME=wps-global \
+    -p CRUNCHYDB_USER=wps-crunchydb-${SUFFIX}-pguser-wps-crunchydb-${SUFFIX} \
     -p JOB_NAME=${JOB_NAME} \
     -p IMAGE_NAMESPACE=${IMAGE_NAMESPACE} \
     -p APP_LABEL=${APP_NAME}-${SUFFIX} \

openshift/templates/backup-s3-postgres-cronjob.yaml

@@ -32,24 +32,13 @@ parameters:
     description: "The TAG name for this environment, e.g., dev, test, prod"
     required: true
     value: "dev"
-  - name: "DATABASE_SERVICE_NAME"
-    displayName: "Database Service Name"
-    description: "The name of the database service."
-    required: true
   - name: DATABASE_USER
     value: wps
     required: true
-  - name: "DATABASE_DEPLOYMENT_NAME"
-    displayName: "Database Deployment Name"
-    description: "The name associated to the database deployment resources.  In particular, this is used to wire up the credentials associated to the database."
-    required: true
-  - name: DATABASE_PASSWORD_KEY_NAME
-    displayName: Database Password Key Name
-    description:
-      The database password key name stored in database deployment resources
-      specified by DATABASE_DEPLOYMENT_NAME.
+  - name: "CRUNCHYDB_USER"
+    displayName: "CrunchyDB user details"
+    description: "User details for the database"
     required: true
-    value: app-db-password
   - name: "PG_PORT"
     displayName: "Postgres port"
     required: true
@@ -99,7 +88,7 @@ parameters:
     value: wps-global
   - name: CLUSTER_NAME
     description: |
-      The name of the patroni cluster to connect to.
+      The name of the postgres cluster to connect to.
     displayName: Server Name
     required: true
 objects:
@@ -140,18 +129,21 @@ objects:
                   # image: backup
                   env:
                     - name: PG_HOSTNAME
-                      value: ${CLUSTER_NAME}-leader
+                      value: ${CLUSTER_NAME}-primary
                     - name: PG_PORT
                       value: ${PG_PORT}
                     - name: PG_DATABASE
                       value: ${PG_DATABASE}
                     - name: PG_USER
-                      value: ${DATABASE_USER}
+                      valueFrom:
+                        secretKeyRef:
+                          name: "${CRUNCHYDB_USER}"
+                          key: "user"
                     - name: PG_PASSWORD
                       valueFrom:
                         secretKeyRef:
-                          name: "${DATABASE_DEPLOYMENT_NAME}"
-                          key: "${DATABASE_PASSWORD_KEY_NAME}"
+                          name: "${CRUNCHYDB_USER}"
+                          key: "password"
                     - name: AWS_HOSTNAME
                       valueFrom:
                         secretKeyRef:
@@ -172,15 +164,6 @@ objects:
                         secretKeyRef:
                           name: ${GLOBAL_NAME}
                           key: object-store-bucket
-                    - name: POSTGRESQL_USER
-                      value: ${DATABASE_USER}
-                    - name: POSTGRESQL_PASSWORD
-                      valueFrom:
-                        secretKeyRef:
-                          name: "${DATABASE_DEPLOYMENT_NAME}"
-                          key: "${DATABASE_PASSWORD_KEY_NAME}"
-                    - name: POSTGRESQL_DATABASE
-                      value: ${PG_DATABASE}
               restartPolicy: "Never"
               terminationGracePeriodSeconds: 30
               activeDeadlineSeconds: 1600