deps: Migrate to postgres 16 (#10536)

.github/workflows/ci.yml

@@ -209,7 +209,7 @@ jobs:
           - 6379:6379
         options: --entrypoint redis-server
       postgres:
-        image: postgres:15.10
+        image: postgres:16.6
         env:
           POSTGRES_USER: postgres
           POSTGRES_DB: postgres
@@ -263,7 +263,7 @@ jobs:
           - 6379:6379
         options: --entrypoint redis-server
       postgres:
-        image: postgres:15.10
+        image: postgres:16.6
         env:
           POSTGRES_USER: postgres
           POSTGRES_DB: postgres
@@ -326,7 +326,7 @@ jobs:
           - 6379:6379
         options: --entrypoint redis-server
       postgres:
-        image: postgres:15.10
+        image: postgres:16.6
         env:
           POSTGRES_USER: postgres
           POSTGRES_DB: postgres
@@ -368,7 +368,7 @@ jobs:
           - 6379:6379
         options: --entrypoint redis-server
       postgres:
-        image: postgres:15.10
+        image: postgres:16.6
         env:
           POSTGRES_USER: postgres
           POSTGRES_DB: postgres

.github/workflows/e2e.yml

@@ -48,7 +48,7 @@ jobs:
           - 6379:6379
         options: --entrypoint redis-server
       postgres:
-        image: postgres:15.10
+        image: postgres:16.6
         env:
           POSTGRES_USER: postgres
           POSTGRES_DB: postgres

docs/postgres.md

@@ -1,8 +1,6 @@
 # PostgreSQL Database
 
-You need to have PostgreSQL > 15.x.
-
-In production, we're currently running 16.4.
+You need to have PostgreSQL > 16.x.
 
 ## Installation
 
@@ -57,7 +55,7 @@ If you don't want to run a local instance of PostgreSQL in your computer, you ca
 Create and run the container:
 
 ```
-docker run -p 5432:5432 -e POSTGRES_HOST_AUTH_METHOD=trust -d --name opencollective-postgres --shm-size=1g --memory=4g --cpus=2  postgres:15
+docker run -p 5432:5432 -e POSTGRES_HOST_AUTH_METHOD=trust -d --name opencollective-postgres --shm-size=1g --memory=4g --cpus=2  postgres:16
 ```
 
 Set the necessary environment variables:

scripts/common/confirm.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+echo -n "$@"
+read -e answer
+for response in y Y yes YES Yes Sure sure SURE OK ok Ok; do
+  if [ "$answer" == "$response" ]; then
+    exit 0
+  fi
+done
+
+# Any answer other than the list above is considered a "no" answer
+exit 1

scripts/db_migrate_opencollective_dvl.sh

@@ -4,13 +4,28 @@
 
 PG_DATABASE=opencollective_dvl
 DUMPFILE="test/dbdumps/$PG_DATABASE.pgsql"
-./scripts/db_restore.sh -d $PG_DATABASE -U opencollective -f $DUMPFILE
+
+set -e
+
+if [ "$RESTORE" != "false" ]; then
+  echo "Restoring $PG_DATABASE to latest snapshot"
+
+  # Add a confirmation before running the restore
+  ./scripts/common/confirm.sh "The next command will drop the database $PG_DATABASE in order to make a fresh dump without polluted data. You can avoid this by passing RESTORE=false. Are you sure you want to drop $PG_DATABASE? (yes/no) " || exit 1
+  ./scripts/db_restore.sh -d $PG_DATABASE -U opencollective -f $DUMPFILE
+fi
+
 echo "Migrating $PG_DATABASE"
 DEBUG=psql PG_DATABASE=$PG_DATABASE npm run db:migrate
 PG_DATABASE=$PG_DATABASE npm run db:sanitize
-pg_dump -O -F t $PG_DATABASE >$DUMPFILE
-# Uncomment the line below to use a specific version of pg_dump (using docker)
-# (sudo docker run --rm --network host postgres:14.5 pg_dump -O -F t -h localhost -p 5432 -U opencollective $PG_DATABASE) >$DUMPFILE
+
+# Run docker if $USE_DOCKER is set
+if [ "$USE_DOCKER" = "true" ]; then
+  echo "Using docker to run pg_dump"
+  (./scripts/dev/run-docker.sh run --rm --network host postgres:16 pg_dump -O -F t -h localhost -p 5432 -U opencollective $PG_DATABASE) >$DUMPFILE
+else
+  pg_dump -O -F t $PG_DATABASE >$DUMPFILE
+fi
 
 echo "$DUMPFILE migrated. Please commit it and push it."
 echo ""

scripts/db_restore.sh

@@ -36,6 +36,7 @@ done
 LOCALDBUSER=${LOCALDBUSER:-"opencollective"}
 LOCALDBNAME=${LOCALDBNAME:-"opencollective_dvl"}
 DBDUMP_FILE=${DBDUMP_FILE:-"test/dbdumps/opencollective_dvl.pgsql"}
+LOCAL_FILE=$DBDUMP_FILE
 
 echo "LOCALDBUSER=$LOCALDBUSER"
 echo "LOCALDBNAME=$LOCALDBNAME"
@@ -52,39 +53,57 @@ if [ -z "$LOCALDBNAME" ]; then usage; fi
 # where pg_stat_activity.datname = '$LOCALDBNAME'
 # EOF
 
+# Adapt commands based on docker usage
+CMD_DROP_DB="dropdb"
+CMD_CREATE_DB="createdb"
+CMD_PSQL="psql"
+CMD_PG_RESTORE="pg_restore"
+
+if [ "$USE_DOCKER" = "true" ]; then
+  echo "Using docker to run Postgres commands"
+  CMD_DOCKER="./scripts/dev/run-docker.sh run --rm --network host"
+  CMD_DROP_DB="$CMD_DOCKER postgres:16 dropdb"
+  CMD_CREATE_DB="$CMD_DOCKER postgres:16 createdb"
+  CMD_PSQL="$CMD_DOCKER postgres:16 psql"
+  CMD_PG_RESTORE="$CMD_DOCKER -v ./$(dirname "$DBDUMP_FILE"):/dbdumps postgres:16 pg_restore"
+  LOCAL_FILE="/dbdumps/$(basename "$DBDUMP_FILE")"
+fi
+
+set -e
+
 echo "Dropping '$LOCALDBNAME'"
-dropdb -U postgres -h localhost --if-exists $LOCALDBNAME
+$CMD_DROP_DB -U postgres -h localhost --if-exists $LOCALDBNAME
 
 echo "Creating '$LOCALDBNAME'"
-createdb -U postgres -h localhost $LOCALDBNAME 2>/dev/null
+$CMD_CREATE_DB -U postgres -h localhost $LOCALDBNAME 2>/dev/null
 
 # When restoring old backups, you may need to enable Postgis
 if [ "$USE_POSTGIS" = "1" ]; then
   echo "Enabling Postgis"
-  psql "${LOCALDBNAME}" -c "CREATE EXTENSION postgis;"
-  psql "${LOCALDBNAME}" -c "ALTER TABLE public.spatial_ref_sys OWNER TO ${LOCALDBUSER};"
-  psql "${LOCALDBNAME}" -c "GRANT SELECT, INSERT ON TABLE public.spatial_ref_sys TO public;"
+  $CMD_PSQL "${LOCALDBNAME}" -c "CREATE EXTENSION postgis;"
+  $CMD_PSQL "${LOCALDBNAME}" -c "ALTER TABLE public.spatial_ref_sys OWNER TO ${LOCALDBUSER};"
+  $CMD_PSQL "${LOCALDBNAME}" -c "GRANT SELECT, INSERT ON TABLE public.spatial_ref_sys TO public;"
 fi
 
 # cool trick: all stdout ignored in this block
 {
   set +e
   # We make sure the user $LOCALDBUSER has access; could fail
-  psql -U postgres -h localhost "${LOCALDBNAME}" -c "CREATE ROLE ${LOCALDBUSER} WITH login;" 2>/dev/null
+  $CMD_PSQL -U postgres -h localhost "${LOCALDBNAME}" -c "CREATE ROLE ${LOCALDBUSER} WITH login;" 2>/dev/null
   set -e
 } | tee >/dev/null
 
 # Update table permissions
 echo "Updating table permissions"
-psql -U postgres -h localhost $LOCALDBNAME -c "GRANT ALL ON SCHEMA public TO ${LOCALDBUSER};"
+$CMD_PSQL -U postgres -h localhost $LOCALDBNAME -c "GRANT ALL ON SCHEMA public TO ${LOCALDBUSER};"
 
 # The first time we run it, we will trigger FK constraints errors
 set +e
-pg_restore -U postgres -h localhost --no-acl --no-owner --role=${LOCALDBUSER} -n public -O -c -d "${LOCALDBNAME}" "${DBDUMP_FILE}" 2>/dev/null
+$CMD_PG_RESTORE -U postgres -h localhost --no-acl --no-owner --role=${LOCALDBUSER} -n public -O -c -d "${LOCALDBNAME}" "${LOCAL_FILE}" 2>/dev/null
 set -e
 
 # So we run it twice :-)
-pg_restore -U postgres -h localhost --no-acl --no-owner --role=${LOCALDBUSER} -n public -O -c -d "${LOCALDBNAME}" "${DBDUMP_FILE}"
+$CMD_PG_RESTORE -U postgres -h localhost --no-acl --no-owner --role=${LOCALDBUSER} -n public -O -c -d "${LOCALDBNAME}" "${LOCAL_FILE}"
 
 echo "DB restored to postgres://localhost/${LOCALDBNAME}"
 

scripts/dev/run-docker.sh

@@ -8,8 +8,8 @@ if ! command -v docker &>/dev/null; then
 fi
 
 if docker info &>/dev/null; then
-  docker $@
+  docker "$@"
 else
-  echo "Docker requires root privileges. Running with sudo..."
-  sudo docker $@
+  echo "Docker requires root privileges. Running with sudo: docker $@"
+  sudo docker "$@"
 fi