Merge pull request #635 from Aiven-Open/support_pg17

Support pg17

.github/workflows/build.yml

@@ -77,7 +77,7 @@ jobs:
           wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
           sudo apt-get update
           # Setup build deps
-          sudo apt-get install -y libsnappy-dev postgresql-10 postgresql-11 postgresql-12 postgresql-13 postgresql-14 postgresql-15 postgresql-16
+          sudo apt-get install -y libsnappy-dev postgresql-12 postgresql-13 postgresql-14 postgresql-15 postgresql-16 postgresql-17
           # Setup common python dependencies
           python -m pip install --upgrade pip
           pip install .

README.rst

@@ -133,7 +133,7 @@ python 3.10, 3.11 and 3.12 virtual environments and installations of postgresql
 By default vagrant up will start a Virtualbox environment. The Vagrantfile will also work for libvirt, just prefix
 ``VAGRANT_DEFAULT_PROVIDER=libvirt`` to the ``vagrant up`` command.
 
-Any combination of Python (3.10, 3.11 and 3.12) and Postgresql (12, 13, 14, 15 and 16)
+Any combination of Python (3.10, 3.11 and 3.12) and Postgresql (12, 13, 14, 15, 16 and 17)
 
 Bring up vagrant instance and connect via ssh::
 

Vagrantfile

@@ -39,7 +39,7 @@ Vagrant.configure("2") do |config|
         sed -i "s/^#create_main_cluster.*/create_main_cluster=false/g" /etc/postgresql-common/createcluster.conf
 
         apt-get install -y python{3.10,3.11,3.12} python{3.10,3.11,3.12}-dev python{3.10,3.11,3.12}-venv
-        apt-get install -y postgresql-{12,13,14,15,16} postgresql-server-dev-{12,13,14,15,16}
+        apt-get install -y postgresql-{12,13,14,15,16,17} postgresql-server-dev-{12,13,14,15,16,17}
 
         username="$(< /dev/urandom tr -dc a-z | head -c${1:-32};echo;)"
         password=$(< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c${1:-32};echo;)

docs/development.rst

@@ -54,12 +54,12 @@ Vagrant
 =======
 
 The Vagrantfile can be used to setup a vagrant development environment.   The vagrant environment has
-python 3.10, 3.11 and 3.12 virtual environments and installations of postgresql 12, 13, 14, 15 and 16.
+python 3.10, 3.11 and 3.12 virtual environments and installations of postgresql 12, 13, 14, 15, 16 and 17.
 
 By default vagrant up will start a Virtualbox environment. The Vagrantfile will also work for libvirt, just prefix
 ``VAGRANT_DEFAULT_PROVIDER=libvirt`` to the ``vagrant up`` command.
 
-Any combination of Python (3.10, 3.11 and 3.12) and Postgresql (12, 13, 14, 15 and 16)
+Any combination of Python (3.10, 3.11 and 3.12) and Postgresql (12, 13, 14, 15, 16 and 17)
 
 Bring up vagrant instance and connect via ssh::
 

pghoard/config.py

@@ -17,7 +17,7 @@
 from pghoard.common import (extract_pg_command_version_string, pg_major_version, pg_version_string_to_number)
 from pghoard.postgres_command import PGHOARD_HOST, PGHOARD_PORT
 
-SUPPORTED_VERSIONS = ["16", "15", "14", "13", "12", "11", "10", "9.6", "9.5", "9.4", "9.3"]
+SUPPORTED_VERSIONS = ["17", "16", "15", "14", "13", "12", "11", "10", "9.6", "9.5", "9.4", "9.3"]
 
 
 def get_cpu_count():

pghoard/wal.py

@@ -32,6 +32,7 @@
     0xD10D: 140000,
     0xD110: 150000,
     0xD113: 160000,
+    0xD116: 170000,
 }
 WAL_MAGIC_BY_VERSION = {value: key for key, value in WAL_MAGIC.items()}
 

test/conftest.py

@@ -35,7 +35,7 @@
 
 logutil.configure_logging()
 
-DEFAULT_PG_VERSIONS = ["16", "15", "14", "13", "12"]
+DEFAULT_PG_VERSIONS = ["17", "16", "15", "14", "13", "12"]
 
 
 def port_is_listening(hostname: str, port: int, timeout: float = 0.5) -> bool:

test/test_webserver.py

@@ -242,7 +242,7 @@ def _switch_wal(self, db, count):
         conn.close()
         return start_wal, end_wal
 
-    def test_archive_sync(self, db, pghoard):
+    def test_archive_sync(self, db, pghoard, pg_version: str):
         log = logging.getLogger("test_archive_sync")
         store = pghoard.transfer_agents[0].get_object_storage(pghoard.test_site)
 
@@ -273,7 +273,11 @@ def list_archive(folder):
         # cluster between all tests)
         pg_wal_dir = get_pg_wal_directory(pghoard.config["backup_sites"][pghoard.test_site])
         pg_wals = {f for f in os.listdir(pg_wal_dir) if wal.WAL_RE.match(f) and f > start_wal}
-        assert len(pg_wals) >= 4
+
+        # consider changes in pg_walfile_name, as pg_walfile_name(pg_current_wal_lsn()) might return
+        # previous walfile name and not current.
+        expected_min_wals = 4 if int(pg_version) < 17 else 3
+        assert len(pg_wals) >= expected_min_wals
 
         # create a couple of "recycled" xlog files that we must ignore
         last_wal = sorted(pg_wals)[-1]
@@ -291,7 +295,7 @@ def write_dummy_wal(inc):
         # check what we have archived, there should be at least the three
         # above WALs that are NOT there at the moment
         archived_wals = set(list_archive("xlog"))
-        assert len(pg_wals - archived_wals) >= 4
+        assert len(pg_wals - archived_wals) >= expected_min_wals
         # now perform an archive sync
         arsy = ArchiveSync()
         arsy.run(["--site", pghoard.test_site, "--config", pghoard.config_path])
@@ -329,7 +333,7 @@ def write_dummy_wal(inc):
             "restore_command = 'false'",
         ]
         if Version(db.pgver).major >= 12:
-            with open(os.path.join(db.pgdata, "standby.signal"), "w") as fp:
+            with open(os.path.join(db.pgdata, "standby.signal"), "w"):
                 pass
 
             recovery_conf_path = "postgresql.auto.conf"
@@ -339,7 +343,7 @@ def write_dummy_wal(inc):
             recovery_conf_path = "recovery.conf"
             open_mode = "w"
 
-        with open(os.path.join(db.pgdata, recovery_conf_path), open_mode) as fp:
+        with open(os.path.join(db.pgdata, recovery_conf_path), open_mode) as fp:  # type: ignore
             fp.write("\n".join(recovery_conf) + "\n")
 
         # start PG and promote it