From 318a965719626a744eea7eb45031c9a06600ef23 Mon Sep 17 00:00:00 2001 From: Kathia Barahona Date: Tue, 19 Nov 2024 17:58:15 +0100 Subject: [PATCH] Add support for PG17 In PostgreSQL versions prior to 17, the `pg_walfile_name` function could return the previous WAL file if the provided LSN fell at the boundary between WAL segments. This was fixed in PG17, and now gets the actual wal file name instead. This change affects the way we are retrieving the backup_end_wal_segment for the backups metadata. Therefore, instead of relying on our old behavior, we are now using the LSN yield by pg_backup_stop. --- .github/workflows/build.yml | 2 +- README.rst | 2 +- Vagrantfile | 2 +- docs/development.rst | 4 ++-- pghoard/config.py | 2 +- pghoard/wal.py | 1 + test/conftest.py | 2 +- test/test_webserver.py | 14 +++++++++----- 8 files changed, 17 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ee378ab0..2d6e3ec0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -77,7 +77,7 @@ jobs: wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - sudo apt-get update # Setup build deps - sudo apt-get install -y libsnappy-dev postgresql-10 postgresql-11 postgresql-12 postgresql-13 postgresql-14 postgresql-15 postgresql-16 + sudo apt-get install -y libsnappy-dev postgresql-12 postgresql-13 postgresql-14 postgresql-15 postgresql-16 postgresql-17 # Setup common python dependencies python -m pip install --upgrade pip pip install . diff --git a/README.rst b/README.rst index 242244b5..565e80fe 100644 --- a/README.rst +++ b/README.rst @@ -133,7 +133,7 @@ python 3.10, 3.11 and 3.12 virtual environments and installations of postgresql By default vagrant up will start a Virtualbox environment. The Vagrantfile will also work for libvirt, just prefix ``VAGRANT_DEFAULT_PROVIDER=libvirt`` to the ``vagrant up`` command. -Any combination of Python (3.10, 3.11 and 3.12) and Postgresql (12, 13, 14, 15 and 16) +Any combination of Python (3.10, 3.11 and 3.12) and Postgresql (12, 13, 14, 15, 16 and 17) Bring up vagrant instance and connect via ssh:: diff --git a/Vagrantfile b/Vagrantfile index 489c428d..82cb4c90 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -39,7 +39,7 @@ Vagrant.configure("2") do |config| sed -i "s/^#create_main_cluster.*/create_main_cluster=false/g" /etc/postgresql-common/createcluster.conf apt-get install -y python{3.10,3.11,3.12} python{3.10,3.11,3.12}-dev python{3.10,3.11,3.12}-venv - apt-get install -y postgresql-{12,13,14,15,16} postgresql-server-dev-{12,13,14,15,16} + apt-get install -y postgresql-{12,13,14,15,16,17} postgresql-server-dev-{12,13,14,15,16,17} username="$(< /dev/urandom tr -dc a-z | head -c${1:-32};echo;)" password=$(< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c${1:-32};echo;) diff --git a/docs/development.rst b/docs/development.rst index 576e01ec..273a62d7 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -54,12 +54,12 @@ Vagrant ======= The Vagrantfile can be used to setup a vagrant development environment. The vagrant environment has -python 3.10, 3.11 and 3.12 virtual environments and installations of postgresql 12, 13, 14, 15 and 16. +python 3.10, 3.11 and 3.12 virtual environments and installations of postgresql 12, 13, 14, 15, 16 and 17. By default vagrant up will start a Virtualbox environment. The Vagrantfile will also work for libvirt, just prefix ``VAGRANT_DEFAULT_PROVIDER=libvirt`` to the ``vagrant up`` command. -Any combination of Python (3.10, 3.11 and 3.12) and Postgresql (12, 13, 14, 15 and 16) +Any combination of Python (3.10, 3.11 and 3.12) and Postgresql (12, 13, 14, 15, 16 and 17) Bring up vagrant instance and connect via ssh:: diff --git a/pghoard/config.py b/pghoard/config.py index 058b03c8..a79a6484 100644 --- a/pghoard/config.py +++ b/pghoard/config.py @@ -17,7 +17,7 @@ from pghoard.common import (extract_pg_command_version_string, pg_major_version, pg_version_string_to_number) from pghoard.postgres_command import PGHOARD_HOST, PGHOARD_PORT -SUPPORTED_VERSIONS = ["16", "15", "14", "13", "12", "11", "10", "9.6", "9.5", "9.4", "9.3"] +SUPPORTED_VERSIONS = ["17", "16", "15", "14", "13", "12", "11", "10", "9.6", "9.5", "9.4", "9.3"] def get_cpu_count(): diff --git a/pghoard/wal.py b/pghoard/wal.py index 58f55ede..8e850a80 100644 --- a/pghoard/wal.py +++ b/pghoard/wal.py @@ -32,6 +32,7 @@ 0xD10D: 140000, 0xD110: 150000, 0xD113: 160000, + 0xD116: 170000, } WAL_MAGIC_BY_VERSION = {value: key for key, value in WAL_MAGIC.items()} diff --git a/test/conftest.py b/test/conftest.py index cabeed52..a98ccece 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -35,7 +35,7 @@ logutil.configure_logging() -DEFAULT_PG_VERSIONS = ["16", "15", "14", "13", "12"] +DEFAULT_PG_VERSIONS = ["17", "16", "15", "14", "13", "12"] def port_is_listening(hostname: str, port: int, timeout: float = 0.5) -> bool: diff --git a/test/test_webserver.py b/test/test_webserver.py index 6825e8de..7c7f0976 100644 --- a/test/test_webserver.py +++ b/test/test_webserver.py @@ -242,7 +242,7 @@ def _switch_wal(self, db, count): conn.close() return start_wal, end_wal - def test_archive_sync(self, db, pghoard): + def test_archive_sync(self, db, pghoard, pg_version: str): log = logging.getLogger("test_archive_sync") store = pghoard.transfer_agents[0].get_object_storage(pghoard.test_site) @@ -273,7 +273,11 @@ def list_archive(folder): # cluster between all tests) pg_wal_dir = get_pg_wal_directory(pghoard.config["backup_sites"][pghoard.test_site]) pg_wals = {f for f in os.listdir(pg_wal_dir) if wal.WAL_RE.match(f) and f > start_wal} - assert len(pg_wals) >= 4 + + # consider changes in pg_walfile_name, as pg_walfile_name(pg_current_wal_lsn()) might return + # previous walfile name and not current. + expected_min_wals = 4 if int(pg_version) < 17 else 3 + assert len(pg_wals) >= expected_min_wals # create a couple of "recycled" xlog files that we must ignore last_wal = sorted(pg_wals)[-1] @@ -291,7 +295,7 @@ def write_dummy_wal(inc): # check what we have archived, there should be at least the three # above WALs that are NOT there at the moment archived_wals = set(list_archive("xlog")) - assert len(pg_wals - archived_wals) >= 4 + assert len(pg_wals - archived_wals) >= expected_min_wals # now perform an archive sync arsy = ArchiveSync() arsy.run(["--site", pghoard.test_site, "--config", pghoard.config_path]) @@ -329,7 +333,7 @@ def write_dummy_wal(inc): "restore_command = 'false'", ] if Version(db.pgver).major >= 12: - with open(os.path.join(db.pgdata, "standby.signal"), "w") as fp: + with open(os.path.join(db.pgdata, "standby.signal"), "w"): pass recovery_conf_path = "postgresql.auto.conf" @@ -339,7 +343,7 @@ def write_dummy_wal(inc): recovery_conf_path = "recovery.conf" open_mode = "w" - with open(os.path.join(db.pgdata, recovery_conf_path), open_mode) as fp: + with open(os.path.join(db.pgdata, recovery_conf_path), open_mode) as fp: # type: ignore fp.write("\n".join(recovery_conf) + "\n") # start PG and promote it