From aea4ba847a8ebaa452424ce7654f5127e37efb17 Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Sun, 3 Mar 2024 11:48:15 +0100 Subject: [PATCH 01/14] postgresql: remove thisAttr argument by calling tests directly Previously, it was not possible to run tests on an overridden derivation, because the derivation under test was always pulled from pkgs. With this change, the following will return the same test: postgresql_jit.tests and (postgresql.override { jitSupport = true; }).tests --- nixos/tests/postgresql-jit.nix | 15 +- nixos/tests/postgresql-wal-receiver.nix | 213 ++++++++++++------------ pkgs/servers/sql/postgresql/default.nix | 1 - pkgs/servers/sql/postgresql/generic.nix | 14 +- 4 files changed, 132 insertions(+), 111 deletions(-) diff --git a/nixos/tests/postgresql-jit.nix b/nixos/tests/postgresql-jit.nix index baf26b8da2b39..f4b1d07a7faf8 100644 --- a/nixos/tests/postgresql-jit.nix +++ b/nixos/tests/postgresql-jit.nix @@ -1,6 +1,7 @@ { system ? builtins.currentSystem , config ? {} , pkgs ? import ../.. { inherit system config; } +, package ? null }: with import ../lib/testing-python.nix { inherit system pkgs; }; @@ -9,14 +10,17 @@ let inherit (pkgs) lib; packages = builtins.attrNames (import ../../pkgs/servers/sql/postgresql pkgs); - mkJitTest = packageName: makeTest { - name = "${packageName}"; + mkJitTestFromName = name: + mkJitTest pkgs.${name}; + + mkJitTest = package: makeTest { + name = package.name; meta.maintainers = with lib.maintainers; [ ma27 ]; nodes.machine = { pkgs, lib, ... }: { services.postgresql = { + inherit package; enable = true; enableJIT = true; - package = pkgs.${packageName}; initialScript = pkgs.writeText "init.sql" '' create table demo (id int); insert into demo (id) select generate_series(1, 5); @@ -45,4 +49,7 @@ let ''; }; in -lib.genAttrs packages mkJitTest +if package == null then + lib.genAttrs packages mkJitTestFromName +else + mkJitTest package diff --git a/nixos/tests/postgresql-wal-receiver.nix b/nixos/tests/postgresql-wal-receiver.nix index b0bd7711dbcd9..33c3945f0f790 100644 --- a/nixos/tests/postgresql-wal-receiver.nix +++ b/nixos/tests/postgresql-wal-receiver.nix @@ -1,6 +1,7 @@ { system ? builtins.currentSystem, config ? {}, - pkgs ? import ../.. { inherit system config; } + pkgs ? import ../.. { inherit system config; }, + package ? null }: with import ../lib/testing-python.nix { inherit system pkgs; }; @@ -9,111 +10,117 @@ let lib = pkgs.lib; # Makes a test for a PostgreSQL package, given by name and looked up from `pkgs`. - makePostgresqlWalReceiverTest = postgresqlPackage: + makeTestAttribute = name: { - name = postgresqlPackage; - value = - let - pkg = pkgs."${postgresqlPackage}"; - postgresqlDataDir = "/var/lib/postgresql/${pkg.psqlSchema}"; - replicationUser = "wal_receiver_user"; - replicationSlot = "wal_receiver_slot"; - replicationConn = "postgresql://${replicationUser}@localhost"; - baseBackupDir = "/tmp/pg_basebackup"; - walBackupDir = "/tmp/pg_wal"; - atLeast12 = lib.versionAtLeast pkg.version "12.0"; - - recoveryFile = if atLeast12 - then pkgs.writeTextDir "recovery.signal" "" - else pkgs.writeTextDir "recovery.conf" "restore_command = 'cp ${walBackupDir}/%f %p'"; - - in makeTest { - name = "postgresql-wal-receiver-${postgresqlPackage}"; - meta.maintainers = with lib.maintainers; [ pacien ]; - - nodes.machine = { ... }: { - services.postgresql = { - package = pkg; - enable = true; - settings = lib.mkMerge [ - { - wal_level = "archive"; # alias for replica on pg >= 9.6 - max_wal_senders = 10; - max_replication_slots = 10; - } - (lib.mkIf atLeast12 { - restore_command = "cp ${walBackupDir}/%f %p"; - recovery_end_command = "touch recovery.done"; - }) - ]; - authentication = '' - host replication ${replicationUser} all trust - ''; - initialScript = pkgs.writeText "init.sql" '' - create user ${replicationUser} replication; - select * from pg_create_physical_replication_slot('${replicationSlot}'); - ''; - }; - - services.postgresqlWalReceiver.receivers.main = { - postgresqlPackage = pkg; - connection = replicationConn; - slot = replicationSlot; - directory = walBackupDir; - }; - # This is only to speedup test, it isn't time racing. Service is set to autorestart always, - # default 60sec is fine for real system, but is too much for a test - systemd.services.postgresql-wal-receiver-main.serviceConfig.RestartSec = lib.mkForce 5; + inherit name; + value = makePostgresqlWalReceiverTest pkgs."${name}"; + }; + + makePostgresqlWalReceiverTest = pkg: + let + postgresqlDataDir = "/var/lib/postgresql/${pkg.psqlSchema}"; + replicationUser = "wal_receiver_user"; + replicationSlot = "wal_receiver_slot"; + replicationConn = "postgresql://${replicationUser}@localhost"; + baseBackupDir = "/tmp/pg_basebackup"; + walBackupDir = "/tmp/pg_wal"; + atLeast12 = lib.versionAtLeast pkg.version "12.0"; + + recoveryFile = if atLeast12 + then pkgs.writeTextDir "recovery.signal" "" + else pkgs.writeTextDir "recovery.conf" "restore_command = 'cp ${walBackupDir}/%f %p'"; + + in makeTest { + name = "postgresql-wal-receiver-${pkg.name}"; + meta.maintainers = with lib.maintainers; [ pacien ]; + + nodes.machine = { ... }: { + services.postgresql = { + package = pkg; + enable = true; + settings = lib.mkMerge [ + { + wal_level = "archive"; # alias for replica on pg >= 9.6 + max_wal_senders = 10; + max_replication_slots = 10; + } + (lib.mkIf atLeast12 { + restore_command = "cp ${walBackupDir}/%f %p"; + recovery_end_command = "touch recovery.done"; + }) + ]; + authentication = '' + host replication ${replicationUser} all trust + ''; + initialScript = pkgs.writeText "init.sql" '' + create user ${replicationUser} replication; + select * from pg_create_physical_replication_slot('${replicationSlot}'); + ''; }; - testScript = '' - # make an initial base backup - machine.wait_for_unit("postgresql") - machine.wait_for_unit("postgresql-wal-receiver-main") - # WAL receiver healthchecks PG every 5 seconds, so let's be sure they have connected each other - # required only for 9.4 - machine.sleep(5) - machine.succeed( - "${pkg}/bin/pg_basebackup --dbname=${replicationConn} --pgdata=${baseBackupDir}" - ) - - # create a dummy table with 100 records - machine.succeed( - "sudo -u postgres psql --command='create table dummy as select * from generate_series(1, 100) as val;'" - ) - - # stop postgres and destroy data - machine.systemctl("stop postgresql") - machine.systemctl("stop postgresql-wal-receiver-main") - machine.succeed("rm -r ${postgresqlDataDir}/{base,global,pg_*}") - - # restore the base backup - machine.succeed( - "cp -r ${baseBackupDir}/* ${postgresqlDataDir} && chown postgres:postgres -R ${postgresqlDataDir}" - ) - - # prepare WAL and recovery - machine.succeed("chmod a+rX -R ${walBackupDir}") - machine.execute( - "for part in ${walBackupDir}/*.partial; do mv $part ''${part%%.*}; done" - ) # make use of partial segments too - machine.succeed( - "cp ${recoveryFile}/* ${postgresqlDataDir}/ && chmod 666 ${postgresqlDataDir}/recovery*" - ) - - # replay WAL - machine.systemctl("start postgresql") - machine.wait_for_file("${postgresqlDataDir}/recovery.done") - machine.systemctl("restart postgresql") - machine.wait_for_unit("postgresql") - - # check that our records have been restored - machine.succeed( - "test $(sudo -u postgres psql --pset='pager=off' --tuples-only --command='select count(distinct val) from dummy;') -eq 100" - ) - ''; + services.postgresqlWalReceiver.receivers.main = { + postgresqlPackage = pkg; + connection = replicationConn; + slot = replicationSlot; + directory = walBackupDir; + }; + # This is only to speedup test, it isn't time racing. Service is set to autorestart always, + # default 60sec is fine for real system, but is too much for a test + systemd.services.postgresql-wal-receiver-main.serviceConfig.RestartSec = lib.mkForce 5; }; + + testScript = '' + # make an initial base backup + machine.wait_for_unit("postgresql") + machine.wait_for_unit("postgresql-wal-receiver-main") + # WAL receiver healthchecks PG every 5 seconds, so let's be sure they have connected each other + # required only for 9.4 + machine.sleep(5) + machine.succeed( + "${pkg}/bin/pg_basebackup --dbname=${replicationConn} --pgdata=${baseBackupDir}" + ) + + # create a dummy table with 100 records + machine.succeed( + "sudo -u postgres psql --command='create table dummy as select * from generate_series(1, 100) as val;'" + ) + + # stop postgres and destroy data + machine.systemctl("stop postgresql") + machine.systemctl("stop postgresql-wal-receiver-main") + machine.succeed("rm -r ${postgresqlDataDir}/{base,global,pg_*}") + + # restore the base backup + machine.succeed( + "cp -r ${baseBackupDir}/* ${postgresqlDataDir} && chown postgres:postgres -R ${postgresqlDataDir}" + ) + + # prepare WAL and recovery + machine.succeed("chmod a+rX -R ${walBackupDir}") + machine.execute( + "for part in ${walBackupDir}/*.partial; do mv $part ''${part%%.*}; done" + ) # make use of partial segments too + machine.succeed( + "cp ${recoveryFile}/* ${postgresqlDataDir}/ && chmod 666 ${postgresqlDataDir}/recovery*" + ) + + # replay WAL + machine.systemctl("start postgresql") + machine.wait_for_file("${postgresqlDataDir}/recovery.done") + machine.systemctl("restart postgresql") + machine.wait_for_unit("postgresql") + + # check that our records have been restored + machine.succeed( + "test $(sudo -u postgres psql --pset='pager=off' --tuples-only --command='select count(distinct val) from dummy;') -eq 100" + ) + ''; }; -# Maps the generic function over all attributes of PostgreSQL packages -in builtins.listToAttrs (map makePostgresqlWalReceiverTest (builtins.attrNames (import ../../pkgs/servers/sql/postgresql pkgs))) +in +if package == null then + # all-tests.nix: Maps the generic function over all attributes of PostgreSQL packages + builtins.listToAttrs (map makeTestAttribute (builtins.attrNames (import ../../pkgs/servers/sql/postgresql pkgs))) +else + # Called directly from .tests + makePostgresqlWalReceiverTest package diff --git a/pkgs/servers/sql/postgresql/default.nix b/pkgs/servers/sql/postgresql/default.nix index d11a2d06b2d26..ff9f4f4150a22 100644 --- a/pkgs/servers/sql/postgresql/default.nix +++ b/pkgs/servers/sql/postgresql/default.nix @@ -15,7 +15,6 @@ let in self.lib.nameValuePair attrName (import path { inherit jitSupport self; - thisAttr = attrName; }) ) versions; diff --git a/pkgs/servers/sql/postgresql/generic.nix b/pkgs/servers/sql/postgresql/generic.nix index 7f9e561f877ba..863d8c3b4e2f5 100644 --- a/pkgs/servers/sql/postgresql/generic.nix +++ b/pkgs/servers/sql/postgresql/generic.nix @@ -19,7 +19,7 @@ let , version, hash, muslPatches # for tests - , testers, nixosTests, thisAttr + , testers, nixosTests # JIT , jitSupport @@ -254,10 +254,18 @@ let this.pkgs; tests = { - postgresql = nixosTests.postgresql-wal-receiver.${thisAttr}; + postgresql-wal-receiver = import ../../../../nixos/tests/postgresql-wal-receiver.nix { + inherit (stdenv) system; + pkgs = self; + package = this; + }; pkg-config = testers.testMetaPkgConfig finalAttrs.finalPackage; } // lib.optionalAttrs jitSupport { - postgresql-jit = nixosTests.postgresql-jit.${thisAttr}; + postgresql-jit = import ../../../../nixos/tests/postgresql-jit.nix { + inherit (stdenv) system; + pkgs = self; + package = this; + }; }; }; From 29d5adb676dd70c326b799f87ea295ffd2b0e34e Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Sun, 3 Mar 2024 11:55:01 +0100 Subject: [PATCH 02/14] postgresql: change pname for postgresql_jit attributes to postgresql-jit This is for consistency between the top-level attribute and the respective package name, which makes it easier to track derivations. --- pkgs/servers/sql/postgresql/generic.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkgs/servers/sql/postgresql/generic.nix b/pkgs/servers/sql/postgresql/generic.nix index 863d8c3b4e2f5..375492e317859 100644 --- a/pkgs/servers/sql/postgresql/generic.nix +++ b/pkgs/servers/sql/postgresql/generic.nix @@ -46,7 +46,8 @@ let stdenv' = if jitSupport then llvmPackages.stdenv else stdenv; in stdenv'.mkDerivation (finalAttrs: { - inherit pname version; + inherit version; + pname = pname + lib.optionalString jitSupport "-jit"; src = fetchurl { url = "mirror://postgresql/source/v${version}/${pname}-${version}.tar.bz2"; From 605618c2e8ba6257fd17479f76e0d08682c4bc7f Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Sat, 2 Mar 2024 21:21:50 +0100 Subject: [PATCH 03/14] postgresql: clean up patches More cosmetic than anything else. The description in findstring.patch was duplicated, postgresql-9.6.1 references confusing. Timestamps are not needed, that's what git blame is for. --- .../patches/disable-resolve_symlinks.patch | 4 ++-- .../sql/postgresql/patches/findstring.patch | 14 -------------- .../postgresql/patches/hardcode-pgxs-path.patch | 5 ++--- .../sql/postgresql/patches/less-is-more.patch | 5 ++--- .../postgresql/patches/locale-binary-path.patch | 2 -- .../postgresql/patches/socketdir-in-run-13.patch | 6 ++---- .../sql/postgresql/patches/socketdir-in-run.patch | 2 -- .../patches/specify_pkglibdir_at_runtime.patch | 5 ++--- 8 files changed, 10 insertions(+), 33 deletions(-) diff --git a/pkgs/servers/sql/postgresql/patches/disable-resolve_symlinks.patch b/pkgs/servers/sql/postgresql/patches/disable-resolve_symlinks.patch index fadeea90ac4b6..83e4834a504e2 100644 --- a/pkgs/servers/sql/postgresql/patches/disable-resolve_symlinks.patch +++ b/pkgs/servers/sql/postgresql/patches/disable-resolve_symlinks.patch @@ -1,5 +1,5 @@ ---- a/src/common/exec.c 2014-09-04 20:19:12.236057588 +0200 -+++ b/src/common/exec.c 2014-09-04 20:19:50.550251633 +0200 +--- a/src/common/exec.c ++++ b/src/common/exec.c @@ -218,6 +218,9 @@ static int resolve_symlinks(char *path) diff --git a/pkgs/servers/sql/postgresql/patches/findstring.patch b/pkgs/servers/sql/postgresql/patches/findstring.patch index 959bf6a6caa4c..efb400b7116b9 100644 --- a/pkgs/servers/sql/postgresql/patches/findstring.patch +++ b/pkgs/servers/sql/postgresql/patches/findstring.patch @@ -10,20 +10,6 @@ $out/share. To fix this, we just look for postgres or psql in the part after the / using make's notdir. --- -From: Matthew Bauer -Date: Wed, 29 May 2019 22:51:52 -0400 -Subject: [PATCH] Add /postgresql suffix for Nix outputs - -Nix outputs put the `name' in each store path like -/nix/store/...-. This was confusing the Postgres make script -because it thought its data directory already had postgresql in its -directory. This lead to Postgres installing all of its fils in -$out/share. To fix this, we just look for postgres or psql in the part -after the / using make's notdir. - ---- -diff --git a/src/Makefile.global.in b/src/Makefile.global.in -index b9d86acaa9..bce05464c3 100644 --- a/src/Makefile.global.in +++ b/src/Makefile.global.in @@ -102,15 +102,15 @@ datarootdir := @datarootdir@ diff --git a/pkgs/servers/sql/postgresql/patches/hardcode-pgxs-path.patch b/pkgs/servers/sql/postgresql/patches/hardcode-pgxs-path.patch index 6cd449769baac..ffe071e65cfe3 100644 --- a/pkgs/servers/sql/postgresql/patches/hardcode-pgxs-path.patch +++ b/pkgs/servers/sql/postgresql/patches/hardcode-pgxs-path.patch @@ -1,6 +1,5 @@ -diff -Naur postgresql-9.6.1-orig/src/common/config_info.c postgresql-9.6.1/src/common/config_info.c ---- postgresql-9.6.1-orig/src/common/config_info.c 2016-11-22 21:39:29.231929261 +0100 -+++ postgresql-9.6.1/src/common/config_info.c 2016-11-22 23:36:53.685163543 +0100 +--- a/src/common/config_info.c ++++ b/src/common/config_info.c @@ -118,7 +118,10 @@ i++; diff --git a/pkgs/servers/sql/postgresql/patches/less-is-more.patch b/pkgs/servers/sql/postgresql/patches/less-is-more.patch index f14af9dc22073..a72d1a28f9cea 100644 --- a/pkgs/servers/sql/postgresql/patches/less-is-more.patch +++ b/pkgs/servers/sql/postgresql/patches/less-is-more.patch @@ -1,6 +1,5 @@ -diff -Naur postgresql-9.6.1-orig/src/include/fe_utils/print.h postgresql-9.6.1/src/include/fe_utils/print.h ---- postgresql-9.6.1-orig/src/include/fe_utils/print.h 2016-11-22 21:39:29.148932827 +0100 -+++ postgresql-9.6.1/src/include/fe_utils/print.h 2016-11-22 21:39:36.283626258 +0100 +--- a/src/include/fe_utils/print.h ++++ b/src/include/fe_utils/print.h @@ -18,7 +18,7 @@ /* This is not a particularly great place for this ... */ diff --git a/pkgs/servers/sql/postgresql/patches/locale-binary-path.patch b/pkgs/servers/sql/postgresql/patches/locale-binary-path.patch index 08e90bce750ca..8068683ab64bf 100644 --- a/pkgs/servers/sql/postgresql/patches/locale-binary-path.patch +++ b/pkgs/servers/sql/postgresql/patches/locale-binary-path.patch @@ -1,5 +1,3 @@ -diff --git a/src/backend/commands/collationcmds.c b/src/backend/commands/collationcmds.c -index fcfc02d..d011394 100644 --- a/src/backend/commands/collationcmds.c +++ b/src/backend/commands/collationcmds.c @@ -611,7 +611,7 @@ pg_import_system_collations(PG_FUNCTION_ARGS) diff --git a/pkgs/servers/sql/postgresql/patches/socketdir-in-run-13.patch b/pkgs/servers/sql/postgresql/patches/socketdir-in-run-13.patch index 72c778b0758ef..fd808b6098249 100644 --- a/pkgs/servers/sql/postgresql/patches/socketdir-in-run-13.patch +++ b/pkgs/servers/sql/postgresql/patches/socketdir-in-run-13.patch @@ -1,7 +1,5 @@ -diff --git i/src/include/pg_config_manual.h w/src/include/pg_config_manual.h -index 8f3ec6bde1..4fc01e4a0a 100644 ---- i/src/include/pg_config_manual.h -+++ w/src/include/pg_config_manual.h +--- a/src/include/pg_config_manual.h ++++ b/src/include/pg_config_manual.h @@ -201,7 +201,7 @@ * support them yet. */ diff --git a/pkgs/servers/sql/postgresql/patches/socketdir-in-run.patch b/pkgs/servers/sql/postgresql/patches/socketdir-in-run.patch index 969f80ff8fc7f..4932ef69ee360 100644 --- a/pkgs/servers/sql/postgresql/patches/socketdir-in-run.patch +++ b/pkgs/servers/sql/postgresql/patches/socketdir-in-run.patch @@ -1,5 +1,3 @@ -diff --git a/src/include/pg_config_manual.h b/src/include/pg_config_manual.h -index 743401cb96..be5c5f61d2 100644 --- a/src/include/pg_config_manual.h +++ b/src/include/pg_config_manual.h @@ -179,7 +179,7 @@ diff --git a/pkgs/servers/sql/postgresql/patches/specify_pkglibdir_at_runtime.patch b/pkgs/servers/sql/postgresql/patches/specify_pkglibdir_at_runtime.patch index fe95d2ee99f06..b94fc9efcbff8 100644 --- a/pkgs/servers/sql/postgresql/patches/specify_pkglibdir_at_runtime.patch +++ b/pkgs/servers/sql/postgresql/patches/specify_pkglibdir_at_runtime.patch @@ -1,6 +1,5 @@ -diff -ur postgresql-9.5.3-orig/src/port/path.c postgresql-9.5.3/src/port/path.c ---- postgresql-9.5.3-orig/src/port/path.c 2016-05-09 22:50:23.000000000 +0200 -+++ postgresql-9.5.3/src/port/path.c 2016-08-29 22:44:10.507377613 +0200 +--- a/src/port/path.c ++++ b/src/port/path.c @@ -714,7 +714,11 @@ void get_lib_path(const char *my_exec_path, char *ret_path) From 154215cfd8f81f76e8a9f054ef52679564bf40c1 Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Tue, 5 Mar 2024 15:15:07 +0100 Subject: [PATCH 04/14] postgresql: rename patches Trying to understand what each patch does made me come up with some more descriptive names: - Renaming the two disable-xxx patches to a common names makes it immediately clear that one replaces the other depending on version number. - findstring was really not descriptive at all. - hardcode-pgxs-path will be extended with more paths for split outputs in a later commit. Renaming here already to allow git to better track renames. Finally replacing HARDCODED_PGXS_PATH with $out/lib in the last patch, makes it easier to understand what the end result will look like when reading the patch. --- pkgs/servers/sql/postgresql/generic.nix | 11 +++++------ ...-pgxs-path.patch => paths-for-split-outputs.patch} | 8 +++----- ...tring.patch => paths-with-postgresql-suffix.patch} | 4 ---- ...exec_path.patch => relative-to-symlinks-16+.patch} | 5 +++-- ...olve_symlinks.patch => relative-to-symlinks.patch} | 5 +++-- ...dir-in-run-13.patch => socketdir-in-run-13+.patch} | 0 6 files changed, 14 insertions(+), 19 deletions(-) rename pkgs/servers/sql/postgresql/patches/{hardcode-pgxs-path.patch => paths-for-split-outputs.patch} (57%) rename pkgs/servers/sql/postgresql/patches/{findstring.patch => paths-with-postgresql-suffix.patch} (91%) rename pkgs/servers/sql/postgresql/patches/{disable-normalize_exec_path.patch => relative-to-symlinks-16+.patch} (75%) rename pkgs/servers/sql/postgresql/patches/{disable-resolve_symlinks.patch => relative-to-symlinks.patch} (70%) rename pkgs/servers/sql/postgresql/patches/{socketdir-in-run-13.patch => socketdir-in-run-13+.patch} (100%) diff --git a/pkgs/servers/sql/postgresql/generic.nix b/pkgs/servers/sql/postgresql/generic.nix index 375492e317859..e1781352bbc5b 100644 --- a/pkgs/servers/sql/postgresql/generic.nix +++ b/pkgs/servers/sql/postgresql/generic.nix @@ -111,12 +111,11 @@ let ++ lib.optionals stdenv'.isLinux [ "--with-pam" ]; patches = [ - (if atLeast "16" then ./patches/disable-normalize_exec_path.patch - else ./patches/disable-resolve_symlinks.patch) + (if atLeast "16" then ./patches/relative-to-symlinks-16+.patch else ./patches/relative-to-symlinks.patch) ./patches/less-is-more.patch - ./patches/hardcode-pgxs-path.patch + ./patches/paths-for-split-outputs.patch ./patches/specify_pkglibdir_at_runtime.patch - ./patches/findstring.patch + ./patches/paths-with-postgresql-suffix.patch (substituteAll { src = ./patches/locale-binary-path.patch; @@ -127,7 +126,7 @@ let # Using fetchurl instead of fetchpatch on purpose: https://github.com/NixOS/nixpkgs/issues/240141 map fetchurl (lib.attrValues muslPatches) ) ++ lib.optionals stdenv'.isLinux [ - (if atLeast "13" then ./patches/socketdir-in-run-13.patch else ./patches/socketdir-in-run.patch) + (if atLeast "13" then ./patches/socketdir-in-run-13+.patch else ./patches/socketdir-in-run.patch) ]; installTargets = [ "install-world" ]; @@ -136,7 +135,7 @@ let postPatch = '' # Hardcode the path to pgxs so pg_config returns the path in $out - substituteInPlace "src/common/config_info.c" --replace HARDCODED_PGXS_PATH "$out/lib" + substituteInPlace "src/common/config_info.c" --subst-var out '' + lib.optionalString jitSupport '' # Force lookup of jit stuff in $out instead of $lib substituteInPlace src/backend/jit/jit.c --replace pkglib_path \"$out/lib\" diff --git a/pkgs/servers/sql/postgresql/patches/hardcode-pgxs-path.patch b/pkgs/servers/sql/postgresql/patches/paths-for-split-outputs.patch similarity index 57% rename from pkgs/servers/sql/postgresql/patches/hardcode-pgxs-path.patch rename to pkgs/servers/sql/postgresql/patches/paths-for-split-outputs.patch index ffe071e65cfe3..2134f7e81a870 100644 --- a/pkgs/servers/sql/postgresql/patches/hardcode-pgxs-path.patch +++ b/pkgs/servers/sql/postgresql/patches/paths-for-split-outputs.patch @@ -1,13 +1,11 @@ --- a/src/common/config_info.c +++ b/src/common/config_info.c -@@ -118,7 +118,10 @@ +@@ -118,7 +118,7 @@ i++; configdata[i].name = pstrdup("PGXS"); -+ strlcpy(path, "HARDCODED_PGXS_PATH", sizeof(path)); -+/* commented out to be able to point to nix $out path - get_pkglib_path(my_exec_path, path); -+*/ ++ strlcpy(path, "@out@/lib", sizeof(path)); +- get_pkglib_path(my_exec_path, path); strlcat(path, "/pgxs/src/makefiles/pgxs.mk", sizeof(path)); cleanup_path(path); configdata[i].setting = pstrdup(path); diff --git a/pkgs/servers/sql/postgresql/patches/findstring.patch b/pkgs/servers/sql/postgresql/patches/paths-with-postgresql-suffix.patch similarity index 91% rename from pkgs/servers/sql/postgresql/patches/findstring.patch rename to pkgs/servers/sql/postgresql/patches/paths-with-postgresql-suffix.patch index efb400b7116b9..04d2f556e0bff 100644 --- a/pkgs/servers/sql/postgresql/patches/findstring.patch +++ b/pkgs/servers/sql/postgresql/patches/paths-with-postgresql-suffix.patch @@ -1,7 +1,3 @@ -From: Matthew Bauer -Date: Wed, 29 May 2019 22:51:52 -0400 -Subject: [PATCH] Add /postgresql suffix for Nix outputs - Nix outputs put the `name' in each store path like /nix/store/...-. This was confusing the Postgres make script because it thought its data directory already had postgresql in its diff --git a/pkgs/servers/sql/postgresql/patches/disable-normalize_exec_path.patch b/pkgs/servers/sql/postgresql/patches/relative-to-symlinks-16+.patch similarity index 75% rename from pkgs/servers/sql/postgresql/patches/disable-normalize_exec_path.patch rename to pkgs/servers/sql/postgresql/patches/relative-to-symlinks-16+.patch index 349fd4203348a..996072ebd2fbd 100644 --- a/pkgs/servers/sql/postgresql/patches/disable-normalize_exec_path.patch +++ b/pkgs/servers/sql/postgresql/patches/relative-to-symlinks-16+.patch @@ -1,10 +1,11 @@ +On NixOS we *want* stuff relative to symlinks. +--- --- a/src/common/exec.c +++ b/src/common/exec.c -@@ -238,6 +238,9 @@ +@@ -238,6 +238,8 @@ static int normalize_exec_path(char *path) { -+ // On NixOS we *want* stuff relative to symlinks. + return 0; + /* diff --git a/pkgs/servers/sql/postgresql/patches/disable-resolve_symlinks.patch b/pkgs/servers/sql/postgresql/patches/relative-to-symlinks.patch similarity index 70% rename from pkgs/servers/sql/postgresql/patches/disable-resolve_symlinks.patch rename to pkgs/servers/sql/postgresql/patches/relative-to-symlinks.patch index 83e4834a504e2..c9b199baf960c 100644 --- a/pkgs/servers/sql/postgresql/patches/disable-resolve_symlinks.patch +++ b/pkgs/servers/sql/postgresql/patches/relative-to-symlinks.patch @@ -1,10 +1,11 @@ +On NixOS we *want* stuff relative to symlinks. +--- --- a/src/common/exec.c +++ b/src/common/exec.c -@@ -218,6 +218,9 @@ +@@ -218,6 +218,8 @@ static int resolve_symlinks(char *path) { -+ // On NixOS we *want* stuff relative to symlinks. + return 0; + #ifdef HAVE_READLINK diff --git a/pkgs/servers/sql/postgresql/patches/socketdir-in-run-13.patch b/pkgs/servers/sql/postgresql/patches/socketdir-in-run-13+.patch similarity index 100% rename from pkgs/servers/sql/postgresql/patches/socketdir-in-run-13.patch rename to pkgs/servers/sql/postgresql/patches/socketdir-in-run-13+.patch From c3c52231f02e9ea1cb5a77128734719c1738d845 Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Wed, 6 Mar 2024 19:07:41 +0100 Subject: [PATCH 05/14] postgresql: remove left-over LC_ALL environment variable This was introduced in 65fd8f3f48fed5e2a4b82da7e354fac53ae72099 around 20 years ago as LANG=en_US, later changed to LC_ALL=en_US and LC_ALL=C to avoid bash warnings. This might have been relevant to run initdb during the check phase, but the PostgreSQL docs [1] say this today: For implementation reasons, setting LC_ALL does not work for this purpose Therefore it can be removed safely. [1]: https://www.postgresql.org/docs/current/regress-run.html#REGRESS-RUN-LOCALE --- pkgs/servers/sql/postgresql/generic.nix | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkgs/servers/sql/postgresql/generic.nix b/pkgs/servers/sql/postgresql/generic.nix index e1781352bbc5b..2608bfb0207e7 100644 --- a/pkgs/servers/sql/postgresql/generic.nix +++ b/pkgs/servers/sql/postgresql/generic.nix @@ -131,8 +131,6 @@ let installTargets = [ "install-world" ]; - LC_ALL = "C"; - postPatch = '' # Hardcode the path to pgxs so pg_config returns the path in $out substituteInPlace "src/common/config_info.c" --subst-var out From 6c1e2a44013ca3c1d209914cc186cbe1785a2945 Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Wed, 6 Mar 2024 22:06:36 +0100 Subject: [PATCH 06/14] postgresql: remove preConfigure CC variable This seems to have been done better by now. There are no references kept and the build does not fail when removing it. --- pkgs/servers/sql/postgresql/generic.nix | 3 --- 1 file changed, 3 deletions(-) diff --git a/pkgs/servers/sql/postgresql/generic.nix b/pkgs/servers/sql/postgresql/generic.nix index 2608bfb0207e7..a69ec29b4226e 100644 --- a/pkgs/servers/sql/postgresql/generic.nix +++ b/pkgs/servers/sql/postgresql/generic.nix @@ -90,9 +90,6 @@ let env.NIX_CFLAGS_COMPILE = "-I${libxml2.dev}/include/libxml2"; - # Otherwise it retains a reference to compiler and fails; see #44767. TODO: better. - preConfigure = "CC=${stdenv'.cc.targetPrefix}cc"; - configureFlags = [ "--with-openssl" "--with-libxml" From 4adcc41cd99cfcd128fff3d65b2580ff00d8e42b Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Wed, 6 Mar 2024 19:17:06 +0100 Subject: [PATCH 07/14] postgresql: move explicit libxml2 include behind version flag This was introduced in #44083 to fix cross building, where xml2-config wouldn't run on the host platform. This was fixed upstream two years later [1], so that from v13 on pkg-config is used before xml2-config is. Once v12 is EOL, we can remove this entirely. [1]: https://github.com/postgres/postgres/commit/0bc8cebdb889368abdf224aeac8bc197fe4c9ae6 --- pkgs/servers/sql/postgresql/generic.nix | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkgs/servers/sql/postgresql/generic.nix b/pkgs/servers/sql/postgresql/generic.nix index a69ec29b4226e..4d6c1926dfd51 100644 --- a/pkgs/servers/sql/postgresql/generic.nix +++ b/pkgs/servers/sql/postgresql/generic.nix @@ -88,7 +88,9 @@ let buildFlags = [ "world" ]; - env.NIX_CFLAGS_COMPILE = "-I${libxml2.dev}/include/libxml2"; + # Makes cross-compiling work when xml2-config can't be executed on the host. + # Fixed upstream in https://github.com/postgres/postgres/commit/0bc8cebdb889368abdf224aeac8bc197fe4c9ae6 + env.NIX_CFLAGS_COMPILE = lib.optionalString (olderThan "13") "-I${libxml2.dev}/include/libxml2"; configureFlags = [ "--with-openssl" From 9f90ad30fa9b61a985693ae938e901948e6c8c2e Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Wed, 6 Mar 2024 21:42:37 +0100 Subject: [PATCH 08/14] postgresql: enable parallel building on darwin This was disabled to fix #51093. The related discussion upstream revealed that this has been fixed in [1] for v12+. Since v12 is our oldest supported version right now, we should be able to safely enable parallel building on darwin again. Furthermore, it seems that this kind of issue was also fixed in #51221 and #51408 for the general case. [1]: https://github.com/postgres/postgres/commit/826eff57c4c23f77314ba7151d3dc506ce0fa24c --- pkgs/servers/sql/postgresql/generic.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/servers/sql/postgresql/generic.nix b/pkgs/servers/sql/postgresql/generic.nix index 4d6c1926dfd51..48f5ff4fc9cca 100644 --- a/pkgs/servers/sql/postgresql/generic.nix +++ b/pkgs/servers/sql/postgresql/generic.nix @@ -82,7 +82,7 @@ let ] ++ lib.optionals jitSupport [ llvmPackages.llvm.dev nukeReferences patchelf ]; - enableParallelBuilding = !stdenv'.isDarwin; + enableParallelBuilding = true; separateDebugInfo = true; From ec7b4581d7e4cab10b4f3184ae5965cb2046a385 Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Thu, 7 Mar 2024 08:48:40 +0100 Subject: [PATCH 09/14] pkgsMusl.postgresql: mark v12 and v13 with JIT as broken Those currently don't build on master, not sure whether they ever built correctly in the first place. --- pkgs/servers/sql/postgresql/generic.nix | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkgs/servers/sql/postgresql/generic.nix b/pkgs/servers/sql/postgresql/generic.nix index 48f5ff4fc9cca..d2979f624bda7 100644 --- a/pkgs/servers/sql/postgresql/generic.nix +++ b/pkgs/servers/sql/postgresql/generic.nix @@ -285,7 +285,9 @@ let # resulting LLVM IR isn't platform-independent this doesn't give you much. # In fact, I tried to test the result in a VM-test, but as soon as JIT was used to optimize # a query, postgres would coredump with `Illegal instruction`. - broken = jitSupport && (stdenv.hostPlatform != stdenv.buildPlatform); + broken = (jitSupport && stdenv.hostPlatform != stdenv.buildPlatform) + # Allmost all tests fail FATAL errors for v12 and v13 + || (jitSupport && stdenv.hostPlatform.isMusl && olderThan "14"); }; }); From c6017567e02b632a25c4b4c9e8aa46c36f67d009 Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Sun, 10 Mar 2024 16:23:23 +0100 Subject: [PATCH 10/14] git-blame-ignore-revs: add postgresql cleanup --- .git-blame-ignore-revs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index 219efb6e2a548..432191b92de47 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -105,3 +105,6 @@ fb0e5be84331188a69b3edd31679ca6576edb75a # {pkgs/development/cuda-modules,pkgs/test/cuda,pkgs/top-level/cuda-packages.nix}: reformat all CUDA files with nixfmt-rfc-style 2023-03-01 802a1b4d3338f24cbc4efd704616654456d75a94 + +# postgresql: move packages.nix to ext/default.nix +719034f6f6749d624faa28dff259309fc0e3e730 From 1682b4cc393dffbadedf8b869a3942965b056743 Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Sun, 10 Mar 2024 18:16:41 +0100 Subject: [PATCH 11/14] nixos/postgresql: fix enableJIT = false The enableJIT = true case was fixed in #221851 or e2fb65175228a992f196f3b1700a53e18602e7f6 respectively. However this did not take the case into consideration, when doing this: services.postgresql = { enable = true; enableJIT = false; package = pkgs.postgresql_15_jit; }; If enableJIT is treated as the source of truth, then this should indeed cause JIT to be disabled, which this commit does. --- nixos/modules/services/databases/postgresql.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixos/modules/services/databases/postgresql.nix b/nixos/modules/services/databases/postgresql.nix index d3fd6db6aea15..3f6448ac50f87 100644 --- a/nixos/modules/services/databases/postgresql.nix +++ b/nixos/modules/services/databases/postgresql.nix @@ -14,7 +14,7 @@ let # package = pkgs.postgresql_; # }; # works. - base = if cfg.enableJIT then cfg.package.withJIT else cfg.package; + base = if cfg.enableJIT then cfg.package.withJIT else cfg.package.withoutJIT; in if cfg.extraPlugins == [] then base From 30fa4dca882a91a18629fb93b31f8c4ee79285c8 Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Sat, 16 Mar 2024 14:36:36 +0100 Subject: [PATCH 12/14] pkgsMusl.postgresql: remove icu-collations-hack patch This was introduced in #228349, but doesn't seem necessary to build for pkgsMusl. In fact the patch itself seems to be very specific about a stripped down icu package in Alpine Linux, which does not apply to nixpkgs. --- pkgs/servers/sql/postgresql/12.nix | 6 ------ pkgs/servers/sql/postgresql/13.nix | 4 ---- pkgs/servers/sql/postgresql/14.nix | 4 ---- pkgs/servers/sql/postgresql/15.nix | 6 ------ pkgs/servers/sql/postgresql/16.nix | 6 ------ pkgs/servers/sql/postgresql/generic.nix | 2 +- 6 files changed, 1 insertion(+), 27 deletions(-) diff --git a/pkgs/servers/sql/postgresql/12.nix b/pkgs/servers/sql/postgresql/12.nix index 9e0388be04513..d29fc7683048f 100644 --- a/pkgs/servers/sql/postgresql/12.nix +++ b/pkgs/servers/sql/postgresql/12.nix @@ -1,10 +1,4 @@ import ./generic.nix { version = "12.18"; hash = "sha256-T5kZcl2UHOmGjgf+HtHTqGdIWZtIM4ZUdYOSi3TDkYo="; - muslPatches = { - icu-collations-hack = { - url = "https://git.alpinelinux.org/aports/plain/testing/postgresql12/icu-collations-hack.patch?id=d5227c91adda59d4e7f55f13468f0314e8869174"; - hash = "sha256-wuwjvGHArkRNwFo40g3p43W32OrJohretlt6iSRlJKg="; - }; - }; } diff --git a/pkgs/servers/sql/postgresql/13.nix b/pkgs/servers/sql/postgresql/13.nix index a4870812acdb7..c81e15bc7f685 100644 --- a/pkgs/servers/sql/postgresql/13.nix +++ b/pkgs/servers/sql/postgresql/13.nix @@ -2,10 +2,6 @@ import ./generic.nix { version = "13.14"; hash = "sha256-uN8HhVGJiWC9UA3F04oXfpkFN234H+fytmChQH+mpe0="; muslPatches = { - icu-collations-hack = { - url = "https://git.alpinelinux.org/aports/plain/main/postgresql14/icu-collations-hack.patch?id=56999e6d0265ceff5c5239f85fdd33e146f06cb7"; - hash = "sha256-wuwjvGHArkRNwFo40g3p43W32OrJohretlt6iSRlJKg="; - }; disable-test-collate-icu-utf8 = { url = "https://git.alpinelinux.org/aports/plain/main/postgresql13/disable-test-collate.icu.utf8.patch?id=69faa146ec9fff3b981511068f17f9e629d4688b"; hash = "sha256-jS/qxezaiaKhkWeMCXwpz1SDJwUWn9tzN0uKaZ3Ph2Y="; diff --git a/pkgs/servers/sql/postgresql/14.nix b/pkgs/servers/sql/postgresql/14.nix index 2de876cf4ad6c..c96ddd3ddc808 100644 --- a/pkgs/servers/sql/postgresql/14.nix +++ b/pkgs/servers/sql/postgresql/14.nix @@ -2,10 +2,6 @@ import ./generic.nix { version = "14.11"; hash = "sha256-pnC9fc4i3K1Cl7JhE2s7HUoJpvVBcZViqhTKY78paKg="; muslPatches = { - icu-collations-hack = { - url = "https://git.alpinelinux.org/aports/plain/main/postgresql14/icu-collations-hack.patch?id=56999e6d0265ceff5c5239f85fdd33e146f06cb7"; - hash = "sha256-wuwjvGHArkRNwFo40g3p43W32OrJohretlt6iSRlJKg="; - }; disable-test-collate-icu-utf8 = { url = "https://git.alpinelinux.org/aports/plain/main/postgresql14/disable-test-collate.icu.utf8.patch?id=56999e6d0265ceff5c5239f85fdd33e146f06cb7"; hash = "sha256-jXe23AxnFjEl+TZQm4R7rStk2Leo08ctxMNmu1xr5zM="; diff --git a/pkgs/servers/sql/postgresql/15.nix b/pkgs/servers/sql/postgresql/15.nix index f633dc9750850..00dfc0cbbc19d 100644 --- a/pkgs/servers/sql/postgresql/15.nix +++ b/pkgs/servers/sql/postgresql/15.nix @@ -1,10 +1,4 @@ import ./generic.nix { version = "15.6"; hash = "sha256-hFUUbtnGnJOlfelUrq0DAsr60DXCskIXXWqh4X68svs="; - muslPatches = { - icu-collations-hack = { - url = "https://git.alpinelinux.org/aports/plain/main/postgresql15/icu-collations-hack.patch?id=f424e934e6d076c4ae065ce45e734aa283eecb9c"; - hash = "sha256-HgtmhF4OJYU9macGJbTB9PjQi/yW7c3Akm3U0niWs8I="; - }; - }; } diff --git a/pkgs/servers/sql/postgresql/16.nix b/pkgs/servers/sql/postgresql/16.nix index 6a6420643b31a..cdc37b7c62f5f 100644 --- a/pkgs/servers/sql/postgresql/16.nix +++ b/pkgs/servers/sql/postgresql/16.nix @@ -1,10 +1,4 @@ import ./generic.nix { version = "16.2"; hash = "sha256-RG6IKU28LJCFq0twYaZG+mBLS+wDUh1epnHC5a2bKVI="; - muslPatches = { - icu-collations-hack = { - url = "https://git.alpinelinux.org/aports/plain/main/postgresql16/icu-collations-hack.patch?id=08a24be262339fd093e641860680944c3590238e"; - hash = "sha256-+urQdVIlADLdDPeT68XYv5rljhbK8M/7mPZn/cF+FT0="; - }; - }; } diff --git a/pkgs/servers/sql/postgresql/generic.nix b/pkgs/servers/sql/postgresql/generic.nix index d2979f624bda7..450179731a7d9 100644 --- a/pkgs/servers/sql/postgresql/generic.nix +++ b/pkgs/servers/sql/postgresql/generic.nix @@ -16,7 +16,7 @@ let , self, newScope, buildEnv # source specification - , version, hash, muslPatches + , version, hash, muslPatches ? {} # for tests , testers, nixosTests From 67510f6baacdcb005d106fbc1f22e4f7e343a410 Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Sun, 17 Mar 2024 15:39:16 +0100 Subject: [PATCH 13/14] nixosTests.postgresql-wal-receiver: remove left-over v12 conditionals We don't have postgresql < 12 in nixpkgs anymore. --- nixos/tests/postgresql-wal-receiver.nix | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/nixos/tests/postgresql-wal-receiver.nix b/nixos/tests/postgresql-wal-receiver.nix index 33c3945f0f790..ab2ab4ad0d4fa 100644 --- a/nixos/tests/postgresql-wal-receiver.nix +++ b/nixos/tests/postgresql-wal-receiver.nix @@ -24,11 +24,8 @@ let replicationConn = "postgresql://${replicationUser}@localhost"; baseBackupDir = "/tmp/pg_basebackup"; walBackupDir = "/tmp/pg_wal"; - atLeast12 = lib.versionAtLeast pkg.version "12.0"; - recoveryFile = if atLeast12 - then pkgs.writeTextDir "recovery.signal" "" - else pkgs.writeTextDir "recovery.conf" "restore_command = 'cp ${walBackupDir}/%f %p'"; + recoveryFile = pkgs.writeTextDir "recovery.signal" ""; in makeTest { name = "postgresql-wal-receiver-${pkg.name}"; @@ -38,17 +35,13 @@ let services.postgresql = { package = pkg; enable = true; - settings = lib.mkMerge [ - { - wal_level = "archive"; # alias for replica on pg >= 9.6 - max_wal_senders = 10; - max_replication_slots = 10; - } - (lib.mkIf atLeast12 { - restore_command = "cp ${walBackupDir}/%f %p"; - recovery_end_command = "touch recovery.done"; - }) - ]; + settings = { + max_replication_slots = 10; + max_wal_senders = 10; + recovery_end_command = "touch recovery.done"; + restore_command = "cp ${walBackupDir}/%f %p"; + wal_level = "archive"; # alias for replica on pg >= 9.6 + }; authentication = '' host replication ${replicationUser} all trust ''; From 81201ce88947c945794d37115de2982eb0ba97ae Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Fri, 15 Mar 2024 18:44:26 +0100 Subject: [PATCH 14/14] pkgsMusl.postgresql: add TODO to run all regression tests This was fixed upstream as a result of this discussion: https://www.postgresql.org/message-id/flat/ef1b47df-0808-4bd7-b08c-5153d5d75f4c%40technowledgy.de --- pkgs/servers/sql/postgresql/generic.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/pkgs/servers/sql/postgresql/generic.nix b/pkgs/servers/sql/postgresql/generic.nix index 450179731a7d9..8c27ce07c9c02 100644 --- a/pkgs/servers/sql/postgresql/generic.nix +++ b/pkgs/servers/sql/postgresql/generic.nix @@ -199,6 +199,7 @@ let # autodetection doesn't seem to able to find this, but it's there. checkTarget = "check"; + # TODO: Remove after the next set of minor releases on May 9th 2024 preCheck = # On musl, comment skip the following tests, because they break due to # ! ERROR: could not load library "/build/postgresql-11.5/tmp_install/nix/store/...-postgresql-11.5-lib/lib/libpqwalreceiver.so": Error loading shared library libpq.so.5: No such file or directory (needed by /build/postgresql-11.5/tmp_install/nix/store/...-postgresql-11.5-lib/lib/libpqwalreceiver.so)