etc-overlay: avoid rebuilding the initrd every time the etc contents …

…change

Before this change, the hash of the etc metadata image was included in
the mount unit that's responsible for mounting this metadata image in the
initrd.
And because this metadata image changes with every change to the etc
contents, the initrd would be rebuild every time as well.
This can lead to a lot of rebuilds (especially when revision info is
included in /etc/os-release) and all these initrd archives use up a lot of
space on the ESP.

With this change, we instead include a symlink to the metadata image in the
top-level directory, in the same way as we already do for things like init and
prepare-root, and we deduce the store path from the init= kernel parameter,
in the same way as we already do to find the path to init and prepare-root.

Doing so avoids rebuilding the initrd all the time.

nixos/modules/system/activation/top-level.nix

@@ -20,6 +20,11 @@ let
  ''}
 
  ln -s ${config.system.build.etc}/etc $out/etc
+
+ ${lib.optionalString config.system.etc.overlay.enable ''
+ ln -s ${config.system.build.etcMetadataImage} $out/etc-metadata-image
+ ''}
+
  ln -s ${config.system.path} $out/sw
  ln -s "$systemd" $out/systemd
 

nixos/modules/system/etc/etc-activation.nix

@@ -1,4 +1,4 @@
-{ config, lib, ... }:
+{ config, lib, pkgs, ... }:
 
 {
 
@@ -34,12 +34,23 @@
  mounts = [
  {
  where = "/run/etc-metadata";
- what = "/sysroot${config.system.build.etcMetadataImage}";
+ what = "/tmp/etc-metadata-image";
  type = "erofs";
  options = "loop";
- unitConfig.RequiresMountsFor = [
- "/sysroot/nix/store"
- ];
+ unitConfig = {
+ # Since this unit depends on the nix store being mounted, it cannot depend
+ # on local-fs.target, because if it did, we'd have local-fs.target
+ # ordered after the nix store mount which would cause things like
+ # network.target to only become active after the nix store has been mounted.
+ # This breaks for instance setups where sshd needs to be up before
+ # any encrypted disks can be mounted.
+ DefaultDependencies = false;
+ RequiresMountsFor = [
+ "/sysroot/nix/store"
+ ];
+ };
+ requires = [ "find-etc-metadata-image.service" ];
+ after = [ "local-fs-pre.target" "find-etc-metadata-image.service" ];
  }
  {
  where = "/sysroot/etc";
@@ -67,20 +78,66 @@
  ];
  }
  ];
- services = lib.mkIf config.system.etc.overlay.mutable {
- rw-etc = {
- unitConfig = {
- DefaultDependencies = false;
- RequiresMountsFor = "/sysroot";
+ services = lib.mkMerge [
+ (lib.mkIf config.system.etc.overlay.mutable {
+ rw-etc = {
+ unitConfig = {
+ DefaultDependencies = false;
+ RequiresMountsFor = "/sysroot";
+ };
+ serviceConfig = {
+ Type = "oneshot";
+ ExecStart = ''
+ /bin/mkdir -p -m 0755 /sysroot/.rw-etc/upper /sysroot/.rw-etc/work
+ '';
+ };
  };
- serviceConfig = {
- Type = "oneshot";
- ExecStart = ''
- /bin/mkdir -p -m 0755 /sysroot/.rw-etc/upper /sysroot/.rw-etc/work
+ })
+ {
+ find-etc-metadata-image = {
+ description = "Find the path to the etc metadata image";
+ before = [ "shutdown.target" ];
+ conflicts = [ "shutdown.target" ];
+ unitConfig = {
+ DefaultDependencies = false;
+ RequiresMountsFor = "/sysroot/nix/store";
+ };
+ serviceConfig = {
+ Type = "oneshot";
+ RemainAfterExit = true;
+ };
+
+ script = /* bash */ ''
+ set -uo pipefail
+
+ # Figure out what closure to boot
+ closure=
+ for o in $(< /proc/cmdline); do
+ case $o in
+ init=*)
+ IFS='=' read -r -a initParam <<< "$o"
+ closure="''${initParam[1]}"
+ ;;
+ esac
+ done
+
+ # Sanity check
+ if [ -z "''${closure:-}" ]; then
+ echo 'No init= parameter on the kernel command line' >&2
+ exit 1
+ fi
+
+ # Resolve symlinks in the init parameter
+ closure="$(chroot /sysroot ${lib.getExe' pkgs.coreutils "realpath"} "$closure")"
+ # Assume the directory containing the init script is the closure.
+ closure="$(dirname "$closure")"
+
+ metadata_image="$(chroot /sysroot ${lib.getExe' pkgs.coreutils "realpath"} "$closure/etc-metadata-image")"
+ ln -s "/sysroot$metadata_image" /tmp/etc-metadata-image
  '';
  };
- };
- };
+ }
+ ];
  };
 
  })

nixos/tests/activation/etc-overlay-immutable.nix

@@ -4,7 +4,7 @@
 
  meta.maintainers = with lib.maintainers; [ nikstur ];
 
- nodes.machine = { pkgs, ... }: {
+ nodes.machine = { lib, config, pkgs, ... }: {
  system.etc.overlay.enable = true;
  system.etc.overlay.mutable = false;
 
@@ -23,6 +23,15 @@
  specialisation.new-generation.configuration = {
  environment.etc."newgen".text = "newgen";
  };
+
+ assertions = lib.mkIf (lib.length (lib.attrNames config.specialisation) > 0) [
+ {
+ assertion =
+ config.system.build.initialRamdisk.drvPath ==
+ config.specialisation.new-generation.configuration.system.build.initialRamdisk.drvPath;
+ message = "The initrd for the base system and the specialisation are not the same!";
+ }
+ ];
  };
 
  testScript = ''

nixos/tests/activation/etc-overlay-mutable.nix

@@ -4,7 +4,7 @@
 
  meta.maintainers = with lib.maintainers; [ nikstur ];
 
- nodes.machine = { pkgs, ... }: {
+ nodes.machine = { lib, config, pkgs, ... }: {
  system.etc.overlay.enable = true;
  system.etc.overlay.mutable = true;
 
@@ -15,6 +15,15 @@
  specialisation.new-generation.configuration = {
  environment.etc."newgen".text = "newgen";
  };
+
+ assertions = lib.mkIf (lib.length (lib.attrNames config.specialisation) > 0) [
+ {
+ assertion =
+ config.system.build.initialRamdisk.drvPath ==
+ config.specialisation.new-generation.configuration.system.build.initialRamdisk.drvPath;
+ message = "The initrd for the base system and the specialisation are not the same!";
+ }
+ ];
  };
 
  testScript = ''