pkgs/top-level: make package sets composable

lib/systems/default.nix

@@ -6,9 +6,9 @@ let
     filterAttrs
     foldl
     hasInfix
+    isAttrs
     isFunction
     isList
-    isString
     mapAttrs
     optional
     optionalAttrs
@@ -55,24 +55,34 @@ let
   */
   flakeExposed = import ./flake-systems.nix { };
 
+  # Turn localSystem or crossSystem, which could be system-string or attrset, into
+  # attrset.
+  asAttrs = systemOrArgs:
+    if isAttrs systemOrArgs then systemOrArgs else { system = systemOrArgs; };
+
   # Elaborate a `localSystem` or `crossSystem` so that it contains everything
   # necessary.
   #
   # `parsed` is inferred from args, both because there are two options with one
   # clearly preferred, and to prevent cycles. A simpler fixed point where the RHS
   # always just used `final.*` would fail on both counts.
-  elaborate = args': let
-    args = if isString args' then { system = args'; }
-           else args';
+  elaborate = systemOrArgs: let
+    allArgs = asAttrs systemOrArgs;
+
+    # Those two will always be derived from "config", if given, so they should NOT
+    # be overridden further down with "// args".
+    args = builtins.removeAttrs allArgs [ "parsed" "system" ];
 
     # TODO: deprecate args.rustc in favour of args.rust after 23.05 is EOL.
     rust = args.rust or args.rustc or {};
 
     final = {
       # Prefer to parse `config` as it is strictly more informative.
-      parsed = parse.mkSystemFromString (if args ? config then args.config else args.system);
-      # Either of these can be losslessly-extracted from `parsed` iff parsing succeeds.
+      parsed = parse.mkSystemFromString (args.config or allArgs.system);
+      # This can be losslessly-extracted from `parsed` iff parsing succeeds.
       system = parse.doubleFromSystem final.parsed;
+      # TODO: This currently can't be losslessly-extracted from `parsed`, for example
+      # because of -mingw32.
       config = parse.tripleFromSystem final.parsed;
       # Determine whether we can execute binaries built for the provided platform.
       canExecute = platform:
@@ -433,6 +443,7 @@ in
 {
   inherit
     architectures
+    asAttrs
     doubles
     elaborate
     equals

pkgs/test/top-level/default.nix

@@ -44,4 +44,85 @@ lib.recurseIntoAttrs {
     assert lib.all (p: p.buildPlatform == p.hostPlatform) pkgsLocal;
     assert lib.all (p: p.buildPlatform != p.hostPlatform) pkgsCross;
     pkgs.emptyFile;
+
+  composePackageSets = with pkgs;
+    let
+      # To silence platform specific evaluation errors
+      discardEvaluationErrors = e:
+        let res = builtins.tryEval e; in
+        res.success == res.value;
+
+      # Basic test for idempotency of the package set, i.e:
+      # Applying the same package set twice should work and
+      # not change anything.
+      isIdempotent = set: discardEvaluationErrors (
+        pkgs.${set}.stdenv == pkgs.${set}.${set}.stdenv
+      );
+
+      # Some package sets should be noops in certain circumstances.
+      # This is very similar to the idempotency test, but not going
+      # via the super' overlay.
+      isNoop = parent: child: discardEvaluationErrors (
+        (lib.getAttrFromPath parent pkgs).stdenv ==
+        (lib.getAttrFromPath parent pkgs).${child}.stdenv
+      );
+
+      allMuslExamples = builtins.attrNames (lib.filterAttrs
+        (_: { config, ... }: lib.hasSuffix "-musl" config)
+        lib.systems.examples);
+
+      allLLVMExamples = builtins.attrNames (lib.filterAttrs
+        (_: { useLLVM ? false, ... }: useLLVM)
+        lib.systems.examples);
+
+      # A package set should only change specific configuration, but needs
+      # to keep all other configuration from previous layers in place.
+      # Each package set has one or more key characteristics for which we
+      # test here. Those should be kept, even when applying the "set" package
+      # set.
+      isComposable = set:
+        discardEvaluationErrors (pkgsCross.mingwW64.${set}.stdenv.hostPlatform.config == "x86_64-w64-mingw32") &&
+        discardEvaluationErrors (pkgsCross.mingwW64.${set}.stdenv.hostPlatform.libc == "msvcrt") &&
+        discardEvaluationErrors (pkgsCross.ppc64-musl.${set}.stdenv.hostPlatform.gcc.abi == "elfv2") &&
+        discardEvaluationErrors (builtins.elem "trivialautovarinit" pkgs.pkgsExtraHardening.${set}.stdenv.cc.defaultHardeningFlags) &&
+        discardEvaluationErrors (pkgs.pkgsLLVM.${set}.stdenv.hostPlatform.useLLVM) &&
+        discardEvaluationErrors (pkgs.pkgsArocc.${set}.stdenv.hostPlatform.useArocc) &&
+        discardEvaluationErrors (pkgs.pkgsZig.${set}.stdenv.hostPlatform.useZig) &&
+        discardEvaluationErrors (pkgs.pkgsLinux.${set}.stdenv.buildPlatform.isLinux) &&
+        discardEvaluationErrors (pkgs.pkgsMusl.${set}.stdenv.hostPlatform.isMusl) &&
+        discardEvaluationErrors (pkgs.pkgsStatic.${set}.stdenv.hostPlatform.isStatic) &&
+        discardEvaluationErrors (pkgs.pkgsi686Linux.${set}.stdenv.hostPlatform.isx86_32) &&
+        discardEvaluationErrors (pkgs.pkgsx86_64Darwin.${set}.stdenv.hostPlatform.isx86_64);
+    in
+      # Appends same flags over and over again
+      # assert isIdempotent "pkgsExtraHardening";
+      assert isIdempotent "pkgsLLVM";
+      assert isIdempotent "pkgsArocc";
+      assert isIdempotent "pkgsZig";
+      assert isIdempotent "pkgsLinux";
+      assert isIdempotent "pkgsMusl";
+      assert isIdempotent "pkgsStatic";
+      assert isIdempotent "pkgsi686Linux";
+      assert isIdempotent "pkgsx86_64Darwin";
+
+      assert isNoop [ "pkgsStatic" ] "pkgsMusl";
+      assert lib.all (sys: isNoop [ "pkgsCross" sys ] "pkgsMusl") allMuslExamples;
+      assert lib.all (sys: isNoop [ "pkgsCross" sys ] "pkgsLLVM") allLLVMExamples;
+
+      assert isComposable "pkgsExtraHardening";
+      assert isComposable "pkgsLLVM";
+      assert isComposable "pkgsArocc";
+      # TODO: unexpected argument 'bintools' - uncomment once https://github.com/NixOS/nixpkgs/pull/331011 is done
+      # assert isComposable "pkgsZig";
+      assert isComposable "pkgsMusl";
+      assert isComposable "pkgsStatic";
+      assert isComposable "pkgsi686Linux";
+
+      # Special cases regarding buildPlatform vs hostPlatform
+      assert discardEvaluationErrors (pkgsCross.gnu64.pkgsMusl.stdenv.hostPlatform.isMusl);
+      assert discardEvaluationErrors (pkgsCross.gnu64.pkgsi686Linux.stdenv.hostPlatform.isx86_32);
+      assert discardEvaluationErrors (pkgsCross.mingwW64.pkgsLinux.stdenv.hostPlatform.isLinux);
+      assert discardEvaluationErrors (pkgsCross.aarch64-darwin.pkgsx86_64Darwin.stdenv.hostPlatform.isx86_64);
+
+      emptyFile;
 }

pkgs/top-level/default.nix

@@ -44,7 +44,9 @@
 
 let # Rename the function arguments
   config0 = config;
-  crossSystem0 = crossSystem;
+  isCross = args ? "crossSystem" && crossSystem != null;
+  crossSystem0 = if crossSystem == null then localSystem else crossSystem;
+  localSystem0 = localSystem;
 
 in let
   lib = import ../../lib;
@@ -58,7 +60,7 @@ in let
     lib.foldr (x: throwIfNot (lib.isFunction x) "All crossOverlays passed to nixpkgs must be functions.") (r: r) crossOverlays
     ;
 
-  localSystem = lib.systems.elaborate args.localSystem;
+  localSystem = lib.systems.elaborate localSystem0;
 
   # Condition preserves sharing which in turn affects equality.
   #
@@ -74,7 +76,7 @@ in let
   # inferred from the system double in `localSystem`.
   crossSystem =
     let system = lib.systems.elaborate crossSystem0; in
-    if crossSystem0 == null || lib.systems.equals system localSystem
+    if lib.systems.equals system localSystem
     then localSystem
     else system;
 
@@ -101,22 +103,18 @@ in let
   config = lib.showWarnings configEval.config.warnings configEval.config;
 
   # A few packages make a new package set to draw their dependencies from.
-  # (Currently to get a cross tool chain, or forced-i686 package.) Rather than
-  # give `all-packages.nix` all the arguments to this function, even ones that
-  # don't concern it, we give it this function to "re-call" nixpkgs, inheriting
-  # whatever arguments it doesn't explicitly provide. This way,
-  # `all-packages.nix` doesn't know more than it needs too.
+  # Rather than give `all-packages.nix` all the arguments to this function,
+  # even ones that don't concern it, we give it this function to "re-call"
+  # nixpkgs, inheriting whatever arguments it doesn't explicitly provide. This
+  # way, `all-packages.nix` doesn't know more than it needs to.
   #
   # It's OK that `args` doesn't include default arguments from this file:
-  # they'll be deterministically inferred. In fact we must *not* include them,
+  # They'll be deterministically inferred. In fact we must *not* include them,
   # because it's important that if some parameter which affects the default is
   # substituted with a different argument, the default is re-inferred.
   #
-  # To put this in concrete terms, this function is basically just used today to
-  # use package for a different platform for the current platform (namely cross
-  # compiling toolchains and 32-bit packages on x86_64). In both those cases we
-  # want the provided non-native `localSystem` argument to affect the stdenv
-  # chosen.
+  # To put this in concrete terms, we want the provided non-native `localSystem`
+  # and `crossSystem` arguments to affect the stdenv chosen.
   #
   # NB!!! This thing gets its `config` argument from `args`, i.e. it's actually
   # `config0`. It is important to keep it to `config0` format (as opposed to the
@@ -125,7 +123,19 @@ in let
   # via `evalModules` is not idempotent. In other words, if you add `config` to
   # `newArgs`, expect strange very hard to debug errors! (Yes, I'm speaking from
   # experience here.)
-  nixpkgsFun = newArgs: import ./. (args // newArgs);
+  nixpkgsFun = newArgs: import ./. (
+    args // lib.optionalAttrs (newArgs ? "localSystem" && !isCross) {
+      localSystem = lib.systems.asAttrs localSystem0 // newArgs.localSystem;
+    } // lib.optionalAttrs (newArgs ? "localSystem" && isCross) {
+      # Redirect "localSystem" from stage.nix package sets to crossSystem when we
+      # are already cross. Otherwise pkgsCross.X.pkgsY will modify the localSystem,
+      # which is more than unexpected.
+      crossSystem = lib.systems.asAttrs crossSystem0 // newArgs.localSystem;
+    } // lib.optionalAttrs (newArgs ? "crossSystem") {
+      crossSystem = lib.systems.asAttrs crossSystem0 // newArgs.crossSystem;
+    } // lib.optionalAttrs (newArgs ? "overlays") {
+      overlays = overlays ++ newArgs.overlays;
+    });
 
   # Partially apply some arguments for building bootstraping stage pkgs
   # sets. Only apply arguments which no stdenv would want to override.