-
-
Notifications
You must be signed in to change notification settings - Fork 14k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
CDI: Add
mounts, mount-nvidia-binaries and `mount-nvidia-docker-1…
…-directories`
Add three options to `hardware.nvidia-container-toolkit`:
- `mounts`: list of mounts that allow to mount arbitrary paths on the
CDI enabled containers.
- `mount-nvidia-binaries`: this option allows users to avoid mounting
nvidia binaries on the container.
- `mount-nvidia-docker-1-directories`: this option allows users to
avoid mounting `/usr/local/nvidia/lib{,64}` on containers.
Remove `cdi.static` and `cdi.dynamic.nvidia.enable` attributes.- Loading branch information
Showing
6 changed files
with
161 additions
and
111 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
60 changes: 0 additions & 60 deletions
60
nixos/modules/services/hardware/nvidia-container-toolkit-cdi-generator/cdi-generate.nix
This file was deleted.
Oops, something went wrong.
35 changes: 35 additions & 0 deletions
35
nixos/modules/services/hardware/nvidia-container-toolkit/cdi-generate.nix
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,35 @@ | ||
| { | ||
| glibc, | ||
| jq, | ||
| lib, | ||
| mounts, | ||
| nvidia-container-toolkit, | ||
| nvidia-driver, | ||
| runtimeShell, | ||
| writeScriptBin, | ||
| }: let | ||
| mkMount = {hostPath, containerPath, mountOptions}: { | ||
| inherit hostPath containerPath; | ||
| options = mountOptions; | ||
| }; | ||
| jqAddMountExpression = ".containerEdits.mounts[.containerEdits.mounts | length] |= . +"; | ||
| allJqMounts = lib.concatMap | ||
| (mount: | ||
| ["${lib.getExe jq} '${jqAddMountExpression} ${builtins.toJSON (mkMount mount)}'"]) | ||
| mounts; | ||
| in | ||
| writeScriptBin "nvidia-cdi-generator" | ||
| '' | ||
| #! ${runtimeShell} | ||
| function cdiGenerate { | ||
| ${lib.getExe' nvidia-container-toolkit "nvidia-ctk"} cdi generate \ | ||
| --format json \ | ||
| --ldconfig-path ${lib.getExe' glibc "ldconfig"} \ | ||
| --library-search-path ${lib.getLib nvidia-driver}/lib \ | ||
| --nvidia-ctk-path ${lib.getExe' nvidia-container-toolkit "nvidia-ctk"} | ||
| } | ||
| cdiGenerate | \ | ||
| ${lib.concatStringsSep " | " allJqMounts} > $RUNTIME_DIRECTORY/nvidia-container-toolkit.json | ||
| '' |
122 changes: 122 additions & 0 deletions
122
nixos/modules/services/hardware/nvidia-container-toolkit/default.nix
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,122 @@ | ||
| { config, lib, pkgs, ... }: | ||
|
|
||
| { | ||
| imports = [ | ||
| (lib.mkRenamedOptionModule | ||
| [ "virtualisation" "containers" "cdi" "dynamic" "nvidia" "enable" ] | ||
| [ "hardware" "nvidia-container-toolkit" "enable" ]) | ||
| ]; | ||
|
|
||
| options = let | ||
| mountType = { | ||
| options = { | ||
| hostPath = lib.mkOption { | ||
| type = lib.types.str; | ||
| description = lib.mdDoc "Host path."; | ||
| }; | ||
| containerPath = lib.mkOption { | ||
| type = lib.types.str; | ||
| description = lib.mdDoc "Container path."; | ||
| }; | ||
| mountOptions = lib.mkOption { | ||
| default = [ "ro" "nosuid" "nodev" "bind" ]; | ||
| type = lib.types.listOf lib.types.str; | ||
| description = lib.mdDoc "Mount options."; | ||
| }; | ||
| }; | ||
| }; | ||
| in { | ||
|
|
||
| hardware.nvidia-container-toolkit = { | ||
| enable = lib.mkOption { | ||
| default = false; | ||
| type = lib.types.bool; | ||
| description = lib.mdDoc '' | ||
| Enable dynamic CDI configuration for NVidia devices by running | ||
| nvidia-container-toolkit on boot. | ||
| ''; | ||
| }; | ||
|
|
||
| mounts = lib.mkOption { | ||
| type = lib.types.listOf (lib.types.submodule mountType); | ||
| default = []; | ||
| description = lib.mdDoc "Mounts to be added to every container under the Nvidia CDI profile."; | ||
| }; | ||
|
|
||
| mount-nvidia-executables = lib.mkOption { | ||
| default = true; | ||
| type = lib.types.bool; | ||
| description = lib.mdDoc '' | ||
| Mount executables nvidia-smi, nvidia-cuda-mps-control, nvidia-cuda-mps-server, | ||
| nvidia-debugdump, nvidia-powerd and nvidia-ctk on containers. | ||
| ''; | ||
| }; | ||
|
|
||
| mount-nvidia-docker-1-directories = lib.mkOption { | ||
| default = true; | ||
| type = lib.types.bool; | ||
| description = lib.mdDoc '' | ||
| Mount nvidia-docker-1 directories on containers: /usr/local/nvidia/lib and | ||
| /usr/local/nvidia/lib64. | ||
| ''; | ||
| }; | ||
|
|
||
| }; | ||
|
|
||
| }; | ||
|
|
||
| config = { | ||
|
|
||
| hardware.nvidia-container-toolkit.mounts = let | ||
| nvidia-driver = config.hardware.nvidia.package; | ||
| in (lib.mkMerge [ | ||
| [{ hostPath = pkgs.addDriverRunpath.driverLink; | ||
| containerPath = pkgs.addDriverRunpath.driverLink; } | ||
| { hostPath = "${lib.getLib pkgs.glibc}/lib"; | ||
| containerPath = "${lib.getLib pkgs.glibc}/lib"; } | ||
| { hostPath = "${lib.getLib pkgs.glibc}/lib64"; | ||
| containerPath = "${lib.getLib pkgs.glibc}/lib64"; }] | ||
| (lib.mkIf config.hardware.nvidia-container-toolkit.mount-nvidia-executables | ||
| [{ hostPath = lib.getExe' nvidia-driver "nvidia-cuda-mps-control"; | ||
| containerPath = "/usr/bin/nvidia-cuda-mps-control"; } | ||
| { hostPath = lib.getExe' nvidia-driver "nvidia-cuda-mps-server"; | ||
| containerPath = "/usr/bin/nvidia-cuda-mps-server"; } | ||
| { hostPath = lib.getExe' nvidia-driver "nvidia-debugdump"; | ||
| containerPath = "/usr/bin/nvidia-debugdump"; } | ||
| { hostPath = lib.getExe' nvidia-driver "nvidia-powerd"; | ||
| containerPath = "/usr/bin/nvidia-powerd"; } | ||
| { hostPath = lib.getExe' nvidia-driver "nvidia-smi"; | ||
| containerPath = "/usr/bin/nvidia-smi"; }]) | ||
| # nvidia-docker 1.0 uses /usr/local/nvidia/lib{,64} | ||
| # e.g. | ||
| # - https://gitlab.com/nvidia/container-images/cuda/-/blob/e3ff10eab3a1424fe394899df0e0f8ca5a410f0f/dist/12.3.1/ubi9/base/Dockerfile#L44 | ||
| # - https://github.com/NVIDIA/nvidia-docker/blob/01d2c9436620d7dde4672e414698afe6da4a282f/src/nvidia/volumes.go#L104-L173 | ||
| (lib.mkIf config.hardware.nvidia-container-toolkit.mount-nvidia-docker-1-directories | ||
| [{ hostPath = "${lib.getLib nvidia-driver}/lib"; | ||
| containerPath = "/usr/local/nvidia/lib"; } | ||
| { hostPath = "${lib.getLib nvidia-driver}/lib"; | ||
| containerPath = "/usr/local/nvidia/lib64"; }]) | ||
| ]); | ||
|
|
||
| systemd.services.nvidia-container-toolkit-cdi-generator = lib.mkIf config.hardware.nvidia-container-toolkit.enable { | ||
| description = "Container Device Interface (CDI) for Nvidia generator"; | ||
| wantedBy = [ "multi-user.target" ]; | ||
| after = [ "systemd-udev-settle.service" ]; | ||
| serviceConfig = { | ||
| RuntimeDirectory = "cdi"; | ||
| RemainAfterExit = true; | ||
| ExecStart = | ||
| let | ||
| script = pkgs.callPackage ./cdi-generate.nix { | ||
| inherit (config.hardware.nvidia-container-toolkit) mounts; | ||
| nvidia-driver = config.hardware.nvidia.package; | ||
| }; | ||
| in | ||
| lib.getExe script; | ||
| Type = "oneshot"; | ||
| }; | ||
| }; | ||
|
|
||
| }; | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters