From 39198cfdf0a7906227a651bc501aae77402d2c76 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 27 Oct 2026 19:16:51 +0000 Subject: [PATCH] Add 'void' volume-mount device in CDI spec This is analogous to us configuring NVIDIA_VISIBLE_DEVICES=void in the CDI spec to avoid NVIDIA Container Runtime from injecting devices. When the volume-mounts strategy is enabled, e.g. when accept-nvidia-visible-devices-as-volume-mounts=true in NVIDIA Container Runtime, we should also prevent NVIDIA Container Runtime from injecting any devices specified through volume mounts. Signed-off-by: Christopher Desiniotis (cherry picked from commit 78f297067e4403ee5ba209c0931e723531f9b08b) --- .../toolkit/toolkit_test.go | 7 ++++ cmd/nvidia-ctk/cdi/generate/generate_test.go | 35 +++++++++++++++++++ internal/config/image/cuda_image.go | 5 +++ internal/config/image/cuda_image_test.go | 19 ++++++++++ internal/modifier/csv_test.go | 1 + pkg/nvcdi/lib-imex_test.go | 8 +++++ pkg/nvcdi/lib_test.go | 16 ++++++++- pkg/nvcdi/wrapper.go | 11 ++++++ 8 files changed, 101 insertions(+), 1 deletion(-) diff --git a/cmd/nvidia-ctk-installer/toolkit/toolkit_test.go b/cmd/nvidia-ctk-installer/toolkit/toolkit_test.go index 6af955cfd..e138ab6a7 100644 --- a/cmd/nvidia-ctk-installer/toolkit/toolkit_test.go +++ b/cmd/nvidia-ctk-installer/toolkit/toolkit_test.go @@ -137,6 +137,13 @@ containerEdits: - nodev - rbind - rprivate + - hostPath: /dev/null + containerPath: /var/run/nvidia-container-devices/void + options: + - ro + - nosuid + - nodev + - bind `, }, } diff --git a/cmd/nvidia-ctk/cdi/generate/generate_test.go b/cmd/nvidia-ctk/cdi/generate/generate_test.go index 6cac036c2..451d798ec 100644 --- a/cmd/nvidia-ctk/cdi/generate/generate_test.go +++ b/cmd/nvidia-ctk/cdi/generate/generate_test.go @@ -194,6 +194,13 @@ containerEdits: - nodev - rbind - rprivate + - hostPath: /dev/null + containerPath: /var/run/nvidia-container-devices/void + options: + - ro + - nosuid + - nodev + - bind `, }, { @@ -312,6 +319,13 @@ containerEdits: - nodev - rbind - rprivate + - hostPath: /dev/null + containerPath: /var/run/nvidia-container-devices/void + options: + - ro + - nosuid + - nodev + - bind `, }, { @@ -419,6 +433,13 @@ containerEdits: - nodev - rbind - rprivate + - hostPath: /dev/null + containerPath: /var/run/nvidia-container-devices/void + options: + - ro + - nosuid + - nodev + - bind `, }, { @@ -478,6 +499,13 @@ containerEdits: - nodev - rbind - rprivate + - hostPath: /dev/null + containerPath: /var/run/nvidia-container-devices/void + options: + - ro + - nosuid + - nodev + - bind `, }, { @@ -563,6 +591,13 @@ containerEdits: - nodev - rbind - rprivate + - hostPath: /dev/null + containerPath: /var/run/nvidia-container-devices/void + options: + - ro + - nosuid + - nodev + - bind `, }, } diff --git a/internal/config/image/cuda_image.go b/internal/config/image/cuda_image.go index a08968fda..5272eac74 100644 --- a/internal/config/image/cuda_image.go +++ b/internal/config/image/cuda_image.go @@ -35,6 +35,8 @@ const ( volumeMountDevicePrefixCDI = "cdi/" volumeMountDevicePrefixImex = "imex/" + + volumeMountDeviceVoid = "void" ) // CUDA represents a CUDA image that can be used for GPU computing. This wraps @@ -331,10 +333,13 @@ func (i CUDA) visibleDevicesFromEnvVar() []string { } // visibleDevicesFromMounts returns the set of visible devices requested as mounts. +// If the void device is requested, no devices are returned. func (i CUDA) visibleDevicesFromMounts() []string { var devices []string for _, device := range i.requestsFromMounts() { switch { + case device == volumeMountDeviceVoid: + return nil case strings.HasPrefix(device, volumeMountDevicePrefixImex): continue case strings.HasPrefix(device, volumeMountDevicePrefixCDI): diff --git a/internal/config/image/cuda_image_test.go b/internal/config/image/cuda_image_test.go index 5133cc308..8ee6d2f77 100644 --- a/internal/config/image/cuda_image_test.go +++ b/internal/config/image/cuda_image_test.go @@ -496,6 +496,11 @@ func TestGetVisibleDevicesFromMounts(t *testing.T) { mounts: makeTestMounts("GPU0", "imex/0", "GPU1"), expectedDevices: []string{"GPU0", "GPU1"}, }, + { + description: "void device results in no devices", + mounts: makeTestMounts("GPU0", "void", "GPU1"), + expectedDevices: nil, + }, } for _, tc := range tests { t.Run(tc.description, func(t *testing.T) { @@ -599,6 +604,20 @@ func TestVisibleDevices(t *testing.T) { acceptMounts: false, expectedDevices: nil, }, + { + description: "Void mount device", + mountDevices: makeTestMounts("GPU0", "GPU1", "void"), + acceptMounts: true, + expectedDevices: nil, + }, + { + description: "Void mount device, unprivileged, accept unprivileged", + mountDevices: makeTestMounts("GPU0", "GPU1", "void"), + envvarDevices: "all", + acceptUnprivileged: true, + acceptMounts: true, + expectedDevices: []string{"all"}, + }, // New test cases for visibleEnvVars functionality { description: "preferred env var set and present in env, privileged", diff --git a/internal/modifier/csv_test.go b/internal/modifier/csv_test.go index 994ceb65f..5b009b129 100644 --- a/internal/modifier/csv_test.go +++ b/internal/modifier/csv_test.go @@ -72,6 +72,7 @@ func TestNewCSVModifier(t *testing.T) { Env: []string{"NVIDIA_VISIBLE_DEVICES=void"}, }, Mounts: []specs.Mount{ + {Source: "/dev/null", Destination: "/var/run/nvidia-container-devices/void", Options: []string{"ro", "nosuid", "nodev", "bind"}}, {Source: "/usr/lib/aarch64-linux-gnu/nvidia/libcuda.so.1.1", Destination: "/usr/lib/aarch64-linux-gnu/nvidia/libcuda.so.1.1", Options: []string{"ro", "nosuid", "nodev", "rbind", "rprivate"}}, {Source: "/usr/lib/aarch64-linux-gnu/nvidia/libnvidia-ml.so.1", Destination: "/usr/lib/aarch64-linux-gnu/nvidia/libnvidia-ml.so.1", Options: []string{"ro", "nosuid", "nodev", "rbind", "rprivate"}}, }, diff --git a/pkg/nvcdi/lib-imex_test.go b/pkg/nvcdi/lib-imex_test.go index aebe71745..0cdf3f275 100644 --- a/pkg/nvcdi/lib-imex_test.go +++ b/pkg/nvcdi/lib-imex_test.go @@ -60,6 +60,14 @@ devices: containerEdits: env: - NVIDIA_VISIBLE_DEVICES=void + mounts: + - hostPath: /dev/null + containerPath: /var/run/nvidia-container-devices/void + options: + - ro + - nosuid + - nodev + - bind ` expectedSpec = strings.ReplaceAll(expectedSpec, "{{ .hostRoot }}", hostRoot) diff --git a/pkg/nvcdi/lib_test.go b/pkg/nvcdi/lib_test.go index 69d711bf2..e08163f7e 100644 --- a/pkg/nvcdi/lib_test.go +++ b/pkg/nvcdi/lib_test.go @@ -39,6 +39,12 @@ func TestNew(t *testing.T) { lookupRoot := filepath.Join(moduleRoot, "testdata", "lookup") + voidMount := &specs.Mount{ + HostPath: "/dev/null", + ContainerPath: "/var/run/nvidia-container-devices/void", + Options: []string{"ro", "nosuid", "nodev", "bind"}, + } + testCases := []struct { description string mode string @@ -82,6 +88,7 @@ func TestNew(t *testing.T) { Env: []string{ "NVIDIA_VISIBLE_DEVICES=void", }, + Mounts: []*specs.Mount{voidMount}, }, }, }, @@ -110,6 +117,7 @@ func TestNew(t *testing.T) { Env: []string{ "NVIDIA_VISIBLE_DEVICES=void", }, + Mounts: []*specs.Mount{voidMount}, }, }, }, @@ -137,6 +145,7 @@ func TestNew(t *testing.T) { Env: []string{ "NVIDIA_VISIBLE_DEVICES=void", }, + Mounts: []*specs.Mount{voidMount}, }, }, }, @@ -162,6 +171,7 @@ func TestNew(t *testing.T) { Env: []string{ "NVIDIA_VISIBLE_DEVICES=void", }, + Mounts: []*specs.Mount{voidMount}, }, }, }, @@ -195,6 +205,7 @@ func TestNew(t *testing.T) { Mounts: []*specs.Mount{ {ContainerPath: "/lib/x86_64-linux-gnu/libcuda.so.999.88.77", HostPath: "/lib/x86_64-linux-gnu/libcuda.so.999.88.77", Options: []string{"ro", "nosuid", "nodev", "rbind", "rprivate"}}, {ContainerPath: "/lib/x86_64-linux-gnu/vdpau/libvdpau_nvidia.so.999.88.77", HostPath: "/lib/x86_64-linux-gnu/vdpau/libvdpau_nvidia.so.999.88.77", Options: []string{"ro", "nosuid", "nodev", "rbind", "rprivate"}}, + voidMount, }, Hooks: []*specs.Hook{ { @@ -250,6 +261,7 @@ func TestNew(t *testing.T) { Mounts: []*specs.Mount{ {ContainerPath: "/lib/x86_64-linux-gnu/libcuda.so.999.88.77", HostPath: "/lib/x86_64-linux-gnu/libcuda.so.999.88.77", Options: []string{"ro", "nosuid", "nodev", "rbind", "rprivate"}}, {ContainerPath: "/lib/x86_64-linux-gnu/vdpau/libvdpau_nvidia.so.999.88.77", HostPath: "/lib/x86_64-linux-gnu/vdpau/libvdpau_nvidia.so.999.88.77", Options: []string{"ro", "nosuid", "nodev", "rbind", "rprivate"}}, + voidMount, }, Hooks: []*specs.Hook{ { @@ -317,7 +329,8 @@ func TestNew(t *testing.T) { }, }, ContainerEdits: specs.ContainerEdits{ - Env: []string{"NVIDIA_VISIBLE_DEVICES=void"}, + Env: []string{"NVIDIA_VISIBLE_DEVICES=void"}, + Mounts: []*specs.Mount{voidMount}, }, }, }, @@ -347,6 +360,7 @@ func TestNew(t *testing.T) { ContainerEdits: specs.ContainerEdits{ Env: []string{"NVIDIA_VISIBLE_DEVICES=void"}, Mounts: []*specs.Mount{ + voidMount, {HostPath: "/usr/lib/aarch64-linux-gnu/nvidia/libcuda.so.1.1", ContainerPath: "/usr/lib/aarch64-linux-gnu/nvidia/libcuda.so.1.1", Options: []string{"ro", "nosuid", "nodev", "rbind", "rprivate"}}, {HostPath: "/usr/lib/aarch64-linux-gnu/nvidia/libnvidia-ml.so.1", ContainerPath: "/usr/lib/aarch64-linux-gnu/nvidia/libnvidia-ml.so.1", Options: []string{"ro", "nosuid", "nodev", "rbind", "rprivate"}}, }, diff --git a/pkg/nvcdi/wrapper.go b/pkg/nvcdi/wrapper.go index 9d4520798..0698cc7a2 100644 --- a/pkg/nvcdi/wrapper.go +++ b/pkg/nvcdi/wrapper.go @@ -18,6 +18,7 @@ package nvcdi import ( "fmt" + "path/filepath" "tags.cncf.io/container-device-interface/pkg/cdi" "tags.cncf.io/container-device-interface/specs-go" @@ -96,7 +97,17 @@ func (m *wrapper) GetCommonEdits() (*cdi.ContainerEdits, error) { if err != nil { return nil, err } + // Configure NVIDIA_VISIBLE_DEVICES=void to prevent duplicate device + // injection from NVIDIA Container Runtime. edits.Env = append(edits.Env, image.EnvVarNvidiaVisibleDevices+"=void") + // Configure a void volume mount device to prevent duplicate device + // injection from NVIDIA Container Runtime when the 'volume-mounts' + // device strategy is enabled. + edits.Mounts = append(edits.Mounts, &specs.Mount{ + HostPath: "/dev/null", + ContainerPath: filepath.Join(image.DeviceListAsVolumeMountsRoot, "void"), + Options: []string{"ro", "nosuid", "nodev", "bind"}, + }) return edits, nil }