[release-1.20] Add 'void' volume-mount device in CDI spec

cmd/nvidia-ctk-installer/toolkit/toolkit_test.go

@@ -137,6 +137,13 @@ containerEdits:
             - nodev
             - rbind
             - rprivate
+        - hostPath: /dev/null
+          containerPath: /var/run/nvidia-container-devices/void
+          options:
+            - ro
+            - nosuid
+            - nodev
+            - bind
 `,
 		},
 	}

cmd/nvidia-ctk/cdi/generate/generate_test.go

@@ -194,6 +194,13 @@ containerEdits:
             - nodev
             - rbind
             - rprivate
+        - hostPath: /dev/null
+          containerPath: /var/run/nvidia-container-devices/void
+          options:
+            - ro
+            - nosuid
+            - nodev
+            - bind
 `,
 		},
 		{
@@ -312,6 +319,13 @@ containerEdits:
             - nodev
             - rbind
             - rprivate
+        - hostPath: /dev/null
+          containerPath: /var/run/nvidia-container-devices/void
+          options:
+            - ro
+            - nosuid
+            - nodev
+            - bind
 `,
 		},
 		{
@@ -419,6 +433,13 @@ containerEdits:
             - nodev
             - rbind
             - rprivate
+        - hostPath: /dev/null
+          containerPath: /var/run/nvidia-container-devices/void
+          options:
+            - ro
+            - nosuid
+            - nodev
+            - bind
 `,
 		},
 		{
@@ -478,6 +499,13 @@ containerEdits:
             - nodev
             - rbind
             - rprivate
+        - hostPath: /dev/null
+          containerPath: /var/run/nvidia-container-devices/void
+          options:
+            - ro
+            - nosuid
+            - nodev
+            - bind
 `,
 		},
 		{
@@ -563,6 +591,13 @@ containerEdits:
             - nodev
             - rbind
             - rprivate
+        - hostPath: /dev/null
+          containerPath: /var/run/nvidia-container-devices/void
+          options:
+            - ro
+            - nosuid
+            - nodev
+            - bind
 `,
 		},
 	}

internal/config/image/cuda_image.go

@@ -35,6 +35,8 @@ const (
 
 	volumeMountDevicePrefixCDI  = "cdi/"
 	volumeMountDevicePrefixImex = "imex/"
+
+	volumeMountDeviceVoid = "void"
 )
 
 // CUDA represents a CUDA image that can be used for GPU computing. This wraps
@@ -331,10 +333,13 @@ func (i CUDA) visibleDevicesFromEnvVar() []string {
 }
 
 // visibleDevicesFromMounts returns the set of visible devices requested as mounts.
+// If the void device is requested, no devices are returned.
 func (i CUDA) visibleDevicesFromMounts() []string {
 	var devices []string
 	for _, device := range i.requestsFromMounts() {
 		switch {
+		case device == volumeMountDeviceVoid:
+			return nil
 		case strings.HasPrefix(device, volumeMountDevicePrefixImex):
 			continue
 		case strings.HasPrefix(device, volumeMountDevicePrefixCDI):

internal/config/image/cuda_image_test.go

@@ -496,6 +496,11 @@ func TestGetVisibleDevicesFromMounts(t *testing.T) {
 			mounts:          makeTestMounts("GPU0", "imex/0", "GPU1"),
 			expectedDevices: []string{"GPU0", "GPU1"},
 		},
+		{
+			description:     "void device results in no devices",
+			mounts:          makeTestMounts("GPU0", "void", "GPU1"),
+			expectedDevices: nil,
+		},
 	}
 	for _, tc := range tests {
 		t.Run(tc.description, func(t *testing.T) {
@@ -599,6 +604,20 @@ func TestVisibleDevices(t *testing.T) {
 			acceptMounts:       false,
 			expectedDevices:    nil,
 		},
+		{
+			description:     "Void mount device",
+			mountDevices:    makeTestMounts("GPU0", "GPU1", "void"),
+			acceptMounts:    true,
+			expectedDevices: nil,
+		},
+		{
+			description:        "Void mount device, unprivileged, accept unprivileged",
+			mountDevices:       makeTestMounts("GPU0", "GPU1", "void"),
+			envvarDevices:      "all",
+			acceptUnprivileged: true,
+			acceptMounts:       true,
+			expectedDevices:    []string{"all"},
+		},
 		// New test cases for visibleEnvVars functionality
 		{
 			description:                   "preferred env var set and present in env, privileged",

internal/modifier/csv_test.go

@@ -72,6 +72,7 @@ func TestNewCSVModifier(t *testing.T) {
 					Env: []string{"NVIDIA_VISIBLE_DEVICES=void"},
 				},
 				Mounts: []specs.Mount{
+					{Source: "/dev/null", Destination: "/var/run/nvidia-container-devices/void", Options: []string{"ro", "nosuid", "nodev", "bind"}},
 					{Source: "/usr/lib/aarch64-linux-gnu/nvidia/libcuda.so.1.1", Destination: "/usr/lib/aarch64-linux-gnu/nvidia/libcuda.so.1.1", Options: []string{"ro", "nosuid", "nodev", "rbind", "rprivate"}},
 					{Source: "/usr/lib/aarch64-linux-gnu/nvidia/libnvidia-ml.so.1", Destination: "/usr/lib/aarch64-linux-gnu/nvidia/libnvidia-ml.so.1", Options: []string{"ro", "nosuid", "nodev", "rbind", "rprivate"}},
 				},

pkg/nvcdi/lib-imex_test.go

@@ -60,6 +60,14 @@ devices:
 containerEdits:
     env:
         - NVIDIA_VISIBLE_DEVICES=void
+    mounts:
+        - hostPath: /dev/null
+          containerPath: /var/run/nvidia-container-devices/void
+          options:
+            - ro
+            - nosuid
+            - nodev
+            - bind
 `
 	expectedSpec = strings.ReplaceAll(expectedSpec, "{{ .hostRoot }}", hostRoot)
 

pkg/nvcdi/lib_test.go

@@ -39,6 +39,12 @@ func TestNew(t *testing.T) {
 
 	lookupRoot := filepath.Join(moduleRoot, "testdata", "lookup")
 
+	voidMount := &specs.Mount{
+		HostPath:      "/dev/null",
+		ContainerPath: "/var/run/nvidia-container-devices/void",
+		Options:       []string{"ro", "nosuid", "nodev", "bind"},
+	}
+
 	testCases := []struct {
 		description       string
 		mode              string
@@ -82,6 +88,7 @@ func TestNew(t *testing.T) {
 					Env: []string{
 						"NVIDIA_VISIBLE_DEVICES=void",
 					},
+					Mounts: []*specs.Mount{voidMount},
 				},
 			},
 		},
@@ -110,6 +117,7 @@ func TestNew(t *testing.T) {
 					Env: []string{
 						"NVIDIA_VISIBLE_DEVICES=void",
 					},
+					Mounts: []*specs.Mount{voidMount},
 				},
 			},
 		},
@@ -137,6 +145,7 @@ func TestNew(t *testing.T) {
 					Env: []string{
 						"NVIDIA_VISIBLE_DEVICES=void",
 					},
+					Mounts: []*specs.Mount{voidMount},
 				},
 			},
 		},
@@ -162,6 +171,7 @@ func TestNew(t *testing.T) {
 					Env: []string{
 						"NVIDIA_VISIBLE_DEVICES=void",
 					},
+					Mounts: []*specs.Mount{voidMount},
 				},
 			},
 		},
@@ -195,6 +205,7 @@ func TestNew(t *testing.T) {
 					Mounts: []*specs.Mount{
 						{ContainerPath: "/lib/x86_64-linux-gnu/libcuda.so.999.88.77", HostPath: "/lib/x86_64-linux-gnu/libcuda.so.999.88.77", Options: []string{"ro", "nosuid", "nodev", "rbind", "rprivate"}},
 						{ContainerPath: "/lib/x86_64-linux-gnu/vdpau/libvdpau_nvidia.so.999.88.77", HostPath: "/lib/x86_64-linux-gnu/vdpau/libvdpau_nvidia.so.999.88.77", Options: []string{"ro", "nosuid", "nodev", "rbind", "rprivate"}},
+						voidMount,
 					},
 					Hooks: []*specs.Hook{
 						{
@@ -250,6 +261,7 @@ func TestNew(t *testing.T) {
 					Mounts: []*specs.Mount{
 						{ContainerPath: "/lib/x86_64-linux-gnu/libcuda.so.999.88.77", HostPath: "/lib/x86_64-linux-gnu/libcuda.so.999.88.77", Options: []string{"ro", "nosuid", "nodev", "rbind", "rprivate"}},
 						{ContainerPath: "/lib/x86_64-linux-gnu/vdpau/libvdpau_nvidia.so.999.88.77", HostPath: "/lib/x86_64-linux-gnu/vdpau/libvdpau_nvidia.so.999.88.77", Options: []string{"ro", "nosuid", "nodev", "rbind", "rprivate"}},
+						voidMount,
 					},
 					Hooks: []*specs.Hook{
 						{
@@ -317,7 +329,8 @@ func TestNew(t *testing.T) {
 					},
 				},
 				ContainerEdits: specs.ContainerEdits{
-					Env: []string{"NVIDIA_VISIBLE_DEVICES=void"},
+					Env:    []string{"NVIDIA_VISIBLE_DEVICES=void"},
+					Mounts: []*specs.Mount{voidMount},
 				},
 			},
 		},
@@ -347,6 +360,7 @@ func TestNew(t *testing.T) {
 				ContainerEdits: specs.ContainerEdits{
 					Env: []string{"NVIDIA_VISIBLE_DEVICES=void"},
 					Mounts: []*specs.Mount{
+						voidMount,
 						{HostPath: "/usr/lib/aarch64-linux-gnu/nvidia/libcuda.so.1.1", ContainerPath: "/usr/lib/aarch64-linux-gnu/nvidia/libcuda.so.1.1", Options: []string{"ro", "nosuid", "nodev", "rbind", "rprivate"}},
 						{HostPath: "/usr/lib/aarch64-linux-gnu/nvidia/libnvidia-ml.so.1", ContainerPath: "/usr/lib/aarch64-linux-gnu/nvidia/libnvidia-ml.so.1", Options: []string{"ro", "nosuid", "nodev", "rbind", "rprivate"}},
 					},

pkg/nvcdi/wrapper.go

@@ -18,6 +18,7 @@ package nvcdi
 
 import (
 	"fmt"
+	"path/filepath"
 
 	"tags.cncf.io/container-device-interface/pkg/cdi"
 	"tags.cncf.io/container-device-interface/specs-go"
@@ -96,7 +97,17 @@ func (m *wrapper) GetCommonEdits() (*cdi.ContainerEdits, error) {
 	if err != nil {
 		return nil, err
 	}
+	// Configure NVIDIA_VISIBLE_DEVICES=void to prevent duplicate device
+	// injection from NVIDIA Container Runtime.
 	edits.Env = append(edits.Env, image.EnvVarNvidiaVisibleDevices+"=void")
+	// Configure a void volume mount device to prevent duplicate device
+	// injection from NVIDIA Container Runtime when the 'volume-mounts'
+	// device strategy is enabled.
+	edits.Mounts = append(edits.Mounts, &specs.Mount{
+		HostPath:      "/dev/null",
+		ContainerPath: filepath.Join(image.DeviceListAsVolumeMountsRoot, "void"),
+		Options:       []string{"ro", "nosuid", "nodev", "bind"},
+	})
 
 	return edits, nil
 }