feat(vgpu-manager): default Fabric Manager to vGPU multitenancy mode

controllers/object_controls.go

@@ -182,6 +182,17 @@ const (
 	OpenKernelModulesEnabledEnvName = "OPEN_KERNEL_MODULES_ENABLED"
 	// KernelModuleTypeEnvName is the name of the driver-container envvar to set the desired kernel module type
 	KernelModuleTypeEnvName = "KERNEL_MODULE_TYPE"
+	// FabricModeEnvName is the name of the vGPU-manager-container envvar that
+	// selects the NVIDIA Fabric Manager mode (the FABRIC_MODE key in
+	// fabricmanager.cfg) on NVSwitch systems.
+	FabricModeEnvName = "FABRIC_MODE"
+	// FabricModeVGPUMultitenancy is the FABRIC_MODE value that starts Fabric
+	// Manager in vGPU multitenancy mode, required for SR-IOV vGPU on NVSwitch
+	// (HGX) systems. The Fabric Manager User Guide documents FABRIC_MODE=0
+	// (bare metal or full passthrough virtualization) as the default,
+	// FABRIC_MODE=1 as Shared NVSwitch multi-tenancy, and FABRIC_MODE=2 as
+	// vGPU multitenancy.
+	FabricModeVGPUMultitenancy = "2"
 	// MPSRootEnvName is the name of the envvar for configuring the MPS root
 	MPSRootEnvName = "MPS_ROOT"
 	// DefaultMPSRoot is the default MPS root path on the host
@@ -3865,6 +3876,18 @@ func transformVGPUManagerContainer(obj *appsv1.DaemonSet, config *gpuv1.ClusterP
 		}
 	}
 
+	// On NVSwitch (HGX) systems, SR-IOV vGPU requires NVIDIA Fabric Manager to
+	// run in vGPU multitenancy mode (FABRIC_MODE=2) rather than the bare-metal
+	// default (FABRIC_MODE=0); otherwise a whole-card vGPU guest cannot
+	// initialize CUDA. Default the vGPU Manager container to FABRIC_MODE=2 so
+	// its entrypoint can configure fabricmanager.cfg accordingly on NVSwitch
+	// hosts. Non-NVSwitch vGPU hosts do not start Fabric Manager, so the value
+	// is ignored there. A user-provided FABRIC_MODE in VGPUManager.Env is
+	// applied above and takes precedence.
+	if getContainerEnv(container, FabricModeEnvName) == "" {
+		setContainerEnv(container, FabricModeEnvName, FabricModeVGPUMultitenancy)
+	}
+
 	// mount any custom kernel module configuration parameters at /drivers
 	if config.VGPUManager.KernelModuleConfig != nil && config.VGPUManager.KernelModuleConfig.Name != "" {
 		volumeMounts, itemsToInclude, err := createConfigMapVolumeMounts(n, config.VGPUManager.KernelModuleConfig.Name, driversDir)

controllers/transforms_test.go

@@ -4464,6 +4464,81 @@ func TestTransformVGPUManager(t *testing.T) {
 	}
 }
 
+// TestTransformVGPUManagerFabricMode verifies that the vGPU Manager container is
+// defaulted to NVIDIA Fabric Manager vGPU multitenancy mode (FABRIC_MODE=2),
+// which SR-IOV vGPU on NVSwitch (HGX) systems requires, and that a
+// user-provided FABRIC_MODE in VGPUManager.Env is preserved.
+func TestTransformVGPUManagerFabricMode(t *testing.T) {
+	node := &corev1.Node{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "test-node",
+			Labels: map[string]string{
+				nfdOSReleaseIDLabelKey: "ubuntu",
+				nfdOSVersionIDLabelKey: "24.04",
+				nfdKernelLabelKey:      "6.8.0-60-generic",
+				commonGPULabelKey:      "true",
+			},
+		},
+	}
+	mockClient := fake.NewFakeClient(node)
+
+	baseSpec := func() *gpuv1.ClusterPolicySpec {
+		return &gpuv1.ClusterPolicySpec{
+			VGPUManager: gpuv1.VGPUManagerSpec{
+				Repository:      "nvcr.io/nvidia",
+				Image:           "vgpu-manager",
+				Version:         "550.90.07",
+				ImagePullPolicy: "IfNotPresent",
+				DriverManager: gpuv1.DriverManagerSpec{
+					Repository:      "nvcr.io/nvidia/cloud-native",
+					Image:           "k8s-driver-manager",
+					ImagePullPolicy: "IfNotPresent",
+					Version:         "v0.8.0",
+				},
+			},
+		}
+	}
+
+	testCases := []struct {
+		description        string
+		env                []gpuv1.EnvVar
+		expectedFabricMode string
+	}{
+		{
+			description:        "defaults to vGPU multitenancy mode",
+			env:                nil,
+			expectedFabricMode: "2",
+		},
+		{
+			description:        "user-provided FABRIC_MODE is preserved",
+			env:                []gpuv1.EnvVar{{Name: "FABRIC_MODE", Value: "0"}},
+			expectedFabricMode: "0",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			ds := NewDaemonset().
+				WithInitContainer(corev1.Container{Name: "k8s-driver-manager"}).
+				WithContainer(corev1.Container{Name: "nvidia-vgpu-manager-ctr"})
+			cpSpec := baseSpec()
+			cpSpec.VGPUManager.Env = tc.env
+
+			clusterPolicyCtrl := ClusterPolicyController{
+				logger:            ctrl.Log.WithName("test"),
+				client:            mockClient,
+				operatorNamespace: "test-ns",
+			}
+			err := TransformVGPUManager(ds.DaemonSet, cpSpec, clusterPolicyCtrl)
+			require.NoError(t, err)
+
+			container := findContainerByName(ds.Spec.Template.Spec.Containers, "nvidia-vgpu-manager-ctr")
+			require.NotNil(t, container)
+			require.Equal(t, tc.expectedFabricMode, getContainerEnv(container, "FABRIC_MODE"))
+		})
+	}
+}
+
 func TestTransformDriverWithAdditionalConfig(t *testing.T) {
 	node := &corev1.Node{
 		ObjectMeta: metav1.ObjectMeta{