From cec9863fc9029faf25a209e95e166942d39631c2 Mon Sep 17 00:00:00 2001 From: Abrar Shivani Date: Fri, 21 Aug 2026 16:25:10 -0700 Subject: [PATCH 1/6] Mount host /lib/modules for precompiled drivers on SUSE getDriverAdditionalConfigs placed the lib-modules volume and mount inside the "if !cr.Spec.UsePrecompiledDrivers()" branch, guarded by a check for UsePrecompiledDrivers. That inner condition can never hold there, so an NVIDIADriver with usePrecompiled: true on sles or sl-micro rendered a driver pod without /run/host/lib/modules. Move the mount out to its own block after the non-precompiled branch. The comment above it goes as well, since the guard already says precompiled and SUSE and the volume literals already say host module tree. The equivalent code in the ClusterPolicy path sits before the early return for precompiled drivers and works as intended, so only the NVIDIADriver path was affected. That is also why this went unnoticed: the tests added with the mount only covered the ClusterPolicy path. Add a table-driven test that renders the DaemonSet and asserts the volume on the pod spec and the mount on nvidia-driver-ctr. Going through the render catches a template change that stopped threading AdditionalConfigs into the pod as well as the Go-side bug. Fixes #2776 Signed-off-by: Abrar Shivani (cherry picked from commit 77d1673507dde030d733b41add997d5cc408abf0) --- internal/state/driver_test.go | 140 +++++++++++++++++++++++++++++++ internal/state/driver_volumes.go | 36 ++++---- 2 files changed, 157 insertions(+), 19 deletions(-) diff --git a/internal/state/driver_test.go b/internal/state/driver_test.go index a3d89e25a5..92907fb62c 100644 --- a/internal/state/driver_test.go +++ b/internal/state/driver_test.go @@ -91,6 +91,33 @@ func getYAMLString(objs []*unstructured.Unstructured) (string, error) { return sb.String(), nil } +func findVolumeByName(volumes []corev1.Volume, name string) *corev1.Volume { + for i := range volumes { + if volumes[i].Name == name { + return &volumes[i] + } + } + return nil +} + +func findVolumeMountByName(volumeMounts []corev1.VolumeMount, name string) *corev1.VolumeMount { + for i := range volumeMounts { + if volumeMounts[i].Name == name { + return &volumeMounts[i] + } + } + return nil +} + +func findContainerByName(containers []corev1.Container, name string) *corev1.Container { + for i := range containers { + if containers[i].Name == name { + return &containers[i] + } + } + return nil +} + func hasSubscriptionVolumeMount(volumeMounts []corev1.VolumeMount) bool { for _, volumeMount := range volumeMounts { if strings.HasPrefix(volumeMount.Name, "subscription-config-") { @@ -657,6 +684,119 @@ func TestDriverAdditionalConfigsSubscriptionMounts(t *testing.T) { } } +func TestDriverPrecompiledLibModules(t *testing.T) { + const ( + libModulesVolumeName = "lib-modules" + driverContainerName = "nvidia-driver-ctr" + precompiledKernelVersion = "5.14.21-150500.55.44-default" + ) + + state, err := NewStateDriver( + fake.NewClientBuilder().WithScheme(scheme.Scheme).Build(), + "test-ns", + scheme.Scheme, + manifestDir) + require.NoError(t, err) + stateDriver, ok := state.(*stateDriver) + require.True(t, ok) + + clusterInfo := testClusterInfo{runtime: consts.Containerd} + + testCases := []struct { + description string + osRelease string + osVersion string + usePrecompiled bool + expectLibModulesMounted bool + }{ + { + description: "sles with precompiled drivers mounts host /lib/modules", + osRelease: "sles", + osVersion: "15.6", + usePrecompiled: true, + expectLibModulesMounted: true, + }, + { + description: "sl-micro with precompiled drivers mounts host /lib/modules", + osRelease: "sl-micro", + osVersion: "6.0", + usePrecompiled: true, + expectLibModulesMounted: true, + }, + { + description: "sles without precompiled drivers does not mount host /lib/modules", + osRelease: "sles", + osVersion: "15.6", + usePrecompiled: false, + expectLibModulesMounted: false, + }, + { + description: "ubuntu with precompiled drivers does not mount host /lib/modules", + osRelease: "ubuntu", + osVersion: "22.04", + usePrecompiled: true, + expectLibModulesMounted: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + driver := &nvidiav1alpha1.NVIDIADriver{} + driver.Spec.UsePrecompiled = ptr.To(tc.usePrecompiled) + + additionalConfigs, err := stateDriver.getDriverAdditionalConfigs( + context.Background(), + driver, + clusterInfo, + nodePool{osRelease: tc.osRelease, osVersion: tc.osVersion}, + ) + require.NoError(t, err) + + renderData := getMinimalDriverRenderData() + renderData.Driver.Spec.UsePrecompiled = ptr.To(tc.usePrecompiled) + renderData.AdditionalConfigs = additionalConfigs + if tc.usePrecompiled { + renderData.Precompiled = &precompiledSpec{ + KernelVersion: precompiledKernelVersion, + SanitizedKernelVersion: precompiledKernelVersion, + } + } + + objs, err := stateDriver.renderer.RenderObjects( + &render.TemplatingData{ + Data: renderData, + }) + require.NoError(t, err) + + ds, err := getDaemonsetFromObjects(objs) + require.NoError(t, err) + + libModulesVolume := findVolumeByName(ds.Spec.Template.Spec.Volumes, libModulesVolumeName) + + driverContainer := findContainerByName(ds.Spec.Template.Spec.Containers, driverContainerName) + require.NotNil(t, driverContainer) + + libModulesMount := findVolumeMountByName(driverContainer.VolumeMounts, libModulesVolumeName) + + if !tc.expectLibModulesMounted { + assert.Nil(t, libModulesVolume, "unexpected lib-modules volume on the driver pod spec") + assert.Nil(t, libModulesMount, "unexpected lib-modules volume mount on nvidia-driver-ctr") + return + } + + require.NotNil(t, libModulesVolume, "expected a lib-modules volume on the driver pod spec") + require.NotNil(t, libModulesVolume.HostPath) + assert.Equal(t, "/lib/modules", libModulesVolume.HostPath.Path) + require.NotNil(t, libModulesVolume.HostPath.Type) + assert.Equal(t, corev1.HostPathDirectory, *libModulesVolume.HostPath.Type) + + require.NotNil(t, libModulesMount, "expected a lib-modules volume mount on nvidia-driver-ctr") + assert.Equal(t, "/run/host/lib/modules", libModulesMount.MountPath) + assert.True(t, libModulesMount.ReadOnly) + }) + } +} + func TestDriverConfigPathHelpers(t *testing.T) { repoConfigPath, err := getRepoConfigPath("rhel") require.NoError(t, err) diff --git a/internal/state/driver_volumes.go b/internal/state/driver_volumes.go index f0aeb2f5c3..34f60ea609 100644 --- a/internal/state/driver_volumes.go +++ b/internal/state/driver_volumes.go @@ -220,29 +220,27 @@ func (s *stateDriver) getDriverAdditionalConfigs(ctx context.Context, cr *v1alph additionalCfgs.Volumes = append(additionalCfgs.Volumes, subscriptionVol) } } + } - // Mount /lib/modules for precompiled drivers on SUSE distributions. - // Those containers need access to host /lib/modules at runtime. - if cr.Spec.UsePrecompiledDrivers() && (pool.osRelease == "sles" || pool.osRelease == "sl-micro") { - logger.Info("Mounting /lib/modules into the driver container") - libModulesVolMount := corev1.VolumeMount{ - Name: "lib-modules", - MountPath: "/run/host/lib/modules", - ReadOnly: true, - } - additionalCfgs.VolumeMounts = append(additionalCfgs.VolumeMounts, libModulesVolMount) + if cr.Spec.UsePrecompiledDrivers() && (pool.osRelease == "sles" || pool.osRelease == "sl-micro") { + logger.Info("Mounting /lib/modules into the driver pod") + libModulesVolMount := corev1.VolumeMount{ + Name: "lib-modules", + MountPath: "/run/host/lib/modules", + ReadOnly: true, + } + additionalCfgs.VolumeMounts = append(additionalCfgs.VolumeMounts, libModulesVolMount) - libModulesVol := corev1.Volume{ - Name: "lib-modules", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/lib/modules", - Type: ptr.To(corev1.HostPathDirectory), - }, + libModulesVol := corev1.Volume{ + Name: "lib-modules", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/lib/modules", + Type: ptr.To(corev1.HostPathDirectory), }, - } - additionalCfgs.Volumes = append(additionalCfgs.Volumes, libModulesVol) + }, } + additionalCfgs.Volumes = append(additionalCfgs.Volumes, libModulesVol) } // mount any custom kernel module configuration parameters at /drivers From 316f55b6247cd86541c6121bd6e29f3dae33eed3 Mon Sep 17 00:00:00 2001 From: Karthikeyan Valliyurnatt Date: Fri, 21 Aug 2026 15:03:29 -0400 Subject: [PATCH 2/6] Avoid advancing state on cleanup errors Signed-off-by: Karthikeyan Valliyurnatt (cherry picked from commit bae2eb694cf2c8e5d613d21b0a10ad5373cf2b9a) --- controllers/state_manager.go | 2 +- controllers/state_manager_test.go | 74 +++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 1 deletion(-) diff --git a/controllers/state_manager.go b/controllers/state_manager.go index d49c311967..eeac1bdce8 100644 --- a/controllers/state_manager.go +++ b/controllers/state_manager.go @@ -965,13 +965,13 @@ func (n *ClusterPolicyController) step() (gpuv1.State, error) { if (n.stateNames[n.idx] == "state-driver" || n.stateNames[n.idx] == "state-vgpu-manager") && n.singleton.Spec.Driver.UseNvidiaDriverCRDType() { n.logger.Info("NVIDIADriver CRD is enabled, cleaning up all NVIDIA driver daemonsets owned by ClusterPolicy") - n.idx++ // Cleanup all driver daemonsets owned by ClusterPolicy while keeping the // running driver pods available until NVIDIADriver rolls replacements. err := n.cleanupAllDriverDaemonSets(n.ctx, metav1.DeletePropagationOrphan) if err != nil { return gpuv1.NotReady, fmt.Errorf("failed to cleanup all NVIDIA driver daemonsets owned by ClusterPolicy: %w", err) } + n.idx++ return gpuv1.Disabled, nil } diff --git a/controllers/state_manager_test.go b/controllers/state_manager_test.go index 73f4b7579e..f21241301a 100644 --- a/controllers/state_manager_test.go +++ b/controllers/state_manager_test.go @@ -165,6 +165,80 @@ func TestGetGPUNodeOSInfoListError(t *testing.T) { require.Contains(t, err.Error(), "unable to list nodes with GPU present") } +func TestStep(t *testing.T) { + expectedErr := errors.New("step failed") + driverCRDPolicy := &gpuv1.ClusterPolicy{ + Spec: gpuv1.ClusterPolicySpec{ + Driver: gpuv1.DriverSpec{UseNvidiaDriverCRD: ptr.To(true)}, + }, + } + + testCases := []struct { + name string + controller ClusterPolicyController + expectedStatus gpuv1.State + expectedIndex int + expectedError error + }{ + { + name: "normal state advances on success", + controller: ClusterPolicyController{ + controls: []controlFunc{{func(ClusterPolicyController) (gpuv1.State, error) { return gpuv1.Ready, nil }}}, + stateNames: []string{"test-state"}, + }, + expectedStatus: gpuv1.Ready, + expectedIndex: 1, + }, + { + name: "normal state does not advance on error", + controller: ClusterPolicyController{ + controls: []controlFunc{{func(ClusterPolicyController) (gpuv1.State, error) { return gpuv1.NotReady, expectedErr }}}, + stateNames: []string{"test-state"}, + }, + expectedStatus: gpuv1.NotReady, + expectedIndex: 0, + expectedError: expectedErr, + }, + { + name: "driver cleanup advances on success", + controller: ClusterPolicyController{ + ctx: context.Background(), + client: errorListClient{}, + singleton: driverCRDPolicy, + stateNames: []string{"state-driver"}, + }, + expectedStatus: gpuv1.Disabled, + expectedIndex: 1, + }, + { + name: "driver cleanup does not advance on error", + controller: ClusterPolicyController{ + ctx: context.Background(), + client: errorListClient{err: expectedErr}, + singleton: driverCRDPolicy, + stateNames: []string{"state-driver"}, + }, + expectedStatus: gpuv1.NotReady, + expectedIndex: 0, + expectedError: expectedErr, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + status, err := tc.controller.step() + + if tc.expectedError != nil { + require.ErrorIs(t, err, tc.expectedError) + } else { + require.NoError(t, err) + } + require.Equal(t, tc.expectedStatus, status) + require.Equal(t, tc.expectedIndex, tc.controller.idx) + }) + } +} + func TestGetGPUNodeOSInfoNoGPUNodes(t *testing.T) { scheme := runtime.NewScheme() require.NoError(t, corev1.AddToScheme(scheme)) From 37a3f2b9b0eb9064f855ad80ca2ab3e2ab6ef4b9 Mon Sep 17 00:00:00 2001 From: Harshal Patil <12152047+harche@users.noreply.github.com> Date: Wed, 26 Aug 2026 16:11:48 -0400 Subject: [PATCH 3/6] Skip device plugin alert when devicePlugin is disabled in ClusterPolicy When devicePlugin.enabled is set to false in the ClusterPolicy, the nvidia-node-status-exporter still monitors the device_plugin_devices_total metric which reports 0 (since no device plugin pods are running). This triggers a false positive GPUOperatorNodeDeploymentFailed alert. Fix: The operator now injects a DEVICE_PLUGIN_ENABLED env var into the node-status-exporter daemonset based on the ClusterPolicy. When set to "false", the exporter skips device plugin validation entirely, so the metric is never emitted and the alert does not fire. Fixes: https://github.com/NVIDIA/gpu-operator/issues/2237 Signed-off-by: Harshal Patil <12152047+harche@users.noreply.github.com> (cherry picked from commit 9b079ef4df6a96bd24a388d27f1ce230db5264ae) --- .../0800_prometheus_rule_openshift.yaml | 5 ++- cmd/nvidia-validator/metrics.go | 10 ++++- controllers/object_controls.go | 8 ++++ controllers/transforms_test.go | 29 +++++++++++++++ tests/e2e/helpers/clusterpolicy.go | 12 ++++++ tests/e2e/suites/clusterpolicy_test.go | 37 +++++++++++++++++++ 6 files changed, 99 insertions(+), 2 deletions(-) diff --git a/assets/state-node-status-exporter/0800_prometheus_rule_openshift.yaml b/assets/state-node-status-exporter/0800_prometheus_rule_openshift.yaml index ab7237fd4d..c899541071 100644 --- a/assets/state-node-status-exporter/0800_prometheus_rule_openshift.yaml +++ b/assets/state-node-status-exporter/0800_prometheus_rule_openshift.yaml @@ -10,7 +10,10 @@ spec: - name: Alert on node deployment failure rules: - alert: GPUOperatorNodeDeploymentFailed - # There is no GPU exposed on the node, + # There is no GPU exposed on the node. + # When the device plugin is intentionally disabled in the ClusterPolicy + # (devicePlugin.enabled: false), the metric is set to -1, so this + # alert will not fire in that case. expr: | gpu_operator_node_device_plugin_devices_total == 0 for: 30m diff --git a/cmd/nvidia-validator/metrics.go b/cmd/nvidia-validator/metrics.go index 4105dd1667..2084c99f4e 100644 --- a/cmd/nvidia-validator/metrics.go +++ b/cmd/nvidia-validator/metrics.go @@ -306,7 +306,15 @@ func (nm *NodeMetrics) Run() error { go nm.watchStatusFile(&nm.cudaReady, cudaStatusFile) go nm.watchDriverValidation() - go nm.watchDevicePluginValidation() + if os.Getenv("DEVICE_PLUGIN_ENABLED") != "false" { + go nm.watchDevicePluginValidation() + } else { + // Set to -1 so the alert (expr: == 0) does not fire. + // The gauge is auto-registered by promauto and defaults to 0, + // which would be a false positive. + nm.deviceCount.Set(-1) + log.Info("metrics: DevicePlugin is disabled in ClusterPolicy, skipping device plugin validation") + } go nm.watchNVIDIAPCI() log.Printf("Running the metrics server, listening on :%d/metrics", nm.port) diff --git a/controllers/object_controls.go b/controllers/object_controls.go index 0113a43b66..a4afe67c81 100644 --- a/controllers/object_controls.go +++ b/controllers/object_controls.go @@ -141,6 +141,8 @@ const ( NvidiaDisableRequireEnvName = "NVIDIA_DISABLE_REQUIRE" // GDSEnabledEnvName is the env name to enable GDS support with device-plugin GDSEnabledEnvName = "GDS_ENABLED" + // DevicePluginEnabledEnvName indicates whether the device plugin is enabled in the ClusterPolicy + DevicePluginEnabledEnvName = "DEVICE_PLUGIN_ENABLED" // MOFEDEnabledEnvName is the env name to enable MOFED devices injection with device-plugin MOFEDEnabledEnvName = "MOFED_ENABLED" // GDRCopyEnabledEnvName is the envvar that enables injection of the GDRCopy device node with the device-plugin @@ -2568,6 +2570,12 @@ func TransformNodeStatusExporter(obj *appsv1.DaemonSet, config *gpuv1.ClusterPol obj.Spec.Template.Spec.Containers[0].Args = config.NodeStatusExporter.Args } + devicePluginEnabled := "true" + if !config.DevicePlugin.IsEnabled() { + devicePluginEnabled = "false" + } + setContainerEnv(&(obj.Spec.Template.Spec.Containers[0]), DevicePluginEnabledEnvName, devicePluginEnabled) + // set/append environment variables for exporter container if len(config.NodeStatusExporter.Env) > 0 { for _, env := range config.NodeStatusExporter.Env { diff --git a/controllers/transforms_test.go b/controllers/transforms_test.go index aa309a3229..ff831e7667 100644 --- a/controllers/transforms_test.go +++ b/controllers/transforms_test.go @@ -3334,6 +3334,35 @@ func TestTransformNodeStatusExporter(t *testing.T) { Name: "dummy", Image: "nvcr.io/nvidia/cloud-native/node-status-exporter:v1.0.0", ImagePullPolicy: corev1.PullIfNotPresent, + Env: []corev1.EnvVar{ + {Name: DevicePluginEnabledEnvName, Value: "true"}, + }, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: rootUID, + }, + }), + }, + { + description: "node status exporter with device plugin disabled", + ds: NewDaemonset(). + WithContainer(corev1.Container{Name: "dummy"}), + cpSpec: &gpuv1.ClusterPolicySpec{ + NodeStatusExporter: gpuv1.NodeStatusExporterSpec{ + Repository: "nvcr.io/nvidia/cloud-native", + Image: "node-status-exporter", + Version: "v1.0.0", + ImagePullPolicy: "IfNotPresent", + }, + DevicePlugin: gpuv1.DevicePluginSpec{Enabled: newBoolPtr(false)}, + }, + expectedDs: NewDaemonset(). + WithContainer(corev1.Container{ + Name: "dummy", + Image: "nvcr.io/nvidia/cloud-native/node-status-exporter:v1.0.0", + ImagePullPolicy: corev1.PullIfNotPresent, + Env: []corev1.EnvVar{ + {Name: DevicePluginEnabledEnvName, Value: "false"}, + }, SecurityContext: &corev1.SecurityContext{ RunAsUser: rootUID, }, diff --git a/tests/e2e/helpers/clusterpolicy.go b/tests/e2e/helpers/clusterpolicy.go index c75c3473a1..fba7cf1b7d 100644 --- a/tests/e2e/helpers/clusterpolicy.go +++ b/tests/e2e/helpers/clusterpolicy.go @@ -106,6 +106,18 @@ func (h *ClusterPolicyClient) DisableGFD(ctx context.Context, name string) error }) } +func (h *ClusterPolicyClient) EnableDevicePlugin(ctx context.Context, name string) error { + return h.modify(ctx, name, func(clusterPolicy *nvidiav1.ClusterPolicy) { + clusterPolicy.Spec.DevicePlugin.Enabled = ptr.To(true) + }) +} + +func (h *ClusterPolicyClient) DisableDevicePlugin(ctx context.Context, name string) error { + return h.modify(ctx, name, func(clusterPolicy *nvidiav1.ClusterPolicy) { + clusterPolicy.Spec.DevicePlugin.Enabled = ptr.To(false) + }) +} + func (h *ClusterPolicyClient) SetMIGStrategy(ctx context.Context, name, strategy string) error { return h.modify(ctx, name, func(clusterPolicy *nvidiav1.ClusterPolicy) { clusterPolicy.Spec.MIG.Strategy = nvidiav1.MIGStrategy(strategy) diff --git a/tests/e2e/suites/clusterpolicy_test.go b/tests/e2e/suites/clusterpolicy_test.go index 92cdbde439..3b7db0331c 100644 --- a/tests/e2e/suites/clusterpolicy_test.go +++ b/tests/e2e/suites/clusterpolicy_test.go @@ -328,6 +328,43 @@ var _ = Describe("ClusterPolicy Management", Label("clusterPolicy"), func() { }) }) + // test_device_plugin_disabled_env - Verify DEVICE_PLUGIN_ENABLED env var propagation + When("Disabling device plugin", Label("device-plugin", "toggle"), func() { + It("should set DEVICE_PLUGIN_ENABLED=false on node-status-exporter when device plugin is disabled", func(ctx context.Context) { + clusterPolicy := getClusterPolicyOrSkip(ctx, clusterPolicyClient, policyName) + originalState := clusterPolicy.Spec.DevicePlugin.Enabled + DeferCleanup(func(ctx context.Context) { + if originalState == nil || *originalState { + _ = clusterPolicyClient.EnableDevicePlugin(ctx, policyName) + waitForDaemonSetReady(ctx, daemonSetClient, testNamespace, "nvidia-device-plugin-daemonset") + } + }) + + err := clusterPolicyClient.DisableDevicePlugin(ctx, policyName) + Expect(err).NotTo(HaveOccurred(), "Failed to disable device plugin in ClusterPolicy") + + verifyEnvInDaemonSet(ctx, daemonSetClient, testNamespace, + "nvidia-node-status-exporter", "DEVICE_PLUGIN_ENABLED", "false") + }) + + It("should set DEVICE_PLUGIN_ENABLED=true on node-status-exporter when device plugin is re-enabled", func(ctx context.Context) { + clusterPolicy := getClusterPolicyOrSkip(ctx, clusterPolicyClient, policyName) + originalState := clusterPolicy.Spec.DevicePlugin.Enabled + DeferCleanup(func(ctx context.Context) { + if originalState != nil && !*originalState { + _ = clusterPolicyClient.DisableDevicePlugin(ctx, policyName) + } + }) + + err := clusterPolicyClient.EnableDevicePlugin(ctx, policyName) + Expect(err).NotTo(HaveOccurred(), "Failed to enable device plugin in ClusterPolicy") + + verifyEnvInDaemonSet(ctx, daemonSetClient, testNamespace, + "nvidia-node-status-exporter", "DEVICE_PLUGIN_ENABLED", "true") + waitForDaemonSetReady(ctx, daemonSetClient, testNamespace, "nvidia-device-plugin-daemonset") + }) + }) + // test_custom_labels_override - Test custom labels on daemonsets When("Updating daemonset custom labels", Label("labels", "config"), func() { It("should apply custom labels to all operand pods", func(ctx context.Context) { From 23464863fc4491f2fb6f6ef9b7a5b47bd41842c8 Mon Sep 17 00:00:00 2001 From: dentinyhao Date: Mon, 24 Aug 2026 19:18:57 -0700 Subject: [PATCH 4/6] Stop defaulting deprecated cdi.default in ClusterPolicy Signed-off-by: dentinyhao (cherry picked from commit 62de6b5eb19980044cfee1d5d69588c8eca6e274) --- api/nvidia/v1/clusterpolicy_types.go | 1 - bundle/manifests/nvidia.com_clusterpolicies.yaml | 1 - config/crd/bases/nvidia.com_clusterpolicies.yaml | 1 - deployments/gpu-operator/crds/nvidia.com_clusterpolicies.yaml | 1 - 4 files changed, 4 deletions(-) diff --git a/api/nvidia/v1/clusterpolicy_types.go b/api/nvidia/v1/clusterpolicy_types.go index 234b5eba36..8c54b44b46 100644 --- a/api/nvidia/v1/clusterpolicy_types.go +++ b/api/nvidia/v1/clusterpolicy_types.go @@ -1966,7 +1966,6 @@ type CDIConfigSpec struct { // Deprecated: This field is no longer used. Setting cdi.enabled=true will configure CDI as the default mechanism for making GPUs accessible to containers. // +kubebuilder:validation:Optional - // +kubebuilder:default=false // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.displayName="Deprecated: This field is no longer used" // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.x-descriptors="urn:alm:descriptor:com.tectonic.ui:booleanSwitch,urn:alm:descriptor:com.tectonic.ui:hidden" diff --git a/bundle/manifests/nvidia.com_clusterpolicies.yaml b/bundle/manifests/nvidia.com_clusterpolicies.yaml index f589282b20..9b80c2cd14 100644 --- a/bundle/manifests/nvidia.com_clusterpolicies.yaml +++ b/bundle/manifests/nvidia.com_clusterpolicies.yaml @@ -139,7 +139,6 @@ spec: used in the cluster properties: default: - default: false description: 'Deprecated: This field is no longer used. Setting cdi.enabled=true will configure CDI as the default mechanism for making GPUs accessible to containers.' diff --git a/config/crd/bases/nvidia.com_clusterpolicies.yaml b/config/crd/bases/nvidia.com_clusterpolicies.yaml index f589282b20..9b80c2cd14 100644 --- a/config/crd/bases/nvidia.com_clusterpolicies.yaml +++ b/config/crd/bases/nvidia.com_clusterpolicies.yaml @@ -139,7 +139,6 @@ spec: used in the cluster properties: default: - default: false description: 'Deprecated: This field is no longer used. Setting cdi.enabled=true will configure CDI as the default mechanism for making GPUs accessible to containers.' diff --git a/deployments/gpu-operator/crds/nvidia.com_clusterpolicies.yaml b/deployments/gpu-operator/crds/nvidia.com_clusterpolicies.yaml index f589282b20..9b80c2cd14 100644 --- a/deployments/gpu-operator/crds/nvidia.com_clusterpolicies.yaml +++ b/deployments/gpu-operator/crds/nvidia.com_clusterpolicies.yaml @@ -139,7 +139,6 @@ spec: used in the cluster properties: default: - default: false description: 'Deprecated: This field is no longer used. Setting cdi.enabled=true will configure CDI as the default mechanism for making GPUs accessible to containers.' From 17159841dadfda826d1e5f223bafc017c15ad265 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 3 Sep 2026 14:55:37 -0500 Subject: [PATCH 5/6] revert the CUDA samples version bump to v13.2 (#2848) (cherry picked from commit bf1ce0407125c09b95a49371c60ab820967ba423) Signed-off-by: Tariq Ibrahim Co-authored-by: Tariq Ibrahim --- docker/Dockerfile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 6baf8ab035..f3c982a8ae 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -12,7 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG CUDA_SAMPLES_VERSION=13.2 +# R580 is the minimum driver branch supported by gpu-operator and +# its backcompat stretches all the way back to the Maxwell architecure. +# CUDA SAMPLES v12.9 is used as that is the latest version with Maxwell support. +ARG CUDA_SAMPLES_VERSION=12.9 FROM golang:1.26.6@sha256:0d1d3a794be25f809dd2cb3160d8c73276c4056a9f8242a138e908ddeee7b6b6 AS builder @@ -64,7 +67,7 @@ WORKDIR /build ARG SAMPLE_NAME=vectorAdd RUN curl -L https://codeload.github.com/NVIDIA/cuda-samples/tar.gz/refs/tags/v${CUDA_SAMPLES_VERSION} | \ - tar -xzvf - --strip-components=1 --wildcards */${SAMPLE_NAME}/* --wildcards */Common/* --wildcards */cmake/* && \ + tar -xzvf - --strip-components=1 --wildcards */${SAMPLE_NAME}/* --wildcards */Common/* && \ cd $(find /build/Samples -iname "${SAMPLE_NAME}") && \ cmake . && \ make && \ From 6e7ae70959203b2cc274de0913eb9a41bdf9821d Mon Sep 17 00:00:00 2001 From: Karthik Date: Fri, 4 Sep 2026 18:51:56 -0400 Subject: [PATCH 6/6] Sync ComputeDomain CRD with DRA driver v0.5.0 schema (cherry picked from commit 08c40bc479192e3d5a82b7fd41d7e85a7197741f) Signed-off-by: Tariq Ibrahim --- bundle/manifests/resource.nvidia.com_computedomains.yaml | 5 +++-- .../crds/resource.nvidia.com_computedomains.yaml | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/bundle/manifests/resource.nvidia.com_computedomains.yaml b/bundle/manifests/resource.nvidia.com_computedomains.yaml index 5a28ae17ca..0811b3ce2c 100644 --- a/bundle/manifests/resource.nvidia.com_computedomains.yaml +++ b/bundle/manifests/resource.nvidia.com_computedomains.yaml @@ -66,12 +66,13 @@ spec: - resourceClaimTemplate type: object numNodes: + default: 0 description: |- Intended number of IMEX daemons (i.e., individual compute nodes) in the ComputeDomain. Must be zero or greater. With `featureGates.IMEXDaemonsWithDNSNames=true` (the default), this is - recommended to be set to zero. Workload must implement and consult its + recommended to be set to zero (default). Workload must implement and consult its own source of truth for the number of workers online before trying to share GPU memory (and hence triggering IMEX interaction). When non-zero, `numNodes` is used only for automatically updating the global @@ -91,10 +92,10 @@ spec: The `numNodes` parameter is deprecated and will be removed in the next API version. + minimum: 0 type: integer required: - channel - - numNodes type: object x-kubernetes-validations: - message: A computeDomain.spec is immutable diff --git a/deployments/gpu-operator/crds/resource.nvidia.com_computedomains.yaml b/deployments/gpu-operator/crds/resource.nvidia.com_computedomains.yaml index 5a28ae17ca..0811b3ce2c 100644 --- a/deployments/gpu-operator/crds/resource.nvidia.com_computedomains.yaml +++ b/deployments/gpu-operator/crds/resource.nvidia.com_computedomains.yaml @@ -66,12 +66,13 @@ spec: - resourceClaimTemplate type: object numNodes: + default: 0 description: |- Intended number of IMEX daemons (i.e., individual compute nodes) in the ComputeDomain. Must be zero or greater. With `featureGates.IMEXDaemonsWithDNSNames=true` (the default), this is - recommended to be set to zero. Workload must implement and consult its + recommended to be set to zero (default). Workload must implement and consult its own source of truth for the number of workers online before trying to share GPU memory (and hence triggering IMEX interaction). When non-zero, `numNodes` is used only for automatically updating the global @@ -91,10 +92,10 @@ spec: The `numNodes` parameter is deprecated and will be removed in the next API version. + minimum: 0 type: integer required: - channel - - numNodes type: object x-kubernetes-validations: - message: A computeDomain.spec is immutable