diff --git a/api/nvidia/v1/clusterpolicy_types.go b/api/nvidia/v1/clusterpolicy_types.go index 234b5eba36..8c54b44b46 100644 --- a/api/nvidia/v1/clusterpolicy_types.go +++ b/api/nvidia/v1/clusterpolicy_types.go @@ -1966,7 +1966,6 @@ type CDIConfigSpec struct { // Deprecated: This field is no longer used. Setting cdi.enabled=true will configure CDI as the default mechanism for making GPUs accessible to containers. // +kubebuilder:validation:Optional - // +kubebuilder:default=false // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.displayName="Deprecated: This field is no longer used" // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.x-descriptors="urn:alm:descriptor:com.tectonic.ui:booleanSwitch,urn:alm:descriptor:com.tectonic.ui:hidden" diff --git a/assets/state-node-status-exporter/0800_prometheus_rule_openshift.yaml b/assets/state-node-status-exporter/0800_prometheus_rule_openshift.yaml index ab7237fd4d..c899541071 100644 --- a/assets/state-node-status-exporter/0800_prometheus_rule_openshift.yaml +++ b/assets/state-node-status-exporter/0800_prometheus_rule_openshift.yaml @@ -10,7 +10,10 @@ spec: - name: Alert on node deployment failure rules: - alert: GPUOperatorNodeDeploymentFailed - # There is no GPU exposed on the node, + # There is no GPU exposed on the node. + # When the device plugin is intentionally disabled in the ClusterPolicy + # (devicePlugin.enabled: false), the metric is set to -1, so this + # alert will not fire in that case. expr: | gpu_operator_node_device_plugin_devices_total == 0 for: 30m diff --git a/bundle/manifests/nvidia.com_clusterpolicies.yaml b/bundle/manifests/nvidia.com_clusterpolicies.yaml index f589282b20..9b80c2cd14 100644 --- a/bundle/manifests/nvidia.com_clusterpolicies.yaml +++ b/bundle/manifests/nvidia.com_clusterpolicies.yaml @@ -139,7 +139,6 @@ spec: used in the cluster properties: default: - default: false description: 'Deprecated: This field is no longer used. Setting cdi.enabled=true will configure CDI as the default mechanism for making GPUs accessible to containers.' diff --git a/bundle/manifests/resource.nvidia.com_computedomains.yaml b/bundle/manifests/resource.nvidia.com_computedomains.yaml index 5a28ae17ca..0811b3ce2c 100644 --- a/bundle/manifests/resource.nvidia.com_computedomains.yaml +++ b/bundle/manifests/resource.nvidia.com_computedomains.yaml @@ -66,12 +66,13 @@ spec: - resourceClaimTemplate type: object numNodes: + default: 0 description: |- Intended number of IMEX daemons (i.e., individual compute nodes) in the ComputeDomain. Must be zero or greater. With `featureGates.IMEXDaemonsWithDNSNames=true` (the default), this is - recommended to be set to zero. Workload must implement and consult its + recommended to be set to zero (default). Workload must implement and consult its own source of truth for the number of workers online before trying to share GPU memory (and hence triggering IMEX interaction). When non-zero, `numNodes` is used only for automatically updating the global @@ -91,10 +92,10 @@ spec: The `numNodes` parameter is deprecated and will be removed in the next API version. + minimum: 0 type: integer required: - channel - - numNodes type: object x-kubernetes-validations: - message: A computeDomain.spec is immutable diff --git a/cmd/nvidia-validator/metrics.go b/cmd/nvidia-validator/metrics.go index 4105dd1667..2084c99f4e 100644 --- a/cmd/nvidia-validator/metrics.go +++ b/cmd/nvidia-validator/metrics.go @@ -306,7 +306,15 @@ func (nm *NodeMetrics) Run() error { go nm.watchStatusFile(&nm.cudaReady, cudaStatusFile) go nm.watchDriverValidation() - go nm.watchDevicePluginValidation() + if os.Getenv("DEVICE_PLUGIN_ENABLED") != "false" { + go nm.watchDevicePluginValidation() + } else { + // Set to -1 so the alert (expr: == 0) does not fire. + // The gauge is auto-registered by promauto and defaults to 0, + // which would be a false positive. + nm.deviceCount.Set(-1) + log.Info("metrics: DevicePlugin is disabled in ClusterPolicy, skipping device plugin validation") + } go nm.watchNVIDIAPCI() log.Printf("Running the metrics server, listening on :%d/metrics", nm.port) diff --git a/config/crd/bases/nvidia.com_clusterpolicies.yaml b/config/crd/bases/nvidia.com_clusterpolicies.yaml index f589282b20..9b80c2cd14 100644 --- a/config/crd/bases/nvidia.com_clusterpolicies.yaml +++ b/config/crd/bases/nvidia.com_clusterpolicies.yaml @@ -139,7 +139,6 @@ spec: used in the cluster properties: default: - default: false description: 'Deprecated: This field is no longer used. Setting cdi.enabled=true will configure CDI as the default mechanism for making GPUs accessible to containers.' diff --git a/controllers/object_controls.go b/controllers/object_controls.go index 0113a43b66..a4afe67c81 100644 --- a/controllers/object_controls.go +++ b/controllers/object_controls.go @@ -141,6 +141,8 @@ const ( NvidiaDisableRequireEnvName = "NVIDIA_DISABLE_REQUIRE" // GDSEnabledEnvName is the env name to enable GDS support with device-plugin GDSEnabledEnvName = "GDS_ENABLED" + // DevicePluginEnabledEnvName indicates whether the device plugin is enabled in the ClusterPolicy + DevicePluginEnabledEnvName = "DEVICE_PLUGIN_ENABLED" // MOFEDEnabledEnvName is the env name to enable MOFED devices injection with device-plugin MOFEDEnabledEnvName = "MOFED_ENABLED" // GDRCopyEnabledEnvName is the envvar that enables injection of the GDRCopy device node with the device-plugin @@ -2568,6 +2570,12 @@ func TransformNodeStatusExporter(obj *appsv1.DaemonSet, config *gpuv1.ClusterPol obj.Spec.Template.Spec.Containers[0].Args = config.NodeStatusExporter.Args } + devicePluginEnabled := "true" + if !config.DevicePlugin.IsEnabled() { + devicePluginEnabled = "false" + } + setContainerEnv(&(obj.Spec.Template.Spec.Containers[0]), DevicePluginEnabledEnvName, devicePluginEnabled) + // set/append environment variables for exporter container if len(config.NodeStatusExporter.Env) > 0 { for _, env := range config.NodeStatusExporter.Env { diff --git a/controllers/state_manager.go b/controllers/state_manager.go index d49c311967..eeac1bdce8 100644 --- a/controllers/state_manager.go +++ b/controllers/state_manager.go @@ -965,13 +965,13 @@ func (n *ClusterPolicyController) step() (gpuv1.State, error) { if (n.stateNames[n.idx] == "state-driver" || n.stateNames[n.idx] == "state-vgpu-manager") && n.singleton.Spec.Driver.UseNvidiaDriverCRDType() { n.logger.Info("NVIDIADriver CRD is enabled, cleaning up all NVIDIA driver daemonsets owned by ClusterPolicy") - n.idx++ // Cleanup all driver daemonsets owned by ClusterPolicy while keeping the // running driver pods available until NVIDIADriver rolls replacements. err := n.cleanupAllDriverDaemonSets(n.ctx, metav1.DeletePropagationOrphan) if err != nil { return gpuv1.NotReady, fmt.Errorf("failed to cleanup all NVIDIA driver daemonsets owned by ClusterPolicy: %w", err) } + n.idx++ return gpuv1.Disabled, nil } diff --git a/controllers/state_manager_test.go b/controllers/state_manager_test.go index 73f4b7579e..f21241301a 100644 --- a/controllers/state_manager_test.go +++ b/controllers/state_manager_test.go @@ -165,6 +165,80 @@ func TestGetGPUNodeOSInfoListError(t *testing.T) { require.Contains(t, err.Error(), "unable to list nodes with GPU present") } +func TestStep(t *testing.T) { + expectedErr := errors.New("step failed") + driverCRDPolicy := &gpuv1.ClusterPolicy{ + Spec: gpuv1.ClusterPolicySpec{ + Driver: gpuv1.DriverSpec{UseNvidiaDriverCRD: ptr.To(true)}, + }, + } + + testCases := []struct { + name string + controller ClusterPolicyController + expectedStatus gpuv1.State + expectedIndex int + expectedError error + }{ + { + name: "normal state advances on success", + controller: ClusterPolicyController{ + controls: []controlFunc{{func(ClusterPolicyController) (gpuv1.State, error) { return gpuv1.Ready, nil }}}, + stateNames: []string{"test-state"}, + }, + expectedStatus: gpuv1.Ready, + expectedIndex: 1, + }, + { + name: "normal state does not advance on error", + controller: ClusterPolicyController{ + controls: []controlFunc{{func(ClusterPolicyController) (gpuv1.State, error) { return gpuv1.NotReady, expectedErr }}}, + stateNames: []string{"test-state"}, + }, + expectedStatus: gpuv1.NotReady, + expectedIndex: 0, + expectedError: expectedErr, + }, + { + name: "driver cleanup advances on success", + controller: ClusterPolicyController{ + ctx: context.Background(), + client: errorListClient{}, + singleton: driverCRDPolicy, + stateNames: []string{"state-driver"}, + }, + expectedStatus: gpuv1.Disabled, + expectedIndex: 1, + }, + { + name: "driver cleanup does not advance on error", + controller: ClusterPolicyController{ + ctx: context.Background(), + client: errorListClient{err: expectedErr}, + singleton: driverCRDPolicy, + stateNames: []string{"state-driver"}, + }, + expectedStatus: gpuv1.NotReady, + expectedIndex: 0, + expectedError: expectedErr, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + status, err := tc.controller.step() + + if tc.expectedError != nil { + require.ErrorIs(t, err, tc.expectedError) + } else { + require.NoError(t, err) + } + require.Equal(t, tc.expectedStatus, status) + require.Equal(t, tc.expectedIndex, tc.controller.idx) + }) + } +} + func TestGetGPUNodeOSInfoNoGPUNodes(t *testing.T) { scheme := runtime.NewScheme() require.NoError(t, corev1.AddToScheme(scheme)) diff --git a/controllers/transforms_test.go b/controllers/transforms_test.go index aa309a3229..ff831e7667 100644 --- a/controllers/transforms_test.go +++ b/controllers/transforms_test.go @@ -3334,6 +3334,35 @@ func TestTransformNodeStatusExporter(t *testing.T) { Name: "dummy", Image: "nvcr.io/nvidia/cloud-native/node-status-exporter:v1.0.0", ImagePullPolicy: corev1.PullIfNotPresent, + Env: []corev1.EnvVar{ + {Name: DevicePluginEnabledEnvName, Value: "true"}, + }, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: rootUID, + }, + }), + }, + { + description: "node status exporter with device plugin disabled", + ds: NewDaemonset(). + WithContainer(corev1.Container{Name: "dummy"}), + cpSpec: &gpuv1.ClusterPolicySpec{ + NodeStatusExporter: gpuv1.NodeStatusExporterSpec{ + Repository: "nvcr.io/nvidia/cloud-native", + Image: "node-status-exporter", + Version: "v1.0.0", + ImagePullPolicy: "IfNotPresent", + }, + DevicePlugin: gpuv1.DevicePluginSpec{Enabled: newBoolPtr(false)}, + }, + expectedDs: NewDaemonset(). + WithContainer(corev1.Container{ + Name: "dummy", + Image: "nvcr.io/nvidia/cloud-native/node-status-exporter:v1.0.0", + ImagePullPolicy: corev1.PullIfNotPresent, + Env: []corev1.EnvVar{ + {Name: DevicePluginEnabledEnvName, Value: "false"}, + }, SecurityContext: &corev1.SecurityContext{ RunAsUser: rootUID, }, diff --git a/deployments/gpu-operator/crds/nvidia.com_clusterpolicies.yaml b/deployments/gpu-operator/crds/nvidia.com_clusterpolicies.yaml index f589282b20..9b80c2cd14 100644 --- a/deployments/gpu-operator/crds/nvidia.com_clusterpolicies.yaml +++ b/deployments/gpu-operator/crds/nvidia.com_clusterpolicies.yaml @@ -139,7 +139,6 @@ spec: used in the cluster properties: default: - default: false description: 'Deprecated: This field is no longer used. Setting cdi.enabled=true will configure CDI as the default mechanism for making GPUs accessible to containers.' diff --git a/deployments/gpu-operator/crds/resource.nvidia.com_computedomains.yaml b/deployments/gpu-operator/crds/resource.nvidia.com_computedomains.yaml index 5a28ae17ca..0811b3ce2c 100644 --- a/deployments/gpu-operator/crds/resource.nvidia.com_computedomains.yaml +++ b/deployments/gpu-operator/crds/resource.nvidia.com_computedomains.yaml @@ -66,12 +66,13 @@ spec: - resourceClaimTemplate type: object numNodes: + default: 0 description: |- Intended number of IMEX daemons (i.e., individual compute nodes) in the ComputeDomain. Must be zero or greater. With `featureGates.IMEXDaemonsWithDNSNames=true` (the default), this is - recommended to be set to zero. Workload must implement and consult its + recommended to be set to zero (default). Workload must implement and consult its own source of truth for the number of workers online before trying to share GPU memory (and hence triggering IMEX interaction). When non-zero, `numNodes` is used only for automatically updating the global @@ -91,10 +92,10 @@ spec: The `numNodes` parameter is deprecated and will be removed in the next API version. + minimum: 0 type: integer required: - channel - - numNodes type: object x-kubernetes-validations: - message: A computeDomain.spec is immutable diff --git a/docker/Dockerfile b/docker/Dockerfile index 6baf8ab035..f3c982a8ae 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -12,7 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG CUDA_SAMPLES_VERSION=13.2 +# R580 is the minimum driver branch supported by gpu-operator and +# its backcompat stretches all the way back to the Maxwell architecure. +# CUDA SAMPLES v12.9 is used as that is the latest version with Maxwell support. +ARG CUDA_SAMPLES_VERSION=12.9 FROM golang:1.26.6@sha256:0d1d3a794be25f809dd2cb3160d8c73276c4056a9f8242a138e908ddeee7b6b6 AS builder @@ -64,7 +67,7 @@ WORKDIR /build ARG SAMPLE_NAME=vectorAdd RUN curl -L https://codeload.github.com/NVIDIA/cuda-samples/tar.gz/refs/tags/v${CUDA_SAMPLES_VERSION} | \ - tar -xzvf - --strip-components=1 --wildcards */${SAMPLE_NAME}/* --wildcards */Common/* --wildcards */cmake/* && \ + tar -xzvf - --strip-components=1 --wildcards */${SAMPLE_NAME}/* --wildcards */Common/* && \ cd $(find /build/Samples -iname "${SAMPLE_NAME}") && \ cmake . && \ make && \ diff --git a/internal/state/driver_test.go b/internal/state/driver_test.go index a3d89e25a5..92907fb62c 100644 --- a/internal/state/driver_test.go +++ b/internal/state/driver_test.go @@ -91,6 +91,33 @@ func getYAMLString(objs []*unstructured.Unstructured) (string, error) { return sb.String(), nil } +func findVolumeByName(volumes []corev1.Volume, name string) *corev1.Volume { + for i := range volumes { + if volumes[i].Name == name { + return &volumes[i] + } + } + return nil +} + +func findVolumeMountByName(volumeMounts []corev1.VolumeMount, name string) *corev1.VolumeMount { + for i := range volumeMounts { + if volumeMounts[i].Name == name { + return &volumeMounts[i] + } + } + return nil +} + +func findContainerByName(containers []corev1.Container, name string) *corev1.Container { + for i := range containers { + if containers[i].Name == name { + return &containers[i] + } + } + return nil +} + func hasSubscriptionVolumeMount(volumeMounts []corev1.VolumeMount) bool { for _, volumeMount := range volumeMounts { if strings.HasPrefix(volumeMount.Name, "subscription-config-") { @@ -657,6 +684,119 @@ func TestDriverAdditionalConfigsSubscriptionMounts(t *testing.T) { } } +func TestDriverPrecompiledLibModules(t *testing.T) { + const ( + libModulesVolumeName = "lib-modules" + driverContainerName = "nvidia-driver-ctr" + precompiledKernelVersion = "5.14.21-150500.55.44-default" + ) + + state, err := NewStateDriver( + fake.NewClientBuilder().WithScheme(scheme.Scheme).Build(), + "test-ns", + scheme.Scheme, + manifestDir) + require.NoError(t, err) + stateDriver, ok := state.(*stateDriver) + require.True(t, ok) + + clusterInfo := testClusterInfo{runtime: consts.Containerd} + + testCases := []struct { + description string + osRelease string + osVersion string + usePrecompiled bool + expectLibModulesMounted bool + }{ + { + description: "sles with precompiled drivers mounts host /lib/modules", + osRelease: "sles", + osVersion: "15.6", + usePrecompiled: true, + expectLibModulesMounted: true, + }, + { + description: "sl-micro with precompiled drivers mounts host /lib/modules", + osRelease: "sl-micro", + osVersion: "6.0", + usePrecompiled: true, + expectLibModulesMounted: true, + }, + { + description: "sles without precompiled drivers does not mount host /lib/modules", + osRelease: "sles", + osVersion: "15.6", + usePrecompiled: false, + expectLibModulesMounted: false, + }, + { + description: "ubuntu with precompiled drivers does not mount host /lib/modules", + osRelease: "ubuntu", + osVersion: "22.04", + usePrecompiled: true, + expectLibModulesMounted: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + driver := &nvidiav1alpha1.NVIDIADriver{} + driver.Spec.UsePrecompiled = ptr.To(tc.usePrecompiled) + + additionalConfigs, err := stateDriver.getDriverAdditionalConfigs( + context.Background(), + driver, + clusterInfo, + nodePool{osRelease: tc.osRelease, osVersion: tc.osVersion}, + ) + require.NoError(t, err) + + renderData := getMinimalDriverRenderData() + renderData.Driver.Spec.UsePrecompiled = ptr.To(tc.usePrecompiled) + renderData.AdditionalConfigs = additionalConfigs + if tc.usePrecompiled { + renderData.Precompiled = &precompiledSpec{ + KernelVersion: precompiledKernelVersion, + SanitizedKernelVersion: precompiledKernelVersion, + } + } + + objs, err := stateDriver.renderer.RenderObjects( + &render.TemplatingData{ + Data: renderData, + }) + require.NoError(t, err) + + ds, err := getDaemonsetFromObjects(objs) + require.NoError(t, err) + + libModulesVolume := findVolumeByName(ds.Spec.Template.Spec.Volumes, libModulesVolumeName) + + driverContainer := findContainerByName(ds.Spec.Template.Spec.Containers, driverContainerName) + require.NotNil(t, driverContainer) + + libModulesMount := findVolumeMountByName(driverContainer.VolumeMounts, libModulesVolumeName) + + if !tc.expectLibModulesMounted { + assert.Nil(t, libModulesVolume, "unexpected lib-modules volume on the driver pod spec") + assert.Nil(t, libModulesMount, "unexpected lib-modules volume mount on nvidia-driver-ctr") + return + } + + require.NotNil(t, libModulesVolume, "expected a lib-modules volume on the driver pod spec") + require.NotNil(t, libModulesVolume.HostPath) + assert.Equal(t, "/lib/modules", libModulesVolume.HostPath.Path) + require.NotNil(t, libModulesVolume.HostPath.Type) + assert.Equal(t, corev1.HostPathDirectory, *libModulesVolume.HostPath.Type) + + require.NotNil(t, libModulesMount, "expected a lib-modules volume mount on nvidia-driver-ctr") + assert.Equal(t, "/run/host/lib/modules", libModulesMount.MountPath) + assert.True(t, libModulesMount.ReadOnly) + }) + } +} + func TestDriverConfigPathHelpers(t *testing.T) { repoConfigPath, err := getRepoConfigPath("rhel") require.NoError(t, err) diff --git a/internal/state/driver_volumes.go b/internal/state/driver_volumes.go index f0aeb2f5c3..34f60ea609 100644 --- a/internal/state/driver_volumes.go +++ b/internal/state/driver_volumes.go @@ -220,29 +220,27 @@ func (s *stateDriver) getDriverAdditionalConfigs(ctx context.Context, cr *v1alph additionalCfgs.Volumes = append(additionalCfgs.Volumes, subscriptionVol) } } + } - // Mount /lib/modules for precompiled drivers on SUSE distributions. - // Those containers need access to host /lib/modules at runtime. - if cr.Spec.UsePrecompiledDrivers() && (pool.osRelease == "sles" || pool.osRelease == "sl-micro") { - logger.Info("Mounting /lib/modules into the driver container") - libModulesVolMount := corev1.VolumeMount{ - Name: "lib-modules", - MountPath: "/run/host/lib/modules", - ReadOnly: true, - } - additionalCfgs.VolumeMounts = append(additionalCfgs.VolumeMounts, libModulesVolMount) + if cr.Spec.UsePrecompiledDrivers() && (pool.osRelease == "sles" || pool.osRelease == "sl-micro") { + logger.Info("Mounting /lib/modules into the driver pod") + libModulesVolMount := corev1.VolumeMount{ + Name: "lib-modules", + MountPath: "/run/host/lib/modules", + ReadOnly: true, + } + additionalCfgs.VolumeMounts = append(additionalCfgs.VolumeMounts, libModulesVolMount) - libModulesVol := corev1.Volume{ - Name: "lib-modules", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/lib/modules", - Type: ptr.To(corev1.HostPathDirectory), - }, + libModulesVol := corev1.Volume{ + Name: "lib-modules", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/lib/modules", + Type: ptr.To(corev1.HostPathDirectory), }, - } - additionalCfgs.Volumes = append(additionalCfgs.Volumes, libModulesVol) + }, } + additionalCfgs.Volumes = append(additionalCfgs.Volumes, libModulesVol) } // mount any custom kernel module configuration parameters at /drivers diff --git a/tests/e2e/helpers/clusterpolicy.go b/tests/e2e/helpers/clusterpolicy.go index c75c3473a1..fba7cf1b7d 100644 --- a/tests/e2e/helpers/clusterpolicy.go +++ b/tests/e2e/helpers/clusterpolicy.go @@ -106,6 +106,18 @@ func (h *ClusterPolicyClient) DisableGFD(ctx context.Context, name string) error }) } +func (h *ClusterPolicyClient) EnableDevicePlugin(ctx context.Context, name string) error { + return h.modify(ctx, name, func(clusterPolicy *nvidiav1.ClusterPolicy) { + clusterPolicy.Spec.DevicePlugin.Enabled = ptr.To(true) + }) +} + +func (h *ClusterPolicyClient) DisableDevicePlugin(ctx context.Context, name string) error { + return h.modify(ctx, name, func(clusterPolicy *nvidiav1.ClusterPolicy) { + clusterPolicy.Spec.DevicePlugin.Enabled = ptr.To(false) + }) +} + func (h *ClusterPolicyClient) SetMIGStrategy(ctx context.Context, name, strategy string) error { return h.modify(ctx, name, func(clusterPolicy *nvidiav1.ClusterPolicy) { clusterPolicy.Spec.MIG.Strategy = nvidiav1.MIGStrategy(strategy) diff --git a/tests/e2e/suites/clusterpolicy_test.go b/tests/e2e/suites/clusterpolicy_test.go index 92cdbde439..3b7db0331c 100644 --- a/tests/e2e/suites/clusterpolicy_test.go +++ b/tests/e2e/suites/clusterpolicy_test.go @@ -328,6 +328,43 @@ var _ = Describe("ClusterPolicy Management", Label("clusterPolicy"), func() { }) }) + // test_device_plugin_disabled_env - Verify DEVICE_PLUGIN_ENABLED env var propagation + When("Disabling device plugin", Label("device-plugin", "toggle"), func() { + It("should set DEVICE_PLUGIN_ENABLED=false on node-status-exporter when device plugin is disabled", func(ctx context.Context) { + clusterPolicy := getClusterPolicyOrSkip(ctx, clusterPolicyClient, policyName) + originalState := clusterPolicy.Spec.DevicePlugin.Enabled + DeferCleanup(func(ctx context.Context) { + if originalState == nil || *originalState { + _ = clusterPolicyClient.EnableDevicePlugin(ctx, policyName) + waitForDaemonSetReady(ctx, daemonSetClient, testNamespace, "nvidia-device-plugin-daemonset") + } + }) + + err := clusterPolicyClient.DisableDevicePlugin(ctx, policyName) + Expect(err).NotTo(HaveOccurred(), "Failed to disable device plugin in ClusterPolicy") + + verifyEnvInDaemonSet(ctx, daemonSetClient, testNamespace, + "nvidia-node-status-exporter", "DEVICE_PLUGIN_ENABLED", "false") + }) + + It("should set DEVICE_PLUGIN_ENABLED=true on node-status-exporter when device plugin is re-enabled", func(ctx context.Context) { + clusterPolicy := getClusterPolicyOrSkip(ctx, clusterPolicyClient, policyName) + originalState := clusterPolicy.Spec.DevicePlugin.Enabled + DeferCleanup(func(ctx context.Context) { + if originalState != nil && !*originalState { + _ = clusterPolicyClient.DisableDevicePlugin(ctx, policyName) + } + }) + + err := clusterPolicyClient.EnableDevicePlugin(ctx, policyName) + Expect(err).NotTo(HaveOccurred(), "Failed to enable device plugin in ClusterPolicy") + + verifyEnvInDaemonSet(ctx, daemonSetClient, testNamespace, + "nvidia-node-status-exporter", "DEVICE_PLUGIN_ENABLED", "true") + waitForDaemonSetReady(ctx, daemonSetClient, testNamespace, "nvidia-device-plugin-daemonset") + }) + }) + // test_custom_labels_override - Test custom labels on daemonsets When("Updating daemonset custom labels", Label("labels", "config"), func() { It("should apply custom labels to all operand pods", func(ctx context.Context) {