Install a DCGM exporter on Nebius clusters

e2e/clouds/__init__.py

@@ -63,10 +63,11 @@ class Cloud:
 
     What the cluster composition (or the cloud's platform) puts on a real GPU
     node, plus the NFD labels NFD's worker would set - the GPU presence labels
-    (gpu.present, pci-10de.present) and the OS release labels
-    (system-os_release.ID, system-os_release.VERSION_ID) the GPU operator's
-    ClusterPolicy controller reads from GPU nodes since v26.7 - pre-set because
-    the worker's pod on a KWOK node is fake and never labels anything.
+    (gpu.present, and pci-10de.present or Nebius's pci-0302_10de.present) and
+    the OS release labels (system-os_release.ID, system-os_release.VERSION_ID)
+    the GPU operator's ClusterPolicy controller reads from GPU nodes since
+    v26.7 - pre-set because the worker's pod on a KWOK node is fake and never
+    labels anything.
     """
 
     taint_value: str
@@ -270,7 +271,8 @@ class Cloud:
             "modelplane.ai/pool": "gpu-fake",
             "modelplane.ai/gpu": "h100",
             "nvidia.com/gpu.present": "true",
-            "feature.node.kubernetes.io/pci-10de.present": "true",
+            # NFD labels real Nebius GPU nodes pci-0302_10de, not pci-10de.
+            "feature.node.kubernetes.io/pci-0302_10de.present": "true",
             # Nebius GPU node groups run Ubuntu 22.04.
             "feature.node.kubernetes.io/system-os_release.ID": "ubuntu",
             "feature.node.kubernetes.io/system-os_release.VERSION_ID": "22.04",
@@ -287,6 +289,7 @@ class Cloud:
             "dra-driver-cpu/mp-dra-driver-cpu": 2,
             "node-feature-discovery/mp-node-feature-discovery-worker": 3,
             "monitoring/mp-kube-prometheus-stack-prometheus-node-exporter": 5,
+            "dcgm-exporter/mp-dcgm-exporter": 2,
         },
     ),
     # The hand-written Vultr half (stacks/clouds/vultr.py) has no NFD: Vultr

e2e/clouds/test_placement.py

@@ -45,6 +45,7 @@
     "kubelet-plugin",
     "^gpu-feature-discovery",
     "^nvidia-dcgm",
+    "dcgm-exporter",
     "^nvidia-mig-manager",
     "^nvidia-operator-validator",
     "^nvidia-cuda-validator",
@@ -85,6 +86,7 @@
             "kubelet-plugin",
             "^gpu-feature-discovery",
             "^nvidia-dcgm",
+            "dcgm-exporter",
             "^nvidia-mig-manager",
             "^nvidia-operator-validator",
             "^nvidia-cuda-validator",

functions/compose-serving-stack/function/collector.py

@@ -335,7 +335,8 @@ def _scrape_configs() -> list[dict[str, Any]]:
                 # A job of its own because nobody annotates a GPU exporter
                 # for scraping: GKE runs a managed DCGM in
                 # gke-managed-system, the NVIDIA GPU operator installs its
-                # own elsewhere, the AMD GPU operator Modelplane installs on
+                # own elsewhere, Nebius gets the standalone dcgm-exporter
+                # chart, the AMD GPU operator Modelplane installs on
                 # Existing runs AMD's device-metrics-exporter, and the
                 # substrate job sees none of them. Without this every
                 # modelplane_gpu_* series is empty.

functions/compose-serving-stack/function/stacks/clouds/nebius.py

@@ -18,9 +18,12 @@
 hand, in the same shape a generator emits. Where a component also
 appears on a generated cloud, this file states the same pin, so one
 review moves both halves when a version changes - regenerate, then
-mirror the shared pins here.
+mirror the shared pins here. The DCGM exporter's metrics CSV goes one
+step further: rather than restating it, this file imports it from the
+shared generated/aicr/dcgm.py, as civo.py does.
 """
 
+from function.stacks.clouds.generated.aicr.dcgm import DCGM_METRICS_CSV
 from function.stacks.components import Chart, Component
 
 # DRA drivers this cloud supports that arrive outside the serving
@@ -163,4 +166,52 @@
             "resources": {"computeDomains": {"enabled": False}},
         },
     ),
+    # Exports DCGM fields for the collector's modelplane-gpu job, which
+    # keeps pods named .*dcgm.* with a port named metrics. Without this every
+    # modelplane_gpu_* series is empty. DCGM runs embedded (the chart
+    # default). The counters are the generated clouds' list, not the
+    # chart's shorter default, so Nebius exports the same DCGM fields as
+    # EKS, GKE, AKS and Civo. The node terms are the DRA kubelet plugin's PCI
+    # terms: Nebius GPU nodes carry pci-0302_10de but not pci-10de. The
+    # collector finds the pod itself, so no ServiceMonitor.
+    Chart(
+        key="dcgm-exporter",
+        release="mp-dcgm-exporter",
+        namespace="dcgm-exporter",
+        chart="dcgm-exporter",
+        repository="https://nvidia.github.io/dcgm-exporter/helm-charts",
+        version="4.8.4",
+        values={
+            "serviceMonitor": {"enabled": False},
+            "customMetrics": DCGM_METRICS_CSV,
+            # Attribute each GPU's series to the pod holding it. Nebius GPUs
+            # are allocated through DRA, which the exporter only maps back to
+            # pods with this set; the chart adds the RBAC it needs.
+            "kubernetesDRA": {"enabled": True},
+            "tolerations": [
+                {
+                    "key": "nvidia.com/gpu",
+                    "operator": "Exists",
+                    "effect": "NoSchedule",
+                },
+            ],
+            # The first three nodeSelectorTerms of dra-driver-nvidia-gpu
+            # v0.5.0's kubeletPlugin affinity. Recheck them when that chart
+            # moves.
+            "affinity": {
+                "nodeAffinity": {
+                    "requiredDuringSchedulingIgnoredDuringExecution": {
+                        "nodeSelectorTerms": [
+                            {"matchExpressions": [{"key": key, "operator": "In", "values": ["true"]}]}
+                            for key in (
+                                "feature.node.kubernetes.io/pci-10de.present",
+                                "feature.node.kubernetes.io/pci-0302_10de.present",
+                                "feature.node.kubernetes.io/pci-0300_10de.present",
+                            )
+                        ],
+                    },
+                },
+            },
+        },
+    ),
 ]

functions/compose-serving-stack/tests/test_fn.py

@@ -7389,6 +7389,7 @@ def test_compose(case: ComposeCase) -> None:
                     "kube-prometheus-stack": _observed_ready(),
                     "node-feature-discovery": _observed_ready(),
                     "nvidia-dra-driver-gpu": _observed_ready(),
+                    "dcgm-exporter": _observed_ready(),
                     "leader-worker-set": _observed_ready(),
                 },
             ),
@@ -7433,6 +7434,7 @@ def test_compose(case: ComposeCase) -> None:
             "kube-prometheus-stack",
             "node-feature-discovery",
             "nvidia-dra-driver-gpu",
+            "dcgm-exporter",
             # The Standard stack.
             "leader-worker-set",
         },
@@ -7501,6 +7503,7 @@ def test_compose(case: ComposeCase) -> None:
                     "kube-prometheus-stack": _observed_ready(),
                     "node-feature-discovery": _observed_ready(),
                     "nvidia-dra-driver-gpu": _observed_ready(),
+                    "dcgm-exporter": _observed_ready(),
                     "grove": _observed_ready(),
                     "kai-queue": _observed_ready(),
                     "kai-queue-root": _observed_ready(),
@@ -7559,6 +7562,7 @@ def test_compose(case: ComposeCase) -> None:
             "kube-prometheus-stack",
             "node-feature-discovery",
             "nvidia-dra-driver-gpu",
+            "dcgm-exporter",
             # The Dynamo stack.
             "grove",
             "kai-queue",

functions/compose-serving-stack/tests/test_stacks.py

@@ -462,6 +462,53 @@ def test_generated_dcgm_csv(cloud: str) -> None:
 """
 
 
+def test_nebius_dcgm_exporter() -> None:
+    """Nebius's dcgm-exporter exports the generated clouds' DCGM fields.
+
+    Nebius installs NVIDIA's standalone dcgm-exporter chart rather than the
+    GPU Operator, so the CSV goes in the chart's customMetrics rather than
+    dcgmExporter.config.data. Left unset, the chart falls back to its own
+    shorter default list and Nebius silently exports fewer fields than every
+    other cloud. Compared against the same literal as Civo's cases, so a
+    generator-moved CSV fails here until a human confirms the change.
+    """
+    joined = stacks.join("Nebius", "Standard")
+    exporter = next(c for c in joined if isinstance(c, stacks.Chart) and c.key == "dcgm-exporter")
+    assert exporter.values is not None
+    assert exporter.values["customMetrics"] == _DCGM_METRICS_CSV
+
+
+def test_nebius_exporter_affinity() -> None:
+    """Nebius's dcgm-exporter schedules by the DRA kubelet plugin's PCI terms.
+
+    Nebius GPU nodes carry pci-0302_10de but not pci-10de, so dropping the
+    0302 term would leave the exporter on no node with every unit test still
+    passing. Spelled out as a literal so a changed term fails here rather
+    than only on a real cluster.
+    """
+    joined = stacks.join("Nebius", "Standard")
+    exporter = next(c for c in joined if isinstance(c, stacks.Chart) and c.key == "dcgm-exporter")
+    assert exporter.values is not None
+    required = exporter.values["affinity"]["nodeAffinity"]["requiredDuringSchedulingIgnoredDuringExecution"]
+    assert [term["matchExpressions"] for term in required["nodeSelectorTerms"]] == [
+        [{"key": f"feature.node.kubernetes.io/{label}.present", "operator": "In", "values": ["true"]}]
+        for label in ("pci-10de", "pci-0302_10de", "pci-0300_10de")
+    ]
+
+
+def test_nebius_exporter_dra() -> None:
+    """Nebius's dcgm-exporter attributes GPU series to DRA-allocated pods.
+
+    Nebius allocates GPUs through DRA ResourceClaims, not the device plugin.
+    Without kubernetesDRA the exporter still exports every GPU's fields, but
+    can't say which pod holds the GPU, and nothing else fails.
+    """
+    joined = stacks.join("Nebius", "Standard")
+    exporter = next(c for c in joined if isinstance(c, stacks.Chart) and c.key == "dcgm-exporter")
+    assert exporter.values is not None
+    assert exporter.values["kubernetesDRA"] == {"enabled": True}
+
+
 # Each case's input is the real joined Civo stack rather than a literal one.
 # The transform finds the gpu-operator and DRA driver charts by key, and where
 # a key matches nothing it rewrites nothing, without complaint. Only the real