diff --git a/e2e/clouds/__init__.py b/e2e/clouds/__init__.py index 0d3a831b7..51ec14334 100644 --- a/e2e/clouds/__init__.py +++ b/e2e/clouds/__init__.py @@ -63,10 +63,11 @@ class Cloud: What the cluster composition (or the cloud's platform) puts on a real GPU node, plus the NFD labels NFD's worker would set - the GPU presence labels - (gpu.present, pci-10de.present) and the OS release labels - (system-os_release.ID, system-os_release.VERSION_ID) the GPU operator's - ClusterPolicy controller reads from GPU nodes since v26.7 - pre-set because - the worker's pod on a KWOK node is fake and never labels anything. + (gpu.present, and pci-10de.present or Nebius's pci-0302_10de.present) and + the OS release labels (system-os_release.ID, system-os_release.VERSION_ID) + the GPU operator's ClusterPolicy controller reads from GPU nodes since + v26.7 - pre-set because the worker's pod on a KWOK node is fake and never + labels anything. """ taint_value: str @@ -270,7 +271,8 @@ class Cloud: "modelplane.ai/pool": "gpu-fake", "modelplane.ai/gpu": "h100", "nvidia.com/gpu.present": "true", - "feature.node.kubernetes.io/pci-10de.present": "true", + # NFD labels real Nebius GPU nodes pci-0302_10de, not pci-10de. + "feature.node.kubernetes.io/pci-0302_10de.present": "true", # Nebius GPU node groups run Ubuntu 22.04. "feature.node.kubernetes.io/system-os_release.ID": "ubuntu", "feature.node.kubernetes.io/system-os_release.VERSION_ID": "22.04", @@ -287,6 +289,7 @@ class Cloud: "dra-driver-cpu/mp-dra-driver-cpu": 2, "node-feature-discovery/mp-node-feature-discovery-worker": 3, "monitoring/mp-kube-prometheus-stack-prometheus-node-exporter": 5, + "dcgm-exporter/mp-dcgm-exporter": 2, }, ), # The hand-written Vultr half (stacks/clouds/vultr.py) has no NFD: Vultr diff --git a/e2e/clouds/test_placement.py b/e2e/clouds/test_placement.py index 273bd71fd..209be1c24 100644 --- a/e2e/clouds/test_placement.py +++ b/e2e/clouds/test_placement.py @@ -45,6 +45,7 @@ "kubelet-plugin", "^gpu-feature-discovery", "^nvidia-dcgm", + "dcgm-exporter", "^nvidia-mig-manager", "^nvidia-operator-validator", "^nvidia-cuda-validator", @@ -85,6 +86,7 @@ "kubelet-plugin", "^gpu-feature-discovery", "^nvidia-dcgm", + "dcgm-exporter", "^nvidia-mig-manager", "^nvidia-operator-validator", "^nvidia-cuda-validator", diff --git a/functions/compose-serving-stack/function/collector.py b/functions/compose-serving-stack/function/collector.py index 13a18dd65..1825f065e 100644 --- a/functions/compose-serving-stack/function/collector.py +++ b/functions/compose-serving-stack/function/collector.py @@ -335,7 +335,8 @@ def _scrape_configs() -> list[dict[str, Any]]: # A job of its own because nobody annotates a GPU exporter # for scraping: GKE runs a managed DCGM in # gke-managed-system, the NVIDIA GPU operator installs its - # own elsewhere, the AMD GPU operator Modelplane installs on + # own elsewhere, Nebius gets the standalone dcgm-exporter + # chart, the AMD GPU operator Modelplane installs on # Existing runs AMD's device-metrics-exporter, and the # substrate job sees none of them. Without this every # modelplane_gpu_* series is empty. diff --git a/functions/compose-serving-stack/function/stacks/clouds/nebius.py b/functions/compose-serving-stack/function/stacks/clouds/nebius.py index ded4548b1..2af72aa85 100644 --- a/functions/compose-serving-stack/function/stacks/clouds/nebius.py +++ b/functions/compose-serving-stack/function/stacks/clouds/nebius.py @@ -18,9 +18,12 @@ hand, in the same shape a generator emits. Where a component also appears on a generated cloud, this file states the same pin, so one review moves both halves when a version changes - regenerate, then -mirror the shared pins here. +mirror the shared pins here. The DCGM exporter's metrics CSV goes one +step further: rather than restating it, this file imports it from the +shared generated/aicr/dcgm.py, as civo.py does. """ +from function.stacks.clouds.generated.aicr.dcgm import DCGM_METRICS_CSV from function.stacks.components import Chart, Component # DRA drivers this cloud supports that arrive outside the serving @@ -163,4 +166,52 @@ "resources": {"computeDomains": {"enabled": False}}, }, ), + # Exports DCGM fields for the collector's modelplane-gpu job, which + # keeps pods named .*dcgm.* with a port named metrics. Without this every + # modelplane_gpu_* series is empty. DCGM runs embedded (the chart + # default). The counters are the generated clouds' list, not the + # chart's shorter default, so Nebius exports the same DCGM fields as + # EKS, GKE, AKS and Civo. The node terms are the DRA kubelet plugin's PCI + # terms: Nebius GPU nodes carry pci-0302_10de but not pci-10de. The + # collector finds the pod itself, so no ServiceMonitor. + Chart( + key="dcgm-exporter", + release="mp-dcgm-exporter", + namespace="dcgm-exporter", + chart="dcgm-exporter", + repository="https://nvidia.github.io/dcgm-exporter/helm-charts", + version="4.8.4", + values={ + "serviceMonitor": {"enabled": False}, + "customMetrics": DCGM_METRICS_CSV, + # Attribute each GPU's series to the pod holding it. Nebius GPUs + # are allocated through DRA, which the exporter only maps back to + # pods with this set; the chart adds the RBAC it needs. + "kubernetesDRA": {"enabled": True}, + "tolerations": [ + { + "key": "nvidia.com/gpu", + "operator": "Exists", + "effect": "NoSchedule", + }, + ], + # The first three nodeSelectorTerms of dra-driver-nvidia-gpu + # v0.5.0's kubeletPlugin affinity. Recheck them when that chart + # moves. + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + {"matchExpressions": [{"key": key, "operator": "In", "values": ["true"]}]} + for key in ( + "feature.node.kubernetes.io/pci-10de.present", + "feature.node.kubernetes.io/pci-0302_10de.present", + "feature.node.kubernetes.io/pci-0300_10de.present", + ) + ], + }, + }, + }, + }, + ), ] diff --git a/functions/compose-serving-stack/tests/test_fn.py b/functions/compose-serving-stack/tests/test_fn.py index 90409cf2a..2b82ccc35 100644 --- a/functions/compose-serving-stack/tests/test_fn.py +++ b/functions/compose-serving-stack/tests/test_fn.py @@ -7389,6 +7389,7 @@ def test_compose(case: ComposeCase) -> None: "kube-prometheus-stack": _observed_ready(), "node-feature-discovery": _observed_ready(), "nvidia-dra-driver-gpu": _observed_ready(), + "dcgm-exporter": _observed_ready(), "leader-worker-set": _observed_ready(), }, ), @@ -7433,6 +7434,7 @@ def test_compose(case: ComposeCase) -> None: "kube-prometheus-stack", "node-feature-discovery", "nvidia-dra-driver-gpu", + "dcgm-exporter", # The Standard stack. "leader-worker-set", }, @@ -7501,6 +7503,7 @@ def test_compose(case: ComposeCase) -> None: "kube-prometheus-stack": _observed_ready(), "node-feature-discovery": _observed_ready(), "nvidia-dra-driver-gpu": _observed_ready(), + "dcgm-exporter": _observed_ready(), "grove": _observed_ready(), "kai-queue": _observed_ready(), "kai-queue-root": _observed_ready(), @@ -7559,6 +7562,7 @@ def test_compose(case: ComposeCase) -> None: "kube-prometheus-stack", "node-feature-discovery", "nvidia-dra-driver-gpu", + "dcgm-exporter", # The Dynamo stack. "grove", "kai-queue", diff --git a/functions/compose-serving-stack/tests/test_stacks.py b/functions/compose-serving-stack/tests/test_stacks.py index 5add66b21..14626d7f4 100644 --- a/functions/compose-serving-stack/tests/test_stacks.py +++ b/functions/compose-serving-stack/tests/test_stacks.py @@ -462,6 +462,53 @@ def test_generated_dcgm_csv(cloud: str) -> None: """ +def test_nebius_dcgm_exporter() -> None: + """Nebius's dcgm-exporter exports the generated clouds' DCGM fields. + + Nebius installs NVIDIA's standalone dcgm-exporter chart rather than the + GPU Operator, so the CSV goes in the chart's customMetrics rather than + dcgmExporter.config.data. Left unset, the chart falls back to its own + shorter default list and Nebius silently exports fewer fields than every + other cloud. Compared against the same literal as Civo's cases, so a + generator-moved CSV fails here until a human confirms the change. + """ + joined = stacks.join("Nebius", "Standard") + exporter = next(c for c in joined if isinstance(c, stacks.Chart) and c.key == "dcgm-exporter") + assert exporter.values is not None + assert exporter.values["customMetrics"] == _DCGM_METRICS_CSV + + +def test_nebius_exporter_affinity() -> None: + """Nebius's dcgm-exporter schedules by the DRA kubelet plugin's PCI terms. + + Nebius GPU nodes carry pci-0302_10de but not pci-10de, so dropping the + 0302 term would leave the exporter on no node with every unit test still + passing. Spelled out as a literal so a changed term fails here rather + than only on a real cluster. + """ + joined = stacks.join("Nebius", "Standard") + exporter = next(c for c in joined if isinstance(c, stacks.Chart) and c.key == "dcgm-exporter") + assert exporter.values is not None + required = exporter.values["affinity"]["nodeAffinity"]["requiredDuringSchedulingIgnoredDuringExecution"] + assert [term["matchExpressions"] for term in required["nodeSelectorTerms"]] == [ + [{"key": f"feature.node.kubernetes.io/{label}.present", "operator": "In", "values": ["true"]}] + for label in ("pci-10de", "pci-0302_10de", "pci-0300_10de") + ] + + +def test_nebius_exporter_dra() -> None: + """Nebius's dcgm-exporter attributes GPU series to DRA-allocated pods. + + Nebius allocates GPUs through DRA ResourceClaims, not the device plugin. + Without kubernetesDRA the exporter still exports every GPU's fields, but + can't say which pod holds the GPU, and nothing else fails. + """ + joined = stacks.join("Nebius", "Standard") + exporter = next(c for c in joined if isinstance(c, stacks.Chart) and c.key == "dcgm-exporter") + assert exporter.values is not None + assert exporter.values["kubernetesDRA"] == {"enabled": True} + + # Each case's input is the real joined Civo stack rather than a literal one. # The transform finds the gpu-operator and DRA driver charts by key, and where # a key matches nothing it rewrites nothing, without complaint. Only the real