From 845adc854ac68bb8fbbee3741b3111da360ea4b7 Mon Sep 17 00:00:00 2001 From: Christopher Haar Date: Thu, 29 Oct 2026 16:47:18 -0700 Subject: [PATCH 01/19] Install each device stack only where its driver is claimed Every cluster got NVIDIA's device stack and no other vendor's. Each cloud's component list pinned the NVIDIA GPU operator or DRA driver unconditionally, nothing checked that the devices a class declares match the stack its cluster gets, and a pool without a GPU couldn't be provisioned at all. compose-inference-cluster now collects the DRA drivers its classes claim onto ServingStack spec.draDrivers, and compose-serving-stack installs a component tagged with drivers only when one of them is listed. A class claiming a driver its cluster's cloud can't serve sets ClusterReady=False with reason UnsupportedDevices. The gate is a fatal result rather than an empty desired state, which Crossplane would answer by garbage collecting a running cluster. This also adds CPU inference and AMD GPUs. A class without an accelerator gets a CPU pool served by dra-driver-cpu, on every cloud. On Existing, a class claiming gpu.amd.com gets the AMD GPU operator in DRA mode, and the collector maps its device-metrics-exporter onto the same modelplane_* names as DCGM. Designed in #432. Signed-off-by: Christopher Haar Signed-off-by: Nic Cope --- CONTRIBUTING.md | 5 +- apis/aksclusters/definition.yaml | 5 +- apis/civoclusters/definition.yaml | 15 +- apis/eksclusters/definition.yaml | 5 +- apis/gkeclusters/definition.yaml | 5 +- apis/inferenceclasses/definition.yaml | 147 ++++- apis/inferenceclusters/definition.yaml | 15 +- apis/metricmappings/definition.yaml | 20 +- apis/modeldeployments/definition.yaml | 2 +- apis/nebiusclusters/definition.yaml | 5 +- apis/servingstacks/definition.yaml | 16 + apis/vultrclusters/definition.yaml | 5 +- design/free-gpu-scheduling.md | 2 +- design/multi-accelerator.md | 18 +- docs/content/architecture/scheduling.md | 2 +- docs/content/platform/telemetry.md | 33 +- .../concepts/inference-class-h100-byo.yaml | 2 +- .../config/vocabularies/Modelplane/accept.txt | 24 + functions/compose-aks-cluster/function/fn.py | 13 + .../compose-aks-cluster/tests/test_fn.py | 80 +++ functions/compose-civo-cluster/function/fn.py | 16 + .../compose-civo-cluster/tests/test_fn.py | 83 +++ functions/compose-eks-cluster/function/fn.py | 18 + .../compose-eks-cluster/tests/test_fn.py | 237 ++++++++ functions/compose-gke-cluster/function/fn.py | 16 + .../compose-gke-cluster/tests/test_fn.py | 102 ++++ .../compose-inference-cluster/function/fn.py | 255 +++++--- .../tests/test_fn.py | 548 +++++++++++++++-- .../compose-model-deployment/function/cel.py | 4 +- .../compose-model-deployment/function/fn.py | 2 +- .../function/scheduling.py | 32 +- .../compose-model-deployment/tests/test_fn.py | 2 +- .../tests/test_scheduling.py | 242 ++++---- .../function/backends/base.py | 75 ++- .../compose-model-replica/function/fn.py | 14 + .../function/observation.py | 4 +- .../tests/test_backends.py | 423 ++++++++++++- .../compose-model-replica/tests/test_fn.py | 14 +- .../compose-nebius-cluster/function/fn.py | 16 + .../compose-nebius-cluster/tests/test_fn.py | 116 ++++ .../function/collector.py | 42 +- .../compose-serving-stack/function/fn.py | 15 +- .../function/stacks/__init__.py | 90 ++- .../function/stacks/clouds/civo.py | 18 +- .../function/stacks/clouds/existing.py | 157 ++++- .../function/stacks/clouds/nebius.py | 17 +- .../function/stacks/clouds/vultr.py | 13 +- .../function/stacks/common.py | 85 ++- .../function/stacks/components.py | 13 + .../function/stacks/metrics.py | 45 +- functions/compose-serving-stack/generate.py | 103 +++- .../tests/test_collector.py | 48 +- .../compose-serving-stack/tests/test_fn.py | 574 ++++++++++++++++-- .../tests/test_metrics.py | 6 +- .../tests/test_stacks.py | 186 ++++++ .../compose-vultr-cluster/function/fn.py | 23 +- .../compose-vultr-cluster/tests/test_fn.py | 77 ++- 57 files changed, 3671 insertions(+), 479 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1f53a7eaf..e2c8ca839 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -429,7 +429,10 @@ checks to trip. In order: `dra` owns the DRA selector paths the upstream values don't. The fork is deleted the day NVIDIA/aicr#2517 lands. 3. Expect `ALLOW`/`DROP` to fail closed on any component the new catalog - adds; classify it with a reason. + adds; classify it with a reason. An `ALLOW` entry also names the DRA + drivers whose device stacks the component belongs to (empty for a + component every cluster needs) - deciding that is part of classifying + it; see design/multi-accelerator.md. 4. Expect `check_namespace_holds` to fail closed if the catalog moved a component's chart namespace: Helm can't move a release across namespaces, so existing clusters would wedge on a fresh-install diff --git a/apis/aksclusters/definition.yaml b/apis/aksclusters/definition.yaml index e8a39647b..3cec68b50 100644 --- a/apis/aksclusters/definition.yaml +++ b/apis/aksclusters/definition.yaml @@ -128,10 +128,13 @@ spec: Determines what workloads this pool runs. System pools host controllers, gateways, and infrastructure. GPU pools host inference workloads and get the NVIDIA - driver installed by AKS. + driver installed by AKS. CPU pools host CPU inference + workloads: no GPU driver, and the modelplane.ai/cpu + label and taint. enum: - System - GPU + - CPU vmSize: type: string description: >- diff --git a/apis/civoclusters/definition.yaml b/apis/civoclusters/definition.yaml index f45cefffc..6200f7336 100644 --- a/apis/civoclusters/definition.yaml +++ b/apis/civoclusters/definition.yaml @@ -51,10 +51,14 @@ spec: description: >- Civo Kubernetes (k3s) version. Civo requires an exact version string including the k3s build suffix (e.g. - 1.34.1-k3s1); list current versions with civo kubernetes + 1.36.0-k3s1); list current versions with civo kubernetes versions. Choose 1.34 or newer so Dynamic Resource - Allocation (how GPUs bind to pods) is generally available. - Defaults to Civo's current default version. + Allocation (how devices bind to pods) is generally + available, and 1.36 or newer for a CPU pool: its cores are + claimed as consumable capacity, which Kubernetes enables by + default from 1.36 (DRAConsumableCapacity). Defaults to + Civo's current default version, which may be older than a + CPU pool needs. minLength: 1 maxLength: 32 credentials: @@ -110,10 +114,13 @@ spec: Determines what workloads this pool runs. System pools host controllers, gateways, and infrastructure. GPU pools host inference workloads and are tainted to - exclude non-GPU pods. + exclude non-GPU pods. CPU pools host CPU inference + workloads: a non-GPU size and the modelplane.ai/cpu + label and taint. enum: - System - GPU + - CPU size: type: string description: >- diff --git a/apis/eksclusters/definition.yaml b/apis/eksclusters/definition.yaml index 1f347efa4..8bfa2dddd 100644 --- a/apis/eksclusters/definition.yaml +++ b/apis/eksclusters/definition.yaml @@ -132,10 +132,13 @@ spec: Determines what workloads this group runs. System groups host controllers, gateways, and infrastructure. GPU groups host inference workloads and use a - GPU-enabled AMI. + GPU-enabled AMI. CPU groups host CPU inference + workloads: a standard AMI, no GPU, and the + modelplane.ai/cpu label and taint. enum: - System - GPU + - CPU instanceType: type: string description: >- diff --git a/apis/gkeclusters/definition.yaml b/apis/gkeclusters/definition.yaml index d5517f26a..588edc7d0 100644 --- a/apis/gkeclusters/definition.yaml +++ b/apis/gkeclusters/definition.yaml @@ -124,10 +124,13 @@ spec: Determines what workloads this pool runs. System pools host controllers, gateways, and infrastructure. GPU pools host inference workloads and are tainted to - exclude non-GPU pods. + exclude non-GPU pods. CPU pools host CPU inference + workloads: a standard node image, no GPU, and the + modelplane.ai/cpu label and taint. enum: - System - GPU + - CPU machineType: type: string description: >- diff --git a/apis/inferenceclasses/definition.yaml b/apis/inferenceclasses/definition.yaml index 44264d65b..b1ead5206 100644 --- a/apis/inferenceclasses/definition.yaml +++ b/apis/inferenceclasses/definition.yaml @@ -22,6 +22,100 @@ spec: spec: type: object required: [devices] + x-kubernetes-validations: + # The pool's role follows accelerator - compose--cluster + # composes a GPU pool when it's set and a CPU pool when it isn't - + # while the scheduler and ServingStack spec.draDrivers follow + # spec.devices. These rules keep the two in step, one pair per + # cloud because CEL can't address the provider's block + # dynamically: a class claiming a GPU driver without accelerator + # would compose a CPU pool whose advertised GPUs nothing can + # allocate, and accelerator on a class claiming only dra.cpu would + # provision GPUs nothing claims. + - rule: >- + !has(self.provisioning) || !has(self.provisioning.gke) || + has(self.provisioning.gke.accelerator) || + !(has(self.devices) && self.devices.exists(d, (!has(d.claim) || d.claim == 'DRA') && d.driver != 'dra.cpu')) + message: >- + spec.provisioning.gke.accelerator is required when a device + claims a DRA driver other than dra.cpu. + - rule: >- + !has(self.provisioning) || !has(self.provisioning.gke) || + !has(self.provisioning.gke.accelerator) || + (has(self.devices) && self.devices.exists(d, (!has(d.claim) || d.claim == 'DRA') && d.driver != 'dra.cpu')) + message: >- + spec.provisioning.gke.accelerator must be omitted when no + device claims a DRA driver other than dra.cpu. + - rule: >- + !has(self.provisioning) || !has(self.provisioning.eks) || + has(self.provisioning.eks.accelerator) || + !(has(self.devices) && self.devices.exists(d, (!has(d.claim) || d.claim == 'DRA') && d.driver != 'dra.cpu')) + message: >- + spec.provisioning.eks.accelerator is required when a device + claims a DRA driver other than dra.cpu. + - rule: >- + !has(self.provisioning) || !has(self.provisioning.eks) || + !has(self.provisioning.eks.accelerator) || + (has(self.devices) && self.devices.exists(d, (!has(d.claim) || d.claim == 'DRA') && d.driver != 'dra.cpu')) + message: >- + spec.provisioning.eks.accelerator must be omitted when no + device claims a DRA driver other than dra.cpu. + - rule: >- + !has(self.provisioning) || !has(self.provisioning.aks) || + has(self.provisioning.aks.accelerator) || + !(has(self.devices) && self.devices.exists(d, (!has(d.claim) || d.claim == 'DRA') && d.driver != 'dra.cpu')) + message: >- + spec.provisioning.aks.accelerator is required when a device + claims a DRA driver other than dra.cpu. + - rule: >- + !has(self.provisioning) || !has(self.provisioning.aks) || + !has(self.provisioning.aks.accelerator) || + (has(self.devices) && self.devices.exists(d, (!has(d.claim) || d.claim == 'DRA') && d.driver != 'dra.cpu')) + message: >- + spec.provisioning.aks.accelerator must be omitted when no + device claims a DRA driver other than dra.cpu. + - rule: >- + !has(self.provisioning) || !has(self.provisioning.nebius) || + has(self.provisioning.nebius.accelerator) || + !(has(self.devices) && self.devices.exists(d, (!has(d.claim) || d.claim == 'DRA') && d.driver != 'dra.cpu')) + message: >- + spec.provisioning.nebius.accelerator is required when a device + claims a DRA driver other than dra.cpu. + - rule: >- + !has(self.provisioning) || !has(self.provisioning.nebius) || + !has(self.provisioning.nebius.accelerator) || + (has(self.devices) && self.devices.exists(d, (!has(d.claim) || d.claim == 'DRA') && d.driver != 'dra.cpu')) + message: >- + spec.provisioning.nebius.accelerator must be omitted when no + device claims a DRA driver other than dra.cpu. + - rule: >- + !has(self.provisioning) || !has(self.provisioning.vultr) || + has(self.provisioning.vultr.accelerator) || + !(has(self.devices) && self.devices.exists(d, (!has(d.claim) || d.claim == 'DRA') && d.driver != 'dra.cpu')) + message: >- + spec.provisioning.vultr.accelerator is required when a device + claims a DRA driver other than dra.cpu. + - rule: >- + !has(self.provisioning) || !has(self.provisioning.vultr) || + !has(self.provisioning.vultr.accelerator) || + (has(self.devices) && self.devices.exists(d, (!has(d.claim) || d.claim == 'DRA') && d.driver != 'dra.cpu')) + message: >- + spec.provisioning.vultr.accelerator must be omitted when no + device claims a DRA driver other than dra.cpu. + - rule: >- + !has(self.provisioning) || !has(self.provisioning.civo) || + has(self.provisioning.civo.accelerator) || + !(has(self.devices) && self.devices.exists(d, (!has(d.claim) || d.claim == 'DRA') && d.driver != 'dra.cpu')) + message: >- + spec.provisioning.civo.accelerator is required when a device + claims a DRA driver other than dra.cpu. + - rule: >- + !has(self.provisioning) || !has(self.provisioning.civo) || + !has(self.provisioning.civo.accelerator) || + (has(self.devices) && self.devices.exists(d, (!has(d.claim) || d.claim == 'DRA') && d.driver != 'dra.cpu')) + message: >- + spec.provisioning.civo.accelerator must be omitted when no + device claims a DRA driver other than dra.cpu. properties: description: type: string @@ -52,7 +146,7 @@ spec: enum: [GKE, EKS, AKS, Nebius, Vultr, Civo] gke: type: object - required: [machineType, accelerator] + required: [machineType] properties: machineType: type: string @@ -68,7 +162,11 @@ spec: pool. Provisioning input only: the scheduler matches against spec.devices, not this block, so count here is the GCP machine's GPU count and need not be restated in - devices. + devices. Set this when and only when a device + claims a DRA driver other than dra.cpu; the API + rejects a mismatch. Omitted, the pool has no GPU: + Modelplane provisions it with a standard node image + and the modelplane.ai/cpu label and taint. required: [type, count] properties: type: @@ -84,7 +182,7 @@ spec: maximum: 16 eks: type: object - required: [instanceType, accelerator] + required: [instanceType] properties: instanceType: type: string @@ -103,7 +201,12 @@ spec: description: >- GPU accelerator to attach when provisioning the node group. Provisioning input only: the scheduler matches - against spec.devices, not this block. + against spec.devices, not this block. Set this when + and only when a device claims a DRA driver other + than dra.cpu; the API rejects a mismatch. Omitted, + the group has no GPU: Modelplane provisions it with + a standard node image and the modelplane.ai/cpu + label and taint. required: [type, count] properties: type: @@ -120,7 +223,7 @@ spec: maximum: 16 aks: type: object - required: [vmSize, accelerator] + required: [vmSize] properties: vmSize: type: string @@ -139,7 +242,12 @@ spec: description: >- GPU accelerator to attach when provisioning the node pool. Provisioning input only: the scheduler matches - against spec.devices, not this block. + against spec.devices, not this block. Set this when + and only when a device claims a DRA driver other + than dra.cpu; the API rejects a mismatch. Omitted, + the pool has no GPU: Modelplane provisions it with a + standard node image and the modelplane.ai/cpu label + and taint. required: [type, count] properties: type: @@ -156,7 +264,7 @@ spec: maximum: 16 nebius: type: object - required: [platform, preset, accelerator] + required: [platform, preset] properties: platform: type: string @@ -197,7 +305,12 @@ spec: description: >- GPU accelerator to attach when provisioning the node group. Provisioning input only: the scheduler matches - against spec.devices, not this block. + against spec.devices, not this block. Set this when + and only when a device claims a DRA driver other + than dra.cpu; the API rejects a mismatch. Omitted, + the group has no GPU: Modelplane provisions it with + a standard node image and the modelplane.ai/cpu + label and taint. required: [type, count] properties: type: @@ -214,7 +327,7 @@ spec: maximum: 16 vultr: type: object - required: [plan, accelerator] + required: [plan] properties: plan: type: string @@ -229,7 +342,12 @@ spec: description: >- GPU accelerator to attach when provisioning the node pool. Provisioning input only: the scheduler matches - against spec.devices, not this block. + against spec.devices, not this block. Set this when + and only when a device claims a DRA driver other + than dra.cpu; the API rejects a mismatch. Omitted, + the pool has no GPU: Modelplane provisions it with a + standard node image and the modelplane.ai/cpu label + and taint. required: [type, count] properties: type: @@ -246,7 +364,7 @@ spec: maximum: 16 civo: type: object - required: [size, accelerator] + required: [size] properties: size: type: string @@ -261,7 +379,12 @@ spec: description: >- GPU accelerator to attach when provisioning the node pool. Provisioning input only: the scheduler matches - against spec.devices, not this block. + against spec.devices, not this block. Set this when + and only when a device claims a DRA driver other + than dra.cpu; the API rejects a mismatch. Omitted, + the pool has no GPU: Modelplane provisions it with a + standard node image and the modelplane.ai/cpu label + and taint. required: [type, count] properties: type: diff --git a/apis/inferenceclusters/definition.yaml b/apis/inferenceclusters/definition.yaml index 74c18fab3..41e34647f 100644 --- a/apis/inferenceclusters/definition.yaml +++ b/apis/inferenceclusters/definition.yaml @@ -333,9 +333,13 @@ spec: exact version string including the k3s build suffix; list current versions with civo kubernetes versions. Choose 1.34 or newer so Dynamic Resource - Allocation (how GPUs bind to pods) is generally - available. Defaults to Civo's current default - version. + Allocation (how devices bind to pods) is generally + available, and 1.36 or newer for a CPU pool: its + cores are claimed as consumable capacity, which + Kubernetes enables by default from 1.36 + (DRAConsumableCapacity). Defaults to Civo's current + default version, which may be older than a CPU + pool needs. minLength: 1 maxLength: 32 credentials: @@ -613,10 +617,11 @@ spec: PVCs on this cluster. ModelCache reads this to target the cache PVC. maxLength: 253 - gpuPools: + schedulablePools: type: array description: >- - Schedulable GPU node pools on this cluster, derived from the + Schedulable node pools on this cluster - GPU and CPU alike; + every pool whose class declares devices - derived from the referenced classes, the per-pool node counts, and what the cluster agent observes of each pool. ModelDeployment scheduling matches against these. diff --git a/apis/metricmappings/definition.yaml b/apis/metricmappings/definition.yaml index 63b512801..467aed7e0 100644 --- a/apis/metricmappings/definition.yaml +++ b/apis/metricmappings/definition.yaml @@ -150,22 +150,30 @@ spec: as the component wrote it. fromUnit: type: string - enum: [Millijoules, Mebibytes, Milliseconds, Nanoseconds, Percent] + enum: [Millijoules, Microjoules, Mebibytes, Milliseconds, Nanoseconds, Percent] description: >- What the component measures this in, when that isn't the unit the name claims. Modelplane converts to the base unit: millijoules and milliseconds are divided by - a thousand, nanoseconds by a billion, percent by a - hundred, and mebibytes multiplied out to bytes. A - histogram is converted whole - its sum, its bounds and - its bucket boundaries - so its quantiles come out in - the target unit too. + a thousand, microjoules by a million, nanoseconds by a + billion, percent by a hundred, and mebibytes multiplied + out to bytes. A histogram is converted whole - its sum, + its bounds and its bucket boundaries - so its quantiles + come out in the target unit too. Percent is for a component that counts a saturation from nought to a hundred where the name says a ratio. Check rather than assume: vLLM publishes kv_cache_usage_perc and the value is a fraction, so a name is no guide. + cumulative: + type: boolean + description: >- + Set when the component reports a cumulative total as a + gauge. Modelplane converts the series to a monotonic + cumulative sum before renaming it, so the exported + type matches the measurement and a modelplane_* name + never carries two types across sources. status: type: object properties: diff --git a/apis/modeldeployments/definition.yaml b/apis/modeldeployments/definition.yaml index 0e9bfb60e..0442ed7e5 100644 --- a/apis/modeldeployments/definition.yaml +++ b/apis/modeldeployments/definition.yaml @@ -458,7 +458,7 @@ spec: pods: what devices each pod needs from its node. The scheduler matches it against a candidate pool's InferenceClass devices (surfaced on - InferenceCluster status.gpuPools) and places the + InferenceCluster status.schedulablePools) and places the member on a pool that satisfies it, preferring one pool for the whole engine and splitting members across pools only when no pool satisfies diff --git a/apis/nebiusclusters/definition.yaml b/apis/nebiusclusters/definition.yaml index 022f29819..171592e49 100644 --- a/apis/nebiusclusters/definition.yaml +++ b/apis/nebiusclusters/definition.yaml @@ -101,10 +101,13 @@ spec: Determines what workloads this group runs. System groups host controllers, gateways, and infrastructure. GPU groups host inference workloads and are tainted to - exclude non-GPU pods. + exclude non-GPU pods. CPU groups host CPU inference + workloads: a CPU platform, no GPU, and the + modelplane.ai/cpu label and taint. enum: - System - GPU + - CPU platform: type: string description: >- diff --git a/apis/servingstacks/definition.yaml b/apis/servingstacks/definition.yaml index 65722e89a..40c30446a 100644 --- a/apis/servingstacks/definition.yaml +++ b/apis/servingstacks/definition.yaml @@ -63,6 +63,22 @@ spec: - Vultr - Civo - Existing + draDrivers: + type: array + description: >- + DRA driver names whose device stacks this cluster needs + (gpu.nvidia.com, dra.cpu, ...), collected by the cluster + composition from the claim: DRA devices of the cluster's + referenced InferenceClasses. A component tagged with a + driver installs only when that driver is listed; untagged + components always install. Absent means the same as + empty: only the untagged components install. + maxItems: 16 + x-kubernetes-list-type: set + items: + type: string + minLength: 1 + maxLength: 253 secrets: type: array description: >- diff --git a/apis/vultrclusters/definition.yaml b/apis/vultrclusters/definition.yaml index 6d065cbeb..26346cdee 100644 --- a/apis/vultrclusters/definition.yaml +++ b/apis/vultrclusters/definition.yaml @@ -110,10 +110,13 @@ spec: Determines what workloads this pool runs. System pools host controllers, gateways, and infrastructure. GPU pools host inference workloads and are tainted to - exclude non-GPU pods. + exclude non-GPU pods. CPU pools host CPU inference + workloads: a non-GPU plan and the modelplane.ai/cpu + label and taint. enum: - System - GPU + - CPU plan: type: string description: >- diff --git a/design/free-gpu-scheduling.md b/design/free-gpu-scheduling.md index ebd13a40d..78c10953a 100644 --- a/design/free-gpu-scheduling.md +++ b/design/free-gpu-scheduling.md @@ -72,7 +72,7 @@ publish: ```yaml # The InferenceCluster. status: - gpuPools: + schedulablePools: - name: gpu-l4x4-a nodes: 1 # The most nodes the pool can have. observedNodes: 1 # New. The nodes the pool has now, ready or not. diff --git a/design/multi-accelerator.md b/design/multi-accelerator.md index fa07031d5..f168fb566 100644 --- a/design/multi-accelerator.md +++ b/design/multi-accelerator.md @@ -168,6 +168,16 @@ what is genuinely CPU-specific: the driver's reserved cores, the CPU is also why there is no opt-in path: a CPU class can run on any cloud, so every cloud filters and `spec.draDrivers` flows everywhere from the start. +One floor is CPU's alone: the driver's grouped mode publishes each NUMA +node's cores as one device whose CPUs are consumable capacity +(KEP-5075), which Kubernetes enables by default from 1.36 and managed +control planes can't enable earlier - on 1.34/1.35 the scheduler +allocates the device but strips the capacity accounting, and the engine +pod fails at NodePrepareResources. So a CPU pool needs Kubernetes 1.36 +on a managed cloud, while GPU pools keep the 1.34/1.35 floors; cluster +manifests for CPU pools pin the version rather than taking the cloud's +default. + **Supported drivers per cloud.** A cloud supports the drivers its component list is tagged with, so in the stacks package the per-cloud table is derived from the tags rather than declared beside them; it can't drift from what the @@ -229,7 +239,13 @@ layers reject the mismatch: "Supported drivers per cloud"). A class is not checked on its own — `compose-inference-class` stays a pass-through — so an unsupported driver surfaces when a cluster references the class, the first moment it could - do harm, and the per-cloud table has a single hand-kept mirror. + do harm, and the per-cloud table has a single hand-kept mirror. The gate + stops with a fatal result, because it can fire on a cluster that is + already running — a shared class edited to claim a new driver — and + everything composes after it: on a bare return Crossplane would + garbage-collect the cloud cluster XR, the ServingStack and their Usages. + A fatal result returns before Crossplane applies desired state or + collects anything, and the condition still lands on the XR. 2. The component join raises if a ServingStack names a driver its cloud neither tags components with nor declares external, and if a component depends on one that filtering could remove from under it: a dependency diff --git a/docs/content/architecture/scheduling.md b/docs/content/architecture/scheduling.md index 57718b2b0..2a7b4fa9e 100644 --- a/docs/content/architecture/scheduling.md +++ b/docs/content/architecture/scheduling.md @@ -100,7 +100,7 @@ reports how many of each pool's nodes have each number of free devices. The ```yaml status: - gpuPools: + schedulablePools: - name: gpu-l4x4-a nodes: 3 # The most nodes the pool can have. observedNodes: 1 # The nodes the pool has now, ready or not. diff --git a/docs/content/platform/telemetry.md b/docs/content/platform/telemetry.md index 99f565e15..6d08d1f2e 100644 --- a/docs/content/platform/telemetry.md +++ b/docs/content/platform/telemetry.md @@ -93,7 +93,7 @@ Some examples of the available metrics: Latency appears twice on purpose. The `frontend_` series are what your caller experienced, measured at the gateway. The engine's own series are what the engine spent. For -example, if the frontend metric is slow and the engine isn't, you can +example, if the frontend metric is slow and the engine isn't, you can troubleshoot routing, queueing, or networking issues instead of the model. Saturation gauges are per replica, so how you combine them decides what you see. Average @@ -223,7 +223,7 @@ Anything else the exporter takes goes under `config`, passed through as you wrot Modelplane doesn't define a schema for an exporter's settings so anything under the `config` is passed to the collector exactly as written. TLS, retries, queueing, compression and headers all work and any new settings in the collector -are respected and the sink keeps working. +are respected and the sink keeps working. To use an authentication scheme Modelplane doesn't compose, define the extension yourself under `spec.extensions`. Then reference it by its key from the sink's @@ -459,12 +459,16 @@ spec: ## GPUs -The GPU series come from a [DCGM -exporter](https://github.com/NVIDIA/dcgm-exporter), not from the engine: a GPU +The GPU series come from a vendor's GPU exporter, not from the engine: a GPU belongs to a node, and its series have a `node` label rather than a -deployment. The collector discovers the exporter by its pod - an `app` or -`app.kubernetes.io/name` label containing `dcgm`, with a container port named -`metrics` - and scrapes it directly, not through Prometheus. +deployment. On NVIDIA pools that's a [DCGM +exporter](https://github.com/NVIDIA/dcgm-exporter); on AMD pools it's the AMD +GPU operator's +[device-metrics-exporter](https://github.com/ROCm/device-metrics-exporter), +which the serving stack installs with the AMD stack. The collector discovers +either by its pod - an `app` or `app.kubernetes.io/name` label containing +`dcgm` or `metrics-exporter`, with a container port named `metrics` or +`exporter-port` - and scrapes it directly, not through Prometheus. Where the exporter comes from depends on the cluster: @@ -504,6 +508,21 @@ curl -s localhost:9400/metrics | grep -c -E \ 'DCGM_FI_DEV_FB_(USED|FREE)|DCGM_FI_PROF_(GR_ENGINE_ACTIVE|PIPE_TENSOR_ACTIVE|DRAM_ACTIVE)|DCGM_FI_DEV_(GPU_TEMP|POWER_USAGE|TOTAL_ENERGY_CONSUMPTION)' ``` +On an AMD pool the same `modelplane_*` names come from the +device-metrics-exporter's fields instead. A dashboard built on them needs no +change when a deployment moves between vendors' pools. + +| AMD field | Becomes | Unit | +| --- | --- | --- | +| `gpu_used_vram` | `modelplane_gpu_memory_used_bytes` | MiB, converted to bytes | +| `gpu_free_vram` | `modelplane_gpu_memory_free_bytes` | MiB, converted to bytes | +| `gpu_gfx_activity` | `modelplane_gpu_compute_active_ratio` | percent, converted to a ratio | +| `gpu_umc_activity` | `modelplane_gpu_memory_bandwidth_ratio` | percent, converted to a ratio | +| `gpu_prof_tensor_active_percent` | `modelplane_gpu_tensor_active_ratio` | percent, converted to a ratio | +| `gpu_junction_temperature` | `modelplane_gpu_temperature_celsius` | Celsius | +| `gpu_power_usage` | `modelplane_gpu_power_watts` | watts | +| `gpu_energy_consumed` | `modelplane_energy_joules_total` | microjoules, converted to joules | + If fields are missing, extend the exporter's metrics ConfigMap - on a GPU Operator install that's the `dcgmExporter.config` chart value. The configuration Modelplane installs on the clouds it provisions publishes each diff --git a/docs/manifests/concepts/inference-class-h100-byo.yaml b/docs/manifests/concepts/inference-class-h100-byo.yaml index fc070712d..4c2f32aa3 100644 --- a/docs/manifests/concepts/inference-class-h100-byo.yaml +++ b/docs/manifests/concepts/inference-class-h100-byo.yaml @@ -2,7 +2,7 @@ # # No provisioning block: this class describes hardware that already # exists on a bring-your-own cluster. Modelplane copies the devices block -# onto the cluster's status.gpuPools for the scheduler to match against. +# onto the cluster's status.schedulablePools for the scheduler to match against. apiVersion: modelplane.ai/v1alpha1 kind: InferenceClass metadata: diff --git a/docs/utils/vale/styles/config/vocabularies/Modelplane/accept.txt b/docs/utils/vale/styles/config/vocabularies/Modelplane/accept.txt index e06475704..6c4278346 100644 --- a/docs/utils/vale/styles/config/vocabularies/Modelplane/accept.txt +++ b/docs/utils/vale/styles/config/vocabularies/Modelplane/accept.txt @@ -115,6 +115,18 @@ vLLM [Hh]uggingFace NVIDIA NVLink +AMD +Instinct +ROCm +amdgpu +CUDA +HBM3e +cpuset +device-metrics-exporter +[Mm]icrojoules +MI325X +CPUs +NUMA Envoy Traefik MetalLB @@ -725,6 +737,18 @@ vLLM [Hh]uggingFace NVIDIA NVLink +AMD +Instinct +ROCm +amdgpu +CUDA +HBM3e +cpuset +device-metrics-exporter +[Mm]icrojoules +MI325X +CPUs +NUMA Envoy Traefik MetalLB diff --git a/functions/compose-aks-cluster/function/fn.py b/functions/compose-aks-cluster/function/fn.py index a6d52a6a1..8c36d27cf 100644 --- a/functions/compose-aks-cluster/function/fn.py +++ b/functions/compose-aks-cluster/function/fn.py @@ -103,6 +103,13 @@ _GPU_TAINT = "nvidia.com/gpu=true:NoSchedule" _GPU_TAINT_KEY = "nvidia.com/gpu" +# Label and taint on CPU inference pools. The label is what the serving +# stack's dra-driver-cpu selects its nodes by; the taint keeps system +# workloads off the pool as the GPU taint does on GPU pools, and engine +# pods tolerate it (compose-model-replica). +_LABEL_CPU = "modelplane.ai/cpu" +_CPU_TAINT = "modelplane.ai/cpu=true:NoSchedule" + # Node pool gpuDriver value asking AKS to install the NVIDIA driver on GPU # nodes. The device plugin is not installed in this path - the serving # stack's DRA driver binds GPUs to pods, as on EKS. @@ -410,6 +417,12 @@ def compose_node_pools(self) -> None: _LABEL_POOL: pool.name, } fp.nodeTaints = [_GPU_TAINT] + elif pool.role == "CPU": + fp.nodeLabels = { + _LABEL_POOL: pool.name, + _LABEL_CPU: "true", + } + fp.nodeTaints = [_CPU_TAINT] resource.update( self.rsp.desired.resources[f"nodepool-{pool.name}"], diff --git a/functions/compose-aks-cluster/tests/test_fn.py b/functions/compose-aks-cluster/tests/test_fn.py index 4263875bc..512e2d9da 100644 --- a/functions/compose-aks-cluster/tests/test_fn.py +++ b/functions/compose-aks-cluster/tests/test_fn.py @@ -212,6 +212,41 @@ def _nodepool_gpu(*, cred_kind: str, cred_name: str, zones: list[str] | None, re return fnv1.Resource(resource=resource.dict_to_struct(nodepool), ready=ready) +def _nodepool_cpu(*, cred_kind: str, cred_name: str) -> fnv1.Resource: + """The composed cpupool node pool, with the CPU label and taint and no GPU driver.""" + return fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "containerservice.azure.m.upbound.io/v1beta1", + "kind": "KubernetesClusterNodePool", + "metadata": {"annotations": {"crossplane.io/external-name": "cpupool"}}, + "spec": { + "providerConfigRef": {"kind": cred_kind, "name": cred_name}, + "managementPolicies": ["Observe", "Create", "Update", "Delete"], + "initProvider": {"nodeCount": 1}, + "forProvider": { + "kubernetesClusterIdSelector": {"matchControllerRef": True}, + "vnetSubnetIdSelector": {"matchControllerRef": True}, + "mode": "User", + "vmSize": "Standard_D8s_v5", + "osDiskSizeGb": 100, + "orchestratorVersion": "1.34", + "autoScalingEnabled": True, + "minCount": 0, + "maxCount": 4, + "nodeLabels": { + "modelplane.ai/pool": "cpupool", + "modelplane.ai/cpu": "true", + }, + "nodeTaints": ["modelplane.ai/cpu=true:NoSchedule"], + "upgradeSettings": {"maxSurge": "10%"}, + }, + }, + } + ), + ) + + def _provider_config(*, api_version: str) -> fnv1.Resource: """A Ready ProviderConfig that reaches the cluster through its kubeconfig Secret.""" return fnv1.Resource( @@ -293,6 +328,51 @@ def _to_dict(msg: message.Message) -> dict: context=structpb.Struct(), ), ), + Case( + name="FirstPassCPUPool", + reason="A CPU-role node pool composes with the modelplane.ai/pool and modelplane.ai/cpu labels and the CPU taint, and no GPU driver.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + node_pools=[ + v1alpha1.NodePool( + name="cpupool", + role="CPU", + vmSize="Standard_D8s_v5", + nodeCount=1, + minNodeCount=0, + maxNodeCount=4, + ), + ], + ), + ), + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_xr(), + resources={ + "resource-group": _resource_group( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "virtual-network": _virtual_network( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "subnet": _subnet( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "cluster": _cluster( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "nodepool-cpupool": _nodepool_cpu(cred_kind="ClusterProviderConfig", cred_name="default"), + "provider-config-kubernetes": _provider_config(api_version="kubernetes.m.crossplane.io/v1alpha1"), + "provider-config-helm": _provider_config(api_version="helm.m.crossplane.io/v1beta1"), + }, + ), + context=structpb.Struct(), + ), + ), Case( name="NodePoolZones", reason="An AKSCluster passes a pool's zones through to its KubernetesClusterNodePool.", diff --git a/functions/compose-civo-cluster/function/fn.py b/functions/compose-civo-cluster/function/fn.py index 0ba30cade..081c931dc 100644 --- a/functions/compose-civo-cluster/function/fn.py +++ b/functions/compose-civo-cluster/function/fn.py @@ -96,6 +96,12 @@ _GPU_TAINT_VALUE = "true" _GPU_TAINT_EFFECT = "NoSchedule" +# Label and taint on CPU inference pools. The label is what the serving +# stack's dra-driver-cpu selects its nodes by; the taint keeps system +# workloads off the pool as the GPU taint does on GPU pools, and engine +# pods tolerate it (compose-model-replica). +_LABEL_CPU = "modelplane.ai/cpu" + # Civo's default marketplace applications include the Traefik ingress # controller, which would compete with the serving stack's Envoy Gateway # for LoadBalancer traffic. A "-" prefix removes a default application at @@ -430,6 +436,8 @@ def _node_pool(self, pool: v1alpha1.NodePool) -> nodepoolv1beta1.NodePool: labels = {_LABEL_POOL: pool.name} if pool.role == "GPU" and pool.gpu: labels[_LABEL_GPU] = pool.gpu.acceleratorType + elif pool.role == "CPU": + labels[_LABEL_CPU] = "true" fp = nodepoolv1beta1.ForProvider( label=pool.name, @@ -451,6 +459,14 @@ def _node_pool(self, pool: v1alpha1.NodePool) -> nodepoolv1beta1.NodePool: effect=_GPU_TAINT_EFFECT, ), ] + elif pool.role == "CPU": + fp.taint = [ + nodepoolv1beta1.TaintItem( + key=_LABEL_CPU, + value="true", + effect=_GPU_TAINT_EFFECT, + ), + ] spec = nodepoolv1beta1.Spec( providerConfigRef=nodepoolv1beta1.ProviderConfigRef( diff --git a/functions/compose-civo-cluster/tests/test_fn.py b/functions/compose-civo-cluster/tests/test_fn.py index c0ae9ae55..92edc33a0 100644 --- a/functions/compose-civo-cluster/tests/test_fn.py +++ b/functions/compose-civo-cluster/tests/test_fn.py @@ -238,6 +238,47 @@ def _node_pool_gpu( ) +def _node_pool_cpu( + *, + management_policies: list[str] | None, + init_node_count: int | None, + node_count: int | None, + ready: fnv1.Ready, +) -> fnv1.Resource: + """The composed NodePool for the cpu-pool pool, with the CPU label and taint and no GPU label.""" + for_provider: dict[str, Any] = { + "label": "cpu-pool", + "size": "g4s.kube.large", + "region": "LON1", + "labels": { + "modelplane.ai/pool": "cpu-pool", + "modelplane.ai/cpu": "true", + }, + "clusterIdSelector": {"matchControllerRef": True}, + "taint": [{"key": "modelplane.ai/cpu", "value": "true", "effect": "NoSchedule"}], + } + if node_count is not None: + for_provider["nodeCount"] = node_count + spec: dict[str, Any] = { + "providerConfigRef": {"kind": "ClusterProviderConfig", "name": "default"}, + "forProvider": for_provider, + } + if management_policies is not None: + spec["managementPolicies"] = management_policies + if init_node_count is not None: + spec["initProvider"] = {"nodeCount": init_node_count} + return fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "kubernetes.civo.m.upbound.io/v1beta1", + "kind": "NodePool", + "spec": spec, + } + ), + ready=ready, + ) + + def _provider_config() -> fnv1.Resource: """The composed provider-helm ProviderConfig for the cluster, which is always ready.""" return fnv1.Resource( @@ -381,6 +422,48 @@ def _to_dict(msg: message.Message) -> dict: context=structpb.Struct(), ), ), + Case( + name="ClusterReadyCPUPool", + reason="A CPU-role pool composes with the modelplane.ai/pool and modelplane.ai/cpu labels and the CPU taint, and no GPU label.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + node_pools=[ + v1alpha1.NodePool( + name="cpu-pool", + role="CPU", + size="g4s.kube.large", + maxNodeCount=4, + ), + ], + ), + resources={ + "cluster": _observed_cluster(cred_kind="ClusterProviderConfig", cred_name="default", ready=True), + }, + ), + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_xr(), + resources={ + "network": _network(cred_kind="ClusterProviderConfig", cred_name="default"), + "firewall": _firewall(cred_kind="ClusterProviderConfig", cred_name="default"), + "cluster": _cluster(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), + "node-pool-cpu-pool": _node_pool_cpu( + management_policies=["Observe", "Create", "Update", "Delete"], + init_node_count=1, + node_count=None, + ready=fnv1.READY_UNSPECIFIED, + ), + "provider-config-helm": _provider_config(), + "release-cluster-autoscaler": _autoscaler_release(autoscaling_groups=[]), + }, + ), + context=structpb.Struct(), + ), + ), # The autoscaler addresses node groups by the pool ID the provider writes # to the NodePool's external-name annotation once the pool exists. The pool # leaves nodeCount at its default of 1, so this case can't tell a floor diff --git a/functions/compose-eks-cluster/function/fn.py b/functions/compose-eks-cluster/function/fn.py index 95ac2ec2c..72341c41c 100644 --- a/functions/compose-eks-cluster/function/fn.py +++ b/functions/compose-eks-cluster/function/fn.py @@ -226,6 +226,12 @@ def _namespace(meta: metav1.ObjectMeta | None) -> str: _GPU_TAINT_VALUE = "true" _GPU_TAINT_EFFECT = "NO_SCHEDULE" +# Label and taint on CPU inference node groups. The label is what the +# serving stack's dra-driver-cpu selects its nodes by; the taint keeps +# system workloads off the group as the GPU taint does on GPU groups, +# and engine pods tolerate it (compose-model-replica). +_LABEL_CPU = "modelplane.ai/cpu" + # Kubernetes-API taint effect (as a toleration value, not the EC2 NO_SCHEDULE # form). The EFA DRA driver's DaemonSet must tolerate the GPU taint to run on # the GPU nodes, like the NVIDIA GPU DRA driver does. @@ -910,6 +916,18 @@ def compose_node_groups(self) -> None: effect=_GPU_TAINT_EFFECT, ), ] + elif pool.role == "CPU": + fp.labels = { + _LABEL_POOL: pool.name, + _LABEL_CPU: "true", + } + fp.taint = [ + ngv1beta1.TaintItem( + key=_LABEL_CPU, + value="true", + effect=_GPU_TAINT_EFFECT, + ), + ] resource.update( self.rsp.desired.resources[f"nodegroup-{pool.name}"], diff --git a/functions/compose-eks-cluster/tests/test_fn.py b/functions/compose-eks-cluster/tests/test_fn.py index d08a6a7f6..bd16d8e9c 100644 --- a/functions/compose-eks-cluster/tests/test_fn.py +++ b/functions/compose-eks-cluster/tests/test_fn.py @@ -631,6 +631,50 @@ def _nodegroup_gpu(*, cred_kind: str, cred_name: str) -> fnv1.Resource: ) +def _nodegroup_cpu(*, cred_kind: str, cred_name: str) -> fnv1.Resource: + """The composed cpu-pool node group, on the standard AMI with the CPU label and taint.""" + return fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "eks.aws.m.upbound.io/v1beta1", + "kind": "NodeGroup", + "spec": { + "providerConfigRef": {"kind": cred_kind, "name": cred_name}, + "managementPolicies": ["Observe", "Create", "Update", "Delete"], + "initProvider": {"scalingConfig": {"desiredSize": 1}}, + "forProvider": { + "region": "us-west-2", + "amiType": "AL2023_x86_64_STANDARD", + "instanceTypes": ["m6i.4xlarge"], + "diskSize": 100, + "clusterNameSelector": {"matchControllerRef": True}, + "nodeRoleArnSelector": { + "matchControllerRef": True, + "matchLabels": {"modelplane.ai/iam-role": "node"}, + }, + "subnetIdSelector": { + "matchControllerRef": True, + "matchLabels": {"modelplane.ai/subnet-tier": "private"}, + }, + "scalingConfig": {"minSize": 0, "maxSize": 4}, + "labels": { + "modelplane.ai/pool": "cpu-pool", + "modelplane.ai/cpu": "true", + }, + "taint": [ + { + "key": "modelplane.ai/cpu", + "value": "true", + "effect": "NO_SCHEDULE", + }, + ], + }, + }, + } + ), + ) + + def _launch_template_efa(*, security_groups: list[str] | None) -> fnv1.Resource: """The gpu-h200 EFA launch template, with security_groups on every interface if given.""" # p5en.48xlarge has 16 network cards; one EFA interface per card. @@ -1276,6 +1320,199 @@ def _to_dict(msg: message.Message) -> dict: context=structpb.Struct(), ), ), + Case( + name="FirstPassCPUPool", + reason="A CPU-role node pool composes its node group on the standard AMI with the modelplane.ai/pool and modelplane.ai/cpu labels and the CPU taint.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + node_pools=[ + v1alpha1.NodePool( + name="cpu-pool", + role="CPU", + instanceType="m6i.4xlarge", + nodeCount=1, + minNodeCount=0, + maxNodeCount=4, + ), + ], + ), + ), + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_xr(), + resources={ + "vpc": _vpc(cred_kind="ClusterProviderConfig", cred_name="default"), + "subnet-0": _subnet( + name="test-cluster-subnet-us-west-2a-952dc", + az="us-west-2a", + cidr="10.0.0.0/20", + cred_kind="ClusterProviderConfig", + cred_name="default", + ), + "subnet-1": _subnet( + name="test-cluster-subnet-us-west-2b-2b80f", + az="us-west-2b", + cidr="10.0.16.0/20", + cred_kind="ClusterProviderConfig", + cred_name="default", + ), + "subnet-2": _subnet( + name="test-cluster-subnet-us-west-2c-03273", + az="us-west-2c", + cidr="10.0.32.0/20", + cred_kind="ClusterProviderConfig", + cred_name="default", + ), + "private-subnet-0": _private_subnet( + name="test-cluster-private-subnet-us-west-2a-6a89f", + az="us-west-2a", + cidr="10.0.48.0/20", + cred_kind="ClusterProviderConfig", + cred_name="default", + ), + "private-subnet-1": _private_subnet( + name="test-cluster-private-subnet-us-west-2b-b7832", + az="us-west-2b", + cidr="10.0.64.0/20", + cred_kind="ClusterProviderConfig", + cred_name="default", + ), + "private-subnet-2": _private_subnet( + name="test-cluster-private-subnet-us-west-2c-ef57d", + az="us-west-2c", + cidr="10.0.80.0/20", + cred_kind="ClusterProviderConfig", + cred_name="default", + ), + "internet-gateway": _internet_gateway(cred_kind="ClusterProviderConfig", cred_name="default"), + "nat-eip": _nat_eip(cred_kind="ClusterProviderConfig", cred_name="default"), + "nat-gateway": _nat_gateway(cred_kind="ClusterProviderConfig", cred_name="default"), + "route-table": _route_table(cred_kind="ClusterProviderConfig", cred_name="default"), + "route-default": _route_default(cred_kind="ClusterProviderConfig", cred_name="default"), + "private-route-table": _private_route_table(cred_kind="ClusterProviderConfig", cred_name="default"), + "private-route-default": _private_route_default( + cred_kind="ClusterProviderConfig", cred_name="default" + ), + "route-table-association-0": _route_table_association( + az="us-west-2a", cred_kind="ClusterProviderConfig", cred_name="default" + ), + "route-table-association-1": _route_table_association( + az="us-west-2b", cred_kind="ClusterProviderConfig", cred_name="default" + ), + "route-table-association-2": _route_table_association( + az="us-west-2c", cred_kind="ClusterProviderConfig", cred_name="default" + ), + "private-route-table-association-0": _private_route_table_association( + az="us-west-2a", cred_kind="ClusterProviderConfig", cred_name="default" + ), + "private-route-table-association-1": _private_route_table_association( + az="us-west-2b", cred_kind="ClusterProviderConfig", cred_name="default" + ), + "private-route-table-association-2": _private_route_table_association( + az="us-west-2c", cred_kind="ClusterProviderConfig", cred_name="default" + ), + "iam-role-cluster": _cluster_role(cred_kind="ClusterProviderConfig", cred_name="default"), + "iam-attach-cluster-policy": _role_policy_attachment( + role="cluster", + arn="arn:aws:iam::aws:policy/AmazonEKSClusterPolicy", + cred_kind="ClusterProviderConfig", + cred_name="default", + ), + "iam-role-node": _node_role(cred_kind="ClusterProviderConfig", cred_name="default"), + "iam-attach-node-worker": _role_policy_attachment( + role="node", + arn="arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy", + cred_kind="ClusterProviderConfig", + cred_name="default", + ), + "iam-attach-node-cni": _role_policy_attachment( + role="node", + arn="arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy", + cred_kind="ClusterProviderConfig", + cred_name="default", + ), + "iam-attach-node-ecr": _role_policy_attachment( + role="node", + arn="arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly", + cred_kind="ClusterProviderConfig", + cred_name="default", + ), + "cluster": _eks_cluster( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "cluster-auth": _cluster_auth( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "nodegroup-system": _nodegroup_system(cred_kind="ClusterProviderConfig", cred_name="default"), + "nodegroup-cpu-pool": _nodegroup_cpu(cred_kind="ClusterProviderConfig", cred_name="default"), + "addon-vpc-cni": _addon(name="vpc-cni", cred_kind="ClusterProviderConfig", cred_name="default"), + "addon-kube-proxy": _addon( + name="kube-proxy", cred_kind="ClusterProviderConfig", cred_name="default" + ), + "addon-coredns": _addon(name="coredns", cred_kind="ClusterProviderConfig", cred_name="default"), + "efs-filesystem": _efs_filesystem( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "efs-security-group": _efs_security_group(cred_kind="ClusterProviderConfig", cred_name="default"), + "efs-security-group-ingress": _efs_security_group_ingress( + cred_kind="ClusterProviderConfig", cred_name="default" + ), + "efs-mount-target-0": _efs_mount_target( + subnet_name="test-cluster-private-subnet-us-west-2a-6a89f", + cred_kind="ClusterProviderConfig", + cred_name="default", + ), + "efs-mount-target-1": _efs_mount_target( + subnet_name="test-cluster-private-subnet-us-west-2b-b7832", + cred_kind="ClusterProviderConfig", + cred_name="default", + ), + "efs-mount-target-2": _efs_mount_target( + subnet_name="test-cluster-private-subnet-us-west-2c-ef57d", + cred_kind="ClusterProviderConfig", + cred_name="default", + ), + "iam-role-efs-csi": _pod_identity_role( + role="efs-csi", cred_kind="ClusterProviderConfig", cred_name="default" + ), + "iam-attach-efs-csi": _role_policy_attachment( + role="efs-csi", + arn="arn:aws:iam::aws:policy/service-role/AmazonEFSCSIDriverPolicy", + cred_kind="ClusterProviderConfig", + cred_name="default", + ), + "addon-eks-pod-identity-agent": _addon( + name="eks-pod-identity-agent", cred_kind="ClusterProviderConfig", cred_name="default" + ), + "pod-identity-efs-csi": _pod_identity_association( + cred_kind="ClusterProviderConfig", cred_name="default" + ), + "addon-aws-efs-csi-driver": _addon( + name="aws-efs-csi-driver", cred_kind="ClusterProviderConfig", cred_name="default" + ), + "iam-policy-cluster-autoscaler": _autoscaler_policy( + cred_kind="ClusterProviderConfig", cred_name="default" + ), + "iam-role-cluster-autoscaler": _pod_identity_role( + role="cluster-autoscaler", cred_kind="ClusterProviderConfig", cred_name="default" + ), + "iam-attach-cluster-autoscaler": _autoscaler_attachment( + cred_kind="ClusterProviderConfig", cred_name="default" + ), + "pod-identity-cluster-autoscaler": _autoscaler_pod_identity( + cred_kind="ClusterProviderConfig", cred_name="default" + ), + "provider-config-kubernetes": _provider_config(api_version="kubernetes.m.crossplane.io/v1alpha1"), + "provider-config-helm": _provider_config(api_version="helm.m.crossplane.io/v1beta1"), + }, + ), + context=structpb.Struct(), + ), + ), # The observed filesystem id lets the function compose the StorageClass # Object, which is ready, against the cluster's own ProviderConfig. The # observed cluster lets it compose the autoscaler Helm release, since diff --git a/functions/compose-gke-cluster/function/fn.py b/functions/compose-gke-cluster/function/fn.py index 18d550b99..e5a48445f 100644 --- a/functions/compose-gke-cluster/function/fn.py +++ b/functions/compose-gke-cluster/function/fn.py @@ -65,6 +65,14 @@ _LABEL_GPU = "modelplane.ai/gpu" _LABEL_POOL = "modelplane.ai/pool" +# Label and taint on CPU inference pools. The label is what the serving +# stack's dra-driver-cpu selects its nodes by; the taint keeps system +# workloads off the pool as the GPU taint does on GPU pools, and engine +# pods tolerate it (compose-model-replica). GKE auto-taints GPU pools +# with nvidia.com/gpu but knows nothing of CPU inference pools, so the +# taint is set explicitly here. +_LABEL_CPU = "modelplane.ai/cpu" + # GKE's cluster autoscaler uses this label to recognise a node pool as one whose # nodes run the NVIDIA GPU DRA driver, and so models the DRA ResourceSlices such # a node would publish. Without it the autoscaler can't tell that a new node @@ -390,6 +398,14 @@ def compose_node_pools(self) -> None: _LABEL_POOL: pool.name, _LABEL_GPU_DRA_DRIVER: "true", } + elif pool.role == "CPU": + node_config.labels = { + _LABEL_POOL: pool.name, + _LABEL_CPU: "true", + } + node_config.taint = [ + nodepoolv1beta1.TaintItem(key=_LABEL_CPU, value="true", effect="NO_SCHEDULE"), + ] else: node_config.labels = { _LABEL_POOL: pool.name, diff --git a/functions/compose-gke-cluster/tests/test_fn.py b/functions/compose-gke-cluster/tests/test_fn.py index c3ccd534d..fe12aabf4 100644 --- a/functions/compose-gke-cluster/tests/test_fn.py +++ b/functions/compose-gke-cluster/tests/test_fn.py @@ -279,6 +279,46 @@ def _nodepool_gpu(*, cred_kind: str, cred_name: str) -> fnv1.Resource: ) +def _nodepool_cpu(*, cred_kind: str, cred_name: str) -> fnv1.Resource: + """The composed NodePool for the XR's cpu-pool.""" + return fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "container.gcp.m.upbound.io/v1beta1", + "kind": "NodePool", + "spec": { + "providerConfigRef": {"kind": cred_kind, "name": cred_name}, + "forProvider": { + "location": "us-central1", + "clusterSelector": {"matchControllerRef": True}, + "initialNodeCount": 1, + "autoscaling": {"minNodeCount": 0, "maxNodeCount": 8}, + "nodeConfig": { + "machineType": "e2-standard-8", + "diskSizeGb": 100, + "imageType": "COS_CONTAINERD", + "oauthScopes": [ + "https://www.googleapis.com/auth/cloud-platform", + ], + "labels": { + "modelplane.ai/pool": "cpu-pool", + "modelplane.ai/cpu": "true", + }, + "taint": [ + { + "key": "modelplane.ai/cpu", + "value": "true", + "effect": "NO_SCHEDULE", + }, + ], + }, + }, + }, + } + ), + ) + + def _service_account(*, cred_kind: str, cred_name: str, ready: fnv1.Ready) -> fnv1.Resource: """The composed GCP ServiceAccount.""" return fnv1.Resource( @@ -421,6 +461,68 @@ def _to_dict(msg: message.Message) -> dict: ), ), ), + Case( + name="FirstPassCPUPool", + reason="A CPU-role node pool composes with the modelplane.ai/pool and modelplane.ai/cpu labels and the CPU taint, and no guest accelerator.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + node_pools=[ + v1alpha1.NodePool( + name="cpu-pool", + role="CPU", + machineType="e2-standard-8", + ), + ], + ), + ), + required_resources={ + "gcp-provider-config": fnv1.Resources( + items=[_gcp_provider_config(kind="ClusterProviderConfig", name="default", namespace=None)], + ), + }, + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_xr(), + resources={ + "network": _network( + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, + ), + "projectservice-filestore": _projectservice_filestore( + cred_kind="ClusterProviderConfig", + cred_name="default", + ), + "subnet": _subnet(cred_kind="ClusterProviderConfig", cred_name="default"), + "cluster": _cluster(cred_kind="ClusterProviderConfig", cred_name="default"), + "nodepool-system": _nodepool_system(cred_kind="ClusterProviderConfig", cred_name="default"), + "nodepool-cpu-pool": _nodepool_cpu(cred_kind="ClusterProviderConfig", cred_name="default"), + "service-account": _service_account( + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, + ), + "service-account-key": _service_account_key(cred_kind="ClusterProviderConfig", cred_name="default"), + "provider-config-kubernetes": _provider_config(api_version="kubernetes.m.crossplane.io/v1alpha1"), + "provider-config-helm": _provider_config(api_version="helm.m.crossplane.io/v1beta1"), + }, + ), + context=structpb.Struct(), + requirements=fnv1.Requirements( + resources={ + "gcp-provider-config": fnv1.ResourceSelector( + api_version="gcp.m.upbound.io/v1beta1", + kind="ClusterProviderConfig", + match_name="default", + ), + }, + ), + ), + ), # An XR with no composed resources would aggregate to trivially ready, so # the function marks it not ready. Case( diff --git a/functions/compose-inference-cluster/function/fn.py b/functions/compose-inference-cluster/function/fn.py index 87d0fd35a..85368f352 100644 --- a/functions/compose-inference-cluster/function/fn.py +++ b/functions/compose-inference-cluster/function/fn.py @@ -19,11 +19,12 @@ Existing) to determine how the cluster is obtained, then composes a ServingStack on it. -GPU node pools reference InferenceClasses. For provisioned clusters -the class's provisioning block describes how to build the pool; -for BYO (Existing) clusters the class is a pure description of pools -that already exist. Either way, the class's resources block populates -status.gpuPools so the scheduler can match models. +Node pools reference InferenceClasses. For provisioned clusters +the class's provisioning block describes how to build the pool (a GPU +pool when it names an accelerator, a CPU pool when it doesn't); for BYO +(Existing) clusters the class is a pure description of pools that +already exist. Either way, the class's devices populate +status.schedulablePools so the scheduler can match models. For provisioned clusters, a system node pool is injected automatically to host control-plane components (Envoy Gateway, Prometheus, etc.). @@ -83,6 +84,28 @@ CONDITION_REASON_BACKEND_HEALTHY = "BackendHealthy" CONDITION_REASON_INSTALLING = "Installing" CONDITION_REASON_INVALID_NODE_POOL = "InvalidNodePool" +CONDITION_REASON_UNSUPPORTED_DEVICES = "UnsupportedDevices" + +# The DRA drivers each cloud's serving stack supports. A hand-kept +# mirror of supported_drivers() in compose-serving-stack's stacks +# package - functions are self-contained by design, so keep the two in +# sync by hand. A stale mirror fails closed: it rejects a newly +# supported driver until someone updates it, and never admits an +# unsupported one, because the serving stack join's raise backstops +# that direction. EKS's dra.net arrives outside the serving stack +# (compose_eks_cluster installs the EFA DRA driver when a pool asks for +# the fabric). Existing is deliberately absent and never gates: a +# claimed driver beyond the stack's own installs nothing and is never +# rejected - the rule, and why, is stated on the stacks package's +# join() in compose-serving-stack. +_SUPPORTED_DRA_DRIVERS: dict[str, frozenset[str]] = { + CLUSTER_SOURCE_GKE: frozenset({"gpu.nvidia.com", "dra.cpu"}), + CLUSTER_SOURCE_EKS: frozenset({"gpu.nvidia.com", "dra.cpu", "dra.net"}), + CLUSTER_SOURCE_AKS: frozenset({"gpu.nvidia.com", "dra.cpu"}), + CLUSTER_SOURCE_NEBIUS: frozenset({"gpu.nvidia.com", "dra.cpu"}), + CLUSTER_SOURCE_VULTR: frozenset({"gpu.nvidia.com", "dra.cpu"}), + CLUSTER_SOURCE_CIVO: frozenset({"gpu.nvidia.com", "dra.cpu"}), +} # Composed resource key for the backend XR. BACKEND_RESOURCE_KEY = "serving-stack" @@ -318,6 +341,9 @@ def compose(self) -> None: return source = cluster.source + if not self.check_devices_supported(source): + return + if source == CLUSTER_SOURCE_GKE: self.compose_gke(cluster.gke) elif source == CLUSTER_SOURCE_EKS: @@ -546,9 +572,9 @@ def compose_pool_observations(self) -> None: """Compose a PoolObservation on the cluster for each of its GPU pools. The cluster agent reports each pool's nodes and free devices in the - status of its PoolObservation, which gpu_pools publishes. The agent and - the PoolObservation CRD arrive with the serving stack, so they're first - composed once the stack is ready, and kept after. + status of its PoolObservation, which schedulable_pools publishes. The + agent and the PoolObservation CRD arrive with the serving stack, so + they're first composed once the stack is ready, and kept after. provider-kubernetes watches each one, and polls it often in case the watch has stopped (see _POOL_OBSERVATION_POLL_INTERVAL), so the agent's @@ -567,7 +593,7 @@ def compose_pool_observations(self) -> None: ) pc = resource.child_name(_name(self.xr.metadata), "cluster-kubeconfig") for pool in self.xr.spec.nodePools or []: - # Only the pools gpu_pools publishes have devices to observe. + # Only the pools schedulable_pools publishes have devices to observe. cls = self.classes.get(pool.className) if not cls or not cls.spec.devices: continue @@ -677,6 +703,67 @@ def resolve_classes(self) -> bool: return True + def dra_drivers(self) -> list[str]: + """DRA driver names claimed across the referenced classes' devices. + + Collected verbatim onto ServingStack spec.draDrivers, which filters + the cloud's tagged components to the device stacks the cluster's + classes actually claim. Only claim: DRA devices count: a Synthetic + device is described for scheduling and never claimed, and its driver + is only the attribute domain its CEL selectors read, not an + installable stack. Sorted, so the ServingStack spec is stable across + reconciles. + """ + drivers: set[str] = set() + for cls in self.classes.values(): + for device in cls.spec.devices or []: + if (device.claim or "DRA") == "DRA": + drivers.add(device.driver) + return sorted(drivers) + + def check_devices_supported(self, source: Cloud) -> bool: + """Gate on a class claiming a driver the cloud's stack can't serve. + + Checked here, before any cluster or ServingStack is composed, because + the cluster would provision devices nothing can drive. A typo'd or + not-yet-supported driver lands here too; unknown and unsupported are + the same failure. Existing never gates (see _SUPPORTED_DRA_DRIVERS). + Gating is a fatal result, so a cluster already running when a class + turns unsupported holds its existing resources. + """ + supported = _SUPPORTED_DRA_DRIVERS.get(source) + if supported is None: + return True + for name in sorted(self.classes): + cls = self.classes[name] + claimed = {d.driver for d in (cls.spec.devices or []) if (d.claim or "DRA") == "DRA"} + unsupported = claimed - supported + if not unsupported: + continue + msg = ( + f"class {name} claims {', '.join(sorted(unsupported))}; " + f"the {source} serving stack installs {', '.join(sorted(supported))}" + ) + response.set_conditions( + self.rsp, + resource.Condition( + typ=CONDITION_TYPE_CLUSTER_READY, + status="False", + reason=CONDITION_REASON_UNSUPPORTED_DEVICES, + message=msg, + ), + ) + # A fatal result, not a bare return: everything after this gate + # composes the cluster, so returning an empty desired state would + # have Crossplane garbage-collect a running cluster - the cloud + # cluster XR, ServingStack and Usages - the moment a shared class + # starts claiming an unsupported driver. Crossplane stops before + # applying desired state or collecting anything on a fatal result, + # and still applies the condition above. + response.fatal(self.rsp, msg) + return False + return True + def compose_gke(self, gke: v1alpha1.Gke | None) -> None: """Compose an InferenceCluster backed by a Modelplane-provisioned GKE cluster. Composes the GKECluster XR, waits for it to be ready, @@ -714,7 +801,7 @@ def compose_gke(self, gke: v1alpha1.Gke | None) -> None: if not backend_exists: response.normal(self.rsp, "GKE cluster ready, composing backend") - self.write_status(self.gpu_pools()) + self.write_status(self.schedulable_pools()) self.derive_conditions(cluster_ready=gke_ready) def compose_eks(self, eks: v1alpha1.Eks | None) -> None: @@ -755,7 +842,7 @@ def compose_eks(self, eks: v1alpha1.Eks | None) -> None: if not backend_exists: response.normal(self.rsp, "EKS cluster ready, composing backend") - self.write_status(self.gpu_pools()) + self.write_status(self.schedulable_pools()) self.derive_conditions(cluster_ready=eks_ready) def compose_aks(self, aks: v1alpha1.Aks | None) -> None: @@ -794,7 +881,7 @@ def compose_aks(self, aks: v1alpha1.Aks | None) -> None: if not backend_exists: response.normal(self.rsp, "AKS cluster ready, composing backend") - self.write_status(self.gpu_pools()) + self.write_status(self.schedulable_pools()) self.derive_conditions(cluster_ready=aks_ready) def compose_nebius(self, nebius: v1alpha1.Nebius | None) -> None: @@ -843,7 +930,7 @@ def compose_nebius(self, nebius: v1alpha1.Nebius | None) -> None: if not backend_exists: response.normal(self.rsp, "Nebius cluster ready, composing backend") - self.write_status(self.gpu_pools()) + self.write_status(self.schedulable_pools()) self.derive_conditions(cluster_ready=nebius_ready) def compose_vultr(self, vultr: v1alpha1.Vultr | None) -> None: @@ -882,7 +969,7 @@ def compose_vultr(self, vultr: v1alpha1.Vultr | None) -> None: if not backend_exists: response.normal(self.rsp, "Vultr cluster ready, composing backend") - self.write_status(self.gpu_pools()) + self.write_status(self.schedulable_pools()) self.derive_conditions(cluster_ready=vultr_ready) def compose_civo(self, civo: v1alpha1.Civo | None) -> None: @@ -927,7 +1014,7 @@ def compose_civo(self, civo: v1alpha1.Civo | None) -> None: if not backend_exists: response.normal(self.rsp, "Civo cluster ready, composing backend") - self.write_status(self.gpu_pools()) + self.write_status(self.schedulable_pools()) self.derive_conditions(cluster_ready=civo_ready) def compose_existing(self, existing: v1alpha1.Existing | None) -> None: @@ -956,7 +1043,7 @@ def compose_existing(self, existing: v1alpha1.Existing | None) -> None: ) self.compose_serving_stack(backend_secrets, CLUSTER_SOURCE_EXISTING) - self.write_status(self.gpu_pools()) + self.write_status(self.schedulable_pools()) self.derive_conditions(cluster_ready=True) def compose_serving_stack( @@ -971,9 +1058,11 @@ def compose_serving_stack( cloud names the cluster's source (this XR's spec.cluster.source) and selects the component list the serving stack installs, including cloud specifics like where the node image puts the - NVIDIA driver. gpu carries per-pool driver configuration the - component list can't know at build time (see - civo_nvlink_disabled_pools). + NVIDIA driver. spec.draDrivers is always set - an empty list when + no class claims a device - so the stack filters its tagged device + stacks to the drivers the classes claim. gpu carries per-pool + driver configuration the component list can't know at build time + (see civo_nvlink_disabled_pools). """ # The gateway's name and the CAs it should accept client certificates # from. The name is Modelplane's own, derived from this cluster's name; @@ -990,6 +1079,7 @@ def compose_serving_stack( stack=self.xr.spec.stack, cloud=cloud, gateway=gateway, + draDrivers=[ssv1alpha1.DraDriver(d) for d in self.dra_drivers()], ) if gpu is not None: spec.gpu = gpu @@ -1051,14 +1141,14 @@ def compose_cluster_provider_config( ) self.rsp.desired.resources["cluster-provider-config-kubernetes"].ready = fnv1.READY_TRUE - def write_status(self, gpu_pools: list[v1alpha1.GpuPool]) -> None: + def write_status(self, schedulable_pools: list[v1alpha1.SchedulablePool]) -> None: """Write the InferenceCluster status.""" status = v1alpha1.Status( providerConfigRef=v1alpha1.ProviderConfigRef( name=resource.child_name(_name(self.xr.metadata), "cluster-kubeconfig"), ), namespace=_NAMESPACE_SYSTEM, - gpuPools=gpu_pools, + schedulablePools=schedulable_pools, ) cache_storage_class = self.observed_cache_storage_class() if cache_storage_class: @@ -1158,22 +1248,30 @@ def compose_gke_cluster(self, gke: v1alpha1.Gke) -> None: response.warning(self.rsp, msg) return prov = cls.spec.provisioning.gke - gke_node_pools.append( - gkev1alpha1.NodePool( - name=pool.name, - role="GPU", - machineType=prov.machineType, - diskSizeGb=prov.diskSizeGb, - nodeCount=pool.nodeCount, - minNodeCount=pool.minNodeCount, - maxNodeCount=pool.maxNodeCount, - gpu=gkev1alpha1.Gpu( - acceleratorType=prov.accelerator.type, - acceleratorCount=prov.accelerator.count, - ), - zones=[gkev1alpha1.Zone(z) for z in pool.zones or []], - ) + # A class with no accelerator describes a CPU pool: a standard + # node image, no GPU attachment, and the modelplane.ai/cpu + # label and taint composed by compose-gke-cluster. + node_pool = gkev1alpha1.NodePool( + name=pool.name, + role="GPU" if prov.accelerator else "CPU", + machineType=prov.machineType, + diskSizeGb=prov.diskSizeGb, + nodeCount=pool.nodeCount, + minNodeCount=pool.minNodeCount, + maxNodeCount=pool.maxNodeCount, ) + if prov.accelerator: + node_pool.gpu = gkev1alpha1.Gpu( + acceleratorType=prov.accelerator.type, + acceleratorCount=prov.accelerator.count, + ) + # Only set zones when the pool names some: GKE requires a + # non-empty list when zones are set, and resource.update + # serializes with exclude_unset, so leaving it unset keeps an + # empty list out of the GKECluster spec. + if pool.zones: + node_pool.zones = [gkev1alpha1.Zone(z) for z in pool.zones] + gke_node_pools.append(node_pool) gke_spec = gkev1alpha1.Spec( region=gke.region, @@ -1221,19 +1319,26 @@ def compose_eks_cluster(self, eks: v1alpha1.Eks) -> None: response.warning(self.rsp, msg) return prov = cls.spec.provisioning.eks + # A class with no accelerator describes a CPU pool (see the GKE + # builder above). node_pool = eksv1alpha1.NodePool( name=pool.name, - role="GPU", + role="GPU" if prov.accelerator else "CPU", instanceType=prov.instanceType, diskSizeGb=prov.diskSizeGb, nodeCount=pool.nodeCount, minNodeCount=pool.minNodeCount, maxNodeCount=pool.maxNodeCount, - gpu=eksv1alpha1.Gpu( - acceleratorType=prov.accelerator.type, - ), - zones=[eksv1alpha1.Zone(z) for z in pool.zones or []], ) + if prov.accelerator: + node_pool.gpu = eksv1alpha1.Gpu( + acceleratorType=prov.accelerator.type, + ) + # Only set zones when the pool names some, so an unzoned pool's + # EKSCluster spec stays free of zones: [] (resource.update + # serializes with exclude_unset). + if pool.zones: + node_pool.zones = [eksv1alpha1.Zone(z) for z in pool.zones] # Only set capacityBlock when the pool has one. resource.update # serializes with exclude_unset, so leaving it unset keeps it out # of the EKSCluster spec rather than emitting capacityBlock: null. @@ -1293,18 +1398,21 @@ def compose_aks_cluster(self, aks: v1alpha1.Aks) -> None: response.warning(self.rsp, msg) return prov = cls.spec.provisioning.aks + # A class with no accelerator describes a CPU pool (see the GKE + # builder above). node_pool = aksv1alpha1.NodePool( name=pool.name, - role="GPU", + role="GPU" if prov.accelerator else "CPU", vmSize=prov.vmSize, diskSizeGb=prov.diskSizeGb, nodeCount=pool.nodeCount, minNodeCount=pool.minNodeCount, maxNodeCount=pool.maxNodeCount, - gpu=aksv1alpha1.Gpu( - acceleratorType=prov.accelerator.type, - ), ) + if prov.accelerator: + node_pool.gpu = aksv1alpha1.Gpu( + acceleratorType=prov.accelerator.type, + ) # Only set zones when the pool names some: the AKSCluster XRD # requires a non-empty list, and many GPU VM sizes aren't zonal. if pool.zones: @@ -1364,18 +1472,22 @@ def compose_nebius_cluster(self, nebius: v1alpha1.Nebius) -> None: response.warning(self.rsp, msg) return prov = cls.spec.provisioning.nebius + # A class with no accelerator describes a CPU pool (see the GKE + # builder above). driversPreset rides the gpu block: a CPU + # platform preinstalls no NVIDIA stack. node_pool = nebiusv1alpha1.NodePool( name=pool.name, - role="GPU", + role="GPU" if prov.accelerator else "CPU", platform=prov.platform, preset=prov.preset, diskSizeGb=prov.diskSizeGb, nodeCount=pool.nodeCount, - gpu=nebiusv1alpha1.Gpu( + ) + if prov.accelerator: + node_pool.gpu = nebiusv1alpha1.Gpu( acceleratorType=prov.accelerator.type, driversPreset=prov.driversPreset, - ), - ) + ) # mk8s node groups are either fixed size or autoscaled, never # both. Only set the autoscaling bounds when the pool opts in, so # fixed-size pools stay fixed (resource.update serializes with @@ -1521,15 +1633,18 @@ def compose_vultr_cluster(self, vultr: v1alpha1.Vultr) -> None: response.warning(self.rsp, msg) return prov = cls.spec.provisioning.vultr + # A class with no accelerator describes a CPU pool (see the GKE + # builder above). node_pool = vultrv1alpha1.NodePool( name=pool.name, - role="GPU", + role="GPU" if prov.accelerator else "CPU", plan=prov.plan, nodeCount=pool.nodeCount, - gpu=vultrv1alpha1.Gpu( - acceleratorType=prov.accelerator.type, - ), ) + if prov.accelerator: + node_pool.gpu = vultrv1alpha1.Gpu( + acceleratorType=prov.accelerator.type, + ) # Only set the autoscaling bounds when the pool opts in, so # fixed-size pools stay fixed (resource.update serializes with # exclude_unset, keeping unset bounds out of the VultrCluster @@ -1639,7 +1754,7 @@ def civo_nvlink_disabled_pools(self) -> list[str]: if not cls or not cls.spec.provisioning or not cls.spec.provisioning.civo: continue accelerator = cls.spec.provisioning.civo.accelerator - if accelerator.type == _ACCELERATOR_H100 and accelerator.count == 1: + if accelerator and accelerator.type == _ACCELERATOR_H100 and accelerator.count == 1: pools.append(pool.name) return pools @@ -1668,15 +1783,18 @@ def compose_civo_cluster(self, civo: v1alpha1.Civo) -> None: response.warning(self.rsp, msg) return prov = cls.spec.provisioning.civo + # A class with no accelerator describes a CPU pool (see the GKE + # builder above). node_pool = civov1alpha1.NodePool( name=pool.name, - role="GPU", + role="GPU" if prov.accelerator else "CPU", size=prov.size, nodeCount=pool.nodeCount, - gpu=civov1alpha1.Gpu( - acceleratorType=prov.accelerator.type, - ), ) + if prov.accelerator: + node_pool.gpu = civov1alpha1.Gpu( + acceleratorType=prov.accelerator.type, + ) # Only set the autoscaling bounds when the pool opts in, so # fixed-size pools stay fixed (resource.update serializes with # exclude_unset, keeping unset bounds out of the CivoCluster @@ -1931,17 +2049,18 @@ def resolve_gke_backend_secrets(self, *, gke_ready: bool, backend_exists: bool) return None - def gpu_pools(self) -> list[v1alpha1.GpuPool]: - """Derive status.gpuPools from each node pool's class and observation. + def schedulable_pools(self) -> list[v1alpha1.SchedulablePool]: + """Derive status.schedulablePools from each pool's class and observation. The class declares the node's devices (DRA-style); the pool declares how many nodes it can have. We copy the class's devices verbatim so ModelDeployment.nodeSelector can match against them, and record the node count. Once the cluster agent has observed the pool, we also record the nodes it has now and how many devices are free on them, which the - scheduler fits replicas against. + scheduler fits replicas against. Nothing here is specific to GPUs: a CPU + pool's dra.cpu device lists like any other. """ - gpu_pools = [] + pools = [] for pool in self.xr.spec.nodePools or []: cls = self.classes.get(pool.className) if not cls or not cls.spec.devices: @@ -1951,7 +2070,7 @@ def gpu_pools(self) -> list[v1alpha1.GpuPool]: # the typed attribute value objects (string/version/bool/int) one-of # clean. by_alias keeps DRA's wire names (bool/int) rather than the # generated bool_/int_ attributes. - gpu_pool = v1alpha1.GpuPool( + schedulable = v1alpha1.SchedulablePool( name=pool.name, nodes=pool.maxNodeCount or pool.nodeCount, devices=[ @@ -1961,17 +2080,17 @@ def gpu_pools(self) -> list[v1alpha1.GpuPool]: ) observed = self.observed_pool(pool.name) if observed is not None: - gpu_pool.observedNodes = observed.nodes.total - gpu_pool.nodesByFreeDevices = [ + schedulable.observedNodes = observed.nodes.total + schedulable.nodesByFreeDevices = [ v1alpha1.NodesByFreeDevice.model_validate(b.model_dump(exclude_none=True)) for b in observed.nodesByFreeDevices or [] ] # The agent reports a time once a claim has allocated devices # on the pool. if observed.lastAllocationTime is not None: - gpu_pool.lastAllocationTime = observed.lastAllocationTime - gpu_pools.append(gpu_pool) - return gpu_pools + schedulable.lastAllocationTime = observed.lastAllocationTime + pools.append(schedulable) + return pools def observed_gke_secrets(self) -> list[gkev1alpha1.Secret] | None: """Read the GKECluster's status.secrets from observed state.""" diff --git a/functions/compose-inference-cluster/tests/test_fn.py b/functions/compose-inference-cluster/tests/test_fn.py index 1da0a9baa..52331afdd 100644 --- a/functions/compose-inference-cluster/tests/test_fn.py +++ b/functions/compose-inference-cluster/tests/test_fn.py @@ -61,12 +61,14 @@ def _inference_cluster(*, cluster: v1alpha1.Cluster, node_pools: list[v1alpha1.N ) -def _desired_inference_cluster(*, gpu_pools: list[dict], cache: dict | None, gateway: dict | None) -> fnv1.Resource: +def _desired_inference_cluster( + *, schedulable_pools: list[dict], cache: dict | None, gateway: dict | None +) -> fnv1.Resource: """The desired InferenceCluster XR's status, with its cache and gateway unless they're None.""" status: dict = { "providerConfigRef": {"name": "test-cluster-cluster-kubeconfig-d0f89"}, "namespace": "modelplane-system", - "gpuPools": gpu_pools, + "schedulablePools": schedulable_pools, } if cache is not None: status["cache"] = cache @@ -101,6 +103,32 @@ def _inference_class(*, name: str, count: int, memory: str, provisioning: dict) ) +def _cpu_inference_class(*, name: str, cpus: str, provisioning: dict) -> fnv1.Resource: + """An InferenceClass of one DRA-claimed CPU device of cpus cores, as its class requirement returns it.""" + return fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "modelplane.ai/v1alpha1", + "kind": "InferenceClass", + "metadata": {"name": name}, + "spec": { + "devices": [ + { + "name": "cpu", + "claim": "DRA", + "driver": "dra.cpu", + "deviceClassName": "dra.cpu", + "count": 1, + "capacity": {"cpu": {"value": cpus}}, + }, + ], + "provisioning": provisioning, + }, + } + ) + ) + + def _inference_gateway(*, name: str, cluster: str, status: dict | None) -> fnv1.Resource: """An InferenceGateway on cluster, as the gateways requirement returns it, with status unless it's None.""" gateway: dict = { @@ -267,6 +295,33 @@ def _vultr_cluster(*, credentials: dict | None, ready: fnv1.Ready) -> fnv1.Resou ) +def _vultr_cluster_cpu(*, ready: fnv1.Ready) -> fnv1.Resource: + """The composed VultrCluster with cpu-pool, a CPU-role pool with no gpu block.""" + return fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "infrastructure.modelplane.ai/v1alpha1", + "kind": "VultrCluster", + "metadata": {"name": "test-cluster", "namespace": "modelplane-system"}, + "spec": { + "region": "ewr", + "kubernetesVersion": "v1.36.2+1", + "nodePools": [ + { + "name": "cpu-pool", + "role": "CPU", + "plan": "vc2-6c-16gb", + "nodeCount": 2, + "maxNodeCount": 4, + }, + ], + }, + } + ), + ready=ready, + ) + + def _civo_cluster(*, node_pools: list[dict], credentials: dict | None, ready: fnv1.Ready) -> fnv1.Resource: """The composed CivoCluster in LON1 with node_pools, using credentials unless they're None.""" spec: dict = {"region": "LON1", "nodePools": node_pools} @@ -309,13 +364,29 @@ def _cluster_provider_config(*, kubeconfig: str, identity: dict | None) -> fnv1. def _serving_stack( - *, cloud: str, secrets: list[dict], client_cas: list[dict] | None, gpu: dict | None, ready: fnv1.Ready + *, + cloud: str, + secrets: list[dict], + client_cas: list[dict] | None, + gpu: dict | None, + ready: fnv1.Ready, + dra_drivers: tuple[str, ...], ) -> fnv1.Resource: - """The composed ServingStack, its gateway accepting client_cas and its spec.gpu set to gpu unless they're None.""" + """The composed ServingStack, its gateway accepting client_cas and its spec.gpu set to gpu unless they're None. + + dra_drivers is the drivers the cluster's classes claim, always stamped + on the spec. + """ gateway: dict = {"hostname": "gateway-test-cluster-09532.modelplane-system.svc.cluster.local"} if client_cas is not None: gateway["clientCAs"] = client_cas - spec: dict = {"cloud": cloud, "gateway": gateway, "stack": "Standard", "secrets": secrets} + spec: dict = { + "cloud": cloud, + "gateway": gateway, + "stack": "Standard", + "secrets": secrets, + "draDrivers": list(dra_drivers), + } if gpu is not None: spec["gpu"] = gpu return fnv1.Resource( @@ -544,7 +615,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[ + schedulable_pools=[ { "name": "l4-pool", "nodes": 4, @@ -569,6 +640,7 @@ def _to_dict(msg: message.Message) -> dict: ), "serving-stack": _serving_stack( cloud="Existing", + dra_drivers=("gpu.nvidia.com",), secrets=[{"type": "Kubeconfig", "name": "my-kubeconfig", "key": "kubeconfig"}], client_cas=None, gpu=None, @@ -648,7 +720,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[ + schedulable_pools=[ { "name": "l4-pool", "nodes": 4, @@ -682,6 +754,7 @@ def _to_dict(msg: message.Message) -> dict: ), "serving-stack": _serving_stack( cloud="Existing", + dra_drivers=("gpu.nvidia.com",), secrets=[ {"type": "Kubeconfig", "name": "my-kubeconfig", "key": "kubeconfig"}, { @@ -778,7 +851,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[ + schedulable_pools=[ { "name": "l4-pool", "nodes": 4, @@ -902,7 +975,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[ + schedulable_pools=[ { "name": "l4-pool", "nodes": 4, @@ -1008,7 +1081,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[ + schedulable_pools=[ { "name": "l4-pool", "nodes": 4, @@ -1033,6 +1106,7 @@ def _to_dict(msg: message.Message) -> dict: ), "serving-stack": _serving_stack( cloud="Existing", + dra_drivers=("gpu.nvidia.com",), secrets=[{"type": "Kubeconfig", "name": "my-kubeconfig", "key": "kubeconfig"}], client_cas=None, gpu=None, @@ -1142,7 +1216,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[ + schedulable_pools=[ { "name": "gpu-l4x4-a", "nodes": 3, @@ -1174,6 +1248,7 @@ def _to_dict(msg: message.Message) -> dict: client_cas=None, gpu=None, ready=fnv1.READY_TRUE, + dra_drivers=("gpu.nvidia.com",), ), "pool-observation-gpu-l4x4-a": _pool_observation(pool="gpu-l4x4-a"), }, @@ -1273,7 +1348,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[ + schedulable_pools=[ { "name": "gpu-l4x4-a", "nodes": 3, @@ -1302,6 +1377,7 @@ def _to_dict(msg: message.Message) -> dict: client_cas=None, gpu=None, ready=fnv1.READY_TRUE, + dra_drivers=("gpu.nvidia.com",), ), "pool-observation-gpu-l4x4-a": _pool_observation(pool="gpu-l4x4-a"), }, @@ -1407,7 +1483,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[ + schedulable_pools=[ { "name": "gpu-l4x4-a", "nodes": 3, @@ -1439,6 +1515,7 @@ def _to_dict(msg: message.Message) -> dict: client_cas=None, gpu=None, ready=fnv1.READY_TRUE, + dra_drivers=("gpu.nvidia.com",), ), "pool-observation-gpu-l4x4-a": _pool_observation(pool="gpu-l4x4-a"), }, @@ -1543,7 +1620,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[ + schedulable_pools=[ { "name": "gpu-l4x4-a", "nodes": 3, @@ -1572,6 +1649,7 @@ def _to_dict(msg: message.Message) -> dict: client_cas=None, gpu=None, ready=fnv1.READY_TRUE, + dra_drivers=("gpu.nvidia.com",), ), "pool-observation-gpu-l4x4-a": _pool_observation(pool="gpu-l4x4-a"), }, @@ -1695,7 +1773,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[ + schedulable_pools=[ { "name": "gpu-l4x4-a", "nodes": 3, @@ -1726,6 +1804,7 @@ def _to_dict(msg: message.Message) -> dict: client_cas=None, gpu=None, ready=fnv1.READY_UNSPECIFIED, + dra_drivers=("gpu.nvidia.com",), ), "pool-observation-gpu-l4x4-a": _pool_observation(pool="gpu-l4x4-a"), }, @@ -1826,7 +1905,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[ + schedulable_pools=[ { "name": "l4-pool", "nodes": 4, @@ -1999,7 +2078,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[ + schedulable_pools=[ { "name": "l4-pool", "nodes": 4, @@ -2174,7 +2253,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[ + schedulable_pools=[ { "name": "l4-pool", "nodes": 4, @@ -2222,6 +2301,7 @@ def _to_dict(msg: message.Message) -> dict: ), "serving-stack": _serving_stack( cloud="GKE", + dra_drivers=("gpu.nvidia.com",), secrets=[ {"type": "Kubeconfig", "name": "test-cluster-kubeconfig-abcde", "key": "kubeconfig"}, { @@ -2367,7 +2447,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[ + schedulable_pools=[ { "name": "l4-pool", "nodes": 4, @@ -2422,6 +2502,7 @@ def _to_dict(msg: message.Message) -> dict: ), "serving-stack": _serving_stack( cloud="EKS", + dra_drivers=("gpu.nvidia.com",), secrets=[{"type": "Kubeconfig", "name": "test-cluster-kubeconfig-abcde", "key": "kubeconfig"}], client_cas=None, gpu=None, @@ -2532,7 +2613,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[ + schedulable_pools=[ { "name": "l4-pool", "nodes": 4, @@ -2686,7 +2767,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[ + schedulable_pools=[ { "name": "l4-pool", "nodes": 4, @@ -2824,7 +2905,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[ + schedulable_pools=[ { "name": "h100-pool", "nodes": 4, @@ -3012,7 +3093,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[ + schedulable_pools=[ { "name": "h100-pool", "nodes": 4, @@ -3083,6 +3164,7 @@ def _to_dict(msg: message.Message) -> dict: ), "serving-stack": _serving_stack( cloud="Nebius", + dra_drivers=("gpu.nvidia.com",), secrets=[ {"type": "Kubeconfig", "name": "test-cluster-kubeconfig-abcde", "key": "kubeconfig"}, { @@ -3183,7 +3265,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[ + schedulable_pools=[ { "name": "h100pool", "nodes": 4, @@ -3361,7 +3443,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[ + schedulable_pools=[ { "name": "h100pool", "nodes": 4, @@ -3423,6 +3505,7 @@ def _to_dict(msg: message.Message) -> dict: ), "serving-stack": _serving_stack( cloud="AKS", + dra_drivers=("gpu.nvidia.com",), secrets=[{"type": "Kubeconfig", "name": "test-cluster-kubeconfig-abcde", "key": "kubeconfig"}], client_cas=None, gpu=None, @@ -3518,7 +3601,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[ + schedulable_pools=[ { "name": "l4-pool", "nodes": 4, @@ -3662,7 +3745,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[ + schedulable_pools=[ { "name": "l4-pool", "nodes": 4, @@ -3710,6 +3793,7 @@ def _to_dict(msg: message.Message) -> dict: ), "serving-stack": _serving_stack( cloud="GKE", + dra_drivers=("gpu.nvidia.com",), secrets=[ {"type": "Kubeconfig", "name": "test-cluster-kubeconfig-abcde", "key": "kubeconfig"}, { @@ -3794,7 +3878,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[ + schedulable_pools=[ { "name": "l40s-pool", "nodes": 4, @@ -3893,7 +3977,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[ + schedulable_pools=[ { "name": "l40s-pool", "nodes": 4, @@ -4028,7 +4112,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[ + schedulable_pools=[ { "name": "l40s-pool", "nodes": 4, @@ -4058,6 +4142,7 @@ def _to_dict(msg: message.Message) -> dict: ), "serving-stack": _serving_stack( cloud="Vultr", + dra_drivers=("gpu.nvidia.com",), secrets=[{"type": "Kubeconfig", "name": "test-cluster-kubeconfig-abcde", "key": "kubeconfig"}], client_cas=None, gpu=None, @@ -4093,6 +4178,148 @@ def _to_dict(msg: message.Message) -> dict: ], ), ), + # A class with no accelerator describes a CPU pool: the VultrCluster pool is + # CPU-role with no gpu block, the ServingStack installs the dra.cpu stack, + # and the class's CPU device lists in schedulablePools like any other. + ComposeCase( + name="VultrReadyCPUPool", + reason="Once its VultrCluster is Ready, a Vultr cluster whose pool's class names no accelerator composes a CPU-role pool, a ServingStack claiming dra.cpu and the Usage.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_inference_cluster( + cluster=v1alpha1.Cluster(source="Vultr", vultr=v1alpha1.Vultr(region="ewr")), + node_pools=[ + v1alpha1.NodePool(name="cpu-pool", className="cpu-vultr", nodeCount=2, maxNodeCount=4), + ], + ), + resources={ + "vultr-cluster": fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "infrastructure.modelplane.ai/v1alpha1", + "kind": "VultrCluster", + "metadata": {"name": "test-cluster", "namespace": "modelplane-system"}, + "spec": { + "region": "ewr", + "nodePools": [ + { + "name": "cpu-pool", + "role": "CPU", + "plan": "vc2-6c-16gb", + "nodeCount": 2, + }, + ], + }, + "status": { + "conditions": [ + { + "type": "Ready", + "status": "True", + "reason": "Available", + "lastTransitionTime": "2024-01-01T00:00:00Z", + }, + ], + "secrets": [ + { + "type": "Kubeconfig", + "name": "test-cluster-kubeconfig-abcde", + "key": "kubeconfig", + }, + ], + }, + } + ), + ), + "activation": _observed_activation_policy( + activated=["kubernetes.vke.vultr.m.upbound.io", "kubernetesnodepools.vke.vultr.m.upbound.io"] + ), + }, + ), + required_resources={ + "class-cpu-vultr": fnv1.Resources( + items=[ + _cpu_inference_class( + name="cpu-vultr", + cpus="6", + provisioning={ + "provider": "Vultr", + "vultr": {"plan": "vc2-6c-16gb"}, + }, + ), + ], + ), + }, + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_inference_cluster( + schedulable_pools=[ + { + "name": "cpu-pool", + "nodes": 4, + "devices": [ + { + "name": "cpu", + "claim": "DRA", + "driver": "dra.cpu", + "deviceClassName": "dra.cpu", + "count": 1, + "capacity": {"cpu": {"value": "6"}}, + }, + ], + }, + ], + cache=None, + gateway=None, + ), + resources={ + "activation": _activation_policy( + activate=["kubernetes.vke.vultr.m.upbound.io", "kubernetesnodepools.vke.vultr.m.upbound.io"], + ready=fnv1.READY_TRUE, + ), + "vultr-cluster": _vultr_cluster_cpu(ready=fnv1.READY_TRUE), + "cluster-provider-config-kubernetes": _cluster_provider_config( + kubeconfig="test-cluster-kubeconfig-abcde", identity=None + ), + "serving-stack": _serving_stack( + cloud="Vultr", + secrets=[{"type": "Kubeconfig", "name": "test-cluster-kubeconfig-abcde", "key": "kubeconfig"}], + client_cas=None, + gpu=None, + ready=fnv1.READY_UNSPECIFIED, + dra_drivers=("dra.cpu",), + ), + "usage-vultr-by-backend": _backend_usage(cluster_kind="VultrCluster"), + }, + ), + results=[fnv1.Result(severity=fnv1.SEVERITY_NORMAL, message="Vultr cluster ready, composing backend")], + context=structpb.Struct(), + requirements=fnv1.Requirements( + resources={ + "model-replicas": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", + kind="ModelReplica", + match_labels=fnv1.MatchLabels(labels={"modelplane.ai/cluster": "test-cluster"}), + ), + "model-routes": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", + kind="ModelRoute", + match_labels=fnv1.MatchLabels(labels={"modelplane.ai/cluster": "test-cluster"}), + ), + "model-caches": fnv1.ResourceSelector(api_version="modelplane.ai/v1alpha1", kind="ModelCache"), + "gateways": fnv1.ResourceSelector(api_version="modelplane.ai/v1alpha1", kind="InferenceGateway"), + "class-cpu-vultr": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", kind="InferenceClass", match_name="cpu-vultr" + ), + }, + ), + conditions=[ + fnv1.Condition(type="ClusterReady", status=fnv1.STATUS_CONDITION_TRUE, reason="ClusterRunning"), + fnv1.Condition(type="BackendReady", status=fnv1.STATUS_CONDITION_FALSE, reason="Installing"), + ], + ), + ), # minNodeCount stays unset so the pool's autoscaling floor defaults to its # node count downstream. ComposeCase( @@ -4140,7 +4367,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[ + schedulable_pools=[ { "name": "l40s-pool", "nodes": 4, @@ -4262,7 +4489,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[ + schedulable_pools=[ { "name": "l40s-pool", "nodes": 4, @@ -4418,7 +4645,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[ + schedulable_pools=[ { "name": "l40s-pool", "nodes": 4, @@ -4466,6 +4693,7 @@ def _to_dict(msg: message.Message) -> dict: ), "serving-stack": _serving_stack( cloud="Civo", + dra_drivers=("gpu.nvidia.com",), secrets=[{"type": "Kubeconfig", "name": "test-cluster-kubeconfig-abcde", "key": "kubeconfig"}], client_cas=None, gpu=None, @@ -4606,7 +4834,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[ + schedulable_pools=[ { "name": "h100-pool", "nodes": 1, @@ -4653,6 +4881,7 @@ def _to_dict(msg: message.Message) -> dict: ), "serving-stack": _serving_stack( cloud="Civo", + dra_drivers=("gpu.nvidia.com",), secrets=[{"type": "Kubeconfig", "name": "test-cluster-kubeconfig-abcde", "key": "kubeconfig"}], client_cas=None, gpu={"pools": [{"name": "h100-pool", "disableNvLink": True}]}, @@ -4802,7 +5031,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[ + schedulable_pools=[ { "name": "l4-pool", "nodes": 4, @@ -4834,6 +5063,7 @@ def _to_dict(msg: message.Message) -> dict: ), "serving-stack": _serving_stack( cloud="Existing", + dra_drivers=("gpu.nvidia.com",), secrets=[{"type": "Kubeconfig", "name": "my-kubeconfig", "key": "kubeconfig"}], client_cas=None, gpu=None, @@ -4925,7 +5155,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[ + schedulable_pools=[ { "name": "l4-pool", "nodes": 4, @@ -4952,6 +5182,7 @@ def _to_dict(msg: message.Message) -> dict: ), "serving-stack": _serving_stack( cloud="Existing", + dra_drivers=("gpu.nvidia.com",), secrets=[{"type": "Kubeconfig", "name": "my-kubeconfig", "key": "kubeconfig"}], client_cas=None, gpu=None, @@ -5029,7 +5260,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[ + schedulable_pools=[ { "name": "l4-pool", "nodes": 4, @@ -5056,6 +5287,7 @@ def _to_dict(msg: message.Message) -> dict: ), "serving-stack": _serving_stack( cloud="Existing", + dra_drivers=("gpu.nvidia.com",), secrets=[{"type": "Kubeconfig", "name": "my-kubeconfig", "key": "kubeconfig"}], client_cas=None, gpu=None, @@ -5134,7 +5366,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[ + schedulable_pools=[ { "name": "l4-pool", "nodes": 4, @@ -5160,6 +5392,7 @@ def _to_dict(msg: message.Message) -> dict: ), "serving-stack": _serving_stack( cloud="Existing", + dra_drivers=("gpu.nvidia.com",), secrets=[{"type": "Kubeconfig", "name": "my-kubeconfig", "key": "kubeconfig"}], client_cas=None, gpu=None, @@ -5243,7 +5476,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[ + schedulable_pools=[ { "name": "l4-pool", "nodes": 4, @@ -5268,6 +5501,7 @@ def _to_dict(msg: message.Message) -> dict: ), "serving-stack": _serving_stack( cloud="Existing", + dra_drivers=("gpu.nvidia.com",), secrets=[{"type": "Kubeconfig", "name": "my-kubeconfig", "key": "kubeconfig"}], client_cas=None, gpu=None, @@ -5384,6 +5618,122 @@ def _to_dict(msg: message.Message) -> dict: ], ), ), + # The devices gate fires on a cluster that is already running - a shared + # class edited to claim a driver the cloud can't serve. Everything after + # the gate composes the cluster, so a bare return would empty the desired + # state and Crossplane would garbage-collect the cloud cluster XR, the + # ServingStack and their Usages. The fatal result is what stops that: + # Crossplane returns before applying desired state or collecting anything, + # and still applies the condition. This is the case that regresses if the + # gate goes back to returning without a fatal result. + ComposeCase( + name="UnsupportedDriverHolds", + reason="A running cluster whose class turns unsupported gets a fatal result, so Crossplane holds its resources instead of collecting them.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_inference_cluster( + cluster=v1alpha1.Cluster(source="GKE", gke=v1alpha1.Gke(region="us-central1")), + node_pools=[ + v1alpha1.NodePool(name="l4-pool", className="gpu-l4", nodeCount=2, maxNodeCount=4), + ], + ), + resources={ + "gke-cluster": fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "infrastructure.modelplane.ai/v1alpha1", + "kind": "GKECluster", + "metadata": {"name": "test-cluster", "namespace": "modelplane-system"}, + "status": { + "conditions": [ + { + "type": "Ready", + "status": "True", + "reason": "Available", + "lastTransitionTime": "2026-06-08T00:00:00Z", + }, + ], + }, + } + ), + ), + }, + ), + required_resources={ + "class-gpu-l4": fnv1.Resources( + items=[ + fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "modelplane.ai/v1alpha1", + "kind": "InferenceClass", + "metadata": {"name": "gpu-l4"}, + "spec": { + "devices": [ + { + "name": "gpu", + "claim": "DRA", + "driver": "gpu.amd.com", + "deviceClassName": "gpu.amd.com", + "count": 1, + "capacity": {"memory": {"value": "24Gi"}}, + }, + ], + "provisioning": { + "provider": "GKE", + "gke": { + "machineType": "g2-standard-48", + "diskSizeGb": 100, + "accelerator": {"type": "nvidia-l4", "count": 1}, + }, + }, + }, + } + ) + ), + ], + ), + }, + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State(), + results=[ + fnv1.Result( + severity=fnv1.SEVERITY_FATAL, + message="class gpu-l4 claims gpu.amd.com; the GKE serving stack installs dra.cpu, gpu.nvidia.com", + ), + ], + context=structpb.Struct(), + requirements=fnv1.Requirements( + resources={ + "model-replicas": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", + kind="ModelReplica", + match_labels=fnv1.MatchLabels(labels={"modelplane.ai/cluster": "test-cluster"}), + ), + "model-routes": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", + kind="ModelRoute", + match_labels=fnv1.MatchLabels(labels={"modelplane.ai/cluster": "test-cluster"}), + ), + "model-caches": fnv1.ResourceSelector(api_version="modelplane.ai/v1alpha1", kind="ModelCache"), + "gateways": fnv1.ResourceSelector(api_version="modelplane.ai/v1alpha1", kind="InferenceGateway"), + "class-gpu-l4": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", kind="InferenceClass", match_name="gpu-l4" + ), + }, + ), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="UnsupportedDevices", + message="class gpu-l4 claims gpu.amd.com; the GKE serving stack installs dra.cpu, gpu.nvidia.com", + ), + ], + ), + ), # The hostname gate, which is what keeps a cluster off the schedule until # traffic to it is mutually authenticated in both directions. These cases # observe an existing cluster's ServingStack ready, with whatever it and the @@ -5426,7 +5776,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[], + schedulable_pools=[], cache=None, gateway={ "address": "34.55.100.10", @@ -5441,6 +5791,7 @@ def _to_dict(msg: message.Message) -> dict: ), "serving-stack": _serving_stack( cloud="Existing", + dra_drivers=(), secrets=[{"type": "Kubeconfig", "name": "my-kubeconfig", "key": "kubeconfig"}], client_cas=[{"name": "fleet-0", "certificate": "fleet-ca"}], gpu=None, @@ -5499,7 +5850,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[], cache=None, gateway={"address": "34.55.100.10", "caCertificate": "cluster-ca"} + schedulable_pools=[], cache=None, gateway={"address": "34.55.100.10", "caCertificate": "cluster-ca"} ), resources={ "cluster-provider-config-kubernetes": _cluster_provider_config( @@ -5507,6 +5858,7 @@ def _to_dict(msg: message.Message) -> dict: ), "serving-stack": _serving_stack( cloud="Existing", + dra_drivers=(), secrets=[{"type": "Kubeconfig", "name": "my-kubeconfig", "key": "kubeconfig"}], client_cas=None, gpu=None, @@ -5569,7 +5921,9 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_inference_cluster(gpu_pools=[], cache=None, gateway={"address": "34.55.100.10"}), + composite=_desired_inference_cluster( + schedulable_pools=[], cache=None, gateway={"address": "34.55.100.10"} + ), resources={ "usage-replicas": _guard_cluster_usage(reason="InferenceGateways use this InferenceCluster"), "cluster-provider-config-kubernetes": _cluster_provider_config( @@ -5577,6 +5931,7 @@ def _to_dict(msg: message.Message) -> dict: ), "serving-stack": _serving_stack( cloud="Existing", + dra_drivers=(), secrets=[{"type": "Kubeconfig", "name": "my-kubeconfig", "key": "kubeconfig"}], client_cas=[{"name": "fleet-0", "certificate": "fleet-ca"}], gpu=None, @@ -5638,7 +5993,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_inference_cluster(gpu_pools=[], cache=None, gateway=None), + composite=_desired_inference_cluster(schedulable_pools=[], cache=None, gateway=None), resources={ "usage-replicas": _guard_cluster_usage(reason="InferenceGateways use this InferenceCluster"), "cluster-provider-config-kubernetes": _cluster_provider_config( @@ -5646,6 +6001,7 @@ def _to_dict(msg: message.Message) -> dict: ), "serving-stack": _serving_stack( cloud="Existing", + dra_drivers=(), secrets=[{"type": "Kubeconfig", "name": "my-kubeconfig", "key": "kubeconfig"}], client_cas=[{"name": "fleet-0", "certificate": "fleet-ca"}], gpu=None, @@ -5717,7 +6073,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[], + schedulable_pools=[], cache=None, gateway={ "address": "34.55.100.10", @@ -5732,6 +6088,7 @@ def _to_dict(msg: message.Message) -> dict: ), "serving-stack": _serving_stack( cloud="Existing", + dra_drivers=(), secrets=[{"type": "Kubeconfig", "name": "my-kubeconfig", "key": "kubeconfig"}], client_cas=[ {"name": "aaa", "certificate": "aaa-ca"}, @@ -5807,3 +6164,106 @@ def test_compose(case: ComposeCase) -> None: def test_gateway_hostname(case: GatewayHostnameCase) -> None: """_gateway_hostname derives a cluster's gateway hostname.""" assert fn._gateway_hostname(case.cluster_name) == case.want, case.reason + + +def _composer(*, source: fn.Cloud, devices_by_class: dict[str, list[dict]]) -> fn.Composer: + """A Composer for a test-cluster of the given source whose resolved + classes each declare the given devices, calling the derivation and + gate directly - through RunFunction every combination would need a + whole compose case.""" + cluster = v1alpha1.Cluster(source=source) + req = fnv1.RunFunctionRequest( + observed=fnv1.State(composite=_inference_cluster(cluster=cluster, node_pools=None)), + ) + composer = fn.Composer(req, fnv1.RunFunctionResponse()) + for name, devices in devices_by_class.items(): + composer.classes[name] = fn.iclv1alpha1.InferenceClass.model_validate( + { + "apiVersion": "modelplane.ai/v1alpha1", + "kind": "InferenceClass", + "metadata": {"name": name}, + "spec": {"devices": devices}, + } + ) + return composer + + +def test_dra_drivers_sorted() -> None: + """dra_drivers collects each claim: DRA device's driver once, sorted.""" + composer = _composer( + source="Existing", + devices_by_class={ + "nvidia-a": [{"name": "gpu", "claim": "DRA", "driver": "gpu.nvidia.com", "deviceClassName": "g"}], + "nvidia-b": [{"name": "gpu", "claim": "DRA", "driver": "gpu.nvidia.com", "deviceClassName": "g"}], + "cpu": [{"name": "cpu", "claim": "DRA", "driver": "dra.cpu", "deviceClassName": "dra.cpu"}], + }, + ) + assert composer.dra_drivers() == ["dra.cpu", "gpu.nvidia.com"] + + +def test_dra_drivers_synthetic() -> None: + """A Synthetic device is never claimed: its driver names the attribute + domain its CEL selectors read, not an installable stack, so it stays + off spec.draDrivers. An unset claim defaults to DRA and counts.""" + composer = _composer( + source="Existing", + devices_by_class={ + "efa": [ + {"name": "gpu", "driver": "gpu.nvidia.com", "deviceClassName": "g"}, + {"name": "fabric", "claim": "Synthetic", "driver": "fabric.example.com"}, + ], + }, + ) + assert composer.dra_drivers() == ["gpu.nvidia.com"] + + +def test_unsupported_devices() -> None: + """A class claiming a driver its cloud can't serve gates ClusterReady + with UnsupportedDevices and the design's message, and stops with a + fatal result so a running cluster holds its existing resources + rather than being garbage-collected.""" + composer = _composer( + source="Vultr", + devices_by_class={ + "amd-mi355x-8x": [ + {"name": "gpu", "claim": "DRA", "driver": "gpu.amd.com", "deviceClassName": "gpu.amd.com"}, + ], + }, + ) + assert composer.check_devices_supported("Vultr") is False + conditions = {c.type: c for c in composer.rsp.conditions} + assert conditions["ClusterReady"].status == fnv1.STATUS_CONDITION_FALSE + assert conditions["ClusterReady"].reason == "UnsupportedDevices" + assert conditions["ClusterReady"].message == ( + "class amd-mi355x-8x claims gpu.amd.com; the Vultr serving stack installs dra.cpu, gpu.nvidia.com" + ) + assert [(r.severity, r.message) for r in composer.rsp.results] == [ + ( + fnv1.SEVERITY_FATAL, + "class amd-mi355x-8x claims gpu.amd.com; the Vultr serving stack installs dra.cpu, gpu.nvidia.com", + ), + ] + + +def test_external_driver() -> None: + """dra.net is declared external on EKS - compose_eks_cluster installs + the EFA driver - so it passes there and gates everywhere else.""" + devices = { + "efa": [{"name": "efa", "claim": "DRA", "driver": "dra.net", "deviceClassName": "efa.networking.k8s.aws"}], + } + assert _composer(source="EKS", devices_by_class=devices).check_devices_supported("EKS") is True + assert _composer(source="GKE", devices_by_class=devices).check_devices_supported("GKE") is False + + +def test_existing_never_gates() -> None: + """Existing is BYO: any driver is treated as externally installed, so + nothing gates - a typo'd driver surfaces when its ResourceClaim can't + allocate, not as a condition.""" + composer = _composer( + source="Existing", + devices_by_class={ + "byo": [{"name": "gpu", "claim": "DRA", "driver": "gpu.example.com", "deviceClassName": "g"}], + }, + ) + assert composer.check_devices_supported("Existing") is True + assert list(composer.rsp.conditions) == [] diff --git a/functions/compose-model-deployment/function/cel.py b/functions/compose-model-deployment/function/cel.py index a4efea60f..d8aafe335 100644 --- a/functions/compose-model-deployment/function/cel.py +++ b/functions/compose-model-deployment/function/cel.py @@ -181,8 +181,8 @@ def __init__(self, expr: str) -> None: def matches(self, device: dict) -> bool: """Evaluate the selector against one device. - `device` is the raw dict from a pool's status.gpuPools devices - entry, shaped like: + `device` is the raw dict from a pool's status.schedulablePools + devices entry, shaped like: device = { "driver": "", diff --git a/functions/compose-model-deployment/function/fn.py b/functions/compose-model-deployment/function/fn.py index 5e7cb38b3..0c3c285e7 100644 --- a/functions/compose-model-deployment/function/fn.py +++ b/functions/compose-model-deployment/function/fn.py @@ -177,7 +177,7 @@ def _inference_cluster( ic.status = ic.status or icv1alpha1.Status() ic.status.providerConfigRef = ic.status.providerConfigRef or icv1alpha1.ProviderConfigRef() ic.status.gateway = ic.status.gateway or icv1alpha1.Gateway() - ic.status.gpuPools = ic.status.gpuPools or [] + ic.status.schedulablePools = ic.status.schedulablePools or [] return ic diff --git a/functions/compose-model-deployment/function/scheduling.py b/functions/compose-model-deployment/function/scheduling.py index 9d3a06d1b..6af7d4af0 100644 --- a/functions/compose-model-deployment/function/scheduling.py +++ b/functions/compose-model-deployment/function/scheduling.py @@ -136,9 +136,9 @@ def _published_count(value: int | None) -> int: Cluster status counts (a pool's node count, a device's per-node count) are unconstrained on the wire: the InferenceCluster XRD puts no minimum on - status.gpuPools[].nodes or [].devices[].count, so a published value can be 0 - (an autoscaled-to-zero pool's node count), None, or - in a malformed status - - negative. All must read as "none available". The load-bearing case is a + status.schedulablePools[].nodes or [].devices[].count, so a published value + can be 0 (an autoscaled-to-zero pool's node count), None, or - in a + malformed status - negative. All must read as "none available". The load-bearing case is a device count of 0: it differs from a deployment-side count, which the XRD floors at 1, so `x or 1` is a safe default-for-None there but would wrongly turn a published 0 into 1 here. Flooring None and negatives to 0 is defensive @@ -395,7 +395,7 @@ def _device_satisfies(device: icv1alpha1.Device, programs: list[cel.Program]) -> return all(p.matches(raw) for p in programs) -def _match_member(pool: icv1alpha1.GpuPool, member: _CompiledMember) -> list[DeviceRequest] | None: +def _match_member(pool: icv1alpha1.SchedulablePool, member: _CompiledMember) -> list[DeviceRequest] | None: """Match a member's requests against a pool. Returns the resolved claim: DRA DeviceRequests when the pool satisfies @@ -478,16 +478,16 @@ def _observed_claim(member: mrv1alpha1.Member) -> _Claim: return claim -def _gpu_pools(cluster: icv1alpha1.InferenceCluster) -> list[icv1alpha1.GpuPool]: - """The cluster's published GPU pools, empty if it has none yet.""" +def _schedulable_pools(cluster: icv1alpha1.InferenceCluster) -> list[icv1alpha1.SchedulablePool]: + """The cluster's published schedulable pools, empty if it has none yet.""" if not cluster.status: return [] - return cluster.status.gpuPools or [] + return cluster.status.schedulablePools or [] -def _pool_by_name(cluster: icv1alpha1.InferenceCluster, pool_name: str) -> icv1alpha1.GpuPool | None: +def _pool_by_name(cluster: icv1alpha1.InferenceCluster, pool_name: str) -> icv1alpha1.SchedulablePool | None: """The cluster's published pool with this name, or None.""" - for pool in _gpu_pools(cluster): + for pool in _schedulable_pools(cluster): if (pool.name or "") == pool_name: return pool return None @@ -566,7 +566,7 @@ def place(self, pods: list[_Claim]) -> bool: return fit -def _pool_capacity(pool: icv1alpha1.GpuPool) -> _Pool: +def _pool_capacity(pool: icv1alpha1.SchedulablePool) -> _Pool: """A pool's free capacity, before any replica is charged to it. A node the pool hasn't got yet will have every device its class declares @@ -609,7 +609,7 @@ class _Ledger: pools: dict[tuple[str, str], _Pool] -def _settled(replica: mrv1alpha1.ModelReplica, pool: icv1alpha1.GpuPool) -> bool: +def _settled(replica: mrv1alpha1.ModelReplica, pool: icv1alpha1.SchedulablePool) -> bool: """Whether a pool's histogram accounts for the devices a replica's pods hold on it. The cluster agent records when it first saw each claim allocated. It @@ -661,7 +661,7 @@ def _build_ledger( that doesn't fit - it's Pending. """ published = { - (_name(cluster.metadata), pool.name or ""): pool for cluster in clusters for pool in _gpu_pools(cluster) + (_name(cluster.metadata), pool.name or ""): pool for cluster in clusters for pool in _schedulable_pools(cluster) } pools = {key: _pool_capacity(pool) for key, pool in published.items()} @@ -876,7 +876,9 @@ def _place_engines( cluster_name = _name(cluster.metadata) # Discarded wholesale when any member fails to place, so partial placement # never leaks into the real ledger. - trial = {(pool.name or ""): ledger.pools[(cluster_name, pool.name or "")].copy() for pool in _gpu_pools(cluster)} + trial = { + (pool.name or ""): ledger.pools[(cluster_name, pool.name or "")].copy() for pool in _schedulable_pools(cluster) + } placed: dict[str, EnginePlacement] = {} for engine in sorted(engines, key=_largest_pod): members = _place_engine(cluster, engine, trial) @@ -924,7 +926,7 @@ def _place_engine( Places pods on trial as it goes; on failure the caller discards the whole trial, so partial placement doesn't leak. """ - pools = _gpu_pools(cluster) + pools = _schedulable_pools(cluster) # Whether each member's requests resolve a claimable device on any pool of # this cluster, ignoring capacity. Used to refuse a placement that would # strand a claim-capable member on a pool where it resolves only synthetic @@ -948,7 +950,7 @@ def _place_engine( def _match_members( - pool: icv1alpha1.GpuPool, engine: _CompiledEngine, claimable: list[bool] + pool: icv1alpha1.SchedulablePool, engine: _CompiledEngine, claimable: list[bool] ) -> list[MemberPlacement] | None: """Match every member of an engine against one pool. diff --git a/functions/compose-model-deployment/tests/test_fn.py b/functions/compose-model-deployment/tests/test_fn.py index 683b48ad2..c0e77f000 100644 --- a/functions/compose-model-deployment/tests/test_fn.py +++ b/functions/compose-model-deployment/tests/test_fn.py @@ -173,7 +173,7 @@ def _cluster( } ], "providerConfigRef": {"name": name}, - "gpuPools": [ + "schedulablePools": [ { "name": "default", "nodes": nodes, diff --git a/functions/compose-model-deployment/tests/test_scheduling.py b/functions/compose-model-deployment/tests/test_scheduling.py index 44d54fe17..cdcc90785 100644 --- a/functions/compose-model-deployment/tests/test_scheduling.py +++ b/functions/compose-model-deployment/tests/test_scheduling.py @@ -125,7 +125,7 @@ def _cluster( name: str, gateway_hostname: str | None, ready: bool, - pools: list[icv1alpha1.GpuPool], + pools: list[icv1alpha1.SchedulablePool], taints: list[icv1alpha1.Taint] | None, placement_labels: dict[str, str] | None, ) -> icv1alpha1.InferenceCluster: @@ -157,7 +157,7 @@ def _cluster( # InferenceGateway addresses a cluster by name. gateway=icv1alpha1.Gateway(address="10.0.0.1", hostname=gateway_hostname), providerConfigRef=icv1alpha1.ProviderConfigRef(name=name), - gpuPools=pools, + schedulablePools=pools, ), ) @@ -292,7 +292,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -346,7 +346,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=False, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -379,7 +379,7 @@ def _candidate( gateway_hostname=None, ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -417,7 +417,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -450,7 +450,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -462,7 +462,7 @@ def _candidate( gateway_hostname="cluster-b.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -541,7 +541,7 @@ def _candidate( gateway_hostname=None, ready=False, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -620,7 +620,7 @@ def _candidate( gateway_hostname="cluster-b.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -699,7 +699,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -711,7 +711,7 @@ def _candidate( gateway_hostname="cluster-b.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -810,7 +810,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=1, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -889,7 +889,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -963,7 +963,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -975,7 +975,7 @@ def _candidate( gateway_hostname="cluster-b.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -1049,7 +1049,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=4, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -1061,7 +1061,7 @@ def _candidate( gateway_hostname="cluster-b.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=4, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -1155,7 +1155,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=4, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -1167,7 +1167,7 @@ def _candidate( gateway_hostname="cluster-b.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=1, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -1261,7 +1261,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=4, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -1273,7 +1273,7 @@ def _candidate( gateway_hostname="cluster-b.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=4, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -1372,7 +1372,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=4, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -1515,7 +1515,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=4, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -1527,7 +1527,7 @@ def _candidate( gateway_hostname="cluster-b.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=4, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -1683,7 +1683,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=6, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -1823,7 +1823,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=4, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -1835,7 +1835,7 @@ def _candidate( gateway_hostname="cluster-b.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=4, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -1982,7 +1982,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=4, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -2105,7 +2105,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -2117,7 +2117,7 @@ def _candidate( gateway_hostname="cluster-b.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -2220,7 +2220,7 @@ def _candidate( gateway_hostname="cluster-c.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -2232,7 +2232,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -2244,7 +2244,7 @@ def _candidate( gateway_hostname="cluster-b.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -2318,7 +2318,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=1, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -2376,7 +2376,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=1, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -2458,7 +2458,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="frontier", nodes=1, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")], @@ -2542,10 +2542,10 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="a", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ), - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="b", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ), ], @@ -2652,7 +2652,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -2685,7 +2685,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -2764,7 +2764,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -2776,7 +2776,7 @@ def _candidate( gateway_hostname="cluster-b.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -2856,7 +2856,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="frontier", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")], @@ -2912,7 +2912,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="frontier", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")], @@ -2947,7 +2947,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="frontier", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=4, memory="141Gi")], @@ -2985,7 +2985,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="frontier", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=0, memory="141Gi")], @@ -3020,7 +3020,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="frontier", nodes=0, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")], @@ -3058,7 +3058,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="frontier", nodes=2, devices=[ @@ -3120,7 +3120,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="frontier", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")], @@ -3159,7 +3159,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="frontier", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")], @@ -3199,7 +3199,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="frontier", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=8, memory="141Gi")], @@ -3237,7 +3237,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="frontier", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=8, memory="141Gi")], @@ -3302,7 +3302,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="dev", nodes=2, devices=[ @@ -3310,7 +3310,7 @@ def _candidate( _nic_device(link_type="gpudirect-tcpx"), ], ), - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="frontier", nodes=2, devices=[ @@ -3371,7 +3371,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="frontier", nodes=2, devices=[ @@ -3409,7 +3409,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="frontier", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")], @@ -3495,7 +3495,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="a", nodes=2, devices=[ @@ -3503,7 +3503,7 @@ def _candidate( _nic_device(link_type="gpudirect-tcpx"), ], ), - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="b", nodes=2, devices=[ @@ -3590,7 +3590,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="a", nodes=2, devices=[ @@ -3598,7 +3598,7 @@ def _candidate( _nic_device(link_type="infiniband"), ], ), - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="b", nodes=2, devices=[ @@ -3685,7 +3685,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="a", nodes=2, devices=[ @@ -3751,7 +3751,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="frontier", nodes=2, devices=[ @@ -3839,7 +3839,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="frontier", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")], @@ -3964,10 +3964,10 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="a", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=4, memory="141Gi")] ), - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="b", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=8, memory="141Gi")] ), ], @@ -4028,10 +4028,10 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="small", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ), - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="big", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="200Gi")] ), ], @@ -4104,10 +4104,10 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="small", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ), - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="big", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="200Gi")] ), ], @@ -4145,10 +4145,10 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="small", nodes=8, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="40Gi")] ), - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="big", nodes=1, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ), ], @@ -4186,10 +4186,10 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="small", nodes=8, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="40Gi")] ), - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="big", nodes=1, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ), ], @@ -4201,7 +4201,7 @@ def _candidate( gateway_hostname="cluster-b.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="big", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -4274,7 +4274,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="a", nodes=2, devices=[ @@ -4282,7 +4282,7 @@ def _candidate( _gpu_device(name="syn", claim="Synthetic", count=1, memory="200Gi"), ], ), - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="b", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="200Gi")] ), ], @@ -4355,7 +4355,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="frontier", nodes=2, devices=[ @@ -4424,7 +4424,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -4458,7 +4458,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="frontier", nodes=1, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")], @@ -4520,7 +4520,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="frontier", nodes=1, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")], @@ -4609,7 +4609,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -4694,7 +4694,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=4, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -4789,7 +4789,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -4801,7 +4801,7 @@ def _candidate( gateway_hostname="cluster-b.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -4855,7 +4855,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -4934,7 +4934,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -4988,7 +4988,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -5000,7 +5000,7 @@ def _candidate( gateway_hostname="cluster-b.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -5079,7 +5079,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -5159,7 +5159,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -5219,7 +5219,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -5231,7 +5231,7 @@ def _candidate( gateway_hostname="cluster-b.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -5310,7 +5310,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -5325,7 +5325,7 @@ def _candidate( gateway_hostname="cluster-b.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -5379,7 +5379,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -5433,7 +5433,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -5466,7 +5466,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -5528,7 +5528,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -5588,7 +5588,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="gpu-l4x4-a", nodes=1, observedNodes=1, @@ -5645,7 +5645,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="gpu-l4x4-a", nodes=1, observedNodes=1, @@ -5684,7 +5684,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="gpu-l4x4-a", nodes=1, observedNodes=1, @@ -5792,7 +5792,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="gpu-l4x4-a", nodes=3, observedNodes=1, @@ -5898,7 +5898,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=1, observedNodes=1, @@ -6010,7 +6010,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, observedNodes=2, @@ -6050,7 +6050,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=1, observedNodes=1, @@ -6117,7 +6117,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=1, observedNodes=1, @@ -6204,7 +6204,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=1, observedNodes=1, @@ -6276,7 +6276,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, observedNodes=2, @@ -6331,7 +6331,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=1, observedNodes=1, @@ -6400,7 +6400,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=1, observedNodes=1, @@ -6474,7 +6474,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=1, observedNodes=1, @@ -6543,7 +6543,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, observedNodes=2, @@ -6623,7 +6623,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=1, observedNodes=1, @@ -6664,7 +6664,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=1, observedNodes=1, @@ -6730,7 +6730,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=1, observedNodes=1, @@ -6801,7 +6801,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="pool-a", nodes=1, observedNodes=1, @@ -6810,7 +6810,7 @@ def _candidate( ], devices=[_gpu_device(name="gpu", claim="DRA", count=4, memory="141Gi")], ), - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="pool-b", nodes=1, observedNodes=1, @@ -6889,7 +6889,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=1, devices=[_gpu_device(name="gpu", claim="DRA", count=8, memory="141Gi")] ) ], @@ -6953,7 +6953,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=1, devices=[_gpu_device(name="gpu", claim="DRA", count=4, memory="141Gi")] ) ], @@ -7028,7 +7028,7 @@ def test_schedule_error() -> None: gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="frontier", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")], diff --git a/functions/compose-model-replica/function/backends/base.py b/functions/compose-model-replica/function/backends/base.py index 1463afd5d..d68e2e31c 100644 --- a/functions/compose-model-replica/function/backends/base.py +++ b/functions/compose-model-replica/function/backends/base.py @@ -807,12 +807,51 @@ def engine_resources(container: v1alpha1.Container, *, claim: bool) -> dict: return resources -# Taint GPU node groups carry so non-GPU pods don't land on them. A pod that -# claims a GPU must tolerate it to schedule there. With GPUs bound via DRA (not -# the device plugin's extended resource), nothing injects this toleration for us -# - the ExtendedResourceToleration admission controller only acts on -# nvidia.com/gpu resource requests, which DRA pods don't make. -_GPU_TOLERATION = {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"} +def cpu_claim_without_request(replica: v1alpha1.ModelReplica) -> str | None: + """Name the first engine/member claiming dra.cpu whose engine sets no cpu request. + + The claim's capacity is sized from that request (see + resource_claim_template), so its absence is a user error the function + stops on, rather than composing a whole-device claim the driver + refuses at pod start - every claimable core taken leaves the node's + shared containers nowhere to run. + """ + for engine in replica.spec.engines: + for member in engine.members: + if not any(r.deviceClassName == CPU_DEVICE_CLASS for r in (member.deviceRequests or [])): + continue + container = engine_container(member) + requests = (container.resources.requests if container.resources else None) or {} + if "cpu" not in requests: + return f"{engine.name}/{member_role(member)}" + return None + + +# The DeviceClass dra-driver-cpu registers, and the qualified name its +# grouped devices hold their cores under. A member claiming through this +# class sizes its claim from the engine's cpu request (see +# resource_claim_template); the names are the driver's contract, kept in +# sync with the dra-driver-cpu pin in compose-serving-stack's +# stacks/common.py. +CPU_DEVICE_CLASS = "dra.cpu" +CPU_CAPACITY = "dra.cpu/cpu" + +# Taints inference node pools carry so unrelated pods don't land on them, one +# per device stack: the cluster compositions taint NVIDIA GPU pools +# nvidia.com/gpu and CPU pools modelplane.ai/cpu, and BYO AMD pools carry +# amd.com/gpu. A pod that claims a device must tolerate its pool's taint to +# schedule there, and with devices bound via DRA (not a device plugin's +# extended resource) nothing injects the toleration for us - the +# ExtendedResourceToleration admission controller only acts on nvidia.com/gpu +# resource requests, which DRA pods don't make. Every engine pod tolerates the +# whole list: tolerating a taint its pool never carries is harmless, and the +# pool nodeSelector is what places the pod, so the accelerator needn't thread +# through placement. +_POOL_TOLERATIONS = [ + {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "amd.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "modelplane.ai/cpu", "operator": "Exists", "effect": "NoSchedule"}, +] # Node label identifying the pool a node belongs to. compose-eks-cluster and # compose-gke-cluster stamp it on every node group they provision; the scheduler @@ -827,9 +866,9 @@ def place_pod(pod_spec: dict, replica: v1alpha1.ModelReplica, engine: v1alpha1.E """Constrain a member's serving pod to the placement the scheduler chose. Pins the pod to its member's scheduled node pool, wires it to claim its - GPUs via DRA through the member's claim, and tolerates the GPU node taint. - Every pod of one member shares this - a native Deployment pod, a - LeaderWorkerSet leader or worker pod, or a Grove leader or worker clique + devices via DRA through the member's claim, and tolerates every inference + pool taint. Every pod of one member shares this - a native Deployment pod, + a LeaderWorkerSet leader or worker pod, or a Grove leader or worker clique pod. The pool nodeSelector is what makes the scheduler's pool choice real: the @@ -846,14 +885,14 @@ def place_pod(pod_spec: dict, replica: v1alpha1.ModelReplica, engine: v1alpha1.E claim. A claimless member - one with no deviceRequests, like a coordinator-only leader - gets no claim at all: only the pool pin places its pods, packed onto the gang's nodes by the cluster's scheduler. It still - tolerates the GPU taint, since the pool it rides along on is a GPU pool. + tolerates the pool taints, since the pool it rides along on carries one. """ pod_spec["nodeSelector"] = {_LABEL_POOL: member.nodePoolName} if member.deviceRequests: pod_spec["resourceClaims"] = [ {"name": _POD_CLAIM_NAME, "resourceClaimTemplateName": claim_template_name(replica, engine, member)} ] - pod_spec.setdefault("tolerations", []).append(_GPU_TOLERATION) + pod_spec.setdefault("tolerations", []).extend(_POOL_TOLERATIONS) def resource_claim_template( @@ -877,6 +916,20 @@ def resource_claim_template( selectors = [{"cel": {"expression": s.cel}} for s in (r.selectors or []) if s.cel] if selectors: exactly["selectors"] = selectors + if r.deviceClassName == CPU_DEVICE_CLASS: + # dra-driver-cpu's grouped device holds its cores as consumable + # capacity, and a claim with no capacity request consumes the + # whole device - which the driver refuses while the node's + # shared containers (every DaemonSet) need somewhere to run. + # So a CPU claim asks for an explicit share: the engine + # container's own cpu request, the number the driver's + # contract already requires there (the kubelet accounts CPUs + # only from requests until KEP-5517). One number, stated + # once, sizes the cgroup accounting and the cpuset alike; + # cpu_claim_without_request gates on it being present. + container = engine_container(member) + assert container.resources is not None and container.resources.requests is not None + exactly["capacity"] = {"requests": {CPU_CAPACITY: str(container.resources.requests["cpu"])}} device_requests.append({"name": r.name, "exactly": exactly}) return wrap_object( diff --git a/functions/compose-model-replica/function/fn.py b/functions/compose-model-replica/function/fn.py index c63984149..fd18e50dc 100644 --- a/functions/compose-model-replica/function/fn.py +++ b/functions/compose-model-replica/function/fn.py @@ -183,6 +183,20 @@ def compose_model_serving(self) -> None: "PrefillDecode serving requires serving.router: EndpointPicker and serving.endpointPicker", ) return + if missing := base.cpu_claim_without_request(self.xr): + # Stop rather than compose a whole-device claim: a CPU claim is + # sized from the engine's cpu request (see + # resource_claim_template), and without one it would consume + # every claimable core, which dra-driver-cpu refuses to prepare + # while the node's shared containers are running. Fatal for the + # same reason as above: existing resources hold. + response.fatal( + self.rsp, + f"a dra.cpu claim is sized from the engine container's cpu request, and {missing} sets none: " + "set resources requests and limits cpu to the cores the claim should take, below the device's " + "total so the node's shared containers keep a core", + ) + return if routing.decode_without_sidecar(self.xr): response.warning( self.rsp, diff --git a/functions/compose-model-replica/function/observation.py b/functions/compose-model-replica/function/observation.py index d6740109b..a36120739 100644 --- a/functions/compose-model-replica/function/observation.py +++ b/functions/compose-model-replica/function/observation.py @@ -106,8 +106,8 @@ def agent_reports(cluster: icv1alpha1.InferenceCluster, replica: v1alpha1.ModelR observedNodes for is one the agent is reporting on. """ pinned = {m.nodePoolName for e in replica.spec.engines for m in e.members} - gpu_pools = (cluster.status.gpuPools if cluster.status else None) or [] - return any(p.name in pinned and p.observedNodes is not None for p in gpu_pools) + pools = (cluster.status.schedulablePools if cluster.status else None) or [] + return any(p.name in pinned and p.observedNodes is not None for p in pools) def pods_scheduled(replica: v1alpha1.ModelReplica, status: rov1alpha1.Status | None) -> resource.Condition: diff --git a/functions/compose-model-replica/tests/test_backends.py b/functions/compose-model-replica/tests/test_backends.py index 743d86430..1c1668a13 100644 --- a/functions/compose-model-replica/tests/test_backends.py +++ b/functions/compose-model-replica/tests/test_backends.py @@ -473,7 +473,9 @@ def _deployment( } ], "tolerations": [ - {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"} + {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "amd.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "modelplane.ai/cpu", "operator": "Exists", "effect": "NoSchedule"}, ], }, }, @@ -538,7 +540,9 @@ def _leader_worker_set( } ], "tolerations": [ - {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"} + {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "amd.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "modelplane.ai/cpu", "operator": "Exists", "effect": "NoSchedule"}, ], }, }, @@ -557,7 +561,9 @@ def _leader_worker_set( } ], "tolerations": [ - {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"} + {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "amd.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "modelplane.ai/cpu", "operator": "Exists", "effect": "NoSchedule"}, ], }, }, @@ -2129,7 +2135,17 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule", - } + }, + { + "key": "amd.com/gpu", + "operator": "Exists", + "effect": "NoSchedule", + }, + { + "key": "modelplane.ai/cpu", + "operator": "Exists", + "effect": "NoSchedule", + }, ], }, }, @@ -2179,7 +2195,17 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule", - } + }, + { + "key": "amd.com/gpu", + "operator": "Exists", + "effect": "NoSchedule", + }, + { + "key": "modelplane.ai/cpu", + "operator": "Exists", + "effect": "NoSchedule", + }, ], }, }, @@ -2353,7 +2379,17 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule", - } + }, + { + "key": "amd.com/gpu", + "operator": "Exists", + "effect": "NoSchedule", + }, + { + "key": "modelplane.ai/cpu", + "operator": "Exists", + "effect": "NoSchedule", + }, ], }, }, @@ -2405,7 +2441,17 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule", - } + }, + { + "key": "amd.com/gpu", + "operator": "Exists", + "effect": "NoSchedule", + }, + { + "key": "modelplane.ai/cpu", + "operator": "Exists", + "effect": "NoSchedule", + }, ], }, }, @@ -2563,7 +2609,17 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule", - } + }, + { + "key": "amd.com/gpu", + "operator": "Exists", + "effect": "NoSchedule", + }, + { + "key": "modelplane.ai/cpu", + "operator": "Exists", + "effect": "NoSchedule", + }, ], }, }, @@ -2613,7 +2669,17 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule", - } + }, + { + "key": "amd.com/gpu", + "operator": "Exists", + "effect": "NoSchedule", + }, + { + "key": "modelplane.ai/cpu", + "operator": "Exists", + "effect": "NoSchedule", + }, ], }, }, @@ -2783,7 +2849,17 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule", - } + }, + { + "key": "amd.com/gpu", + "operator": "Exists", + "effect": "NoSchedule", + }, + { + "key": "modelplane.ai/cpu", + "operator": "Exists", + "effect": "NoSchedule", + }, ], }, }, @@ -2833,7 +2909,17 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule", - } + }, + { + "key": "amd.com/gpu", + "operator": "Exists", + "effect": "NoSchedule", + }, + { + "key": "modelplane.ai/cpu", + "operator": "Exists", + "effect": "NoSchedule", + }, ], }, }, @@ -3003,7 +3089,17 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule", - } + }, + { + "key": "amd.com/gpu", + "operator": "Exists", + "effect": "NoSchedule", + }, + { + "key": "modelplane.ai/cpu", + "operator": "Exists", + "effect": "NoSchedule", + }, ], }, }, @@ -3055,7 +3151,17 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule", - } + }, + { + "key": "amd.com/gpu", + "operator": "Exists", + "effect": "NoSchedule", + }, + { + "key": "modelplane.ai/cpu", + "operator": "Exists", + "effect": "NoSchedule", + }, ], }, }, @@ -3197,7 +3303,17 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule", - } + }, + { + "key": "amd.com/gpu", + "operator": "Exists", + "effect": "NoSchedule", + }, + { + "key": "modelplane.ai/cpu", + "operator": "Exists", + "effect": "NoSchedule", + }, ], }, }, @@ -3247,7 +3363,17 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule", - } + }, + { + "key": "amd.com/gpu", + "operator": "Exists", + "effect": "NoSchedule", + }, + { + "key": "modelplane.ai/cpu", + "operator": "Exists", + "effect": "NoSchedule", + }, ], }, }, @@ -3405,7 +3531,17 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule", - } + }, + { + "key": "amd.com/gpu", + "operator": "Exists", + "effect": "NoSchedule", + }, + { + "key": "modelplane.ai/cpu", + "operator": "Exists", + "effect": "NoSchedule", + }, ], }, }, @@ -3455,7 +3591,17 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule", - } + }, + { + "key": "amd.com/gpu", + "operator": "Exists", + "effect": "NoSchedule", + }, + { + "key": "modelplane.ai/cpu", + "operator": "Exists", + "effect": "NoSchedule", + }, ], }, }, @@ -3635,7 +3781,17 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule", - } + }, + { + "key": "amd.com/gpu", + "operator": "Exists", + "effect": "NoSchedule", + }, + { + "key": "modelplane.ai/cpu", + "operator": "Exists", + "effect": "NoSchedule", + }, ], }, }, @@ -3717,7 +3873,17 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule", - } + }, + { + "key": "amd.com/gpu", + "operator": "Exists", + "effect": "NoSchedule", + }, + { + "key": "modelplane.ai/cpu", + "operator": "Exists", + "effect": "NoSchedule", + }, ], }, }, @@ -3908,7 +4074,17 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule", - } + }, + { + "key": "amd.com/gpu", + "operator": "Exists", + "effect": "NoSchedule", + }, + { + "key": "modelplane.ai/cpu", + "operator": "Exists", + "effect": "NoSchedule", + }, ], }, }, @@ -3990,7 +4166,17 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule", - } + }, + { + "key": "amd.com/gpu", + "operator": "Exists", + "effect": "NoSchedule", + }, + { + "key": "modelplane.ai/cpu", + "operator": "Exists", + "effect": "NoSchedule", + }, ], }, }, @@ -4187,7 +4373,17 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule", - } + }, + { + "key": "amd.com/gpu", + "operator": "Exists", + "effect": "NoSchedule", + }, + { + "key": "modelplane.ai/cpu", + "operator": "Exists", + "effect": "NoSchedule", + }, ], }, }, @@ -4273,7 +4469,17 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule", - } + }, + { + "key": "amd.com/gpu", + "operator": "Exists", + "effect": "NoSchedule", + }, + { + "key": "modelplane.ai/cpu", + "operator": "Exists", + "effect": "NoSchedule", + }, ], }, }, @@ -5219,6 +5425,129 @@ def test_build(case: BuildCase) -> None: assert _sorted(objects=_to_dicts(objects=got)) == _sorted(objects=case.want), case.reason +def _cpu_member(*, cpu_request: str | None) -> v1alpha1.Member: + """A Standalone member claiming one dra.cpu device, its engine requesting cpu_request CPUs (None for none).""" + resources = None + if cpu_request is not None: + resources = v1alpha1.Resources(requests={"cpu": cpu_request}, limits={"cpu": cpu_request}) + return v1alpha1.Member( + role="Standalone", + nodePoolName="cpu-pool", + deviceRequests=[ + v1alpha1.DeviceRequest( + name="cpu", + deviceClassName="dra.cpu", + count=1, + selectors=[v1alpha1.Selector(cel='device.capacity["dra.cpu"].cpu.compareTo(quantity("4")) >= 0')], + ) + ], + template=v1alpha1.Template( + spec=v1alpha1.Spec( + containers=[ + v1alpha1.Container( + name="engine", + image="vllm/vllm-openai-cpu:v0.32.0-x86_64", + resources=resources, + ) + ] + ) + ), + ) + + +def _cpu_replica(*, cpu_request: str | None) -> v1alpha1.ModelReplica: + """A one-engine replica of one _cpu_member.""" + return _replica( + name="r", + namespace="ml-team", + cluster_name="cluster-a", + labels=None, + model_cache_ref=None, + serving=None, + engines=[v1alpha1.Engine(name="main", copies=1, members=[_cpu_member(cpu_request=cpu_request)])], + ) + + +def _gpu_replica() -> v1alpha1.ModelReplica: + """A one-engine replica claiming one 80Gi GPU, as BUILD_CASES' StandaloneEngine does.""" + return _replica( + name="r", + namespace="ml-team", + cluster_name="cluster-a", + labels=None, + model_cache_ref=None, + serving=None, + engines=[ + v1alpha1.Engine( + name="main", + copies=1, + members=[ + v1alpha1.Member( + role="Standalone", + nodePoolName="frontier", + deviceRequests=[ + v1alpha1.DeviceRequest( + name="gpu", + deviceClassName="gpu.nvidia.com", + count=1, + selectors=[ + v1alpha1.Selector( + cel='device.capacity["gpu.nvidia.com"].memory.compareTo(quantity("80Gi")) >= 0' + ) + ], + ) + ], + template=v1alpha1.Template( + spec=v1alpha1.Spec( + containers=[ + v1alpha1.Container( + name="engine", + image="vllm/vllm-openai:latest", + args=["--model=Qwen/Qwen3-0.6B"], + ) + ] + ) + ), + ) + ], + ) + ], + ) + + +def test_cpu_claim_capacity() -> None: + """A dra.cpu claim asks for the engine's cpu request as device capacity, + not the whole device: dra-driver-cpu refuses a claim that would exhaust + the shared CPU pool, so the number the driver's contract already puts + on the container sizes the claim too.""" + replica = _cpu_replica(cpu_request="5") + engine = replica.spec.engines[0] + obj = base.resource_claim_template(replica, engine, engine.members[0], "cluster-a-pc") + got = obj.model_dump(exclude_unset=True, by_alias=True) + requests = got["spec"]["forProvider"]["manifest"]["spec"]["spec"]["devices"]["requests"] + assert requests[0]["exactly"]["capacity"] == {"requests": {"dra.cpu/cpu": "5"}} + + +def test_gpu_claim_uncapped() -> None: + """A GPU claim carries no capacity request: GPUs bind whole devices, and + a capacity request is the CPU driver's contract, not DRA's generally.""" + replica = _gpu_replica() + engine = replica.spec.engines[0] + obj = base.resource_claim_template(replica, engine, engine.members[0], "cluster-a-pc") + got = obj.model_dump(exclude_unset=True, by_alias=True) + requests = got["spec"]["forProvider"]["manifest"]["spec"]["spec"]["devices"]["requests"] + assert "capacity" not in requests[0]["exactly"] + + +def test_cpu_request_missing() -> None: + """cpu_claim_without_request names the member whose dra.cpu claim has no + cpu request to size itself from, and stays quiet when it has one or + when nothing claims CPUs.""" + assert base.cpu_claim_without_request(_cpu_replica(cpu_request=None)) == "main/standalone" + assert base.cpu_claim_without_request(_cpu_replica(cpu_request="5")) is None + assert base.cpu_claim_without_request(_gpu_replica()) is None + + SELECT_BACKEND_CASES = [ SelectBackendCase( name="StandaloneOnStandard", @@ -6602,7 +6931,17 @@ def test_cache_env(case: CacheEnvCase) -> None: "key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule", - } + }, + { + "key": "amd.com/gpu", + "operator": "Exists", + "effect": "NoSchedule", + }, + { + "key": "modelplane.ai/cpu", + "operator": "Exists", + "effect": "NoSchedule", + }, ], }, }, @@ -6652,7 +6991,17 @@ def test_cache_env(case: CacheEnvCase) -> None: "key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule", - } + }, + { + "key": "amd.com/gpu", + "operator": "Exists", + "effect": "NoSchedule", + }, + { + "key": "modelplane.ai/cpu", + "operator": "Exists", + "effect": "NoSchedule", + }, ], }, }, @@ -6899,7 +7248,17 @@ def test_cache_env(case: CacheEnvCase) -> None: "key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule", - } + }, + { + "key": "amd.com/gpu", + "operator": "Exists", + "effect": "NoSchedule", + }, + { + "key": "modelplane.ai/cpu", + "operator": "Exists", + "effect": "NoSchedule", + }, ], }, }, @@ -6949,7 +7308,17 @@ def test_cache_env(case: CacheEnvCase) -> None: "key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule", - } + }, + { + "key": "amd.com/gpu", + "operator": "Exists", + "effect": "NoSchedule", + }, + { + "key": "modelplane.ai/cpu", + "operator": "Exists", + "effect": "NoSchedule", + }, ], }, }, diff --git a/functions/compose-model-replica/tests/test_fn.py b/functions/compose-model-replica/tests/test_fn.py index f37a42ab9..599432ec4 100644 --- a/functions/compose-model-replica/tests/test_fn.py +++ b/functions/compose-model-replica/tests/test_fn.py @@ -216,7 +216,7 @@ def _cluster(*, provider_config_ref: str | None, observed: bool) -> fnv1.Resourc cluster["status"] = { "providerConfigRef": {"name": provider_config_ref}, "gateway": {"address": "10.0.0.1"}, - "gpuPools": [pool], + "schedulablePools": [pool], } return fnv1.Resource(resource=resource.dict_to_struct(cluster)) @@ -282,7 +282,17 @@ def _deployment(*, ready: fnv1.Ready, name: str, claim_template_name: str, label "key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule", - } + }, + { + "key": "amd.com/gpu", + "operator": "Exists", + "effect": "NoSchedule", + }, + { + "key": "modelplane.ai/cpu", + "operator": "Exists", + "effect": "NoSchedule", + }, ], }, }, diff --git a/functions/compose-nebius-cluster/function/fn.py b/functions/compose-nebius-cluster/function/fn.py index 036a8c27b..5ffb82493 100644 --- a/functions/compose-nebius-cluster/function/fn.py +++ b/functions/compose-nebius-cluster/function/fn.py @@ -102,6 +102,12 @@ _GPU_TAINT_VALUE = "true" _GPU_TAINT_EFFECT = "NO_SCHEDULE" +# Label and taint on CPU inference groups. The label is what the serving +# stack's dra-driver-cpu selects its nodes by; the taint keeps system +# workloads off the group as the GPU taint does on GPU groups, and +# engine pods tolerate it (compose-model-replica). +_LABEL_CPU = "modelplane.ai/cpu" + # Management policies that exclude Delete, used for resources installed on the # workload cluster (the RWX StorageClass Object and the CSI driver Helm # Release). These exist only to configure the cluster and are only ever @@ -552,6 +558,8 @@ def compose_node_groups(self) -> None: labels = {_LABEL_POOL: pool.name} if pool.role == "GPU" and pool.gpu: labels[_LABEL_GPU] = pool.gpu.acceleratorType + elif pool.role == "CPU": + labels[_LABEL_CPU] = "true" template = nodegroupv1beta1.Template( resources=nodegroupv1beta1.Resources( @@ -585,6 +593,14 @@ def compose_node_groups(self) -> None: effect=_GPU_TAINT_EFFECT, ), ] + elif pool.role == "CPU": + template.taints = [ + nodegroupv1beta1.Taint( + key=_LABEL_CPU, + value="true", + effect=_GPU_TAINT_EFFECT, + ), + ] fabric = _pool_fabric(pool) if fabric: diff --git a/functions/compose-nebius-cluster/tests/test_fn.py b/functions/compose-nebius-cluster/tests/test_fn.py index 5f61cc045..5f43648f8 100644 --- a/functions/compose-nebius-cluster/tests/test_fn.py +++ b/functions/compose-nebius-cluster/tests/test_fn.py @@ -312,6 +312,55 @@ def _nodegroup_gpu( ) +def _nodegroup_cpu(*, cred_kind: str, cred_name: str, ready: fnv1.Ready) -> fnv1.Resource: + """The composed node group for the cpu-pool pool, with the CPU label and taint and no GPU settings.""" + return fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "mk8s.nebius.m.upbound.io/v1beta1", + "kind": "NodeGroup", + "spec": { + "providerConfigRef": {"kind": cred_kind, "name": cred_name}, + "forProvider": { + "name": "test-cluster-cpu-pool", + "parentIdSelector": {"matchControllerRef": True}, + "version": "1.34", + "template": { + "resources": {"platform": "cpu-d3", "preset": "8vcpu-32gb"}, + "bootDisk": {"sizeGibibytes": 100, "type": "NETWORK_SSD"}, + "networkInterfaces": [ + {"subnetIdSelector": {"matchControllerRef": True}}, + ], + "filesystems": [ + { + "attachMode": "READ_WRITE", + "mountTag": "modelplane-cache", + "existingFilesystem": {"idSelector": {"matchControllerRef": True}}, + }, + ], + "cloudInitUserDataSecretRef": { + "name": "test-cluster-cloud-init-fd2f2", + "key": "userData", + }, + "metadata": { + "labels": { + "modelplane.ai/pool": "cpu-pool", + "modelplane.ai/cpu": "true", + }, + }, + "taints": [ + {"key": "modelplane.ai/cpu", "value": "true", "effect": "NO_SCHEDULE"}, + ], + }, + "autoscaling": {"minNodeCount": 1, "maxNodeCount": 4}, + }, + }, + } + ), + ready=ready, + ) + + def _provider_config(*, api_version: str) -> fnv1.Resource: """A Ready ProviderConfig targeting the cluster as the Nebius service account.""" return fnv1.Resource( @@ -431,6 +480,73 @@ def _to_dict(msg: message.Message) -> dict: ), ), ), + Case( + name="FirstPassCPUPool", + reason="A CPU-role node group composes with the modelplane.ai/pool and modelplane.ai/cpu labels and the CPU taint, and no GPU settings.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + node_pools=[ + v1alpha1.NodePool( + name="cpu-pool", + role="CPU", + platform="cpu-d3", + preset="8vcpu-32gb", + nodeCount=1, + maxNodeCount=4, + ), + ], + ), + ), + required_resources={ + "nebius-provider-config": fnv1.Resources( + items=[_nebius_provider_config(kind="ClusterProviderConfig", name="default", namespace=None)], + ), + }, + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_xr(credentials_secret=True), + resources={ + "network": _network( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "subnet": _subnet( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "cluster": _cluster( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "filesystem": _filesystem( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "cloud-init": _cloud_init_secret(), + "nodegroup-system": _nodegroup_system( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "nodegroup-cpu-pool": _nodegroup_cpu( + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, + ), + "provider-config-kubernetes": _provider_config(api_version="kubernetes.m.crossplane.io/v1alpha1"), + "provider-config-helm": _provider_config(api_version="helm.m.crossplane.io/v1beta1"), + }, + ), + context=structpb.Struct(), + requirements=fnv1.Requirements( + resources={ + "nebius-provider-config": fnv1.ResourceSelector( + api_version="nebius.m.upbound.io/v1beta1", + kind="ClusterProviderConfig", + match_name="default", + ), + }, + ), + ), + ), Case( name="ProviderConfigNotFetched", reason="Before the Nebius provider config is fetched, and with no ProviderConfig observed, a NebiusCluster composes its infrastructure but not the ProviderConfigs, and reports it's waiting.", diff --git a/functions/compose-serving-stack/function/collector.py b/functions/compose-serving-stack/function/collector.py index 8d73b6543..639ee94b2 100644 --- a/functions/compose-serving-stack/function/collector.py +++ b/functions/compose-serving-stack/function/collector.py @@ -139,14 +139,17 @@ def _relabel_pod_identity() -> list[dict[str, Any]]: ] + [{"source_labels": ["__meta_kubernetes_namespace"], "target_label": "namespace"}] -def _dcgm_selector(action: str) -> dict[str, Any]: - """Match a GPU exporter, however it was packaged. - - The two spell it differently - gke-managed-dcgm-exporter and dcgm-exporter - - and put the name on different labels depending on who packaged it, so both - are read and the match is on what they share. One definition, used by the - job that keeps them and the job that has to leave them alone, because the - two drifting apart is how a series gets collected twice. +def _gpu_exporter_selector(action: str) -> dict[str, Any]: + """Match a GPU exporter, whichever vendor's and however it was packaged. + + NVIDIA's DCGM exporters spell it differently - gke-managed-dcgm-exporter + and dcgm-exporter - and put the name on different labels depending on who + packaged it, so both labels are read and the match is on what they share. + The AMD GPU operator names its device-metrics-exporter pods + app.kubernetes.io/name: metrics-exporter, which the same pattern's second + alternative matches. One definition, used by the job that keeps them and + the job that has to leave them alone, because the two drifting apart is + how a series gets collected twice. """ return { "source_labels": [ @@ -154,7 +157,7 @@ def _dcgm_selector(action: str) -> dict[str, Any]: "__meta_kubernetes_pod_label_app", ], "action": action, - "regex": ".*dcgm.*", + "regex": ".*dcgm.*|.*metrics-exporter.*", } @@ -340,8 +343,14 @@ def _scrape_configs() -> list[dict[str, Any]]: # two spell it differently: gke-managed-dcgm-exporter and # dcgm-exporter. Both labels are read, since which one carries # the name depends on who packaged it. - _dcgm_selector("keep"), - {"source_labels": ["__meta_kubernetes_pod_container_port_name"], "action": "keep", "regex": "metrics"}, + _gpu_exporter_selector("keep"), + # The exporters name their metrics port differently: DCGM's + # is metrics, the AMD operator's is exporter-port. + { + "source_labels": ["__meta_kubernetes_pod_container_port_name"], + "action": "keep", + "regex": "metrics|exporter-port", + }, # The node, because a GPU series belongs to hardware rather # than to a deployment. DCGM names the card itself. {"source_labels": ["__meta_kubernetes_pod_node_name"], "target_label": "node"}, @@ -371,7 +380,7 @@ def _scrape_configs() -> list[dict[str, Any]]: "action": "drop", "regex": ".+", }, - _dcgm_selector("drop"), + _gpu_exporter_selector("drop"), # The rest of the same convention, not just the first line of # it. A pod that says scrape me generally also says where: the # cert-manager webhook declares 10250 first and annotates 9402, @@ -411,6 +420,7 @@ def _scrape_configs() -> list[dict[str, Any]]: # literal in that position. _UNIT_FACTOR = { "Millijoules": "0.001", + "Microjoules": "0.000001", "Milliseconds": "0.001", "Nanoseconds": "0.000000001", "Mebibytes": "1048576.0", @@ -424,7 +434,8 @@ def statements(mappings: list[mmv1alpha1.MetricMapping]) -> tuple[list[str], lis OTTL is rendered here rather than written in a MetricMapping so the kind stays a description of what a component emits, and the collector's own configuration language stays Modelplane's problem. It is also the only - place that knows a unit conversion has to run somewhere a rename cannot. + place that knows a unit or type conversion has to run somewhere a + rename cannot. """ extract: list[str] = [] scale: list[str] = [] @@ -444,6 +455,11 @@ def statements(mappings: list[mmv1alpha1.MetricMapping]) -> tuple[list[str], lis source = f"{source}{_PART_SUFFIX[m.part]}" if m.fromUnit: scale.append(f'scale_metric({_UNIT_FACTOR[m.fromUnit]}) where metric.name == "{source}"') + if m.cumulative: + # In the conversion block with the unit scaling: both make + # the series agree with the name it's about to get, and + # both must run while it still answers to the source name. + scale.append(f'convert_gauge_to_sum("cumulative", true) where metric.name == "{source}"') # Labels before the rename, while the series still answers to the # name this mapping selected on. After it, two folded mappings share # one name and a statement could no longer tell them apart. diff --git a/functions/compose-serving-stack/function/fn.py b/functions/compose-serving-stack/function/fn.py index e58cf0b70..fbb6b4b36 100644 --- a/functions/compose-serving-stack/function/fn.py +++ b/functions/compose-serving-stack/function/fn.py @@ -327,9 +327,18 @@ def compose(self) -> None: # The XRD requires and enums both fields, so the join raising means the # API and the stacks package disagree on a value, a broken Modelplane - # build, not a cluster condition. Let it crash rather than dress it up - # as a fatal result. - components = stacks.join(self.xr.spec.cloud, self.xr.spec.stack or "Standard") + # build, not a cluster condition - or, for a draDrivers entry, that + # compose-inference-cluster's hand-kept mirror of the supported + # drivers drifted from the stacks package's: it should have gated the + # cluster with UnsupportedDevices before composing this ServingStack. + # Let it crash rather than dress it up as a fatal result. draDrivers + # absent means the same as empty - the cluster composition always + # sets it - so only the untagged components install. + components = stacks.join( + self.xr.spec.cloud, + self.xr.spec.stack or "Standard", + dra_drivers=[d.root for d in self.xr.spec.draDrivers or []], + ) # Per-pool GPU driver configuration is per-cluster state the cluster # composition projects into spec.gpu, so it can't live in the diff --git a/functions/compose-serving-stack/function/stacks/__init__.py b/functions/compose-serving-stack/function/stacks/__init__.py index 5e803c978..f21adc26f 100644 --- a/functions/compose-serving-stack/function/stacks/__init__.py +++ b/functions/compose-serving-stack/function/stacks/__init__.py @@ -39,6 +39,7 @@ "join", "metrics", "stacks", + "supported_drivers", ] # The cloud halves, keyed by the InferenceCluster's source values. EKS, @@ -59,6 +60,22 @@ "Dynamo": dynamo.COMPONENTS, } +# DRA drivers a cloud supports that arrive outside the serving stack, so +# the filter installs nothing for them. The generated modules' constants +# are emitted by generate.py, where the EKS dra.net declaration lives +# beside the aws-efa DROP entry that records why. Existing is the +# open-ended case, special-cased in join() instead - its docstring +# states the rule. +_EXTERNAL_DRIVERS: dict[Cloud, frozenset[str]] = { + "EKS": frozenset(eks.EXTERNAL_DRIVERS), + "AKS": frozenset(aks.EXTERNAL_DRIVERS), + "GKE": frozenset(gke.EXTERNAL_DRIVERS), + "Nebius": frozenset(), + "Vultr": frozenset(), + "Civo": frozenset(), + "Existing": frozenset(), +} + def clouds() -> list[Cloud]: """The clouds a stack can be joined for.""" @@ -70,14 +87,52 @@ def stacks() -> list[Stack]: return list(_STACKS) -def join(cloud: Cloud, stack: Stack) -> list[Component]: +def supported_drivers(cloud: Cloud) -> frozenset[str]: + """The DRA drivers a cloud's serving stack supports. + + Derived rather than declared: the union of the dra_drivers tags + across the cloud's component list, common.py and every stack half, + plus the drivers the cloud declares installed outside the stack + (_EXTERNAL_DRIVERS), so the table can't drift from what the lists + record. compose-inference-cluster mirrors this by hand in its + _SUPPORTED_DRA_DRIVERS to reject unsupported pairings with a + condition before a ServingStack is ever composed - keep the two in + sync. Existing's tags are returned like any cloud's, but a driver + beyond them is not unsupported there: the user owns the cluster, so + join() never rejects on Existing. + """ + halves = [_CLOUDS[cloud], common.COMPONENTS, *_STACKS.values()] + tagged = {driver for half in halves for c in half for driver in c.dra_drivers} + return frozenset(tagged) | _EXTERNAL_DRIVERS[cloud] + + +def join(cloud: Cloud, stack: Stack, dra_drivers: list[str] | None = None) -> list[Component]: """Join the component lists for a cloud and stack. Fails closed, at import or test time rather than on a cluster: on an - unknown cloud or stack, on a key two lists both produce, and on a + unknown cloud or stack, on a key two lists both produce, on a depends_on edge naming a component the join didn't produce - which catches a generator allowlist that dropped something another - component needs. + component needs - and on a dependency that is neither untagged nor + tagged with every driver its dependent carries, which filtering + could then remove from under it. + + dra_drivers filters the tagged components: a tagged component + survives only when one of its drivers is listed, an untagged one + always does. None disables filtering, for the tests and integrity + checks that inspect the full join; the function always passes a + list. The integrity checks run on the unfiltered join, so a broken + list fails every join for its cloud, not just the driver + combination that trips it. + + A listed driver the cloud neither tags components with nor declares + external raises - except on Existing, where the user owns the + cluster and may have installed any driver, so an unlisted one is + treated as externally installed: it installs nothing and is never + rejected, and a typo surfaces when its ResourceClaim can't + allocate. This docstring is where both that rule and the + dependency-tagging one above are stated; the other mentions point + here. """ if cloud not in _CLOUDS: raise ValueError(f"unknown cloud {cloud!r}; known: {', '.join(_CLOUDS)}") @@ -99,10 +154,31 @@ def join(cloud: Cloud, stack: Stack) -> list[Component]: if duplicates: raise ValueError(f"{cloud}/{stack}: duplicate composed-resource keys {duplicates}") - known = set(keys) + by_key = {c.key: c for c in joined} for c in joined: for dep in c.depends_on: - if dep not in known: + if dep not in by_key: raise ValueError(f"{cloud}/{stack}: {c.key} depends on {dep!r}, which the join did not produce") - - return joined + # The docstring's dependency-tagging rule. The + # `c.dra_drivers and` leg: an untagged dependent is kept by + # every filter, which no tagged dependency survives. + dep_drivers = by_key[dep].dra_drivers + if dep_drivers and not (c.dra_drivers and set(c.dra_drivers) <= set(dep_drivers)): + raise ValueError( + f"{cloud}/{stack}: {c.key} (drivers {c.dra_drivers or 'untagged'}) depends on " + f"{dep!r} (drivers {dep_drivers}), which filtering could remove from under it" + ) + + if dra_drivers is None: + return joined + + if cloud != "Existing": + unknown = set(dra_drivers) - supported_drivers(cloud) + if unknown: + raise ValueError( + f"{cloud}/{stack}: no device stack serves {', '.join(sorted(unknown))}; " + f"the {cloud} serving stack supports {', '.join(sorted(supported_drivers(cloud)))}" + ) + + wanted = set(dra_drivers) + return [c for c in joined if not c.dra_drivers or wanted & set(c.dra_drivers)] diff --git a/functions/compose-serving-stack/function/stacks/clouds/civo.py b/functions/compose-serving-stack/function/stacks/clouds/civo.py index 64277baa8..8c3d367db 100644 --- a/functions/compose-serving-stack/function/stacks/clouds/civo.py +++ b/functions/compose-serving-stack/function/stacks/clouds/civo.py @@ -220,19 +220,21 @@ def transformed(c: Component) -> Component: ], }, }, - # Disable components we don't need for observability. - "grafana": {"enabled": False}, - "alertmanager": {"enabled": False}, # The node-exporter chart's default toleration is keyless - it - # tolerates every taint, landing the pod on tainted GPU nodes by - # accident rather than by intent. Scope it to the GPU taint, + # tolerates every taint, landing the pod on tainted nodes by + # accident rather than by intent. Scope it to the GPU and CPU pool + # taints, # mirroring the generated clouds (TOLERATIONS in generate.py); as # a DaemonSet it still reaches every untainted node. "prometheus-node-exporter": { "tolerations": [ {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "modelplane.ai/cpu", "operator": "Exists", "effect": "NoSchedule"}, ], }, + # Disable components we don't need for observability. + "grafana": {"enabled": False}, + "alertmanager": {"enabled": False}, }, ), Chart( @@ -245,6 +247,10 @@ def transformed(c: Component) -> Component: # gpu-operator below depends on this chart, so its Ready must # mean healthy, not just deployed. wait=True, + # NFD exists here to label GPU nodes for the DRA driver, so it + # carries the GPU stack's driver and stays off clusters whose + # classes claim no GPUs. + dra_drivers=["gpu.nvidia.com"], # The worker must run on the very nodes it is supposed to label: # the DRA driver's kubelet plugin schedules only onto nodes # carrying an NFD GPU label (feature.node.kubernetes.io/pci-10de @@ -287,6 +293,7 @@ def transformed(c: Component) -> Component: # modelplane_energy_joules_total is empty on Civo - the # collector's modelplane-gpu job would find no pod to scrape. depends_on=["node-feature-discovery", "cert-manager"], + dra_drivers=["gpu.nvidia.com"], values={ "ccManager": {"enabled": False}, "cdi": {"default": True, "enabled": True}, @@ -363,6 +370,7 @@ def transformed(c: Component) -> Component: repository="oci://registry.k8s.io/dra-driver-nvidia/charts", version="0.5.0", depends_on=["gpu-operator"], + dra_drivers=["gpu.nvidia.com"], values={ "gpuResourcesEnabledOverride": True, "nvidiaDriverRoot": "/run/nvidia/driver", diff --git a/functions/compose-serving-stack/function/stacks/clouds/existing.py b/functions/compose-serving-stack/function/stacks/clouds/existing.py index 8bcb77226..ecbebf30a 100644 --- a/functions/compose-serving-stack/function/stacks/clouds/existing.py +++ b/functions/compose-serving-stack/function/stacks/clouds/existing.py @@ -20,9 +20,17 @@ component also appears on a generated cloud, this file states the same pin, so one review moves both halves when a version changes - regenerate, then mirror the shared pins here. + +Existing is also the open-ended cloud for devices: a claimed driver +this list doesn't tag installs nothing and rejects nothing - join()'s +docstring states that rule. The list still pins device +stacks like any cloud's, each tagged with its driver so it installs +only when a class claims it: NVIDIA's today, and AMD's - Existing is +the one cloud that installs it, for BYO nodes whose images already +carry the amdgpu kernel driver. """ -from function.stacks.components import Chart, Component +from function.stacks.components import Chart, Component, Manifests COMPONENTS: list[Component] = [ Chart( @@ -90,19 +98,22 @@ ], }, }, - # Disable components we don't need for observability. - "grafana": {"enabled": False}, - "alertmanager": {"enabled": False}, # The node-exporter chart's default toleration is keyless - it - # tolerates every taint, landing the pod on tainted GPU nodes by - # accident rather than by intent. Scope it to the GPU taint, + # tolerates every taint, landing the pod on tainted nodes by + # accident rather than by intent. Scope it to the inference pool + # taints - a BYO pool may carry any of the three - # mirroring the generated clouds (TOLERATIONS in generate.py); as # a DaemonSet it still reaches every untainted node. "prometheus-node-exporter": { "tolerations": [ {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "amd.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "modelplane.ai/cpu", "operator": "Exists", "effect": "NoSchedule"}, ], }, + # Disable components we don't need for observability. + "grafana": {"enabled": False}, + "alertmanager": {"enabled": False}, }, ), Chart( @@ -112,16 +123,24 @@ chart="node-feature-discovery", repository="https://kubernetes-sigs.github.io/node-feature-discovery/charts", version="0.19.0", + # amd-gpu-operator below depends on this chart, so its Ready must + # mean healthy, not just deployed. + wait=True, + # NFD exists here to label GPU nodes for the DRA drivers, so it + # carries both GPU stacks' drivers - NVIDIA's kubelet plugin and + # AMD's operator both find their nodes by NFD labels - and stays + # off clusters whose classes claim no GPUs. + dra_drivers=["gpu.nvidia.com", "gpu.amd.com"], # The worker must run on the very nodes it is supposed to label: # the DRA driver's kubelet plugin schedules only onto nodes # carrying an NFD GPU label (feature.node.kubernetes.io/pci-10de # and friends). The cluster compositions taint GPU nodes with - # nvidia.com/gpu, and the NFD chart's worker tolerates nothing - # by default. Without this toleration the chain breaks silently: - # no worker on the GPU node, no pci-10de label, no kubelet - # plugin, no ResourceSlices, and every GPU ResourceClaim stays - # unallocatable with all components looking healthy. Exists - # matches every taint value. + # nvidia.com/gpu or amd.com/gpu, and the NFD chart's worker + # tolerates nothing by default. Without these tolerations the + # chain breaks silently: no worker on the GPU node, no pci-10de + # label, no kubelet plugin, no ResourceSlices, and every GPU + # ResourceClaim stays unallocatable with all components looking + # healthy. Exists matches every taint value. values={ "worker": { "tolerations": [ @@ -130,6 +149,11 @@ "operator": "Exists", "effect": "NoSchedule", }, + { + "key": "amd.com/gpu", + "operator": "Exists", + "effect": "NoSchedule", + }, ], }, }, @@ -147,9 +171,118 @@ chart="dra-driver-nvidia-gpu", repository="oci://registry.k8s.io/dra-driver-nvidia/charts", version="0.5.0", + dra_drivers=["gpu.nvidia.com"], values={ "gpuResourcesEnabledOverride": True, "resources": {"computeDomains": {"enabled": False}}, }, ), + # The metrics exporter's config, referenced from the DeviceConfig + # below. It exists to turn ProfilerMetrics on: GPU_PROF_* fields ship + # disabled, and tensor activity (MFMA utilization) is one of them - + # without it modelplane_gpu_tensor_active_ratio is NVIDIA-only. The + # hardware runs one profiler instance at a time, so this takes the + # slot application-level profiling would use on these nodes; turn it + # off here to profile. An absent Fields list keeps the exporter's + # default metrics unchanged. Lands in the namespace the chart's + # release creates; provider-kubernetes retries until it exists, like + # the critical-pods quotas in common.py. + Manifests( + key="amd-metrics-exporter-config", + dra_drivers=["gpu.amd.com"], + manifests=[ + { + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": { + "name": "mp-metrics-exporter-config", + "namespace": "kube-amd-gpu", + }, + "data": { + "config.json": '{"GPUConfig": {"ProfilerMetrics": {"all": true}}}', + }, + }, + ], + ), + # The AMD GPU operator, in DRA mode: the chart's default DeviceConfig + # (crds.defaultCR) enables the DRA driver and disables the device + # plugin - the two are mutually exclusive allocators - so the + # gpu.amd.com DeviceClass the chart registers is the sole path to the + # GPUs. Existing is the one cloud that installs it today. + # + # The default DeviceConfig tracks the release only with + # crds.defaultCR.upgrade: the chart renders it on install, marks it + # helm.sh/resource-policy: keep, and skips it on upgrade by default, + # so without the flag a value change below would upgrade the release + # and leave the live DeviceConfig at its install-time spec. + # + # The chart pin pins the operator's own images but not the + # DeviceConfig's, whose DRA driver, node labeller and utility images + # default to floating tags - so each is pinned here. The DRA driver + # is the component the claims go through; on a floating tag any pod + # reschedule could change it under a running cluster. + # + # Driver management is off: a BYO node's image must already carry + # the amdgpu kernel driver, and the chart's in-cluster driver build + # needs a push registry configured (driver.image defaults to a + # docker.io placeholder), without which KMM loops on a driver image + # it can never resolve and the DRA driver - gated on the KMM-ready + # node label - never schedules. KMM must stay installed even with + # the driver unmanaged: the operator watches KMM's kinds at startup + # no matter what, and crashes without their CRDs, taking every + # operand down with it. + # + # The node labeller stays on for scheduling labels; NFD is the + # standalone instance above, so the chart's bundled one is off and + # two instances never race over the same label namespace. Every + # per-node component tolerates the amd.com/gpu taint AMD GPU pools + # carry - the chart defaults tolerate nothing, which strands the + # daemonsets off the very nodes they serve. + Chart( + key="amd-gpu-operator", + release="mp-gpu-operator-charts", + namespace="kube-amd-gpu", + chart="gpu-operator-charts", + repository="https://rocm.github.io/gpu-operator", + version="v1.5.1", + depends_on=["cert-manager", "node-feature-discovery"], + dra_drivers=["gpu.amd.com"], + values={ + "node-feature-discovery": {"enabled": False}, + # Argo-based node remediation pulls a workflow controller the + # serving stack doesn't need. + "remediation": {"enabled": False, "installCRDs": False}, + "crds": {"defaultCR": {"upgrade": True}}, + "deviceConfig": { + "spec": { + "commonConfig": { + "utilsContainer": {"image": "docker.io/rocm/gpu-operator-utils:v1.5.1"}, + }, + "driver": {"enable": False}, + "devicePlugin": { + "enableDevicePlugin": False, + "nodeLabellerImage": "docker.io/rocm/k8s-device-plugin:labeller-1.31.0.11", + "nodeLabellerTolerations": [ + {"key": "amd.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + ], + }, + "metricsExporter": { + # The ConfigMap above: profiler metrics on, for + # the tensor-activity mapping. + "config": {"name": "mp-metrics-exporter-config"}, + "tolerations": [ + {"key": "amd.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + ], + }, + "draDriver": { + "enable": True, + "image": "docker.io/rocm/k8s-gpu-dra-driver:v1.0.1", + "tolerations": [ + {"key": "amd.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + ], + }, + }, + }, + }, + ), ] diff --git a/functions/compose-serving-stack/function/stacks/clouds/nebius.py b/functions/compose-serving-stack/function/stacks/clouds/nebius.py index d19888d23..b3e40ff49 100644 --- a/functions/compose-serving-stack/function/stacks/clouds/nebius.py +++ b/functions/compose-serving-stack/function/stacks/clouds/nebius.py @@ -89,19 +89,21 @@ ], }, }, - # Disable components we don't need for observability. - "grafana": {"enabled": False}, - "alertmanager": {"enabled": False}, # The node-exporter chart's default toleration is keyless - it - # tolerates every taint, landing the pod on tainted GPU nodes by - # accident rather than by intent. Scope it to the GPU taint, + # tolerates every taint, landing the pod on tainted nodes by + # accident rather than by intent. Scope it to the GPU and CPU pool + # taints, # mirroring the generated clouds (TOLERATIONS in generate.py); as # a DaemonSet it still reaches every untainted node. "prometheus-node-exporter": { "tolerations": [ {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "modelplane.ai/cpu", "operator": "Exists", "effect": "NoSchedule"}, ], }, + # Disable components we don't need for observability. + "grafana": {"enabled": False}, + "alertmanager": {"enabled": False}, }, ), Chart( @@ -111,6 +113,10 @@ chart="node-feature-discovery", repository="https://kubernetes-sigs.github.io/node-feature-discovery/charts", version="0.19.0", + # NFD exists here to label GPU nodes for the DRA driver, so it + # carries the GPU stack's driver and stays off clusters whose + # classes claim no GPUs. + dra_drivers=["gpu.nvidia.com"], # The worker must run on the very nodes it is supposed to label: # the DRA driver's kubelet plugin schedules only onto nodes # carrying an NFD GPU label (feature.node.kubernetes.io/pci-10de @@ -146,6 +152,7 @@ chart="dra-driver-nvidia-gpu", repository="oci://registry.k8s.io/dra-driver-nvidia/charts", version="0.5.0", + dra_drivers=["gpu.nvidia.com"], values={ "gpuResourcesEnabledOverride": True, "resources": {"computeDomains": {"enabled": False}}, diff --git a/functions/compose-serving-stack/function/stacks/clouds/vultr.py b/functions/compose-serving-stack/function/stacks/clouds/vultr.py index ab20434ec..e5658c908 100644 --- a/functions/compose-serving-stack/function/stacks/clouds/vultr.py +++ b/functions/compose-serving-stack/function/stacks/clouds/vultr.py @@ -99,19 +99,21 @@ ], }, }, - # Disable components we don't need for observability. - "grafana": {"enabled": False}, - "alertmanager": {"enabled": False}, # The node-exporter chart's default toleration is keyless - it - # tolerates every taint, landing the pod on tainted GPU nodes by - # accident rather than by intent. Scope it to the GPU taint, + # tolerates every taint, landing the pod on tainted nodes by + # accident rather than by intent. Scope it to the GPU and CPU pool + # taints, # mirroring the generated clouds (TOLERATIONS in generate.py); as # a DaemonSet it still reaches every untainted node. "prometheus-node-exporter": { "tolerations": [ {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "modelplane.ai/cpu", "operator": "Exists", "effect": "NoSchedule"}, ], }, + # Disable components we don't need for observability. + "grafana": {"enabled": False}, + "alertmanager": {"enabled": False}, }, ), # Publishes each GPU node's devices as DRA ResourceSlices and @@ -127,6 +129,7 @@ chart="dra-driver-nvidia-gpu", repository="oci://registry.k8s.io/dra-driver-nvidia/charts", version="0.5.0", + dra_drivers=["gpu.nvidia.com"], values={ "gpuResourcesEnabledOverride": True, "resources": {"computeDomains": {"enabled": False}}, diff --git a/functions/compose-serving-stack/function/stacks/common.py b/functions/compose-serving-stack/function/stacks/common.py index 96f68940a..61f18d132 100644 --- a/functions/compose-serving-stack/function/stacks/common.py +++ b/functions/compose-serving-stack/function/stacks/common.py @@ -18,8 +18,9 @@ Dynamo stacks don't differ: the gateway path (Envoy Gateway, Envoy AI Gateway and its CRDs, the Gateway API Inference Extension CRDs, the gateway namespace and EnvoyProxy), the gateway's PKI (a self-signed -Issuer and trust-manager), the DRA critical-pods quota and the cluster -agent. +Issuer and trust-manager), the DRA critical-pods quotas, the cluster +agent, and the device stacks every cloud pins identically - today +dra-driver-cpu, whose nodes look the same whatever provisioned them. The GatewayClass and Gateway are deliberately absent: they read spec.gateway (className, hostname), which is per-cluster configuration, @@ -27,7 +28,7 @@ Versions here are Modelplane's pins: this file carries what no generator resolves - the contract surface a ModelReplica composes -against, whose versions have nothing to do with the hardware (see +against, whose versions have nothing to do with the GPU hardware (see design/serving-stack-generation.md, "Component sources"). """ @@ -67,7 +68,11 @@ # Must match the namespace every cloud half installs the NVIDIA DRA # driver into - generated and hand-written alike - so this quota lands # where the kubelet plugin runs. -_DRA_DRIVER_NAMESPACE = "nvidia-dra-driver" +_NVIDIA_DRA_DRIVER_NAMESPACE = "nvidia-dra-driver" + +# The namespace the dra-driver-cpu component below installs into; its +# critical-pods quota must land in the same place. +_CPU_DRA_DRIVER_NAMESPACE = "dra-driver-cpu" # The cluster agent is Modelplane's own. Its chart and image share one # repository, tagged 0.1.0 and v0.1.0 respectively, and the chart's default @@ -282,17 +287,83 @@ def _crds(filename: str) -> list[dict[str, Any]]: # priority. GKE only admits such pods in a namespace whose # ResourceQuota permits those priority classes; without it the # daemonset gets FailedCreate and never publishes ResourceSlices. - # Laid down everywhere: it only grants headroom, so it's harmless on - # clusters that don't restrict them. + # Laid down wherever the NVIDIA stack installs: it only grants + # headroom, so it's harmless on clusters that don't restrict them, + # but the tag keeps it off clusters where nothing creates the + # nvidia-dra-driver namespace, where the Object could never apply. Manifests( key="dra-driver-critical-pods-quota", + dra_drivers=["gpu.nvidia.com"], + manifests=[ + { + "apiVersion": "v1", + "kind": "ResourceQuota", + "metadata": { + "name": "allow-critical-pods", + "namespace": _NVIDIA_DRA_DRIVER_NAMESPACE, + }, + "spec": { + "hard": {"pods": "1000"}, + "scopeSelector": { + "matchExpressions": [ + { + "operator": "In", + "scopeName": "PriorityClass", + "values": [ + "system-node-critical", + "system-cluster-critical", + ], + }, + ], + }, + }, + }, + ], + ), + # Publishes each CPU node's cores as DRA devices and registers the + # dra.cpu DeviceClass CPU classes claim through: one device per NUMA + # node (the chart's default grouped mode), with the node's + # allocatable CPUs as consumable capacity. A claimed device pins the + # engine container to a cpuset. Pinned here because a CPU node needs + # the same driver whatever provisioned it, so every cloud - Existing + # included - installs the one pin. The nodeSelector keeps it to CPU + # pools, which every cluster composition labels modelplane.ai/cpu; + # the tolerations are set because the chart's default is a keyless + # wildcard, which AKS rejects at admission and the e2e toleration + # audit flags - the CPU taint is the only one the selector can land + # the driver behind. reservedCPUs holds cores 0-1 back for the OS + # and kubelet, so a class's capacity.cpu is the node's vCPUs less + # two - a fixed, documented value rather than a per-pool knob + # someone must keep in step with the class by hand. + Chart( + key="dra-driver-cpu", + release="mp-dra-driver-cpu", + namespace=_CPU_DRA_DRIVER_NAMESPACE, + chart="dra-driver-cpu", + repository="oci://registry.k8s.io/dra-driver-cpu/charts", + version="0.3.0", + dra_drivers=["dra.cpu"], + values={ + "nodeSelector": {"modelplane.ai/cpu": "true"}, + "tolerations": [ + {"key": "modelplane.ai/cpu", "operator": "Exists", "effect": "NoSchedule"}, + ], + "driverConfig": {"reservedCPUs": "0-1"}, + }, + ), + # dra-driver-cpu's daemonset runs at system-node-critical priority + # like the NVIDIA kubelet plugin above, so its namespace needs the + # same quota on GKE. + Manifests( + key="dra-driver-cpu-critical-pods-quota", + dra_drivers=["dra.cpu"], manifests=[ { "apiVersion": "v1", "kind": "ResourceQuota", "metadata": { "name": "allow-critical-pods", - "namespace": _DRA_DRIVER_NAMESPACE, + "namespace": _CPU_DRA_DRIVER_NAMESPACE, }, "spec": { "hard": {"pods": "1000"}, diff --git a/functions/compose-serving-stack/function/stacks/components.py b/functions/compose-serving-stack/function/stacks/components.py index cc6267681..69e22a21f 100644 --- a/functions/compose-serving-stack/function/stacks/components.py +++ b/functions/compose-serving-stack/function/stacks/components.py @@ -56,6 +56,15 @@ class Chart: depends on, so the install gate orders on health rather than deploy - the generator derives it from the dependency edges, and the hand-written files state it where a cross-half edge lands on them. + + `dra_drivers` names the DRA drivers whose device stacks this + component belongs to (gpu.nvidia.com, dra.cpu, ...). An untagged + component is needed regardless of devices and always installs; a + tagged one installs only when one of its drivers is in ServingStack + spec.draDrivers. A component can serve several stacks (NFD labels + nodes for every GPU driver). The tags on a dependency edge are + constrained so filtering can't break it; join()'s docstring states + and its checks enforce that rule. """ key: str @@ -67,6 +76,7 @@ class Chart: wait: bool = False depends_on: list[str] = field(default_factory=list) values: dict[str, Any] | None = None + dra_drivers: list[str] = field(default_factory=list) @dataclass @@ -82,12 +92,15 @@ class Manifests: applied to every doc in the entry (see fn.py's _k8s_object): use it when readiness must reflect a controller-populated status field, and keep an entry to one doc when only that doc has one. + + `dra_drivers` behaves as on Chart. """ key: str manifests: list[dict[str, Any]] depends_on: list[str] = field(default_factory=list) ready: str | None = None + dra_drivers: list[str] = field(default_factory=list) # A plain assignment rather than a `type` statement: the packages diff --git a/functions/compose-serving-stack/function/stacks/metrics.py b/functions/compose-serving-stack/function/stacks/metrics.py index 5d534819c..f234eff0d 100644 --- a/functions/compose-serving-stack/function/stacks/metrics.py +++ b/functions/compose-serving-stack/function/stacks/metrics.py @@ -139,6 +139,44 @@ "DCGM_FI_DEV_TOTAL_ENERGY_CONSUMPTION": "Millijoules", } +# The AMD GPUs, through the AMD GPU operator's device-metrics-exporter, onto +# the same modelplane_gpu_* names the DCGM series map to, so a dashboard reads +# one surface whichever vendor's pool served it. The exporter reports VRAM in +# MiB, activity in percent and energy in microjoules, each renamed onto a name +# that claims a different unit, so each says so. Junction temperature is the +# one the Instinct MI3xx series reports (MI2xx reports edge instead). +# Tensor activity (MFMA utilization, the analog of DCGM's tensor-pipe +# activity) is a GPU_PROF_* field the exporter ships disabled; the stack's +# exporter config turns ProfilerMetrics on to publish it (see the AMD +# chart pin in clouds/existing.py) at the cost of the single profiler +# instance the hardware allows. The +# exporter reports +# its accumulated energy as a gauge where DCGM's is a counter, and +# modelplane_energy_joules_total is a counter, so the energy series is marked +# cumulative and the collector converts it to a monotonic sum - one name, +# one type, whichever vendor feeds it. +_AMD_GPU = { + "gpu_used_vram": "modelplane_gpu_memory_used_bytes", + "gpu_free_vram": "modelplane_gpu_memory_free_bytes", + "gpu_gfx_activity": "modelplane_gpu_compute_active_ratio", + "gpu_umc_activity": "modelplane_gpu_memory_bandwidth_ratio", + "gpu_prof_tensor_active_percent": "modelplane_gpu_tensor_active_ratio", + "gpu_junction_temperature": "modelplane_gpu_temperature_celsius", + "gpu_power_usage": "modelplane_gpu_power_watts", + "gpu_energy_consumed": "modelplane_energy_joules_total", +} + +_AMD_GPU_UNITS = { + "gpu_used_vram": "Mebibytes", + "gpu_free_vram": "Mebibytes", + "gpu_gfx_activity": "Percent", + "gpu_umc_activity": "Percent", + "gpu_prof_tensor_active_percent": "Percent", + "gpu_energy_consumed": "Microjoules", +} + +_AMD_GPU_CUMULATIVE = ("gpu_energy_consumed",) + # How each request vLLM finished ended, by vLLM's finished_reason renamed to # reason: stop, length and abort at v0.9.2, and error and repetition as well @@ -199,10 +237,14 @@ def _mapping( units: dict[str, str] | None = None, tokens: tuple[str, str] | None = None, extra: tuple[dict[str, object], ...] = (), + cumulative: tuple[str, ...] = (), ) -> v1alpha1.MetricMapping: units = units or {} metrics: list[dict[str, object]] = [ - {"from": src, "to": dst} | ({"fromUnit": units[src]} if src in units else {}) for src, dst in pairs.items() + {"from": src, "to": dst} + | ({"fromUnit": units[src]} if src in units else {}) + | ({"cumulative": True} if src in cumulative else {}) + for src, dst in pairs.items() ] if tokens: metrics += _tokens(*tokens) @@ -227,6 +269,7 @@ def mappings() -> list[v1alpha1.MetricMapping]: _mapping("modelplane-sglang", _SGLANG, tokens=_SGLANG_TOKENS, extra=(_SGLANG_RESPONSES,)), _mapping("modelplane-picker", _PICKER), _mapping("modelplane-gpu", _GPU, _GPU_UNITS), + _mapping("modelplane-gpu-amd", _AMD_GPU, _AMD_GPU_UNITS, cumulative=_AMD_GPU_CUMULATIVE), ] diff --git a/functions/compose-serving-stack/generate.py b/functions/compose-serving-stack/generate.py index e8eb2c01f..303e97f42 100644 --- a/functions/compose-serving-stack/generate.py +++ b/functions/compose-serving-stack/generate.py @@ -498,6 +498,14 @@ def tolerations_key(key: str) -> bool: # AKS; GKE's own taint carries present). GPU: list[Values] = [{"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}] +# Tolerate every schedulable pool's taint - GPU and CPU alike - for +# node-reaching DaemonSets that aren't part of any one device stack and +# must observe every node Modelplane schedules models onto. +NODE: list[Values] = [ + {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "modelplane.ai/cpu", "operator": "Exists", "effect": "NoSchedule"}, +] + # Recipe component name -> dotted values path -> the tolerations that # replace a wildcard found there. Like ALLOW/DROP this fails closed: a # wildcard at an unclassified path fails the run, else an aicr bump @@ -519,7 +527,7 @@ def tolerations_key(key: str) -> bool: "gc.tolerations": SYSTEM, }, "kube-prometheus-stack": { - "prometheus-node-exporter.tolerations": GPU, + "prometheus-node-exporter.tolerations": NODE, "alertmanager.alertmanagerSpec.tolerations": SYSTEM, "grafana.tolerations": SYSTEM, "kube-state-metrics.tolerations": SYSTEM, @@ -565,22 +573,27 @@ def tolerations_key(key: str) -> bool: ("gpu-operator", "DaemonSet", "nvidia-toolkit-hardening"): GPU, } -# Recipe component name -> the key we emit it as. The key and the -# mp-prefixed release name are Modelplane's, not AICR's, so a -# component's identity survives upstream renames (see the design's -# "Ordering and identity"). +# Recipe component name -> (the key we emit it as, the DRA drivers +# whose device stacks it belongs to). The key and the mp-prefixed +# release name are Modelplane's, not AICR's, so a component's identity +# survives upstream renames (see the design's "Ordering and identity"). +# The driver list fails closed like the allowlist itself: classifying a +# new component includes deciding which stacks it serves, the review a +# bump already requires - the tag just records it. What the tag means +# for installs lives on Component.dra_drivers and join() in the stacks +# package (and in design/multi-accelerator.md). ALLOW = { - "nfd": "node-feature-discovery", - "cert-manager": "cert-manager", - "kube-prometheus-stack": "kube-prometheus-stack", - "prometheus-operator-crds": "prometheus-operator-crds", - "prometheus-adapter": "prometheus-adapter", - "k8s-ephemeral-storage-metrics": "k8s-ephemeral-storage-metrics", - "gpu-operator": "gpu-operator", - "nvidia-dra-driver-gpu": "nvidia-dra-driver-gpu", - "nvsentinel": "nvsentinel", - "nodewright-operator": "nodewright-operator", - "nodewright-customizations": "nodewright-customizations", + "nfd": ("node-feature-discovery", ["gpu.nvidia.com"]), + "cert-manager": ("cert-manager", []), + "kube-prometheus-stack": ("kube-prometheus-stack", []), + "prometheus-operator-crds": ("prometheus-operator-crds", []), + "prometheus-adapter": ("prometheus-adapter", []), + "k8s-ephemeral-storage-metrics": ("k8s-ephemeral-storage-metrics", []), + "gpu-operator": ("gpu-operator", ["gpu.nvidia.com"]), + "nvidia-dra-driver-gpu": ("nvidia-dra-driver-gpu", ["gpu.nvidia.com"]), + "nvsentinel": ("nvsentinel", ["gpu.nvidia.com"]), + "nodewright-operator": ("nodewright-operator", ["gpu.nvidia.com"]), + "nodewright-customizations": ("nodewright-customizations", ["gpu.nvidia.com"]), } # In the recipe, deliberately not in the stack. Each entry needs a @@ -624,6 +637,16 @@ def tolerations_key(key: str) -> bool: "nodewright-operator": "skyhook", } +# DRA drivers a cloud supports that arrive outside the serving stack, +# emitted into the generated module so the stacks package's derived +# supported-drivers table can't drift from the generated file. EKS's +# dra.net is the aws-efa DROP entry above as the filter sees it: +# compose-eks-cluster installs the EFA DRA driver when a pool asks for +# the fabric, and the kimi-k2 and qwen3-coder recipes claim it today. +EXTERNAL_DRIVERS = { + "eks": ["dra.net"], +} + # Long recipe-carried config blobs keep their content, not their # cosmetics: dcgm-exporter's metrics CSV ships with blank spacer lines # the exporter ignores. Trimming them keeps the generated file @@ -1014,7 +1037,7 @@ def transform(cloud: str, recipe: Values, bundle_dir: pathlib.Path) -> tuple[lis continue if name not in ALLOW: sys.exit(f"unknown component {name!r}: classify it in ALLOW or DROP") - key = ALLOW[name] + key, dra_drivers = ALLOW[name] values = deep_merge(bundle_values(bundle_dir, name), ref.get("overrides")) for path in TRIM.get(name, []): @@ -1048,7 +1071,7 @@ def transform(cloud: str, recipe: Values, bundle_dir: pathlib.Path) -> tuple[lis depends = [] for dep in ref.get("dependencyRefs", []): if dep in ALLOW: - depends.append(ALLOW[dep]) + depends.append(ALLOW[dep][0]) elif dep in DROP: findings.append( f"dropped dep: {name} -> {dep} (dropped; the join with " @@ -1088,13 +1111,16 @@ def transform(cloud: str, recipe: Values, bundle_dir: pathlib.Path) -> tuple[lis findings.append( f"pre-manifests: {name}: {len(manifests)} objects (namespace + {', '.join(f.name for f in pre)})" ) - components.append( - { - "type": "Manifests", - "key": f"{key}-pre-manifests", - "manifests": manifests, - } - ) + pre_entry = { + "type": "Manifests", + "key": f"{key}-pre-manifests", + "manifests": manifests, + } + # The chart's own tags, because the chart depends on this + # entry - join()'s dependency-tagging rule. + if dra_drivers: + pre_entry["dra_drivers"] = dra_drivers + components.append(pre_entry) depends.append(f"{key}-pre-manifests") entry = { @@ -1111,6 +1137,8 @@ def transform(cloud: str, recipe: Values, bundle_dir: pathlib.Path) -> tuple[lis } if depends: entry["depends_on"] = depends + if dra_drivers: + entry["dra_drivers"] = dra_drivers if values: entry["values"] = values components.append(entry) @@ -1132,14 +1160,17 @@ def transform(cloud: str, recipe: Values, bundle_dir: pathlib.Path) -> tuple[lis for doc in manifests: findings += scope_manifest_tolerations(name, doc) findings.append(f"manifests: {name}: {len(manifests)} objects ({', '.join(f.name for f in rendered)})") - components.append( - { - "type": "Manifests", - "key": f"{key}-manifests", - "depends_on": [key], - "manifests": manifests, - } - ) + post_entry = { + "type": "Manifests", + "key": f"{key}-manifests", + "depends_on": [key], + "manifests": manifests, + } + # Depending on a tagged chart requires carrying its tags, so + # the filter keeps or drops the pair together. + if dra_drivers: + post_entry["dra_drivers"] = dra_drivers + components.append(post_entry) # The bundle is consumed fail-closed, like the recipe: every # component directory must belong to a classified component and use @@ -1289,6 +1320,7 @@ def emit( "version", "wait", "depends_on", + "dra_drivers", "values", "manifests", ] @@ -1310,6 +1342,11 @@ def emit( "", f"from function.stacks.components import {', '.join(names)}", "", + "# DRA drivers this cloud supports that arrive outside the serving", + "# stack; the stacks package folds them into its derived", + "# supported-drivers table (see generate.py's EXTERNAL_DRIVERS).", + f"EXTERNAL_DRIVERS: list[str] = {pyfmt(EXTERNAL_DRIVERS.get(cloud, []))}", + "", "COMPONENTS: list[Component] = [", ] for entry in components: diff --git a/functions/compose-serving-stack/tests/test_collector.py b/functions/compose-serving-stack/tests/test_collector.py index 183192334..392b24917 100644 --- a/functions/compose-serving-stack/tests/test_collector.py +++ b/functions/compose-serving-stack/tests/test_collector.py @@ -319,12 +319,12 @@ def _config( "__meta_kubernetes_pod_label_app", ], "action": "keep", - "regex": ".*dcgm.*", + "regex": ".*dcgm.*|.*metrics-exporter.*", }, { "source_labels": ["__meta_kubernetes_pod_container_port_name"], "action": "keep", - "regex": "metrics", + "regex": "metrics|exporter-port", }, {"source_labels": ["__meta_kubernetes_pod_node_name"], "target_label": "node"}, ], @@ -361,7 +361,7 @@ def _config( "__meta_kubernetes_pod_label_app", ], "action": "drop", - "regex": ".*dcgm.*", + "regex": ".*dcgm.*|.*metrics-exporter.*", }, { "source_labels": ["__meta_kubernetes_pod_annotation_prometheus_io_path"], @@ -438,7 +438,9 @@ def _config( # under the default error mode fails the whole batch. A block # ahead of the renames rather than a line, because the # processor finishes a block over every metric before the next - # one starts. + # one starts. AMD's exporter reports its accumulated energy as + # a gauge where the name says counter, so that series is + # converted to a monotonic sum here too. { "context": "metric", "error_mode": "ignore", @@ -446,6 +448,13 @@ def _config( 'scale_metric(1048576.0) where metric.name == "DCGM_FI_DEV_FB_USED"', 'scale_metric(1048576.0) where metric.name == "DCGM_FI_DEV_FB_FREE"', 'scale_metric(0.001) where metric.name == "DCGM_FI_DEV_TOTAL_ENERGY_CONSUMPTION"', + 'scale_metric(1048576.0) where metric.name == "gpu_used_vram"', + 'scale_metric(1048576.0) where metric.name == "gpu_free_vram"', + 'scale_metric(0.01) where metric.name == "gpu_gfx_activity"', + 'scale_metric(0.01) where metric.name == "gpu_umc_activity"', + 'scale_metric(0.01) where metric.name == "gpu_prof_tensor_active_percent"', + 'scale_metric(0.000001) where metric.name == "gpu_energy_consumed"', + 'convert_gauge_to_sum("cumulative", true) where metric.name == "gpu_energy_consumed"', ], }, { @@ -511,6 +520,14 @@ def _config( 'set(metric.name, "modelplane_gpu_temperature_celsius") where metric.name == "DCGM_FI_DEV_GPU_TEMP"', 'set(metric.name, "modelplane_gpu_power_watts") where metric.name == "DCGM_FI_DEV_POWER_USAGE"', 'set(metric.name, "modelplane_energy_joules_total") where metric.name == "DCGM_FI_DEV_TOTAL_ENERGY_CONSUMPTION"', + 'set(metric.name, "modelplane_gpu_memory_used_bytes") where metric.name == "gpu_used_vram"', + 'set(metric.name, "modelplane_gpu_memory_free_bytes") where metric.name == "gpu_free_vram"', + 'set(metric.name, "modelplane_gpu_compute_active_ratio") where metric.name == "gpu_gfx_activity"', + 'set(metric.name, "modelplane_gpu_memory_bandwidth_ratio") where metric.name == "gpu_umc_activity"', + 'set(metric.name, "modelplane_gpu_tensor_active_ratio") where metric.name == "gpu_prof_tensor_active_percent"', + 'set(metric.name, "modelplane_gpu_temperature_celsius") where metric.name == "gpu_junction_temperature"', + 'set(metric.name, "modelplane_gpu_power_watts") where metric.name == "gpu_power_usage"', + 'set(metric.name, "modelplane_energy_joules_total") where metric.name == "gpu_energy_consumed"', ], }, ] @@ -916,7 +933,7 @@ def test_in_flight(case: InFlightCase) -> None: # because what the collector compiles them to is what this case pins. StatementsCase( name="BuiltIn", - reason="The built-in mappings convert DCGM's units, then rename every series they map.", + reason="The built-in mappings convert the GPU exporters' units, then rename every series they map.", mappings=list(stacks.BUILTIN_MAPPINGS), want=( [ @@ -931,6 +948,13 @@ def test_in_flight(case: InFlightCase) -> None: 'scale_metric(1048576.0) where metric.name == "DCGM_FI_DEV_FB_USED"', 'scale_metric(1048576.0) where metric.name == "DCGM_FI_DEV_FB_FREE"', 'scale_metric(0.001) where metric.name == "DCGM_FI_DEV_TOTAL_ENERGY_CONSUMPTION"', + 'scale_metric(1048576.0) where metric.name == "gpu_used_vram"', + 'scale_metric(1048576.0) where metric.name == "gpu_free_vram"', + 'scale_metric(0.01) where metric.name == "gpu_gfx_activity"', + 'scale_metric(0.01) where metric.name == "gpu_umc_activity"', + 'scale_metric(0.01) where metric.name == "gpu_prof_tensor_active_percent"', + 'scale_metric(0.000001) where metric.name == "gpu_energy_consumed"', + 'convert_gauge_to_sum("cumulative", true) where metric.name == "gpu_energy_consumed"', ], [ 'set(datapoint.attributes["direction"], "input") where metric.name == "vllm:request_prompt_tokens_sum"', @@ -993,6 +1017,14 @@ def test_in_flight(case: InFlightCase) -> None: 'set(metric.name, "modelplane_gpu_temperature_celsius") where metric.name == "DCGM_FI_DEV_GPU_TEMP"', 'set(metric.name, "modelplane_gpu_power_watts") where metric.name == "DCGM_FI_DEV_POWER_USAGE"', 'set(metric.name, "modelplane_energy_joules_total") where metric.name == "DCGM_FI_DEV_TOTAL_ENERGY_CONSUMPTION"', + 'set(metric.name, "modelplane_gpu_memory_used_bytes") where metric.name == "gpu_used_vram"', + 'set(metric.name, "modelplane_gpu_memory_free_bytes") where metric.name == "gpu_free_vram"', + 'set(metric.name, "modelplane_gpu_compute_active_ratio") where metric.name == "gpu_gfx_activity"', + 'set(metric.name, "modelplane_gpu_memory_bandwidth_ratio") where metric.name == "gpu_umc_activity"', + 'set(metric.name, "modelplane_gpu_tensor_active_ratio") where metric.name == "gpu_prof_tensor_active_percent"', + 'set(metric.name, "modelplane_gpu_temperature_celsius") where metric.name == "gpu_junction_temperature"', + 'set(metric.name, "modelplane_gpu_power_watts") where metric.name == "gpu_power_usage"', + 'set(metric.name, "modelplane_energy_joules_total") where metric.name == "gpu_energy_consumed"', ], ), ), @@ -1193,7 +1225,7 @@ def test_transform(case: TransformCase) -> None: ( "collector", _deployment( - config_hash="4811f4b31c39a4f1", + config_hash="b18edf7fffe8ea06", volumes=[{"name": "config", "configMap": {"name": "modelplane-collector"}}], volume_mounts=[{"name": "config", "mountPath": "/conf"}], env_from=None, @@ -1238,7 +1270,7 @@ def test_transform(case: TransformCase) -> None: ( "collector", _deployment( - config_hash="978ab36bf03ec977", + config_hash="c26c998b9d08f9ab", volumes=[ {"name": "config", "configMap": {"name": "modelplane-collector"}}, {"name": "credentials-primary", "secret": {"secretName": "telemetry-credentials"}}, @@ -1305,7 +1337,7 @@ def test_transform(case: TransformCase) -> None: ( "collector", _deployment( - config_hash="de542e1c484b7ed1", + config_hash="6069bcd9809fce22", volumes=[ {"name": "config", "configMap": {"name": "modelplane-collector"}}, {"name": "credentials-vendor", "secret": {"secretName": "vendor-token"}}, diff --git a/functions/compose-serving-stack/tests/test_fn.py b/functions/compose-serving-stack/tests/test_fn.py index ae1e46f40..90409cf2a 100644 --- a/functions/compose-serving-stack/tests/test_fn.py +++ b/functions/compose-serving-stack/tests/test_fn.py @@ -95,13 +95,21 @@ def _serving_stack( secrets: list[v1alpha1.Secret], gateway: v1alpha1.Gateway, gpu: v1alpha1.Gpu | None, + dra_drivers: list[str] | None, ) -> fnv1.Resource: - """The observed ServingStack, test-backend in namespace test-ns.""" + """The observed ServingStack, test-backend in namespace test-ns. + + dra_drivers populates spec.draDrivers; None leaves the field absent, + which means the same as empty: only the untagged components install. + """ + spec = v1alpha1.Spec(cloud=cloud, stack=stack, secrets=secrets, gateway=gateway, gpu=gpu) + if dra_drivers is not None: + spec.draDrivers = [v1alpha1.DraDriver(d) for d in dra_drivers] return fnv1.Resource( resource=resource.dict_to_struct( v1alpha1.ServingStack( metadata=metav1.ObjectMeta(name="test-backend", namespace="test-ns"), - spec=v1alpha1.Spec(cloud=cloud, stack=stack, secrets=secrets, gateway=gateway, gpu=gpu), + spec=spec, ).model_dump(exclude_none=True, mode="json", by_alias=True) ) ) @@ -292,8 +300,13 @@ def _cert_manager(*, ready: fnv1.Ready) -> fnv1.Resource: """ -def _kube_prometheus_stack(*, ready: fnv1.Ready) -> fnv1.Resource: - """The composed kube-prometheus-stack Release.""" +def _kube_prometheus_stack(*, ready: fnv1.Ready, node_exporter_tolerations: list[dict]) -> fnv1.Resource: + """The composed kube-prometheus-stack Release. + + node_exporter_tolerations is the inference pool taints the cloud's pin + lets the node exporter tolerate: every taint a BYO pool may carry on + Existing, the NVIDIA and CPU taints elsewhere. + """ return fnv1.Resource( resource=resource.dict_to_struct( { @@ -348,13 +361,9 @@ def _kube_prometheus_stack(*, ready: fnv1.Ready) -> fnv1.Resource: ], } }, + "prometheus-node-exporter": {"tolerations": node_exporter_tolerations}, "grafana": {"enabled": False}, "alertmanager": {"enabled": False}, - "prometheus-node-exporter": { - "tolerations": [ - {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, - ], - }, }, }, }, @@ -364,11 +373,12 @@ def _kube_prometheus_stack(*, ready: fnv1.Ready) -> fnv1.Resource: ) -def _node_feature_discovery(*, ready: fnv1.Ready, wait: bool) -> fnv1.Resource: +def _node_feature_discovery(*, ready: fnv1.Ready, wait: bool, worker_tolerations: list[dict]) -> fnv1.Resource: """The composed node-feature-discovery Release the hand-written clouds pin, waiting for health if wait is set. - Civo's waits and Existing's doesn't. GKE's generated half pins its own, - _gke_node_feature_discovery. + worker_tolerations is the GPU pool taints the cloud's pin lets the + worker tolerate: NVIDIA's and AMD's on Existing, NVIDIA's elsewhere. + GKE's generated half pins its own, _gke_node_feature_discovery. """ for_provider: dict = { "chart": { @@ -377,9 +387,7 @@ def _node_feature_discovery(*, ready: fnv1.Ready, wait: bool) -> fnv1.Resource: "version": "0.19.0", }, "namespace": "node-feature-discovery", - "values": { - "worker": {"tolerations": [{"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}]} - }, + "values": {"worker": {"tolerations": worker_tolerations}}, } if wait: for_provider |= {"wait": True, "waitTimeout": "10m"} @@ -1549,12 +1557,12 @@ def _collector_config( "__meta_kubernetes_pod_label_app", ], "action": "keep", - "regex": ".*dcgm.*", + "regex": ".*dcgm.*|.*metrics-exporter.*", }, { "source_labels": ["__meta_kubernetes_pod_container_port_name"], "action": "keep", - "regex": "metrics", + "regex": "metrics|exporter-port", }, {"source_labels": ["__meta_kubernetes_pod_node_name"], "target_label": "node"}, ], @@ -1587,7 +1595,7 @@ def _collector_config( "__meta_kubernetes_pod_label_app", ], "action": "drop", - "regex": ".*dcgm.*", + "regex": ".*dcgm.*|.*metrics-exporter.*", }, { "source_labels": ["__meta_kubernetes_pod_annotation_prometheus_io_path"], @@ -1654,6 +1662,13 @@ def _collector_config( 'scale_metric(1048576.0) where metric.name == "DCGM_FI_DEV_FB_USED"', 'scale_metric(1048576.0) where metric.name == "DCGM_FI_DEV_FB_FREE"', 'scale_metric(0.001) where metric.name == "DCGM_FI_DEV_TOTAL_ENERGY_CONSUMPTION"', + 'scale_metric(1048576.0) where metric.name == "gpu_used_vram"', + 'scale_metric(1048576.0) where metric.name == "gpu_free_vram"', + 'scale_metric(0.01) where metric.name == "gpu_gfx_activity"', + 'scale_metric(0.01) where metric.name == "gpu_umc_activity"', + 'scale_metric(0.01) where metric.name == "gpu_prof_tensor_active_percent"', + 'scale_metric(0.000001) where metric.name == "gpu_energy_consumed"', + 'convert_gauge_to_sum("cumulative", true) where metric.name == "gpu_energy_consumed"', ], }, { @@ -1719,6 +1734,14 @@ def _collector_config( 'set(metric.name, "modelplane_gpu_temperature_celsius") where metric.name == "DCGM_FI_DEV_GPU_TEMP"', 'set(metric.name, "modelplane_gpu_power_watts") where metric.name == "DCGM_FI_DEV_POWER_USAGE"', 'set(metric.name, "modelplane_energy_joules_total") where metric.name == "DCGM_FI_DEV_TOTAL_ENERGY_CONSUMPTION"', + 'set(metric.name, "modelplane_gpu_memory_used_bytes") where metric.name == "gpu_used_vram"', + 'set(metric.name, "modelplane_gpu_memory_free_bytes") where metric.name == "gpu_free_vram"', + 'set(metric.name, "modelplane_gpu_compute_active_ratio") where metric.name == "gpu_gfx_activity"', + 'set(metric.name, "modelplane_gpu_memory_bandwidth_ratio") where metric.name == "gpu_umc_activity"', + 'set(metric.name, "modelplane_gpu_tensor_active_ratio") where metric.name == "gpu_prof_tensor_active_percent"', + 'set(metric.name, "modelplane_gpu_temperature_celsius") where metric.name == "gpu_junction_temperature"', + 'set(metric.name, "modelplane_gpu_power_watts") where metric.name == "gpu_power_usage"', + 'set(metric.name, "modelplane_energy_joules_total") where metric.name == "gpu_energy_consumed"', ], }, ] @@ -1865,6 +1888,7 @@ def _to_dict(msg: message.Message) -> dict: composite=_serving_stack( cloud="Existing", stack="Dynamo", + dra_drivers=["gpu.nvidia.com"], secrets=[ v1alpha1.Secret(type="Kubeconfig", name="kube-secret", key="kubeconfig"), v1alpha1.Secret(type="GoogleApplicationCredentials", name="sa-secret", key="private_key"), @@ -2038,6 +2062,7 @@ def _to_dict(msg: message.Message) -> dict: composite=_serving_stack( cloud="Existing", stack="Dynamo", + dra_drivers=["gpu.nvidia.com"], secrets=[ v1alpha1.Secret(type="Kubeconfig", name="kube-secret", key="kubeconfig"), v1alpha1.Secret(type="GoogleApplicationCredentials", name="sa-secret", key="private_key"), @@ -2081,8 +2106,22 @@ def _to_dict(msg: message.Message) -> dict: ready=fnv1.READY_TRUE, ), "cert-manager": _cert_manager(ready=fnv1.READY_UNSPECIFIED), - "kube-prometheus-stack": _kube_prometheus_stack(ready=fnv1.READY_UNSPECIFIED), - "node-feature-discovery": _node_feature_discovery(ready=fnv1.READY_UNSPECIFIED, wait=False), + "kube-prometheus-stack": _kube_prometheus_stack( + ready=fnv1.READY_UNSPECIFIED, + node_exporter_tolerations=[ + {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "amd.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "modelplane.ai/cpu", "operator": "Exists", "effect": "NoSchedule"}, + ], + ), + "node-feature-discovery": _node_feature_discovery( + ready=fnv1.READY_UNSPECIFIED, + wait=True, + worker_tolerations=[ + {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "amd.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + ], + ), "nvidia-dra-driver-gpu": _nvidia_dra_driver_gpu(ready=fnv1.READY_UNSPECIFIED), "ai-gateway-crds": _ai_gateway_crds(ready=fnv1.READY_UNSPECIFIED), "gaie-crds-inferenceobjectives.inference.networking.x-k8s.io": _gaie_crd( @@ -2464,6 +2503,7 @@ def _to_dict(msg: message.Message) -> dict: composite=_serving_stack( cloud="Existing", stack="Dynamo", + dra_drivers=["gpu.nvidia.com"], secrets=[ v1alpha1.Secret(type="Kubeconfig", name="kube-secret", key="kubeconfig"), v1alpha1.Secret(type="GoogleApplicationCredentials", name="sa-secret", key="private_key"), @@ -2556,8 +2596,22 @@ def _to_dict(msg: message.Message) -> dict: ready=fnv1.READY_TRUE, ), "cert-manager": _cert_manager(ready=fnv1.READY_TRUE), - "kube-prometheus-stack": _kube_prometheus_stack(ready=fnv1.READY_TRUE), - "node-feature-discovery": _node_feature_discovery(ready=fnv1.READY_TRUE, wait=False), + "kube-prometheus-stack": _kube_prometheus_stack( + ready=fnv1.READY_TRUE, + node_exporter_tolerations=[ + {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "amd.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "modelplane.ai/cpu", "operator": "Exists", "effect": "NoSchedule"}, + ], + ), + "node-feature-discovery": _node_feature_discovery( + ready=fnv1.READY_TRUE, + wait=True, + worker_tolerations=[ + {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "amd.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + ], + ), "nvidia-dra-driver-gpu": _nvidia_dra_driver_gpu(ready=fnv1.READY_TRUE), "envoy-gateway": fnv1.Resource( resource=resource.dict_to_struct( @@ -3209,6 +3263,7 @@ def _to_dict(msg: message.Message) -> dict: composite=_serving_stack( cloud="Nebius", stack="Standard", + dra_drivers=["gpu.nvidia.com"], secrets=[ v1alpha1.Secret(type="Kubeconfig", name="kube-secret", key="kubeconfig"), v1alpha1.Secret( @@ -3338,6 +3393,7 @@ def _to_dict(msg: message.Message) -> dict: composite=_serving_stack( cloud="Existing", stack="Standard", + dra_drivers=["gpu.nvidia.com"], secrets=[v1alpha1.Secret(type="Kubeconfig", name="kube-secret", key="kubeconfig")], gateway=v1alpha1.Gateway( # A full Service FQDN, so the CA certificate's commonName overflows @@ -3372,8 +3428,22 @@ def _to_dict(msg: message.Message) -> dict: "provider-config-kubernetes": _kubernetes_provider_config(identity=None, ready=fnv1.READY_TRUE), "provider-config-helm": _helm_provider_config(identity=None, ready=fnv1.READY_TRUE), "cert-manager": _cert_manager(ready=fnv1.READY_UNSPECIFIED), - "kube-prometheus-stack": _kube_prometheus_stack(ready=fnv1.READY_UNSPECIFIED), - "node-feature-discovery": _node_feature_discovery(ready=fnv1.READY_UNSPECIFIED, wait=False), + "kube-prometheus-stack": _kube_prometheus_stack( + ready=fnv1.READY_UNSPECIFIED, + node_exporter_tolerations=[ + {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "amd.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "modelplane.ai/cpu", "operator": "Exists", "effect": "NoSchedule"}, + ], + ), + "node-feature-discovery": _node_feature_discovery( + ready=fnv1.READY_UNSPECIFIED, + wait=True, + worker_tolerations=[ + {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "amd.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + ], + ), "nvidia-dra-driver-gpu": _nvidia_dra_driver_gpu(ready=fnv1.READY_UNSPECIFIED), "ai-gateway-crds": _ai_gateway_crds(ready=fnv1.READY_UNSPECIFIED), "gaie-crds-inferenceobjectives.inference.networking.x-k8s.io": _gaie_crd( @@ -3509,6 +3579,7 @@ def _to_dict(msg: message.Message) -> dict: composite=_serving_stack( cloud="Existing", stack="Standard", + dra_drivers=["gpu.nvidia.com"], secrets=[v1alpha1.Secret(type="Kubeconfig", name="kube-secret", key="kubeconfig")], gateway=v1alpha1.Gateway( hostname="gateway-test-backend-12345.modelplane-system.svc.cluster.local", @@ -3551,8 +3622,22 @@ def _to_dict(msg: message.Message) -> dict: "provider-config-kubernetes": _kubernetes_provider_config(identity=None, ready=fnv1.READY_TRUE), "provider-config-helm": _helm_provider_config(identity=None, ready=fnv1.READY_TRUE), "cert-manager": _cert_manager(ready=fnv1.READY_UNSPECIFIED), - "kube-prometheus-stack": _kube_prometheus_stack(ready=fnv1.READY_UNSPECIFIED), - "node-feature-discovery": _node_feature_discovery(ready=fnv1.READY_UNSPECIFIED, wait=False), + "kube-prometheus-stack": _kube_prometheus_stack( + ready=fnv1.READY_UNSPECIFIED, + node_exporter_tolerations=[ + {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "amd.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "modelplane.ai/cpu", "operator": "Exists", "effect": "NoSchedule"}, + ], + ), + "node-feature-discovery": _node_feature_discovery( + ready=fnv1.READY_UNSPECIFIED, + wait=True, + worker_tolerations=[ + {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "amd.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + ], + ), "nvidia-dra-driver-gpu": _nvidia_dra_driver_gpu(ready=fnv1.READY_UNSPECIFIED), "ai-gateway-crds": _ai_gateway_crds(ready=fnv1.READY_UNSPECIFIED), "gaie-crds-inferenceobjectives.inference.networking.x-k8s.io": _gaie_crd( @@ -3683,6 +3768,7 @@ def _to_dict(msg: message.Message) -> dict: composite=_serving_stack( cloud="Existing", stack="Standard", + dra_drivers=["gpu.nvidia.com"], secrets=[v1alpha1.Secret(type="Kubeconfig", name="kube-secret", key="kubeconfig")], gateway=v1alpha1.Gateway(hostname="gw.clusters.example.com"), gpu=None, @@ -3701,8 +3787,22 @@ def _to_dict(msg: message.Message) -> dict: "provider-config-kubernetes": _kubernetes_provider_config(identity=None, ready=fnv1.READY_TRUE), "provider-config-helm": _helm_provider_config(identity=None, ready=fnv1.READY_TRUE), "cert-manager": _cert_manager(ready=fnv1.READY_UNSPECIFIED), - "kube-prometheus-stack": _kube_prometheus_stack(ready=fnv1.READY_UNSPECIFIED), - "node-feature-discovery": _node_feature_discovery(ready=fnv1.READY_UNSPECIFIED, wait=False), + "kube-prometheus-stack": _kube_prometheus_stack( + ready=fnv1.READY_UNSPECIFIED, + node_exporter_tolerations=[ + {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "amd.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "modelplane.ai/cpu", "operator": "Exists", "effect": "NoSchedule"}, + ], + ), + "node-feature-discovery": _node_feature_discovery( + ready=fnv1.READY_UNSPECIFIED, + wait=True, + worker_tolerations=[ + {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "amd.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + ], + ), "nvidia-dra-driver-gpu": _nvidia_dra_driver_gpu(ready=fnv1.READY_UNSPECIFIED), "ai-gateway-crds": _ai_gateway_crds(ready=fnv1.READY_UNSPECIFIED), "gaie-crds-inferenceobjectives.inference.networking.x-k8s.io": _gaie_crd( @@ -3827,6 +3927,7 @@ def _to_dict(msg: message.Message) -> dict: composite=_serving_stack( cloud="Civo", stack="Standard", + dra_drivers=["gpu.nvidia.com"], secrets=[v1alpha1.Secret(type="Kubeconfig", name="kube-secret", key="kubeconfig")], gateway=v1alpha1.Gateway(hostname="test-backend.gateways.example.com"), gpu=v1alpha1.Gpu(pools=[v1alpha1.Pool(name="h100-pool", disableNvLink=True)]), @@ -3850,8 +3951,20 @@ def _to_dict(msg: message.Message) -> dict: "provider-config-kubernetes": _kubernetes_provider_config(identity=None, ready=fnv1.READY_TRUE), "provider-config-helm": _helm_provider_config(identity=None, ready=fnv1.READY_TRUE), "cert-manager": _cert_manager(ready=fnv1.READY_TRUE), - "kube-prometheus-stack": _kube_prometheus_stack(ready=fnv1.READY_UNSPECIFIED), - "node-feature-discovery": _node_feature_discovery(ready=fnv1.READY_TRUE, wait=True), + "kube-prometheus-stack": _kube_prometheus_stack( + ready=fnv1.READY_UNSPECIFIED, + node_exporter_tolerations=[ + {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "modelplane.ai/cpu", "operator": "Exists", "effect": "NoSchedule"}, + ], + ), + "node-feature-discovery": _node_feature_discovery( + ready=fnv1.READY_TRUE, + wait=True, + worker_tolerations=[ + {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + ], + ), "gpu-operator": fnv1.Resource( resource=resource.dict_to_struct( { @@ -4255,6 +4368,7 @@ def _to_dict(msg: message.Message) -> dict: composite=_serving_stack( cloud="Civo", stack="Standard", + dra_drivers=["gpu.nvidia.com"], secrets=[v1alpha1.Secret(type="Kubeconfig", name="kube-secret", key="kubeconfig")], gateway=v1alpha1.Gateway(hostname="test-backend.gateways.example.com"), gpu=v1alpha1.Gpu(pools=[v1alpha1.Pool(name="l40s-pool", disableNvLink=False)]), @@ -4274,8 +4388,20 @@ def _to_dict(msg: message.Message) -> dict: "provider-config-kubernetes": _kubernetes_provider_config(identity=None, ready=fnv1.READY_TRUE), "provider-config-helm": _helm_provider_config(identity=None, ready=fnv1.READY_TRUE), "cert-manager": _cert_manager(ready=fnv1.READY_UNSPECIFIED), - "kube-prometheus-stack": _kube_prometheus_stack(ready=fnv1.READY_UNSPECIFIED), - "node-feature-discovery": _node_feature_discovery(ready=fnv1.READY_UNSPECIFIED, wait=True), + "kube-prometheus-stack": _kube_prometheus_stack( + ready=fnv1.READY_UNSPECIFIED, + node_exporter_tolerations=[ + {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "modelplane.ai/cpu", "operator": "Exists", "effect": "NoSchedule"}, + ], + ), + "node-feature-discovery": _node_feature_discovery( + ready=fnv1.READY_UNSPECIFIED, + wait=True, + worker_tolerations=[ + {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + ], + ), "gpu-operator": fnv1.Resource( resource=resource.dict_to_struct( { @@ -4532,6 +4658,7 @@ def _to_dict(msg: message.Message) -> dict: composite=_serving_stack( cloud="GKE", stack="Standard", + dra_drivers=["gpu.nvidia.com"], secrets=[ v1alpha1.Secret(type="Kubeconfig", name="kube-secret", key="kubeconfig"), v1alpha1.Secret(type="GoogleApplicationCredentials", name="sa-secret", key="private_key"), @@ -4631,7 +4758,7 @@ def _to_dict(msg: message.Message) -> dict: service_extensions=None, ), "collector": _collector( - config_hash="522decbfdd3a589c", + config_hash="2fbc7d728e156867", volumes=[{"name": "config", "configMap": {"name": "modelplane-collector"}}], volume_mounts=[{"name": "config", "mountPath": "/conf"}], env_from=None, @@ -4843,6 +4970,7 @@ def _to_dict(msg: message.Message) -> dict: composite=_serving_stack( cloud="GKE", stack="Standard", + dra_drivers=["gpu.nvidia.com"], secrets=[ v1alpha1.Secret(type="Kubeconfig", name="kube-secret", key="kubeconfig"), v1alpha1.Secret(type="GoogleApplicationCredentials", name="sa-secret", key="private_key"), @@ -4996,7 +5124,7 @@ def _to_dict(msg: message.Message) -> dict: ready=fnv1.READY_TRUE, ), "collector": _collector( - config_hash="52b78175754428dd", + config_hash="c5b6d4a261a1e104", volumes=[ {"name": "config", "configMap": {"name": "modelplane-collector"}}, {"name": "credentials-primary", "secret": {"secretName": "telemetry-credentials"}}, @@ -5218,6 +5346,7 @@ def _to_dict(msg: message.Message) -> dict: composite=_serving_stack( cloud="GKE", stack="Standard", + dra_drivers=["gpu.nvidia.com"], secrets=[ v1alpha1.Secret(type="Kubeconfig", name="kube-secret", key="kubeconfig"), v1alpha1.Secret(type="GoogleApplicationCredentials", name="sa-secret", key="private_key"), @@ -5329,7 +5458,7 @@ def _to_dict(msg: message.Message) -> dict: service_extensions=None, ), "collector": _collector( - config_hash="12df8b0b19eb5fce", + config_hash="9623243a4c151500", volumes=[{"name": "config", "configMap": {"name": "modelplane-collector"}}], volume_mounts=[{"name": "config", "mountPath": "/conf"}], env_from=None, @@ -5535,6 +5664,7 @@ def _to_dict(msg: message.Message) -> dict: composite=_serving_stack( cloud="GKE", stack="Standard", + dra_drivers=["gpu.nvidia.com"], secrets=[ v1alpha1.Secret(type="Kubeconfig", name="kube-secret", key="kubeconfig"), v1alpha1.Secret(type="GoogleApplicationCredentials", name="sa-secret", key="private_key"), @@ -5638,7 +5768,7 @@ def _to_dict(msg: message.Message) -> dict: service_extensions=None, ), "collector": _collector( - config_hash="46ed90b25528eb92", + config_hash="39f04258f672d756", volumes=[{"name": "config", "configMap": {"name": "modelplane-collector"}}], volume_mounts=[{"name": "config", "mountPath": "/conf"}], env_from=None, @@ -5855,6 +5985,7 @@ def _to_dict(msg: message.Message) -> dict: composite=_serving_stack( cloud="GKE", stack="Standard", + dra_drivers=["gpu.nvidia.com"], secrets=[ v1alpha1.Secret(type="Kubeconfig", name="kube-secret", key="kubeconfig"), v1alpha1.Secret(type="GoogleApplicationCredentials", name="sa-secret", key="private_key"), @@ -5975,7 +6106,7 @@ def _to_dict(msg: message.Message) -> dict: service_extensions=None, ), "collector": _collector( - config_hash="522decbfdd3a589c", + config_hash="2fbc7d728e156867", volumes=[{"name": "config", "configMap": {"name": "modelplane-collector"}}], volume_mounts=[{"name": "config", "mountPath": "/conf"}], env_from=None, @@ -6213,6 +6344,7 @@ def test_compose(case: ComposeCase) -> None: composite=_serving_stack( cloud="EKS", stack="Standard", + dra_drivers=["gpu.nvidia.com"], secrets=[ v1alpha1.Secret(type="Kubeconfig", name="kube-secret", key="kubeconfig"), v1alpha1.Secret(type="GoogleApplicationCredentials", name="sa-secret", key="private_key"), @@ -6358,6 +6490,7 @@ def test_compose(case: ComposeCase) -> None: composite=_serving_stack( cloud="EKS", stack="Dynamo", + dra_drivers=["gpu.nvidia.com"], secrets=[ v1alpha1.Secret(type="Kubeconfig", name="kube-secret", key="kubeconfig"), v1alpha1.Secret(type="GoogleApplicationCredentials", name="sa-secret", key="private_key"), @@ -6531,6 +6664,7 @@ def test_compose(case: ComposeCase) -> None: composite=_serving_stack( cloud="AKS", stack="Standard", + dra_drivers=["gpu.nvidia.com"], secrets=[ v1alpha1.Secret(type="Kubeconfig", name="kube-secret", key="kubeconfig"), v1alpha1.Secret(type="GoogleApplicationCredentials", name="sa-secret", key="private_key"), @@ -6681,6 +6815,7 @@ def test_compose(case: ComposeCase) -> None: composite=_serving_stack( cloud="AKS", stack="Dynamo", + dra_drivers=["gpu.nvidia.com"], secrets=[ v1alpha1.Secret(type="Kubeconfig", name="kube-secret", key="kubeconfig"), v1alpha1.Secret(type="GoogleApplicationCredentials", name="sa-secret", key="private_key"), @@ -6859,6 +6994,7 @@ def test_compose(case: ComposeCase) -> None: composite=_serving_stack( cloud="GKE", stack="Standard", + dra_drivers=["gpu.nvidia.com"], secrets=[ v1alpha1.Secret(type="Kubeconfig", name="kube-secret", key="kubeconfig"), v1alpha1.Secret(type="GoogleApplicationCredentials", name="sa-secret", key="private_key"), @@ -7015,6 +7151,7 @@ def test_compose(case: ComposeCase) -> None: composite=_serving_stack( cloud="GKE", stack="Dynamo", + dra_drivers=["gpu.nvidia.com"], secrets=[ v1alpha1.Secret(type="Kubeconfig", name="kube-secret", key="kubeconfig"), v1alpha1.Secret(type="GoogleApplicationCredentials", name="sa-secret", key="private_key"), @@ -7199,6 +7336,7 @@ def test_compose(case: ComposeCase) -> None: composite=_serving_stack( cloud="Nebius", stack="Standard", + dra_drivers=["gpu.nvidia.com"], secrets=[ v1alpha1.Secret(type="Kubeconfig", name="kube-secret", key="kubeconfig"), v1alpha1.Secret(type="GoogleApplicationCredentials", name="sa-secret", key="private_key"), @@ -7310,6 +7448,7 @@ def test_compose(case: ComposeCase) -> None: composite=_serving_stack( cloud="Nebius", stack="Dynamo", + dra_drivers=["gpu.nvidia.com"], secrets=[ v1alpha1.Secret(type="Kubeconfig", name="kube-secret", key="kubeconfig"), v1alpha1.Secret(type="GoogleApplicationCredentials", name="sa-secret", key="private_key"), @@ -7449,6 +7588,7 @@ def test_compose(case: ComposeCase) -> None: composite=_serving_stack( cloud="Vultr", stack="Standard", + dra_drivers=["gpu.nvidia.com"], secrets=[ v1alpha1.Secret(type="Kubeconfig", name="kube-secret", key="kubeconfig"), v1alpha1.Secret(type="GoogleApplicationCredentials", name="sa-secret", key="private_key"), @@ -7559,6 +7699,7 @@ def test_compose(case: ComposeCase) -> None: composite=_serving_stack( cloud="Vultr", stack="Dynamo", + dra_drivers=["gpu.nvidia.com"], secrets=[ v1alpha1.Secret(type="Kubeconfig", name="kube-secret", key="kubeconfig"), v1alpha1.Secret(type="GoogleApplicationCredentials", name="sa-secret", key="private_key"), @@ -7697,6 +7838,7 @@ def test_compose(case: ComposeCase) -> None: composite=_serving_stack( cloud="Civo", stack="Standard", + dra_drivers=["gpu.nvidia.com"], secrets=[ v1alpha1.Secret(type="Kubeconfig", name="kube-secret", key="kubeconfig"), v1alpha1.Secret(type="GoogleApplicationCredentials", name="sa-secret", key="private_key"), @@ -7819,6 +7961,7 @@ def test_compose(case: ComposeCase) -> None: composite=_serving_stack( cloud="Civo", stack="Dynamo", + dra_drivers=["gpu.nvidia.com"], secrets=[ v1alpha1.Secret(type="Kubeconfig", name="kube-secret", key="kubeconfig"), v1alpha1.Secret(type="GoogleApplicationCredentials", name="sa-secret", key="private_key"), @@ -7969,6 +8112,7 @@ def test_compose(case: ComposeCase) -> None: composite=_serving_stack( cloud="Existing", stack="Standard", + dra_drivers=["gpu.nvidia.com"], secrets=[ v1alpha1.Secret(type="Kubeconfig", name="kube-secret", key="kubeconfig"), v1alpha1.Secret(type="GoogleApplicationCredentials", name="sa-secret", key="private_key"), @@ -8080,6 +8224,7 @@ def test_compose(case: ComposeCase) -> None: composite=_serving_stack( cloud="Existing", stack="Dynamo", + dra_drivers=["gpu.nvidia.com"], secrets=[ v1alpha1.Secret(type="Kubeconfig", name="kube-secret", key="kubeconfig"), v1alpha1.Secret(type="GoogleApplicationCredentials", name="sa-secret", key="private_key"), @@ -8208,6 +8353,361 @@ def test_compose(case: ComposeCase) -> None: "usage-modelexpress-crds-modelmetadatas.modelexpress.nvidia.com-by-modelexpress-server", }, ), + ComposedResourceKeysCase( + name="ExistingStandardCPUOnly", + reason=( + "With draDrivers [dra.cpu], only the CPU device stack joins the untagged components: " + "dra-driver-cpu and its critical-pods quota install, and the NVIDIA and AMD stacks - " + "NFD included - stay off the cluster." + ), + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_serving_stack( + cloud="Existing", + stack="Standard", + dra_drivers=["dra.cpu"], + secrets=[ + v1alpha1.Secret(type="Kubeconfig", name="kube-secret", key="kubeconfig"), + ], + gateway=v1alpha1.Gateway( + hostname="test-backend.gateways.example.com", + clientCAs=[ + v1alpha1.ClientCA( + name="eu", + certificate="-----BEGIN CERTIFICATE-----\nfleet\n-----END CERTIFICATE-----\n", + ) + ], + ), + gpu=None, + ), + resources={ + "provider-config-kubernetes": _observed_ready(), + "provider-config-helm": _observed_ready(), + "gateway": _observed_ready(), + "gateway-class": _observed_ready(), + "gateway-ca-certificate": _observed_ready(), + "gateway-ca-issuer": _observed_ready(), + "gateway-serving-certificate": _observed_ready(), + "gateway-ca-bundle": _observed_ready(), + "gateway-ca-configmap": _observed_ready(), + "gateway-client-ca-bundle": _observed_ready(), + "gateway-client-auth": _observed_ready(), + "usage-gateway-class-by-gateway": _observed_ready(), + "usage-envoy-gateway-by-gateway-class": _observed_ready(), + "ai-gateway": _observed_ready(), + "ai-gateway-crds": _observed_ready(), + "cluster-agent": _observed_ready(), + "dra-driver-cpu": _observed_ready(), + "dra-driver-cpu-critical-pods-quota": _observed_ready(), + "envoy-gateway": _observed_ready(), + "gaie-crds-inferenceobjectives.inference.networking.x-k8s.io": _observed_ready(), + "gaie-crds-inferencepools.inference.networking.k8s.io": _observed_ready(), + "gaie-crds-inferencepools.inference.networking.x-k8s.io": _observed_ready(), + "gateway-namespace": _observed_ready(), + "gateway-proxy": _observed_ready(), + "gateway-selfsigned-issuer": _observed_ready(), + "trust-manager": _observed_ready(), + "usage-ai-gateway-crds-by-ai-gateway": _observed_ready(), + "usage-cert-manager-by-envoy-gateway": _observed_ready(), + "usage-cert-manager-by-gateway-selfsigned-issuer": _observed_ready(), + "usage-gateway-namespace-by-gateway-proxy": _observed_ready(), + "usage-gateway-namespace-by-cluster-agent": _observed_ready(), + "usage-gateway-namespace-by-gateway-selfsigned-issuer": _observed_ready(), + "usage-gateway-selfsigned-issuer-by-trust-manager": _observed_ready(), + "cert-manager": _observed_ready(), + "kube-prometheus-stack": _observed_ready(), + "leader-worker-set": _observed_ready(), + }, + ), + ), + want={ + # Every cloud and stack. + "provider-config-kubernetes", + "provider-config-helm", + "gateway", + "gateway-class", + "gateway-ca-certificate", + "gateway-ca-issuer", + "gateway-serving-certificate", + "gateway-ca-bundle", + "gateway-ca-configmap", + "gateway-client-ca-bundle", + "gateway-client-auth", + "usage-gateway-class-by-gateway", + "usage-envoy-gateway-by-gateway-class", + # The common components, including the dra.cpu-tagged device + # stack this cluster's classes claim. + "ai-gateway", + "ai-gateway-crds", + "cluster-agent", + "dra-driver-cpu", + "dra-driver-cpu-critical-pods-quota", + "envoy-gateway", + "gaie-crds-inferenceobjectives.inference.networking.x-k8s.io", + "gaie-crds-inferencepools.inference.networking.k8s.io", + "gaie-crds-inferencepools.inference.networking.x-k8s.io", + "gateway-namespace", + "gateway-proxy", + "gateway-selfsigned-issuer", + "trust-manager", + "usage-ai-gateway-crds-by-ai-gateway", + "usage-cert-manager-by-envoy-gateway", + "usage-cert-manager-by-gateway-selfsigned-issuer", + "usage-gateway-namespace-by-gateway-proxy", + "usage-gateway-namespace-by-cluster-agent", + "usage-gateway-namespace-by-gateway-selfsigned-issuer", + "usage-gateway-selfsigned-issuer-by-trust-manager", + # The hand-written Existing half, less its GPU stacks. + "cert-manager", + "kube-prometheus-stack", + # The Standard stack. + "leader-worker-set", + }, + ), + ComposedResourceKeysCase( + name="ExistingStandardAbsent", + reason=( + "With draDrivers absent - the cluster composition always sets it, so absent means the " + "same as empty - every device stack filters out, NVIDIA, AMD and CPU alike, and only " + "the untagged components install." + ), + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_serving_stack( + cloud="Existing", + stack="Standard", + dra_drivers=None, + secrets=[ + v1alpha1.Secret(type="Kubeconfig", name="kube-secret", key="kubeconfig"), + ], + gateway=v1alpha1.Gateway( + hostname="test-backend.gateways.example.com", + clientCAs=[ + v1alpha1.ClientCA( + name="eu", + certificate="-----BEGIN CERTIFICATE-----\nfleet\n-----END CERTIFICATE-----\n", + ) + ], + ), + gpu=None, + ), + resources={ + "provider-config-kubernetes": _observed_ready(), + "provider-config-helm": _observed_ready(), + "gateway": _observed_ready(), + "gateway-class": _observed_ready(), + "gateway-ca-certificate": _observed_ready(), + "gateway-ca-issuer": _observed_ready(), + "gateway-serving-certificate": _observed_ready(), + "gateway-ca-bundle": _observed_ready(), + "gateway-ca-configmap": _observed_ready(), + "gateway-client-ca-bundle": _observed_ready(), + "gateway-client-auth": _observed_ready(), + "usage-gateway-class-by-gateway": _observed_ready(), + "usage-envoy-gateway-by-gateway-class": _observed_ready(), + "ai-gateway": _observed_ready(), + "ai-gateway-crds": _observed_ready(), + "cluster-agent": _observed_ready(), + "amd-gpu-operator": _observed_ready(), + "dra-driver-cpu": _observed_ready(), + "dra-driver-cpu-critical-pods-quota": _observed_ready(), + "dra-driver-critical-pods-quota": _observed_ready(), + "envoy-gateway": _observed_ready(), + "gaie-crds-inferenceobjectives.inference.networking.x-k8s.io": _observed_ready(), + "gaie-crds-inferencepools.inference.networking.k8s.io": _observed_ready(), + "gaie-crds-inferencepools.inference.networking.x-k8s.io": _observed_ready(), + "gateway-namespace": _observed_ready(), + "gateway-proxy": _observed_ready(), + "gateway-selfsigned-issuer": _observed_ready(), + "trust-manager": _observed_ready(), + "usage-ai-gateway-crds-by-ai-gateway": _observed_ready(), + "usage-cert-manager-by-amd-gpu-operator": _observed_ready(), + "usage-cert-manager-by-envoy-gateway": _observed_ready(), + "usage-cert-manager-by-gateway-selfsigned-issuer": _observed_ready(), + "usage-gateway-namespace-by-gateway-proxy": _observed_ready(), + "usage-gateway-namespace-by-cluster-agent": _observed_ready(), + "usage-gateway-namespace-by-gateway-selfsigned-issuer": _observed_ready(), + "usage-gateway-selfsigned-issuer-by-trust-manager": _observed_ready(), + "usage-node-feature-discovery-by-amd-gpu-operator": _observed_ready(), + "cert-manager": _observed_ready(), + "kube-prometheus-stack": _observed_ready(), + "node-feature-discovery": _observed_ready(), + "nvidia-dra-driver-gpu": _observed_ready(), + "leader-worker-set": _observed_ready(), + }, + ), + ), + want={ + # Every cloud and stack. + "provider-config-kubernetes", + "provider-config-helm", + "gateway", + "gateway-class", + "gateway-ca-certificate", + "gateway-ca-issuer", + "gateway-serving-certificate", + "gateway-ca-bundle", + "gateway-ca-configmap", + "gateway-client-ca-bundle", + "gateway-client-auth", + "usage-gateway-class-by-gateway", + "usage-envoy-gateway-by-gateway-class", + # The common components, less every tagged device stack. + "ai-gateway", + "ai-gateway-crds", + "cluster-agent", + "envoy-gateway", + "gaie-crds-inferenceobjectives.inference.networking.x-k8s.io", + "gaie-crds-inferencepools.inference.networking.k8s.io", + "gaie-crds-inferencepools.inference.networking.x-k8s.io", + "gateway-namespace", + "gateway-proxy", + "gateway-selfsigned-issuer", + "trust-manager", + "usage-ai-gateway-crds-by-ai-gateway", + "usage-cert-manager-by-envoy-gateway", + "usage-cert-manager-by-gateway-selfsigned-issuer", + "usage-gateway-namespace-by-gateway-proxy", + "usage-gateway-namespace-by-cluster-agent", + "usage-gateway-namespace-by-gateway-selfsigned-issuer", + "usage-gateway-selfsigned-issuer-by-trust-manager", + # The hand-written Existing half, less its GPU stacks. + "cert-manager", + "kube-prometheus-stack", + # The Standard stack. + "leader-worker-set", + }, + ), + # The other side of the filter from the Absent case: every stack the + # Existing list carries is claimed, the way compose-inference-cluster + # stamps it for a cluster whose classes claim NVIDIA, AMD and CPU + # devices. This is the one golden with the AMD half desired, so a key + # regression there - the operator, its usages, or the exporter config + # that turns profiler metrics on - lands here. + ComposedResourceKeysCase( + name="ExistingStandardAllDrivers", + reason=( + "With every driver the Existing list carries claimed, each device stack joins the " + "untagged components under its pinned key: NVIDIA's, AMD's - exporter config " + "included - and the CPU driver's." + ), + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_serving_stack( + cloud="Existing", + stack="Standard", + dra_drivers=["dra.cpu", "gpu.amd.com", "gpu.nvidia.com"], + secrets=[ + v1alpha1.Secret(type="Kubeconfig", name="kube-secret", key="kubeconfig"), + ], + gateway=v1alpha1.Gateway( + hostname="test-backend.gateways.example.com", + clientCAs=[ + v1alpha1.ClientCA( + name="eu", + certificate="-----BEGIN CERTIFICATE-----\nfleet\n-----END CERTIFICATE-----\n", + ) + ], + ), + gpu=None, + ), + resources={ + "provider-config-kubernetes": _observed_ready(), + "provider-config-helm": _observed_ready(), + "gateway": _observed_ready(), + "gateway-class": _observed_ready(), + "gateway-ca-certificate": _observed_ready(), + "gateway-ca-issuer": _observed_ready(), + "gateway-serving-certificate": _observed_ready(), + "gateway-ca-bundle": _observed_ready(), + "gateway-ca-configmap": _observed_ready(), + "gateway-client-ca-bundle": _observed_ready(), + "gateway-client-auth": _observed_ready(), + "usage-gateway-class-by-gateway": _observed_ready(), + "usage-envoy-gateway-by-gateway-class": _observed_ready(), + "ai-gateway": _observed_ready(), + "ai-gateway-crds": _observed_ready(), + "cluster-agent": _observed_ready(), + "amd-gpu-operator": _observed_ready(), + "amd-metrics-exporter-config": _observed_ready(), + "dra-driver-cpu": _observed_ready(), + "dra-driver-cpu-critical-pods-quota": _observed_ready(), + "dra-driver-critical-pods-quota": _observed_ready(), + "envoy-gateway": _observed_ready(), + "gaie-crds-inferenceobjectives.inference.networking.x-k8s.io": _observed_ready(), + "gaie-crds-inferencepools.inference.networking.k8s.io": _observed_ready(), + "gaie-crds-inferencepools.inference.networking.x-k8s.io": _observed_ready(), + "gateway-namespace": _observed_ready(), + "gateway-proxy": _observed_ready(), + "gateway-selfsigned-issuer": _observed_ready(), + "trust-manager": _observed_ready(), + "usage-ai-gateway-crds-by-ai-gateway": _observed_ready(), + "usage-cert-manager-by-amd-gpu-operator": _observed_ready(), + "usage-cert-manager-by-envoy-gateway": _observed_ready(), + "usage-cert-manager-by-gateway-selfsigned-issuer": _observed_ready(), + "usage-gateway-namespace-by-gateway-proxy": _observed_ready(), + "usage-gateway-namespace-by-cluster-agent": _observed_ready(), + "usage-gateway-namespace-by-gateway-selfsigned-issuer": _observed_ready(), + "usage-gateway-selfsigned-issuer-by-trust-manager": _observed_ready(), + "usage-node-feature-discovery-by-amd-gpu-operator": _observed_ready(), + "cert-manager": _observed_ready(), + "kube-prometheus-stack": _observed_ready(), + "node-feature-discovery": _observed_ready(), + "nvidia-dra-driver-gpu": _observed_ready(), + "leader-worker-set": _observed_ready(), + }, + ), + ), + want={ + # Every cloud and stack. + "provider-config-kubernetes", + "provider-config-helm", + "gateway", + "gateway-class", + "gateway-ca-certificate", + "gateway-ca-issuer", + "gateway-serving-certificate", + "gateway-ca-bundle", + "gateway-ca-configmap", + "gateway-client-ca-bundle", + "gateway-client-auth", + "usage-gateway-class-by-gateway", + "usage-envoy-gateway-by-gateway-class", + # The common components, every device stack claimed. + "ai-gateway", + "ai-gateway-crds", + "cluster-agent", + "dra-driver-cpu", + "dra-driver-cpu-critical-pods-quota", + "dra-driver-critical-pods-quota", + "envoy-gateway", + "gaie-crds-inferenceobjectives.inference.networking.x-k8s.io", + "gaie-crds-inferencepools.inference.networking.k8s.io", + "gaie-crds-inferencepools.inference.networking.x-k8s.io", + "gateway-namespace", + "gateway-proxy", + "gateway-selfsigned-issuer", + "trust-manager", + "usage-ai-gateway-crds-by-ai-gateway", + "usage-cert-manager-by-envoy-gateway", + "usage-cert-manager-by-gateway-selfsigned-issuer", + "usage-gateway-namespace-by-gateway-proxy", + "usage-gateway-namespace-by-cluster-agent", + "usage-gateway-namespace-by-gateway-selfsigned-issuer", + "usage-gateway-selfsigned-issuer-by-trust-manager", + # The hand-written Existing half, both GPU stacks included. + "amd-gpu-operator", + "amd-metrics-exporter-config", + "cert-manager", + "kube-prometheus-stack", + "node-feature-discovery", + "nvidia-dra-driver-gpu", + "usage-cert-manager-by-amd-gpu-operator", + "usage-node-feature-discovery-by-amd-gpu-operator", + # The Standard stack. + "leader-worker-set", + }, + ), ] diff --git a/functions/compose-serving-stack/tests/test_metrics.py b/functions/compose-serving-stack/tests/test_metrics.py index 7e79f22f7..47b1737b9 100644 --- a/functions/compose-serving-stack/tests/test_metrics.py +++ b/functions/compose-serving-stack/tests/test_metrics.py @@ -164,7 +164,11 @@ def test_mappings_scraped() -> None: """Each built-in mapping has a scrape of the component it reads.""" scraped = {case.mapping for case in PUBLISHED_CASES} builtin = {m.metadata.name for m in stacks.BUILTIN_MAPPINGS if m.metadata} - assert scraped == builtin, "a built-in mapping has no scrape to check it against" + # The AMD GPU mapping was written against the device-metrics-exporter's + # documented field list, with no AMD pool to scrape: when one is available, + # scrape its exporter into a fixture, add its PublishedCase, and take this + # exemption out. + assert scraped == builtin - {"modelplane-gpu-amd"}, "a built-in mapping has no scrape to check it against" def test_histogram_layouts() -> None: diff --git a/functions/compose-serving-stack/tests/test_stacks.py b/functions/compose-serving-stack/tests/test_stacks.py index 970509d37..5add66b21 100644 --- a/functions/compose-serving-stack/tests/test_stacks.py +++ b/functions/compose-serving-stack/tests/test_stacks.py @@ -45,6 +45,16 @@ class JoinFailsClosedCase: want: str +@dataclasses.dataclass +class SupportedDriversCase: + """A test case for stacks.supported_drivers.""" + + name: str + reason: str + cloud: str + want: frozenset[str] + + @dataclasses.dataclass class WithNvLinkDisabledCase: """A test case for civo.with_nvlink_disabled.""" @@ -190,6 +200,178 @@ def test_join_fails_closed(case: JoinFailsClosedCase) -> None: stacks.join(case.cloud, case.stack) # ty: ignore[invalid-argument-type] # cases pass values outside the Literals +# The drivers each cloud's serving stack supports, pinned as literals so +# a new dra_drivers tag or external declaration lands here deliberately - +# and moves compose-inference-cluster's hand-kept _SUPPORTED_DRA_DRIVERS +# mirror in the same review. Existing additionally treats any driver +# beyond its tags as externally installed (see test_join_unknown_existing). +SUPPORTED_DRIVERS_CASES = [ + SupportedDriversCase( + name="EKS", + reason="EKS tags the NVIDIA and CPU stacks and declares dra.net installed outside the serving stack.", + cloud="EKS", + want=frozenset({"gpu.nvidia.com", "dra.cpu", "dra.net"}), + ), + SupportedDriversCase( + name="AKS", + reason="AKS tags the NVIDIA and CPU stacks.", + cloud="AKS", + want=frozenset({"gpu.nvidia.com", "dra.cpu"}), + ), + SupportedDriversCase( + name="GKE", + reason="GKE tags the NVIDIA and CPU stacks.", + cloud="GKE", + want=frozenset({"gpu.nvidia.com", "dra.cpu"}), + ), + SupportedDriversCase( + name="Nebius", + reason="Nebius tags the NVIDIA and CPU stacks.", + cloud="Nebius", + want=frozenset({"gpu.nvidia.com", "dra.cpu"}), + ), + SupportedDriversCase( + name="Vultr", + reason="Vultr tags the NVIDIA and CPU stacks.", + cloud="Vultr", + want=frozenset({"gpu.nvidia.com", "dra.cpu"}), + ), + SupportedDriversCase( + name="Civo", + reason="Civo tags the NVIDIA and CPU stacks.", + cloud="Civo", + want=frozenset({"gpu.nvidia.com", "dra.cpu"}), + ), + SupportedDriversCase( + name="Existing", + reason="Existing is the one cloud that tags the AMD stack, beside NVIDIA and CPU.", + cloud="Existing", + want=frozenset({"gpu.nvidia.com", "gpu.amd.com", "dra.cpu"}), + ), +] + + +@pytest.mark.parametrize("case", SUPPORTED_DRIVERS_CASES, ids=lambda case: case.name) +def test_supported_drivers(case: SupportedDriversCase) -> None: + """Each cloud's derived supported drivers match the pinned set.""" + assert stacks.supported_drivers(case.cloud) == case.want, case.reason # ty: ignore[invalid-argument-type] # parametrize erases the Literal + + +@pytest.mark.parametrize("stack", stacks.stacks()) +@pytest.mark.parametrize("cloud", stacks.clouds()) +def test_join_filter_empty(cloud: stacks.Cloud, stack: stacks.Stack) -> None: + """An empty draDrivers list keeps exactly the untagged components.""" + assert stacks.join(cloud, stack, dra_drivers=[]) == [c for c in stacks.join(cloud, stack) if not c.dra_drivers] + + +@pytest.mark.parametrize("stack", stacks.stacks()) +@pytest.mark.parametrize("cloud", stacks.clouds()) +def test_join_filter_all(cloud: stacks.Cloud, stack: stacks.Stack) -> None: + """Listing every tagged driver reproduces the unfiltered join.""" + joined = stacks.join(cloud, stack) + drivers = sorted({d for c in joined for d in c.dra_drivers}) + assert stacks.join(cloud, stack, dra_drivers=drivers) == joined + + +def test_join_filter_nvidia() -> None: + """gpu.nvidia.com alone installs the NVIDIA stack and no other.""" + keys = {c.key for c in stacks.join("Existing", "Standard", dra_drivers=["gpu.nvidia.com"])} + assert {"node-feature-discovery", "nvidia-dra-driver-gpu", "dra-driver-critical-pods-quota"} <= keys + assert {"dra-driver-cpu", "dra-driver-cpu-critical-pods-quota", "amd-gpu-operator"}.isdisjoint(keys) + + +def test_join_filter_cpu() -> None: + """dra.cpu alone installs the CPU driver and neither GPU stack - NFD included.""" + keys = {c.key for c in stacks.join("Existing", "Standard", dra_drivers=["dra.cpu"])} + assert {"dra-driver-cpu", "dra-driver-cpu-critical-pods-quota"} <= keys + assert {"node-feature-discovery", "nvidia-dra-driver-gpu", "amd-gpu-operator"}.isdisjoint(keys) + + +def test_join_filter_amd() -> None: + """gpu.amd.com on Existing installs the AMD operator and the NFD both GPU stacks share.""" + keys = {c.key for c in stacks.join("Existing", "Standard", dra_drivers=["gpu.amd.com"])} + assert {"amd-gpu-operator", "node-feature-discovery"} <= keys + assert {"nvidia-dra-driver-gpu", "dra-driver-critical-pods-quota", "dra-driver-cpu"}.isdisjoint(keys) + + +@pytest.mark.parametrize("cloud", [c for c in stacks.clouds() if c != "Existing"]) +def test_join_unknown_driver(cloud: stacks.Cloud) -> None: + """A driver the cloud neither tags nor declares external fails the join.""" + with pytest.raises(ValueError, match=r"gpu\.example\.com"): + stacks.join(cloud, "Standard", dra_drivers=["gpu.example.com"]) + + +def test_join_unknown_existing() -> None: + """Existing treats an unlisted driver as externally installed: no raise, no tagged component.""" + joined = stacks.join("Existing", "Standard", dra_drivers=["gpu.example.com"]) + assert [c.key for c in joined if c.dra_drivers] == [] + + +def test_join_external_driver() -> None: + """EKS's declared-external dra.net is accepted and installs nothing; elsewhere it raises.""" + joined = stacks.join("EKS", "Standard", dra_drivers=["dra.net"]) + assert [c.key for c in joined if c.dra_drivers] == [] + with pytest.raises(ValueError, match=r"dra\.net"): + stacks.join("GKE", "Standard", dra_drivers=["dra.net"]) + + +def test_join_tagged_dependency(monkeypatch: pytest.MonkeyPatch) -> None: + """A dependency not carrying every driver its dependent does fails every join for the cloud.""" + broken = [ + stacks.Chart( + key="dep", + release="mp-dep", + namespace="d", + chart="dep", + repository="r", + version="1", + wait=True, + dra_drivers=["gpu.nvidia.com"], + ), + stacks.Chart( + key="user", + release="mp-user", + namespace="d", + chart="user", + repository="r", + version="1", + depends_on=["dep"], + dra_drivers=["gpu.nvidia.com", "dra.cpu"], + ), + ] + monkeypatch.setitem(stacks._CLOUDS, "Existing", broken) + with pytest.raises(ValueError, match="filtering could remove"): + stacks.join("Existing", "Standard") + + +def test_join_untagged_dependent(monkeypatch: pytest.MonkeyPatch) -> None: + """An untagged component depending on a tagged one fails: the filter keeps it but can drop its dependency.""" + broken = [ + stacks.Chart( + key="dep", + release="mp-dep", + namespace="d", + chart="dep", + repository="r", + version="1", + wait=True, + dra_drivers=["gpu.nvidia.com"], + ), + stacks.Chart( + key="user", + release="mp-user", + namespace="d", + chart="user", + repository="r", + version="1", + depends_on=["dep"], + ), + ] + monkeypatch.setitem(stacks._CLOUDS, "Existing", broken) + with pytest.raises(ValueError, match="filtering could remove"): + stacks.join("Existing", "Standard") + + @pytest.mark.parametrize("cloud", ["AKS", "EKS", "GKE"]) def test_generated_dcgm_csv(cloud: str) -> None: """Each generated cloud's inline DCGM exporter CSV is the shared one. @@ -305,6 +487,7 @@ def test_generated_dcgm_csv(cloud: str) -> None: version="v26.7.1", wait=True, depends_on=["node-feature-discovery", "cert-manager"], + dra_drivers=["gpu.nvidia.com"], values={ "ccManager": {"enabled": False}, "cdi": {"default": True, "enabled": True}, @@ -354,6 +537,7 @@ def test_generated_dcgm_csv(cloud: str) -> None: repository="oci://registry.k8s.io/dra-driver-nvidia/charts", version="0.5.0", depends_on=["gpu-operator", "nvlink-disabled-driver-h100-pool"], + dra_drivers=["gpu.nvidia.com"], values={ "gpuResourcesEnabledOverride": True, "nvidiaDriverRoot": "/run/nvidia/driver", @@ -413,6 +597,7 @@ def test_generated_dcgm_csv(cloud: str) -> None: version="v26.7.1", wait=True, depends_on=["node-feature-discovery", "cert-manager"], + dra_drivers=["gpu.nvidia.com"], values={ "ccManager": {"enabled": False}, "cdi": {"default": True, "enabled": True}, @@ -459,6 +644,7 @@ def test_generated_dcgm_csv(cloud: str) -> None: repository="oci://registry.k8s.io/dra-driver-nvidia/charts", version="0.5.0", depends_on=["gpu-operator", "nvlink-disabled-driver-pool-a", "nvlink-disabled-driver-pool-b"], + dra_drivers=["gpu.nvidia.com"], values={ "gpuResourcesEnabledOverride": True, "nvidiaDriverRoot": "/run/nvidia/driver", diff --git a/functions/compose-vultr-cluster/function/fn.py b/functions/compose-vultr-cluster/function/fn.py index 830a0a4f4..8db939403 100644 --- a/functions/compose-vultr-cluster/function/fn.py +++ b/functions/compose-vultr-cluster/function/fn.py @@ -85,6 +85,12 @@ _GPU_TAINT_VALUE = "true" _GPU_TAINT_EFFECT = "NoSchedule" +# Label and taint on CPU inference pools. The label is what the serving +# stack's dra-driver-cpu selects its nodes by; the taint keeps system +# workloads off the pool as the GPU taint does on GPU pools, and engine +# pods tolerate it (compose-model-replica). +_LABEL_CPU = "modelplane.ai/cpu" + # Vultr pre-installs the NVIDIA GPU Operator on every VKE GPU cluster. The # operator's validator DaemonSet runs a four-stage init sequence (driver → # CUDA → device-plugin → full validation). When all pods in the DaemonSet @@ -160,7 +166,12 @@ def compose(self) -> None: if self._cluster_ready() or self._dependents_observed(): self.compose_node_pools() self.compose_provider_config() - self.compose_gpu_observer() + # Vultr pre-installs the NVIDIA GPU Operator only on clusters + # with GPU pools, so a CPU-only cluster has no validator + # DaemonSet to observe - composing the observer anyway would + # hold the cluster un-Ready forever. + if any(pool.role == "GPU" for pool in self.xr.spec.nodePools): + self.compose_gpu_observer() self.write_status() self.mark_readiness() @@ -290,6 +301,8 @@ def _node_pool(self, pool: v1alpha1.NodePool) -> vkenodepoolv1beta1.KubernetesNo if pool.role == "GPU" and pool.gpu: labels.append(vkenodepoolv1beta1.Label(key=_LABEL_GPU, value=pool.gpu.acceleratorType)) labels.append(vkenodepoolv1beta1.Label(key=_LABEL_DEVICE_PLUGIN_KEY, value=_LABEL_DEVICE_PLUGIN_VALUE)) + elif pool.role == "CPU": + labels.append(vkenodepoolv1beta1.Label(key=_LABEL_CPU, value="true")) fp = vkenodepoolv1beta1.ForProvider( label=pool.name, @@ -309,6 +322,14 @@ def _node_pool(self, pool: v1alpha1.NodePool) -> vkenodepoolv1beta1.KubernetesNo effect=_GPU_TAINT_EFFECT, ), ] + elif pool.role == "CPU": + fp.taints = [ + vkenodepoolv1beta1.Taint( + key=_LABEL_CPU, + value="true", + effect=_GPU_TAINT_EFFECT, + ), + ] # maxNodeCount opts into VKE's server-side autoscaling. if pool.maxNodeCount is not None: diff --git a/functions/compose-vultr-cluster/tests/test_fn.py b/functions/compose-vultr-cluster/tests/test_fn.py index c6021b0bf..7e735694c 100644 --- a/functions/compose-vultr-cluster/tests/test_fn.py +++ b/functions/compose-vultr-cluster/tests/test_fn.py @@ -179,6 +179,36 @@ def _node_pool_gpu(*, node_quantity: int, autoscaling: dict | None, ready: fnv1. ) +def _node_pool_cpu(*, node_quantity: int, autoscaling: dict | None, ready: fnv1.Ready) -> fnv1.Resource: + """The composed KubernetesNodePool for the cpu-pool pool, with the CPU label and taint and no GPU labels.""" + for_provider: dict[str, Any] = { + "label": "cpu-pool", + "plan": "vc2-6c-16gb", + "nodeQuantity": node_quantity, + "labels": [ + {"key": "modelplane.ai/pool", "value": "cpu-pool"}, + {"key": "modelplane.ai/cpu", "value": "true"}, + ], + "clusterIdSelector": {"matchControllerRef": True}, + "taints": [{"key": "modelplane.ai/cpu", "value": "true", "effect": "NoSchedule"}], + } + if autoscaling is not None: + for_provider.update(autoscaling) + return fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "vke.vultr.m.upbound.io/v1beta1", + "kind": "KubernetesNodePool", + "spec": { + "providerConfigRef": {"kind": "ClusterProviderConfig", "name": "default"}, + "forProvider": for_provider, + }, + } + ), + ready=ready, + ) + + def _provider_config() -> fnv1.Resource: """The composed provider-kubernetes ProviderConfig for the cluster, which is always ready.""" return fnv1.Resource( @@ -319,6 +349,47 @@ def _to_dict(msg: message.Message) -> dict: context=structpb.Struct(), ), ), + Case( + name="ClusterReadyCPUPool", + reason=( + "A CPU-role pool composes with the modelplane.ai/pool and modelplane.ai/cpu labels and the CPU taint, " + "no GPU or device-plugin labels, and no GPU observer - Vultr pre-installs the GPU operator only on " + "clusters with GPU pools, so there is no validator DaemonSet to observe." + ), + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + node_pools=[ + v1alpha1.NodePool( + name="cpu-pool", + role="CPU", + plan="vc2-6c-16gb", + maxNodeCount=4, + ), + ], + ), + resources={ + "cluster": _observed_cluster(ready=True), + }, + ), + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_xr(), + resources={ + "cluster": _cluster(ready=fnv1.READY_TRUE), + "node-pool-cpu-pool": _node_pool_cpu( + node_quantity=1, + autoscaling={"autoScaler": True, "minNodes": 1, "maxNodes": 4}, + ready=fnv1.READY_UNSPECIFIED, + ), + "provider-config-kubernetes": _provider_config(), + }, + ), + context=structpb.Struct(), + ), + ), Case( name="ClusterReadyRegressed", reason="With the cluster's Ready condition regressed to False but its ProviderConfig observed, a VultrCluster keeps the node pool, ProviderConfig and GPU observer composed.", @@ -508,7 +579,10 @@ def _to_dict(msg: message.Message) -> dict: # which of them sets the autoscaler's floor. Case( name="SystemRolePool", - reason="A VultrCluster's System-role pool autoscales from a floor of 2 nodes to maxNodeCount, and gets neither GPU labels nor a taint.", + reason=( + "A VultrCluster's System-role pool autoscales from a floor of 2 nodes to maxNodeCount, and gets " + "neither GPU labels nor a taint - and with no GPU pool there is no GPU observer to compose." + ), req=fnv1.RunFunctionRequest( observed=fnv1.State( composite=_xr( @@ -556,7 +630,6 @@ def _to_dict(msg: message.Message) -> dict: ), ), "provider-config-kubernetes": _provider_config(), - "gpu-observer": _gpu_observer(ready=fnv1.READY_UNSPECIFIED), }, ), context=structpb.Struct(), From f8f061a1d169269459db027c43b42d2a1cada8a5 Mon Sep 17 00:00:00 2001 From: Christopher Haar Date: Thu, 29 Oct 2026 16:47:18 -0700 Subject: [PATCH 02/19] Regenerate schemas and aicr stacks Signed-off-by: Christopher Haar Signed-off-by: Nic Cope --- .../stacks/clouds/generated/aicr/aks.py | 16 +++++++++++- .../stacks/clouds/generated/aicr/eks.py | 15 ++++++++++- .../stacks/clouds/generated/aicr/gke.py | 16 +++++++++++- schemas/.lock.json | 2 +- .../ai/modelplane/inferenceclass/v1alpha1.py | 24 ++++++++--------- .../modelplane/inferencecluster/v1alpha1.py | 26 +++++++++---------- .../infrastructure/akscluster/v1alpha1.py | 4 +-- .../infrastructure/civocluster/v1alpha1.py | 6 ++--- .../infrastructure/ekscluster/v1alpha1.py | 4 +-- .../infrastructure/gkecluster/v1alpha1.py | 4 +-- .../infrastructure/nebiuscluster/v1alpha1.py | 4 +-- .../infrastructure/servingstack/v1alpha1.py | 10 ++++++- .../infrastructure/vultrcluster/v1alpha1.py | 4 +-- .../ai/modelplane/metricmapping/v1alpha1.py | 15 +++++++++-- .../ai/modelplane/modeldeployment/v1alpha1.py | 2 +- 15 files changed, 106 insertions(+), 46 deletions(-) diff --git a/functions/compose-serving-stack/function/stacks/clouds/generated/aicr/aks.py b/functions/compose-serving-stack/function/stacks/clouds/generated/aicr/aks.py index 26c4cf5c3..bf20a2cc4 100644 --- a/functions/compose-serving-stack/function/stacks/clouds/generated/aicr/aks.py +++ b/functions/compose-serving-stack/function/stacks/clouds/generated/aicr/aks.py @@ -24,6 +24,11 @@ from function.stacks.components import Chart, Component, Manifests +# DRA drivers this cloud supports that arrive outside the serving +# stack; the stacks package folds them into its derived +# supported-drivers table (see generate.py's EXTERNAL_DRIVERS). +EXTERNAL_DRIVERS: list[str] = [] + COMPONENTS: list[Component] = [ Chart( key="cert-manager", @@ -61,6 +66,7 @@ repository="https://kubernetes-sigs.github.io/node-feature-discovery/charts", version="0.19.0", wait=True, + dra_drivers=["gpu.nvidia.com"], values={ "gc": {"enable": True, "tolerations": []}, "master": {"enable": True, "tolerations": []}, @@ -88,6 +94,7 @@ chart="nodewright", repository="oci://ghcr.io/nvidia/nodewright/charts", version="v0.19.0", + dra_drivers=["gpu.nvidia.com"], values={ "controllerManager": { "manager": { @@ -190,7 +197,10 @@ }, "prometheus-node-exporter": { "fullnameOverride": "prometheus-node-exporter", - "tolerations": [{"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}], + "tolerations": [ + {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "modelplane.ai/cpu", "operator": "Exists", "effect": "NoSchedule"}, + ], }, "prometheusOperator": {"admissionWebhooks": {"patch": {"tolerations": []}}, "tolerations": []}, "thanosRuler": {"thanosRulerSpec": {"tolerations": []}}, @@ -205,6 +215,7 @@ version="v26.7.1", wait=True, depends_on=["node-feature-discovery", "cert-manager", "kube-prometheus-stack"], + dra_drivers=["gpu.nvidia.com"], values={ "ccManager": {"enabled": False}, "daemonsets": { @@ -325,6 +336,7 @@ Manifests( key="gpu-operator-manifests", depends_on=["gpu-operator"], + dra_drivers=["gpu.nvidia.com"], manifests=[ { "apiVersion": "apps/v1", @@ -452,6 +464,7 @@ repository="oci://registry.k8s.io/dra-driver-nvidia/charts", version="0.5.0", depends_on=["gpu-operator"], + dra_drivers=["gpu.nvidia.com"], values={ "controller": { "affinity": {}, @@ -481,6 +494,7 @@ repository="oci://ghcr.io/nvidia", version="v1.25.0", depends_on=["cert-manager", "gpu-operator", "prometheus-operator-crds"], + dra_drivers=["gpu.nvidia.com"], values={ "fullnameOverride": "nvsentinel", "global": { diff --git a/functions/compose-serving-stack/function/stacks/clouds/generated/aicr/eks.py b/functions/compose-serving-stack/function/stacks/clouds/generated/aicr/eks.py index d49b4dc51..37e522210 100644 --- a/functions/compose-serving-stack/function/stacks/clouds/generated/aicr/eks.py +++ b/functions/compose-serving-stack/function/stacks/clouds/generated/aicr/eks.py @@ -24,6 +24,11 @@ from function.stacks.components import Chart, Component +# DRA drivers this cloud supports that arrive outside the serving +# stack; the stacks package folds them into its derived +# supported-drivers table (see generate.py's EXTERNAL_DRIVERS). +EXTERNAL_DRIVERS: list[str] = ["dra.net"] + COMPONENTS: list[Component] = [ Chart( key="cert-manager", @@ -61,6 +66,7 @@ repository="https://kubernetes-sigs.github.io/node-feature-discovery/charts", version="0.19.0", wait=True, + dra_drivers=["gpu.nvidia.com"], values={ "gc": {"enable": True, "tolerations": []}, "master": {"enable": True, "tolerations": []}, @@ -84,6 +90,7 @@ chart="nodewright", repository="oci://ghcr.io/nvidia/nodewright/charts", version="v0.19.0", + dra_drivers=["gpu.nvidia.com"], values={ "controllerManager": { "manager": { @@ -186,7 +193,10 @@ }, "prometheus-node-exporter": { "fullnameOverride": "prometheus-node-exporter", - "tolerations": [{"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}], + "tolerations": [ + {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "modelplane.ai/cpu", "operator": "Exists", "effect": "NoSchedule"}, + ], }, "prometheusOperator": {"admissionWebhooks": {"patch": {"tolerations": []}}, "tolerations": []}, "thanosRuler": {"thanosRulerSpec": {"tolerations": []}}, @@ -201,6 +211,7 @@ version="v26.7.1", wait=True, depends_on=["node-feature-discovery", "cert-manager", "kube-prometheus-stack"], + dra_drivers=["gpu.nvidia.com"], values={ "ccManager": {"enabled": False}, "daemonsets": {"tolerations": [{"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}]}, @@ -341,6 +352,7 @@ repository="oci://registry.k8s.io/dra-driver-nvidia/charts", version="0.5.0", depends_on=["gpu-operator"], + dra_drivers=["gpu.nvidia.com"], values={ "controller": { "affinity": {}, @@ -368,6 +380,7 @@ repository="oci://ghcr.io/nvidia", version="v1.25.0", depends_on=["cert-manager", "gpu-operator", "prometheus-operator-crds"], + dra_drivers=["gpu.nvidia.com"], values={ "fullnameOverride": "nvsentinel", "global": { diff --git a/functions/compose-serving-stack/function/stacks/clouds/generated/aicr/gke.py b/functions/compose-serving-stack/function/stacks/clouds/generated/aicr/gke.py index 727667de7..8a9a163d3 100644 --- a/functions/compose-serving-stack/function/stacks/clouds/generated/aicr/gke.py +++ b/functions/compose-serving-stack/function/stacks/clouds/generated/aicr/gke.py @@ -24,6 +24,11 @@ from function.stacks.components import Chart, Component, Manifests +# DRA drivers this cloud supports that arrive outside the serving +# stack; the stacks package folds them into its derived +# supported-drivers table (see generate.py's EXTERNAL_DRIVERS). +EXTERNAL_DRIVERS: list[str] = [] + COMPONENTS: list[Component] = [ Chart( key="cert-manager", @@ -61,6 +66,7 @@ repository="https://kubernetes-sigs.github.io/node-feature-discovery/charts", version="0.19.0", wait=True, + dra_drivers=["gpu.nvidia.com"], values={ "gc": {"enable": True, "tolerations": []}, "master": {"enable": True, "tolerations": []}, @@ -84,6 +90,7 @@ chart="nodewright", repository="oci://ghcr.io/nvidia/nodewright/charts", version="v0.19.0", + dra_drivers=["gpu.nvidia.com"], values={ "controllerManager": { "manager": { @@ -187,7 +194,10 @@ }, "prometheus-node-exporter": { "fullnameOverride": "prometheus-node-exporter", - "tolerations": [{"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}], + "tolerations": [ + {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "modelplane.ai/cpu", "operator": "Exists", "effect": "NoSchedule"}, + ], }, "prometheusOperator": {"admissionWebhooks": {"patch": {"tolerations": []}}, "tolerations": []}, "thanosRuler": {"thanosRulerSpec": {"tolerations": []}}, @@ -195,6 +205,7 @@ ), Manifests( key="gpu-operator-pre-manifests", + dra_drivers=["gpu.nvidia.com"], manifests=[ {"apiVersion": "v1", "kind": "Namespace", "metadata": {"name": "gpu-operator"}}, { @@ -225,6 +236,7 @@ version="v26.7.1", wait=True, depends_on=["node-feature-discovery", "cert-manager", "kube-prometheus-stack", "gpu-operator-pre-manifests"], + dra_drivers=["gpu.nvidia.com"], values={ "ccManager": {"enabled": False}, "cdi": {"default": True, "enabled": True}, @@ -371,6 +383,7 @@ repository="oci://registry.k8s.io/dra-driver-nvidia/charts", version="0.5.0", depends_on=["gpu-operator"], + dra_drivers=["gpu.nvidia.com"], values={ "controller": { "affinity": {}, @@ -398,6 +411,7 @@ repository="oci://ghcr.io/nvidia", version="v1.25.0", depends_on=["cert-manager", "gpu-operator", "prometheus-operator-crds"], + dra_drivers=["gpu.nvidia.com"], values={ "fullnameOverride": "nvsentinel", "global": { diff --git a/schemas/.lock.json b/schemas/.lock.json index 4e083b183..26ea5efcb 100644 --- a/schemas/.lock.json +++ b/schemas/.lock.json @@ -1,6 +1,6 @@ { "packages": { - "fs://apis": "dffeedd4cecb877e11fde90a2f3316ba0690935af6c8d5428bf48fcfe4e92d28", + "fs://apis": "172c9217209bee80bac15ff9cb1e98689fb66000e484303c81b2997811fdc8af", "git://https://github.com/crossplane/crossplane/cluster/crds": "90d8b72ad8b829f0bcd7d7d5a98eaa0d579f244a", "git://https://github.com/modelplaneai/cluster-agent/cluster/charts/cluster-agent/files/crds": "ca7631d72ef8550a2d2f415f33db1ec52b20c65c", "xpkg://xpkg.upbound.io/upbound/provider-aws-ec2:v2.8.1": "sha256:ca2e9e3b2e3a8b6ca44a9700d5abf7abd733cfa388d1afe9bb2bf7c847cd39ef", diff --git a/schemas/python/models/ai/modelplane/inferenceclass/v1alpha1.py b/schemas/python/models/ai/modelplane/inferenceclass/v1alpha1.py index 8fadaeaea..96d455664 100644 --- a/schemas/python/models/ai/modelplane/inferenceclass/v1alpha1.py +++ b/schemas/python/models/ai/modelplane/inferenceclass/v1alpha1.py @@ -99,9 +99,9 @@ class Accelerator(BaseModel): class Aks(BaseModel): - accelerator: Accelerator + accelerator: Accelerator | None = None """ - GPU accelerator to attach when provisioning the node pool. Provisioning input only: the scheduler matches against spec.devices, not this block. + GPU accelerator to attach when provisioning the node pool. Provisioning input only: the scheduler matches against spec.devices, not this block. Set this when and only when a device claims a DRA driver other than dra.cpu; the API rejects a mismatch. Omitted, the pool has no GPU: Modelplane provisions it with a standard node image and the modelplane.ai/cpu label and taint. """ diskSizeGb: conint(ge=10) | None = 100 vmSize: constr(min_length=1) @@ -119,9 +119,9 @@ class AcceleratorModel(BaseModel): class Civo(BaseModel): - accelerator: AcceleratorModel + accelerator: AcceleratorModel | None = None """ - GPU accelerator to attach when provisioning the node pool. Provisioning input only: the scheduler matches against spec.devices, not this block. + GPU accelerator to attach when provisioning the node pool. Provisioning input only: the scheduler matches against spec.devices, not this block. Set this when and only when a device claims a DRA driver other than dra.cpu; the API rejects a mismatch. Omitted, the pool has no GPU: Modelplane provisions it with a standard node image and the modelplane.ai/cpu label and taint. """ size: constr(min_length=1, max_length=63) """ @@ -138,9 +138,9 @@ class AcceleratorModel1(BaseModel): class Eks(BaseModel): - accelerator: AcceleratorModel1 + accelerator: AcceleratorModel1 | None = None """ - GPU accelerator to attach when provisioning the node group. Provisioning input only: the scheduler matches against spec.devices, not this block. + GPU accelerator to attach when provisioning the node group. Provisioning input only: the scheduler matches against spec.devices, not this block. Set this when and only when a device claims a DRA driver other than dra.cpu; the API rejects a mismatch. Omitted, the group has no GPU: Modelplane provisions it with a standard node image and the modelplane.ai/cpu label and taint. """ diskSizeGb: conint(ge=10) | None = 100 instanceType: constr(min_length=1) @@ -158,9 +158,9 @@ class AcceleratorModel2(BaseModel): class Gke(BaseModel): - accelerator: AcceleratorModel2 + accelerator: AcceleratorModel2 | None = None """ - GPU accelerator to attach when provisioning the node pool. Provisioning input only: the scheduler matches against spec.devices, not this block, so count here is the GCP machine's GPU count and need not be restated in devices. + GPU accelerator to attach when provisioning the node pool. Provisioning input only: the scheduler matches against spec.devices, not this block, so count here is the GCP machine's GPU count and need not be restated in devices. Set this when and only when a device claims a DRA driver other than dra.cpu; the API rejects a mismatch. Omitted, the pool has no GPU: Modelplane provisions it with a standard node image and the modelplane.ai/cpu label and taint. """ diskSizeGb: conint(ge=10) | None = 100 machineType: constr(min_length=1) @@ -175,9 +175,9 @@ class AcceleratorModel3(BaseModel): class Nebius(BaseModel): - accelerator: AcceleratorModel3 + accelerator: AcceleratorModel3 | None = None """ - GPU accelerator to attach when provisioning the node group. Provisioning input only: the scheduler matches against spec.devices, not this block. + GPU accelerator to attach when provisioning the node group. Provisioning input only: the scheduler matches against spec.devices, not this block. Set this when and only when a device claims a DRA driver other than dra.cpu; the API rejects a mismatch. Omitted, the group has no GPU: Modelplane provisions it with a standard node image and the modelplane.ai/cpu label and taint. """ diskSizeGb: conint(ge=10) | None = 100 driversPreset: constr(min_length=1, max_length=63) | None = 'cuda13.0' @@ -203,9 +203,9 @@ class AcceleratorModel4(BaseModel): class Vultr(BaseModel): - accelerator: AcceleratorModel4 + accelerator: AcceleratorModel4 | None = None """ - GPU accelerator to attach when provisioning the node pool. Provisioning input only: the scheduler matches against spec.devices, not this block. + GPU accelerator to attach when provisioning the node pool. Provisioning input only: the scheduler matches against spec.devices, not this block. Set this when and only when a device claims a DRA driver other than dra.cpu; the API rejects a mismatch. Omitted, the pool has no GPU: Modelplane provisions it with a standard node image and the modelplane.ai/cpu label and taint. """ plan: constr(min_length=1, max_length=63) """ diff --git a/schemas/python/models/ai/modelplane/inferencecluster/v1alpha1.py b/schemas/python/models/ai/modelplane/inferencecluster/v1alpha1.py index 8ea419ad9..822b29bb1 100644 --- a/schemas/python/models/ai/modelplane/inferencecluster/v1alpha1.py +++ b/schemas/python/models/ai/modelplane/inferencecluster/v1alpha1.py @@ -39,7 +39,7 @@ class Civo(BaseModel): """ kubernetesVersion: constr(min_length=1, max_length=32) | None = None """ - Civo Kubernetes (k3s) version. Civo requires an exact version string including the k3s build suffix; list current versions with civo kubernetes versions. Choose 1.34 or newer so Dynamic Resource Allocation (how GPUs bind to pods) is generally available. Defaults to Civo's current default version. + Civo Kubernetes (k3s) version. Civo requires an exact version string including the k3s build suffix; list current versions with civo kubernetes versions. Choose 1.34 or newer so Dynamic Resource Allocation (how devices bind to pods) is generally available, and 1.36 or newer for a CPU pool: its cores are claimed as consumable capacity, which Kubernetes enables by default from 1.36 (DRAConsumableCapacity). Defaults to Civo's current default version, which may be older than a CPU pool needs. """ region: constr(min_length=1, max_length=32) """ @@ -339,6 +339,13 @@ class Gateway(BaseModel): """ +class ProviderConfigRef(BaseModel): + name: str | None = None + """ + Name of the ProviderConfig targeting the remote cluster. Used by ModelReplica to create resources on the cluster. + """ + + class Attributes(BaseModel): bool_: bool | None = Field(None, alias='bool') int_: int | None = Field(None, alias='int') @@ -371,7 +378,7 @@ class NodesByFreeDevice(BaseModel): """ -class GpuPool(BaseModel): +class SchedulablePool(BaseModel): devices: list[Device] | None = Field(None, max_length=16) """ Devices copied from the pool's InferenceClass. ModelDeployment.nodeSelector matches against these. @@ -398,13 +405,6 @@ class GpuPool(BaseModel): """ -class ProviderConfigRef(BaseModel): - name: str | None = None - """ - Name of the ProviderConfig targeting the remote cluster. Used by ModelReplica to create resources on the cluster. - """ - - class Status(BaseModel): cache: CacheModel | None = None """ @@ -418,15 +418,15 @@ class Status(BaseModel): """ This cluster's own gateway: the Envoy gateway the serving stack runs at the cluster's edge, which an InferenceGateway routes to. These fields are how an InferenceGateway reaches it. """ - gpuPools: list[GpuPool] | None = Field(None, max_length=8) - """ - Schedulable GPU node pools on this cluster, derived from the referenced classes, the per-pool node counts, and what the cluster agent observes of each pool. ModelDeployment scheduling matches against these. - """ namespace: str | None = None """ Namespace where the internal XRs (cluster, backend) were created. """ providerConfigRef: ProviderConfigRef | None = None + schedulablePools: list[SchedulablePool] | None = Field(None, max_length=8) + """ + Schedulable node pools on this cluster - GPU and CPU alike; every pool whose class declares devices - derived from the referenced classes, the per-pool node counts, and what the cluster agent observes of each pool. ModelDeployment scheduling matches against these. + """ class InferenceCluster(BaseModel): diff --git a/schemas/python/models/ai/modelplane/infrastructure/akscluster/v1alpha1.py b/schemas/python/models/ai/modelplane/infrastructure/akscluster/v1alpha1.py index f75e5aedb..95ef71792 100644 --- a/schemas/python/models/ai/modelplane/infrastructure/akscluster/v1alpha1.py +++ b/schemas/python/models/ai/modelplane/infrastructure/akscluster/v1alpha1.py @@ -99,9 +99,9 @@ class NodePool(BaseModel): """ Initial number of nodes. """ - role: Literal['System', 'GPU'] + role: Literal['System', 'GPU', 'CPU'] """ - Determines what workloads this pool runs. System pools host controllers, gateways, and infrastructure. GPU pools host inference workloads and get the NVIDIA driver installed by AKS. + Determines what workloads this pool runs. System pools host controllers, gateways, and infrastructure. GPU pools host inference workloads and get the NVIDIA driver installed by AKS. CPU pools host CPU inference workloads: no GPU driver, and the modelplane.ai/cpu label and taint. """ vmSize: constr(min_length=1, max_length=63) """ diff --git a/schemas/python/models/ai/modelplane/infrastructure/civocluster/v1alpha1.py b/schemas/python/models/ai/modelplane/infrastructure/civocluster/v1alpha1.py index d4ce66e24..2b7d90e04 100644 --- a/schemas/python/models/ai/modelplane/infrastructure/civocluster/v1alpha1.py +++ b/schemas/python/models/ai/modelplane/infrastructure/civocluster/v1alpha1.py @@ -76,9 +76,9 @@ class NodePool(BaseModel): """ Number of nodes. Fixed unless maxNodeCount enables autoscaling. """ - role: Literal['System', 'GPU'] + role: Literal['System', 'GPU', 'CPU'] """ - Determines what workloads this pool runs. System pools host controllers, gateways, and infrastructure. GPU pools host inference workloads and are tainted to exclude non-GPU pods. + Determines what workloads this pool runs. System pools host controllers, gateways, and infrastructure. GPU pools host inference workloads and are tainted to exclude non-GPU pods. CPU pools host CPU inference workloads: a non-GPU size and the modelplane.ai/cpu label and taint. """ size: constr(min_length=1, max_length=63) """ @@ -97,7 +97,7 @@ class Spec(BaseModel): """ kubernetesVersion: constr(min_length=1, max_length=32) | None = None """ - Civo Kubernetes (k3s) version. Civo requires an exact version string including the k3s build suffix (e.g. 1.34.1-k3s1); list current versions with civo kubernetes versions. Choose 1.34 or newer so Dynamic Resource Allocation (how GPUs bind to pods) is generally available. Defaults to Civo's current default version. + Civo Kubernetes (k3s) version. Civo requires an exact version string including the k3s build suffix (e.g. 1.36.0-k3s1); list current versions with civo kubernetes versions. Choose 1.34 or newer so Dynamic Resource Allocation (how devices bind to pods) is generally available, and 1.36 or newer for a CPU pool: its cores are claimed as consumable capacity, which Kubernetes enables by default from 1.36 (DRAConsumableCapacity). Defaults to Civo's current default version, which may be older than a CPU pool needs. """ nodePools: list[NodePool] = Field(..., max_length=8, min_length=1) """ diff --git a/schemas/python/models/ai/modelplane/infrastructure/ekscluster/v1alpha1.py b/schemas/python/models/ai/modelplane/infrastructure/ekscluster/v1alpha1.py index 47489cab0..192f72dea 100644 --- a/schemas/python/models/ai/modelplane/infrastructure/ekscluster/v1alpha1.py +++ b/schemas/python/models/ai/modelplane/infrastructure/ekscluster/v1alpha1.py @@ -125,9 +125,9 @@ class NodePool(BaseModel): """ Initial number of nodes. """ - role: Literal['System', 'GPU'] + role: Literal['System', 'GPU', 'CPU'] """ - Determines what workloads this group runs. System groups host controllers, gateways, and infrastructure. GPU groups host inference workloads and use a GPU-enabled AMI. + Determines what workloads this group runs. System groups host controllers, gateways, and infrastructure. GPU groups host inference workloads and use a GPU-enabled AMI. CPU groups host CPU inference workloads: a standard AMI, no GPU, and the modelplane.ai/cpu label and taint. """ zones: list[Zone] | None = Field(None, max_length=8, min_length=1) """ diff --git a/schemas/python/models/ai/modelplane/infrastructure/gkecluster/v1alpha1.py b/schemas/python/models/ai/modelplane/infrastructure/gkecluster/v1alpha1.py index fab5c4dfd..7a408a0d7 100644 --- a/schemas/python/models/ai/modelplane/infrastructure/gkecluster/v1alpha1.py +++ b/schemas/python/models/ai/modelplane/infrastructure/gkecluster/v1alpha1.py @@ -107,9 +107,9 @@ class NodePool(BaseModel): """ Initial number of nodes. """ - role: Literal['System', 'GPU'] + role: Literal['System', 'GPU', 'CPU'] """ - Determines what workloads this pool runs. System pools host controllers, gateways, and infrastructure. GPU pools host inference workloads and are tainted to exclude non-GPU pods. + Determines what workloads this pool runs. System pools host controllers, gateways, and infrastructure. GPU pools host inference workloads and are tainted to exclude non-GPU pods. CPU pools host CPU inference workloads: a standard node image, no GPU, and the modelplane.ai/cpu label and taint. """ zones: list[Zone] | None = Field(None, max_length=8, min_length=1) """ diff --git a/schemas/python/models/ai/modelplane/infrastructure/nebiuscluster/v1alpha1.py b/schemas/python/models/ai/modelplane/infrastructure/nebiuscluster/v1alpha1.py index 013c65d34..4e44a948e 100644 --- a/schemas/python/models/ai/modelplane/infrastructure/nebiuscluster/v1alpha1.py +++ b/schemas/python/models/ai/modelplane/infrastructure/nebiuscluster/v1alpha1.py @@ -114,9 +114,9 @@ class NodePool(BaseModel): """ Resource preset within the platform (e.g. 8gpu-128vcpu-1600gb, 4vcpu-16gb). Determines the GPU, vCPU, and memory shape of each node. """ - role: Literal['System', 'GPU'] + role: Literal['System', 'GPU', 'CPU'] """ - Determines what workloads this group runs. System groups host controllers, gateways, and infrastructure. GPU groups host inference workloads and are tainted to exclude non-GPU pods. + Determines what workloads this group runs. System groups host controllers, gateways, and infrastructure. GPU groups host inference workloads and are tainted to exclude non-GPU pods. CPU groups host CPU inference workloads: a CPU platform, no GPU, and the modelplane.ai/cpu label and taint. """ diff --git a/schemas/python/models/ai/modelplane/infrastructure/servingstack/v1alpha1.py b/schemas/python/models/ai/modelplane/infrastructure/servingstack/v1alpha1.py index c7aab827a..cf1823ab5 100644 --- a/schemas/python/models/ai/modelplane/infrastructure/servingstack/v1alpha1.py +++ b/schemas/python/models/ai/modelplane/infrastructure/servingstack/v1alpha1.py @@ -5,7 +5,7 @@ from typing import Literal -from pydantic import AwareDatetime, BaseModel, Field, constr +from pydantic import AwareDatetime, BaseModel, Field, RootModel, constr from .....io.k8s.apimachinery.pkg.apis.meta import v1 @@ -41,6 +41,10 @@ class Crossplane(BaseModel): resourceRefs: list[ResourceRef] | None = None +class DraDriver(RootModel[constr(min_length=1, max_length=253)]): + root: constr(min_length=1, max_length=253) + + class ClientCA(BaseModel): certificate: constr(min_length=1, max_length=16384) """ @@ -119,6 +123,10 @@ class Spec(BaseModel): """ Configures how Crossplane will reconcile this composite resource """ + draDrivers: list[DraDriver] | None = Field(None, max_length=16) + """ + DRA driver names whose device stacks this cluster needs (gpu.nvidia.com, dra.cpu, ...), collected by the cluster composition from the claim: DRA devices of the cluster's referenced InferenceClasses. A component tagged with a driver installs only when that driver is listed; untagged components always install. Absent means the same as empty: only the untagged components install. + """ gateway: Gateway """ Configuration for the cluster's inference traffic gateway. diff --git a/schemas/python/models/ai/modelplane/infrastructure/vultrcluster/v1alpha1.py b/schemas/python/models/ai/modelplane/infrastructure/vultrcluster/v1alpha1.py index 4e312564d..be03c5199 100644 --- a/schemas/python/models/ai/modelplane/infrastructure/vultrcluster/v1alpha1.py +++ b/schemas/python/models/ai/modelplane/infrastructure/vultrcluster/v1alpha1.py @@ -80,9 +80,9 @@ class NodePool(BaseModel): """ Vultr plan ID for the pool's nodes (e.g. vcg-l40s-16c-180g-48vram, vc2-6c-16gb). This is the instance type used by other clouds; list plans with vultr-cli plans list. """ - role: Literal['System', 'GPU'] + role: Literal['System', 'GPU', 'CPU'] """ - Determines what workloads this pool runs. System pools host controllers, gateways, and infrastructure. GPU pools host inference workloads and are tainted to exclude non-GPU pods. + Determines what workloads this pool runs. System pools host controllers, gateways, and infrastructure. GPU pools host inference workloads and are tainted to exclude non-GPU pods. CPU pools host CPU inference workloads: a non-GPU plan and the modelplane.ai/cpu label and taint. """ diff --git a/schemas/python/models/ai/modelplane/metricmapping/v1alpha1.py b/schemas/python/models/ai/modelplane/metricmapping/v1alpha1.py index f924b9700..7ca6daef4 100644 --- a/schemas/python/models/ai/modelplane/metricmapping/v1alpha1.py +++ b/schemas/python/models/ai/modelplane/metricmapping/v1alpha1.py @@ -65,6 +65,10 @@ class Label(BaseModel): class Metric(BaseModel): + cumulative: bool | None = None + """ + Set when the component reports a cumulative total as a gauge. Modelplane converts the series to a monotonic cumulative sum before renaming it, so the exported type matches the measurement and a modelplane_* name never carries two types across sources. + """ from_: constr(pattern=r'^[a-zA-Z_:][a-zA-Z0-9_:]*$', max_length=255) = Field( ..., alias='from' ) @@ -73,11 +77,18 @@ class Metric(BaseModel): A histogram is named by its base name, without the _count, _sum or _bucket a Prometheus query would use: the collector holds it as one metric, and `part` is what reaches into it. """ fromUnit: ( - Literal['Millijoules', 'Mebibytes', 'Milliseconds', 'Nanoseconds', 'Percent'] + Literal[ + 'Millijoules', + 'Microjoules', + 'Mebibytes', + 'Milliseconds', + 'Nanoseconds', + 'Percent', + ] | None ) = None """ - What the component measures this in, when that isn't the unit the name claims. Modelplane converts to the base unit: millijoules and milliseconds are divided by a thousand, nanoseconds by a billion, percent by a hundred, and mebibytes multiplied out to bytes. A histogram is converted whole - its sum, its bounds and its bucket boundaries - so its quantiles come out in the target unit too. + What the component measures this in, when that isn't the unit the name claims. Modelplane converts to the base unit: millijoules and milliseconds are divided by a thousand, microjoules by a million, nanoseconds by a billion, percent by a hundred, and mebibytes multiplied out to bytes. A histogram is converted whole - its sum, its bounds and its bucket boundaries - so its quantiles come out in the target unit too. Percent is for a component that counts a saturation from nought to a hundred where the name says a ratio. Check rather than assume: vLLM publishes kv_cache_usage_perc and the value is a fraction, so a name is no guide. """ labels: list[Label] | None = Field(None, max_length=16) diff --git a/schemas/python/models/ai/modelplane/modeldeployment/v1alpha1.py b/schemas/python/models/ai/modelplane/modeldeployment/v1alpha1.py index 68d2b0b87..c1632636b 100644 --- a/schemas/python/models/ai/modelplane/modeldeployment/v1alpha1.py +++ b/schemas/python/models/ai/modelplane/modeldeployment/v1alpha1.py @@ -236,7 +236,7 @@ class Worker(BaseModel): class Member(BaseModel): nodeSelector: NodeSelector | None = None """ - The per-node device request for this member's pods: what devices each pod needs from its node. The scheduler matches it against a candidate pool's InferenceClass devices (surfaced on InferenceCluster status.gpuPools) and places the member on a pool that satisfies it, preferring one pool for the whole engine and splitting members across pools only when no pool satisfies them all. claim: DRA requests also become DeviceRequests in the ResourceClaim the member's pods bind GPUs through. A GPU request's count is the GPUs per node. Omitted, the member claims no devices and schedules onto its engine's pool - a coordinator-only leader. At least one member per engine must carry a nodeSelector, and at least one member's requests must resolve to a claimable (claim: DRA) device; an engine that matches only synthetic devices leaves its pods nothing to claim, so the scheduler treats such a pool as ineligible and the deployment reports InsufficientCapacity. + The per-node device request for this member's pods: what devices each pod needs from its node. The scheduler matches it against a candidate pool's InferenceClass devices (surfaced on InferenceCluster status.schedulablePools) and places the member on a pool that satisfies it, preferring one pool for the whole engine and splitting members across pools only when no pool satisfies them all. claim: DRA requests also become DeviceRequests in the ResourceClaim the member's pods bind GPUs through. A GPU request's count is the GPUs per node. Omitted, the member claims no devices and schedules onto its engine's pool - a coordinator-only leader. At least one member per engine must carry a nodeSelector, and at least one member's requests must resolve to a claimable (claim: DRA) device; an engine that matches only synthetic devices leaves its pods nothing to claim, so the scheduler treats such a pool as ineligible and the deployment reports InsufficientCapacity. """ role: Literal['Standalone', 'Leader', 'Worker'] | None = 'Standalone' """ From 2ddbafeb7832a992521ee4f8d1bf5b8f269c72c6 Mon Sep 17 00:00:00 2001 From: Christopher Haar Date: Thu, 29 Oct 2026 16:47:26 -0700 Subject: [PATCH 03/19] Fake a pool per device stack in the cloud e2e legs The cloud legs faked one NVIDIA GPU pool, so nothing checked that the serving stack's driver filter puts each device stack on its own pool and keeps it off the others. Each leg now fakes an NVIDIA and a CPU pool, plus an AMD pool on Existing, and stamps spec.draDrivers the way compose-inference-cluster would. Those legs claim every stack their cloud carries, so a join that ignored the filter would pass them all. A new existing-filtered leg keeps the AMD pool but leaves gpu.amd.com unclaimed, and checks the AMD operands don't exist. Signed-off-by: Christopher Haar Signed-off-by: Nic Cope --- .github/workflows/e2e-clouds.yml | 2 +- e2e/README.md | 51 ++++--- e2e/clouds/__init__.py | 251 +++++++++++++++++++++++++------ e2e/clouds/test_placement.py | 159 +++++++++++++++----- e2e/environment.py | 13 +- e2e/test_legs.py | 24 +++ 6 files changed, 401 insertions(+), 99 deletions(-) diff --git a/.github/workflows/e2e-clouds.yml b/.github/workflows/e2e-clouds.yml index 1aa3f78cb..850c80b5c 100644 --- a/.github/workflows/e2e-clouds.yml +++ b/.github/workflows/e2e-clouds.yml @@ -34,7 +34,7 @@ jobs: strategy: fail-fast: false matrix: - cloud: [eks, aks, gke, civo, nebius, vultr, existing] + cloud: [eks, aks, gke, civo, nebius, vultr, existing, existing-filtered] steps: # Two kind clusters and the serving stack need more disk than the runner diff --git a/e2e/README.md b/e2e/README.md index ea676ce1b..ecefa1891 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -96,21 +96,27 @@ cloud's serving stack — the AICR-generated component lists for EKS/AKS/GKE, or the hand-written halves for Civo, Nebius, Vultr and Existing — onto the same workload kind cluster, with no cloud credentials. The trick is [KWOK](https://kwok.sigs.k8s.io): bring-up applies its pinned `kwok.yaml` and -`stage-fast.yaml` manifests and two fake Nodes labelled and tainted exactly -like that cloud's GPU pools. The real kind node plays the untainted system -pool, so system components install and run for real; the GPU-toleration -DaemonSets fan out to the fake nodes, where KWOK marks their pods Running — -which is all helm `--wait` and the composition's readiness gates need. +`stage-fast.yaml` manifests and two fake Nodes per device pool, labelled and +tainted exactly like that cloud's pools — an NVIDIA GPU pool and a CPU pool +on every leg, plus an AMD GPU pool on Existing, the one cloud that installs +the AMD stack. The real kind node plays the untainted system pool, so system +components install and run for real; each device stack's DaemonSets fan out +to their own pool's fake nodes, where KWOK marks their pods Running — which +is all helm `--wait` and the composition's readiness gates need. The clouds are one data-driven matrix: each leg in `clouds/__init__.py` holds -everything cloud-specific — the fake nodes' labels, the GPU taint value, the -stack's `spec.cloud`, and the DaemonSets expected to fan out to the GPU pool. -Adding fidelity for a cloud means editing its leg, not a copy of the test. +everything cloud-specific — each fake pool's labels, the GPU taint value, the +stack's `spec.cloud`, the `spec.draDrivers` its classes would claim (which +filters the stack to exactly those device stacks), and the DaemonSets +expected to fan out to the pools. Adding fidelity for a cloud means editing +its leg, not a copy of the test. A leg is itself a copy of what its cloud's cluster composition applies, so it can drift from the composition as well as the stack from it. `e2e/test_legs.py` pins each leg's taint value to the `_GPU_TAINT*` constants of the composition -it mirrors (GKE's managed taint and BYO pools' come from outside Modelplane). +it mirrors (GKE's managed taint and BYO pools' come from outside Modelplane), +and the shared CPU taint key to every composition's `_LABEL_CPU` — GKE +included, which sets the CPU taint itself where its GPU taint is managed. It reads the compositions' source and needs no cluster, so it runs as the `e2e-legs` check in `nix flake check` rather than here: a composition-only change triggers the flake checks but not this workflow. The labels are a known @@ -130,15 +136,24 @@ clusters (`--no-apply` stops short of the fake nodes and the XR, `--test` reruns the tests alone). A session fixture waits for the `ServingStack` to go Ready, then the asserts are two-sided: -- **DaemonSet fan-out**: the leg's expected DaemonSets (the DRA kubelet - plugin, NFD worker, node-exporter; toolkit-hardening on AKS) reach their - ready counts — a toleration too *narrow* (e.g. the wrong `Equal` value on - AKS) shows up as a shortfall. -- **Placement/toleration audit**: no pod outside the GPU allowlist carries a - wildcard or `nvidia.com/gpu` toleration, nothing unexpected sits on a fake - GPU node, and GPU-only pods never sit on the system pool — a toleration too - *broad* (the "tolerates every taint" class of bug) fails here even when the - scheduler happened to keep the pod on the system node. +- **DaemonSet fan-out**: the leg's expected DaemonSets (the NVIDIA DRA + kubelet plugin, dra-driver-cpu, NFD worker, node-exporter; the AMD + operands on Existing; toolkit-hardening on AKS) reach their ready counts — + a toleration too *narrow* (e.g. the wrong `Equal` value on AKS), a + selector missing its pool, or a filter that wrongly dropped a claimed + stack shows up as a shortfall. +- **Placement/toleration audit**: no pod outside a pool's allowlist carries + a wildcard or that pool's taint toleration, everything on a fake node + belongs to that node's pool, and pool-only pods (the DRA drivers, the GPU + operands) never sit anywhere else — a toleration too *broad* (the + "tolerates every taint" class of bug) fails here even when the scheduler + happened to keep the pod on the system node. +- **Unclaimed-stack absence**: the legs above claim every stack their cloud + carries, so a join that ignored `spec.draDrivers` would pass them all. + The `existing-filtered` leg is the exclusion side: same pools as + `existing`, AMD fake nodes included, but `gpu.amd.com` left unclaimed — + the AMD DaemonSets must not exist at all, even though the pool is there + and Existing's list pins the stack. What cloud mode does **not** test: AKS's admission rejection of wildcard tolerations (the audit approximates it client-side), real DRA allocation and diff --git a/e2e/clouds/__init__.py b/e2e/clouds/__init__.py index 116cb1074..0d3a831b7 100644 --- a/e2e/clouds/__init__.py +++ b/e2e/clouds/__init__.py @@ -14,9 +14,19 @@ """The cloud legs of the serving-stack e2e matrix. See e2e/README.md. -Each leg shapes two fake, KWOK-managed GPU Nodes the way its cloud's -cluster composition (or platform) shapes real ones - labels and taint - -and names the DaemonSets the serving stack must fan out to them. +Each leg shapes fake, KWOK-managed Nodes the way its cloud's cluster +composition (or platform) shapes real ones - labels and taint - and +names the DaemonSets the serving stack must fan out to them. A leg +carries one pool per device stack its classes would claim: an NVIDIA +GPU pool and a CPU pool on every cloud, plus an AMD GPU pool on +Existing, the one cloud that installs the AMD stack. The ServingStack +under test names those stacks' DRA drivers in spec.draDrivers, the way +compose-inference-cluster stamps them, so the legs exercise the filter: +each device stack must reach its own pool and stay off the others'. +Those legs claim every stack their cloud carries, which only one side +of the filter can fail; the existing-filtered leg keeps the AMD pool +but leaves gpu.amd.com unclaimed, so a join that ignored the filter +fails its absence check rather than passing every leg. Everything cloud-specific lives here, so adding fidelity for a cloud means editing its leg, not a copy of the test. """ @@ -28,19 +38,28 @@ SERVING_STACK = "e2e-cloud" NAMESPACE = "modelplane-system" -# Two fake nodes, so per-node DaemonSet fan-out is visible. +# Two fake nodes per pool, so per-node DaemonSet fan-out is visible. FAKE_NODES = ("kwok-gpu-0", "kwok-gpu-1") +FAKE_CPU_NODES = ("kwok-cpu-0", "kwok-cpu-1") +FAKE_AMD_NODES = ("kwok-amd-0", "kwok-amd-1") + +# The taints the non-NVIDIA pools carry, fixed across clouds: every +# cluster composition taints CPU pools modelplane.ai/cpu=true, and +# amd.com/gpu=true is the convention BYO AMD pools carry (Modelplane +# doesn't provision them; the AMD stack's operands tolerate exactly it). +CPU_TAINT = {"key": "modelplane.ai/cpu", "value": "true", "effect": "NoSchedule"} +AMD_TAINT = {"key": "amd.com/gpu", "value": "true", "effect": "NoSchedule"} @dataclasses.dataclass(frozen=True) class Cloud: - """One cloud's leg: the fake GPU pool's shape, and what must land on it.""" + """One cloud's leg: its fake device pools' shapes, and what must land on them.""" cloud: str """The ServingStack's spec.cloud.""" labels: dict[str, str] - """The cloud's own GPU node labels, stamped on the fake nodes. + """The cloud's own NVIDIA GPU node labels, stamped on the fake GPU nodes. What the cluster composition (or the cloud's platform) puts on a real GPU node, plus the NFD labels NFD's worker would set - the GPU presence labels @@ -53,17 +72,51 @@ class Cloud: taint_value: str """The value of the nvidia.com/gpu NoSchedule taint the cloud's GPU pools carry.""" + cpu_labels: dict[str, str] + """The CPU pool's node labels, stamped on the fake CPU nodes. + + What the cluster composition puts on a real CPU node: the pool label and + modelplane.ai/cpu, which is both what keeps system workloads traceable + and the selector dra-driver-cpu finds its nodes by, plus the cloud's + instance-type label for fidelity. + """ + + dra_drivers: tuple[str, ...] + """The ServingStack's spec.draDrivers: the DRA drivers this leg's pools claim. + + What compose-inference-cluster would collect from the leg's classes; the + join filters the stack's tagged device stacks to exactly these. + """ + expected_daemonsets: dict[str, int] - """The DaemonSets that must fan out to the GPU pool: namespace/name to ready count. + """The DaemonSets that must fan out to the device pools: namespace/name to ready count. - The GPU-reaching DaemonSets from the stack, by the names that cloud's - charts give them: 2 means the fake GPU nodes only, 3 means every node, - the real one included. + The node-reaching DaemonSets from the stack, by the names that cloud's + charts give them, counted across every node their tolerations and + selectors reach: the fake pools they serve plus the real node where they + run everywhere. """ gpu_workloads: tuple[str, ...] = () """Pod-name patterns allowed on the GPU pool beyond the common GPU_WORKLOADS, such as AKS's toolkit hardening.""" + amd_labels: dict[str, str] | None = None + """The AMD GPU pool's node labels, when the leg has one (Existing only). + + The pool label plus the NFD label (feature.node.kubernetes.io/amd-gpu) + the AMD stack's operands select their nodes by - pre-set like the NVIDIA + NFD labels, because nothing real runs on a KWOK node. + """ + + absent_daemonsets: tuple[str, ...] = () + """DaemonSets that must not exist at all: namespace/name of an unclaimed stack's operands. + + The absence side of the driver filter. The pool is on the cluster and + the cloud's list carries the stack; only spec.draDrivers leaving the + driver out keeps these off, so a join that ignored the filter fails + here rather than passing a leg that claims everything. + """ + CLOUDS: dict[str, Cloud] = { # The labels mirror what compose-eks-cluster stamps on GPU node groups @@ -81,10 +134,17 @@ class Cloud: "feature.node.kubernetes.io/system-os_release.VERSION_ID": "2023", }, taint_value="true", + cpu_labels={ + "node.kubernetes.io/instance-type": "m6i.4xlarge", + "modelplane.ai/pool": "cpu-fake", + "modelplane.ai/cpu": "true", + }, + dra_drivers=("dra.cpu", "gpu.nvidia.com"), expected_daemonsets={ "nvidia-dra-driver/nvidia-dra-driver-gpu-kubelet-plugin": 2, + "dra-driver-cpu/mp-dra-driver-cpu": 2, "node-feature-discovery/mp-node-feature-discovery-worker": 3, - "monitoring/prometheus-node-exporter": 3, + "monitoring/prometheus-node-exporter": 5, }, ), # The taint VALUE is load-bearing here: the AKS stack's accelerated @@ -108,10 +168,17 @@ class Cloud: "feature.node.kubernetes.io/system-os_release.VERSION_ID": "22.04", }, taint_value="true", + cpu_labels={ + "node.kubernetes.io/instance-type": "Standard_D8s_v5", + "modelplane.ai/pool": "cpufake", + "modelplane.ai/cpu": "true", + }, + dra_drivers=("dra.cpu", "gpu.nvidia.com"), expected_daemonsets={ "nvidia-dra-driver/nvidia-dra-driver-gpu-kubelet-plugin": 2, + "dra-driver-cpu/mp-dra-driver-cpu": 2, "node-feature-discovery/mp-node-feature-discovery-worker": 3, - "monitoring/prometheus-node-exporter": 3, + "monitoring/prometheus-node-exporter": 5, "gpu-operator/nvidia-toolkit-hardening": 2, }, gpu_workloads=("^nvidia-toolkit-hardening", "^nic-health-monitor"), @@ -143,10 +210,17 @@ class Cloud: "feature.node.kubernetes.io/system-os_release.VERSION_ID": "121", }, taint_value="present", + cpu_labels={ + "node.kubernetes.io/instance-type": "e2-standard-8", + "modelplane.ai/pool": "cpu-fake", + "modelplane.ai/cpu": "true", + }, + dra_drivers=("dra.cpu", "gpu.nvidia.com"), expected_daemonsets={ "nvidia-dra-driver/nvidia-dra-driver-gpu-kubelet-plugin": 2, + "dra-driver-cpu/mp-dra-driver-cpu": 2, "node-feature-discovery/mp-node-feature-discovery-worker": 3, - "monitoring/prometheus-node-exporter": 3, + "monitoring/prometheus-node-exporter": 5, "gpu-operator/nvidia-container-toolkit-daemonset": 2, }, gpu_workloads=("^nvidia-container-toolkit",), @@ -171,10 +245,17 @@ class Cloud: "feature.node.kubernetes.io/system-os_release.VERSION_ID": "22.04", }, taint_value="true", + cpu_labels={ + "node.kubernetes.io/instance-type": "g4s.kube.large", + "modelplane.ai/pool": "cpu-fake", + "modelplane.ai/cpu": "true", + }, + dra_drivers=("dra.cpu", "gpu.nvidia.com"), expected_daemonsets={ "nvidia-dra-driver/dra-driver-nvidia-gpu-kubelet-plugin": 2, + "dra-driver-cpu/mp-dra-driver-cpu": 2, "node-feature-discovery/mp-node-feature-discovery-worker": 3, - "monitoring/mp-kube-prometheus-stack-prometheus-node-exporter": 3, + "monitoring/mp-kube-prometheus-stack-prometheus-node-exporter": 5, }, gpu_workloads=("^nvidia-driver-daemonset",), ), @@ -195,10 +276,17 @@ class Cloud: "feature.node.kubernetes.io/system-os_release.VERSION_ID": "22.04", }, taint_value="true", + cpu_labels={ + "node.kubernetes.io/instance-type": "cpu-d3", + "modelplane.ai/pool": "cpu-fake", + "modelplane.ai/cpu": "true", + }, + dra_drivers=("dra.cpu", "gpu.nvidia.com"), expected_daemonsets={ "nvidia-dra-driver/dra-driver-nvidia-gpu-kubelet-plugin": 2, + "dra-driver-cpu/mp-dra-driver-cpu": 2, "node-feature-discovery/mp-node-feature-discovery-worker": 3, - "monitoring/mp-kube-prometheus-stack-prometheus-node-exporter": 3, + "monitoring/mp-kube-prometheus-stack-prometheus-node-exporter": 5, }, ), # The hand-written Vultr half (stacks/clouds/vultr.py) has no NFD: Vultr @@ -224,9 +312,16 @@ class Cloud: "feature.node.kubernetes.io/system-os_release.VERSION_ID": "22.04", }, taint_value="true", + cpu_labels={ + "node.kubernetes.io/instance-type": "vc2-6c-16gb", + "modelplane.ai/pool": "cpu-fake", + "modelplane.ai/cpu": "true", + }, + dra_drivers=("dra.cpu", "gpu.nvidia.com"), expected_daemonsets={ "nvidia-dra-driver/dra-driver-nvidia-gpu-kubelet-plugin": 2, - "monitoring/mp-kube-prometheus-stack-prometheus-node-exporter": 3, + "dra-driver-cpu/mp-dra-driver-cpu": 2, + "monitoring/mp-kube-prometheus-stack-prometheus-node-exporter": 5, }, ), # The model e2e already installs the Existing stack, but on an untainted @@ -247,43 +342,109 @@ class Cloud: "feature.node.kubernetes.io/system-os_release.VERSION_ID": "22.04", }, taint_value="true", + cpu_labels={ + "modelplane.ai/pool": "cpu-fake", + "modelplane.ai/cpu": "true", + }, + # Existing is the one cloud that installs the AMD stack, so its leg + # is the one with an AMD pool. The operands are named for the AMD + # chart's default DeviceConfig. + amd_labels={ + "modelplane.ai/pool": "amd-fake", + "feature.node.kubernetes.io/amd-gpu": "true", + }, + dra_drivers=("dra.cpu", "gpu.amd.com", "gpu.nvidia.com"), expected_daemonsets={ "nvidia-dra-driver/dra-driver-nvidia-gpu-kubelet-plugin": 2, - "node-feature-discovery/mp-node-feature-discovery-worker": 3, - "monitoring/mp-kube-prometheus-stack-prometheus-node-exporter": 3, + "dra-driver-cpu/mp-dra-driver-cpu": 2, + "kube-amd-gpu/default-dra-driver": 2, + "kube-amd-gpu/default-node-labeller": 2, + "kube-amd-gpu/default-metrics-exporter": 2, + "node-feature-discovery/mp-node-feature-discovery-worker": 5, + "monitoring/mp-kube-prometheus-stack-prometheus-node-exporter": 7, }, ), + # The existing leg with gpu.amd.com left unclaimed: same pools, AMD + # fake nodes included, but spec.draDrivers names only the NVIDIA and + # CPU stacks. Every other leg claims everything its cloud carries, so + # only this one fails a join that ignored the filter: the AMD operands + # must not exist even though their pool is there and Existing's list + # pins them. NFD and the node exporter still reach the AMD nodes - + # their tolerations are static and NFD installs for the NVIDIA claim - + # so their counts match the existing leg's. + "existing-filtered": Cloud( + cloud="Existing", + labels={ + "modelplane.ai/pool": "gpu-fake", + "nvidia.com/gpu.present": "true", + "feature.node.kubernetes.io/pci-10de.present": "true", + # No convention for BYO node OS; Ubuntu 22.04 is a common one. + "feature.node.kubernetes.io/system-os_release.ID": "ubuntu", + "feature.node.kubernetes.io/system-os_release.VERSION_ID": "22.04", + }, + taint_value="true", + cpu_labels={ + "modelplane.ai/pool": "cpu-fake", + "modelplane.ai/cpu": "true", + }, + amd_labels={ + "modelplane.ai/pool": "amd-fake", + "feature.node.kubernetes.io/amd-gpu": "true", + }, + dra_drivers=("dra.cpu", "gpu.nvidia.com"), + expected_daemonsets={ + "nvidia-dra-driver/dra-driver-nvidia-gpu-kubelet-plugin": 2, + "dra-driver-cpu/mp-dra-driver-cpu": 2, + "node-feature-discovery/mp-node-feature-discovery-worker": 5, + "monitoring/mp-kube-prometheus-stack-prometheus-node-exporter": 7, + }, + absent_daemonsets=( + "kube-amd-gpu/default-dra-driver", + "kube-amd-gpu/default-node-labeller", + "kube-amd-gpu/default-metrics-exporter", + ), + ), } -def nodes(cloud: Cloud) -> list[dict[str, typing.Any]]: - """Return the fake GPU Nodes of a cloud's leg, for the workload cluster. +def _node(name: str, labels: dict[str, str], taint: dict[str, str]) -> dict[str, typing.Any]: + """Return one fake Node: a pool's labels, its taint, and KWOK's marks. - The kwok.x-k8s.io/node annotation hands the Nodes to KWOK, which keeps - them Ready and marks every pod scheduled onto them Running. No - nvidia.com/gpu extended resource: the stack is DRA-mode, with the device - plugin disabled everywhere. + The kwok.x-k8s.io/node annotation hands the Node to KWOK, which keeps it + Ready and marks every pod scheduled onto it Running. No extended + resources: the stack is DRA-mode, with every device plugin disabled. """ - return [ - { - "apiVersion": "v1", - "kind": "Node", - "metadata": { - "name": name, - "annotations": {"kwok.x-k8s.io/node": "fake", "node.alpha.kubernetes.io/ttl": "0"}, - # type: kwok is how bring-up and the tests find the fake - # nodes; every cloud's GPU nodes are linux/amd64. - "labels": { - "type": "kwok", - "kubernetes.io/os": "linux", - "kubernetes.io/arch": "amd64", - **cloud.labels, - }, + return { + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "name": name, + "annotations": {"kwok.x-k8s.io/node": "fake", "node.alpha.kubernetes.io/ttl": "0"}, + # type: kwok is how bring-up and the tests find the fake + # nodes; every cloud's inference nodes are linux/amd64. + "labels": { + "type": "kwok", + "kubernetes.io/os": "linux", + "kubernetes.io/arch": "amd64", + **labels, }, - "spec": {"taints": [{"key": "nvidia.com/gpu", "value": cloud.taint_value, "effect": "NoSchedule"}]}, - } - for name in FAKE_NODES - ] + }, + "spec": {"taints": [taint]}, + } + + +def nodes(cloud: Cloud) -> list[dict[str, typing.Any]]: + """Return the fake Nodes of a cloud's leg, for the workload cluster. + + One tainted pool per device stack the leg claims: the cloud's NVIDIA GPU + pool, its CPU pool, and - where the leg declares one - an AMD GPU pool. + """ + gpu_taint = {"key": "nvidia.com/gpu", "value": cloud.taint_value, "effect": "NoSchedule"} + fake = [_node(name, cloud.labels, gpu_taint) for name in FAKE_NODES] + fake += [_node(name, cloud.cpu_labels, CPU_TAINT) for name in FAKE_CPU_NODES] + if cloud.amd_labels is not None: + fake += [_node(name, cloud.amd_labels, AMD_TAINT) for name in FAKE_AMD_NODES] + return fake def serving_stack(cloud: Cloud) -> dict[str, typing.Any]: @@ -305,6 +466,10 @@ def serving_stack(cloud: Cloud) -> dict[str, typing.Any]: "spec": { "cloud": cloud.cloud, "stack": "Standard", + # What compose-inference-cluster would stamp from the leg's + # classes: the join filters the stack's tagged device stacks to + # exactly these, so each pool gets its stack and no other's. + "draDrivers": list(cloud.dra_drivers), "gateway": {"hostname": f"gateway-{SERVING_STACK}.{NAMESPACE}.svc.cluster.local"}, "secrets": [{"type": "Kubeconfig", "name": "local-cluster-kubeconfig", "key": "kubeconfig"}], }, diff --git a/e2e/clouds/test_placement.py b/e2e/clouds/test_placement.py index 40281c6bd..273bd71fd 100644 --- a/e2e/clouds/test_placement.py +++ b/e2e/clouds/test_placement.py @@ -12,27 +12,30 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""The serving stack fans out to the GPU pool, and only the GPU pool. - -Two-sided checks. The fan-out test catches tolerations too narrow to reach -the tainted GPU pool, such as an Equal toleration whose value doesn't match -the taint. The audits catch the opposite class: pods tolerating taints they -shouldn't. Real AKS rejects wildcard tolerations at admission and every cloud -taints its GPU pools, so a system-pool pod carrying a wildcard or -nvidia.com/gpu toleration is a bug even when the scheduler happened to keep -it on the system node. +"""Each device stack fans out to its own pool, and only its own pool. + +Two-sided checks, per pool. The fan-out test catches tolerations too narrow +to reach a tainted pool, such as an Equal toleration whose value doesn't +match the taint, and a filter that wrongly dropped a claimed stack. The +audits catch the opposite class: pods tolerating taints they shouldn't, and +one stack's operands landing on another stack's pool - the NVIDIA kubelet +plugin on a CPU node, or dra-driver-cpu on a GPU node. Real AKS rejects +wildcard tolerations at admission and every cloud taints its inference +pools, so a system-pool pod carrying a wildcard or pool toleration is a bug +even when the scheduler happened to keep it on the system node. """ import re import pytest +from kubernetes import client from e2e import clouds, kube, wait -# Pods expected to carry the GPU toleration and land on the fake GPU nodes: -# the GPU-reaching DaemonSets from generate.py's TOLERATIONS table - the NFD -# worker, node-exporter, DRA kubelet plugin, gpu-operator operands and -# nvsentinel's node monitors (platform-connectors runs on every node; the +# Pods expected to carry the NVIDIA GPU toleration and land on the fake GPU +# nodes: the GPU-reaching DaemonSets from generate.py's TOLERATIONS table - +# the NFD worker, node-exporter, DRA kubelet plugin, gpu-operator operands +# and nvsentinel's node monitors (platform-connectors runs on every node; the # health monitors select on labels nvsentinel's labeler applies at runtime). # Unanchored where the generated and hand-written halves name the same # workload differently. Per-cloud extras come from Cloud.gpu_workloads. @@ -51,9 +54,31 @@ "^platform-connectors", ) -# GPU-only pods must never run on the system pool: they'd only land there by -# a selector or affinity regression. The NFD worker and node-exporter aren't -# here - they legitimately run on every node. +# Pods expected to carry the CPU pool toleration and land on the fake CPU +# nodes: dra-driver-cpu (its pin's nodeSelector and toleration name exactly +# this pool) and the node exporter, which reaches every inference pool. +# NFD is deliberately absent: it exists to label GPU nodes, so it carries +# the GPU drivers' tags and stays off CPU pools. +CPU_WORKLOADS = ( + "dra-driver-cpu", + "prometheus-node-exporter", +) + +# Pods expected to carry the AMD GPU toleration and land on the fake AMD +# nodes (the Existing leg only): the AMD operator's operands, named for its +# chart's default DeviceConfig, plus the NFD worker - it labels both GPU +# vendors' nodes - and the node exporter. +AMD_WORKLOADS = ( + "node-feature-discovery-worker", + "prometheus-node-exporter", + "^default-dra-driver", + "^default-node-labeller", + "^default-metrics-exporter", +) + +# Pool-only pods must never run anywhere but their own pool: they'd only +# land elsewhere by a selector or affinity regression. The NFD worker and +# node-exporter aren't here - they legitimately run on several pools. GPU_ONLY = re.compile( "|".join( ( @@ -69,16 +94,45 @@ ) ) ) +CPU_ONLY = re.compile("dra-driver-cpu") +AMD_ONLY = re.compile("|".join(("^default-dra-driver", "^default-node-labeller", "^default-metrics-exporter"))) + +# The taint each fake pool carries, keyed by the pool the tests report. +POOL_TAINT_KEYS = { + "nvidia": "nvidia.com/gpu", + "cpu": "modelplane.ai/cpu", + "amd": "amd.com/gpu", +} # Cluster infrastructure the serving stack doesn't manage. EXEMPT_NAMESPACES = ("kube-system", "metallb-system", "local-path-storage") def gpu_workloads(cloud: clouds.Cloud) -> re.Pattern[str]: - """Return the pod names allowed to tolerate the GPU taint and run on the fake nodes, for a cloud.""" + """Return the pod names allowed to tolerate the GPU taint and run on the fake GPU nodes, for a cloud.""" return re.compile("|".join(GPU_WORKLOADS + cloud.gpu_workloads)) +def allowed_by_pool(cloud: clouds.Cloud) -> dict[str, re.Pattern[str]]: + """Return each pool's allowed pod names, for a cloud.""" + return { + "nvidia": gpu_workloads(cloud), + "cpu": re.compile("|".join(CPU_WORKLOADS)), + "amd": re.compile("|".join(AMD_WORKLOADS)), + } + + +def fake_pools(workload: kube.Cluster) -> dict[str, str]: + """Return each fake node's pool, from the taint its leg shaped it with.""" + taint_pools = {taint: pool for pool, taint in POOL_TAINT_KEYS.items()} + pools = {} + for node in workload.core.list_node(label_selector="type=kwok", _request_timeout=kube.TIMEOUT_SECONDS).items: + for taint in node.spec.taints or []: + if taint.key in taint_pools: + pools[node.metadata.name] = taint_pools[taint.key] + return pools + + def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: """Parametrize the fan-out test from the leg's expected DaemonSets, so a failure names its DaemonSet.""" if "daemonset" in metafunc.fixturenames: @@ -88,12 +142,13 @@ def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: @pytest.mark.usefixtures("installed") def test_daemonsets_fan_out(workload: kube.Cluster, daemonset: str, want: int) -> None: - """A DaemonSet the cloud expects on its GPU pool reaches its ready count. + """A DaemonSet the cloud expects on its device pools reaches its ready count. Charts without dependents get no helm --wait, so a rollout can lag the ServingStack's Ready condition; poll. KWOK fakes the pods on the fake nodes Running, so a shortfall means the scheduler never placed them - a - toleration too narrow to reach the tainted pool. + toleration too narrow to reach the tainted pool, a selector that misses + its nodes, or a filter that dropped the stack the pool needs. """ namespace, name = daemonset.split("/", 1) @@ -112,13 +167,13 @@ def fanned_out() -> None: @pytest.mark.usefixtures("installed") def test_tolerations_are_scoped(workload: kube.Cluster, cloud: clouds.Cloud) -> None: - """No pod outside the GPU allowlist tolerates the GPU taint, or every taint. + """No pod outside a pool's allowlist tolerates that pool's taint, or every taint. Wherever the scheduler put the pod: a wildcard that happened to stay on the system node still squats on accelerated capacity the moment the real - cloud's autoscaler adds a GPU node. + cloud's autoscaler adds a tainted node. """ - allowed = gpu_workloads(cloud) + allowed = allowed_by_pool(cloud) bad = [] for pod in workload.core.list_pod_for_all_namespaces(_request_timeout=kube.TIMEOUT_SECONDS).items: namespace, name = pod.metadata.namespace, pod.metadata.name @@ -127,27 +182,59 @@ def test_tolerations_are_scoped(workload: kube.Cluster, cloud: clouds.Cloud) -> for toleration in pod.spec.tolerations or []: if toleration.operator == "Exists" and not toleration.key: bad.append(f"{namespace}/{name} tolerates every taint") - elif toleration.key == "nvidia.com/gpu" and not allowed.search(name): - bad.append(f"{namespace}/{name} tolerates the GPU taint") + continue + for pool, taint_key in POOL_TAINT_KEYS.items(): + if toleration.key == taint_key and not allowed[pool].search(name): + bad.append(f"{namespace}/{name} tolerates the {pool} pool's taint") assert bad == [], "pods carrying tolerations the system pool must not have" @pytest.mark.usefixtures("installed") -def test_only_gpu_workloads_on_the_gpu_pool(workload: kube.Cluster, cloud: clouds.Cloud) -> None: - """Everything on a fake GPU node is an expected GPU workload, and GPU-only workloads stay off the system pool.""" - allowed = gpu_workloads(cloud) - fake = { - node.metadata.name - for node in workload.core.list_node(label_selector="type=kwok", _request_timeout=kube.TIMEOUT_SECONDS).items - } - assert fake, "the workload cluster has no fake GPU nodes" +def test_pools_hold_their_own(workload: kube.Cluster, cloud: clouds.Cloud) -> None: + """Everything on a fake node belongs to that node's pool, and pool-only workloads stay there. + + The second side is what gives the filter teeth: dra-driver-cpu off the + CPU pool, or a GPU stack's operand off its GPU pool, means a selector or + toleration regressed - or a device stack installed somewhere its driver + was never claimed. + """ + allowed = allowed_by_pool(cloud) + pools = fake_pools(workload) + assert pools, "the workload cluster has no fake nodes" + only = {"nvidia": GPU_ONLY, "cpu": CPU_ONLY, "amd": AMD_ONLY} bad = [] for pod in workload.core.list_pod_for_all_namespaces(_request_timeout=kube.TIMEOUT_SECONDS).items: namespace, name, node = pod.metadata.namespace, pod.metadata.name, pod.spec.node_name if namespace in EXEMPT_NAMESPACES or node is None: continue - if node in fake and not allowed.search(name): - bad.append(f"{namespace}/{name} doesn't belong on fake GPU node {node}") - elif node not in fake and GPU_ONLY.search(name): - bad.append(f"{namespace}/{name} is GPU-only but sits on system node {node}") + pool = pools.get(node) + if pool is not None and not allowed[pool].search(name): + bad.append(f"{namespace}/{name} doesn't belong on fake {pool} node {node}") + for pool_only, pattern in only.items(): + if pool != pool_only and pattern.search(name): + where = f"fake {pool} node {node}" if pool else f"system node {node}" + bad.append(f"{namespace}/{name} is {pool_only}-only but sits on {where}") assert bad == [], "pods scheduled onto the wrong pool" + + +@pytest.mark.usefixtures("installed") +def test_unclaimed_stack_absent(workload: kube.Cluster, cloud: clouds.Cloud) -> None: + """A stack whose driver no class claims stays off the cluster entirely. + + The existing-filtered leg's teeth: its AMD pool is on the cluster and + Existing's list pins the AMD stack, so only spec.draDrivers leaving + gpu.amd.com out keeps these DaemonSets from existing. Checked once the + stack is Ready, by which point a join that ignored the filter would + have installed them - every other leg claims everything its cloud + carries and lists nothing here. + """ + present = [] + for daemonset in cloud.absent_daemonsets: + namespace, name = daemonset.split("/", 1) + try: + workload.apps.read_namespaced_daemon_set(name, namespace, _request_timeout=kube.TIMEOUT_SECONDS) + present.append(daemonset) + except client.ApiException as e: + if e.status != 404: + raise + assert present == [], "DaemonSets of a stack whose driver no class claims" diff --git a/e2e/environment.py b/e2e/environment.py index 69ce0de7a..a792522cf 100644 --- a/e2e/environment.py +++ b/e2e/environment.py @@ -94,9 +94,20 @@ def up(*, apply_manifests: bool, cloud: clouds.Cloud | None = None) -> None: log.info("Applying %s", MANIFESTS) kubectl(CONTROL_PLANE_CONTEXT, "apply", f"--filename={MANIFESTS}") return - log.info("Applying the fake GPU nodes and the %s ServingStack", cloud.cloud) + log.info("Applying the fake device-pool nodes and the %s ServingStack", cloud.cloud) nodes = {"apiVersion": "v1", "kind": "List", "items": clouds.nodes(cloud)} kubectl(WORKLOAD_CONTEXT, "apply", "--filename=-", stdin=json.dumps(nodes)) + # Apply resets the fake nodes this leg shares with a previous one, but + # never prunes whole objects: a leg with fewer pools reusing a cluster + # (any leg after Existing's AMD pool) would keep stale fake nodes whose + # taints the placement audits then reason about. Delete the leftovers. + wanted = {node["metadata"]["name"] for node in clouds.nodes(cloud)} + observed = output( + "kubectl", f"--context={WORKLOAD_CONTEXT}", + "get", "nodes", "--selector=type=kwok", "--output=json", + ) # fmt: skip + for stale in sorted({node["metadata"]["name"] for node in json.loads(observed)["items"]} - wanted): + kubectl(WORKLOAD_CONTEXT, "delete", "node", stale) kubectl(CONTROL_PLANE_CONTEXT, "apply", "--filename=-", stdin=json.dumps(clouds.serving_stack(cloud))) diff --git a/e2e/test_legs.py b/e2e/test_legs.py index 3e4b07d53..f28e80f14 100644 --- a/e2e/test_legs.py +++ b/e2e/test_legs.py @@ -80,3 +80,27 @@ def test_taint_matches_composition(leg: str) -> None: f"the {leg} leg fakes nvidia.com/gpu={clouds.CLOUDS[leg].taint_value}, " f"but {COMPOSITIONS[leg]} taints GPU pools with {value}" ) + + +# The compositions whose CPU pool label-and-taint the legs' fake CPU nodes +# copy. GKE is included here, unlike COMPOSITIONS: its GPU taint comes from +# GKE itself, but compose-gke-cluster sets the CPU taint explicitly, since +# GKE only auto-taints GPU pools. Still no Existing entry: Modelplane +# doesn't taint BYO pools. +CPU_COMPOSITIONS = {**COMPOSITIONS, "gke": "compose-gke-cluster"} + + +@pytest.mark.parametrize("leg", sorted(CPU_COMPOSITIONS)) +def test_cpu_taint_matches(leg: str) -> None: + """The fake CPU pools' taint key is the one every cluster composition applies. + + The compositions share one key (_LABEL_CPU doubles as the taint key and + the label dra-driver-cpu's nodeSelector reads), so the legs share one + CPU_TAINT rather than faking it per cloud - this pins both halves of + that assumption. + """ + consts = constants(CPU_COMPOSITIONS[leg]) + assert consts["_LABEL_CPU"] == clouds.CPU_TAINT["key"], ( + f"the legs fake the CPU taint {clouds.CPU_TAINT['key']}, " + f"but {CPU_COMPOSITIONS[leg]} labels and taints CPU pools with {consts['_LABEL_CPU']}" + ) From 41cd36ff71ee4b42a8f6f90a7b4ad674f1cf2c25 Mon Sep 17 00:00:00 2001 From: Christopher Haar Date: Thu, 29 Oct 2026 16:47:31 -0700 Subject: [PATCH 04/19] Add Qwen2.5-0.5B recipes for an AMD GPU and for CPUs The recipes covered only NVIDIA GPUs. These add Qwen2.5-0.5B-Instruct on one AMD Instinct MI325X on a bring-your-own cluster, and on CPUs alone on a Civo pool with vLLM's CPU image. The recipe data's gpus list becomes accelerators so it can carry AMD and CPU entries; the docs site reads both (modelplaneai/docs-site#5). Signed-off-by: Christopher Haar Signed-off-by: Nic Cope --- docs/content/recipes/qwen2.5-0.5b-cpu.md | 57 +++++++++++++++++++ docs/content/recipes/qwen2.5-0.5b.md | 47 +++++++++++++++ docs/data/recipes.yaml | 17 +++++- .../qwen2.5-0.5b-cpu/inference-class.yaml | 29 ++++++++++ .../qwen2.5-0.5b-cpu/inference-cluster.yaml | 25 ++++++++ .../qwen2.5-0.5b-cpu/model-deployment.yaml | 56 ++++++++++++++++++ .../qwen2.5-0.5b-cpu/model-service.yaml | 15 +++++ .../recipes/qwen2.5-0.5b/inference-class.yaml | 29 ++++++++++ .../qwen2.5-0.5b/inference-cluster.yaml | 20 +++++++ .../qwen2.5-0.5b/model-deployment.yaml | 36 ++++++++++++ .../recipes/qwen2.5-0.5b/model-service.yaml | 15 +++++ 11 files changed, 345 insertions(+), 1 deletion(-) create mode 100644 docs/content/recipes/qwen2.5-0.5b-cpu.md create mode 100644 docs/content/recipes/qwen2.5-0.5b.md create mode 100644 docs/manifests/recipes/qwen2.5-0.5b-cpu/inference-class.yaml create mode 100644 docs/manifests/recipes/qwen2.5-0.5b-cpu/inference-cluster.yaml create mode 100644 docs/manifests/recipes/qwen2.5-0.5b-cpu/model-deployment.yaml create mode 100644 docs/manifests/recipes/qwen2.5-0.5b-cpu/model-service.yaml create mode 100644 docs/manifests/recipes/qwen2.5-0.5b/inference-class.yaml create mode 100644 docs/manifests/recipes/qwen2.5-0.5b/inference-cluster.yaml create mode 100644 docs/manifests/recipes/qwen2.5-0.5b/model-deployment.yaml create mode 100644 docs/manifests/recipes/qwen2.5-0.5b/model-service.yaml diff --git a/docs/content/recipes/qwen2.5-0.5b-cpu.md b/docs/content/recipes/qwen2.5-0.5b-cpu.md new file mode 100644 index 000000000..6f819d243 --- /dev/null +++ b/docs/content/recipes/qwen2.5-0.5b-cpu.md @@ -0,0 +1,57 @@ +--- +title: Qwen2.5-0.5B on CPU +weight: 14 +description: A 0.5B dense chat model served on CPUs alone, on a Civo CPU pool. +model: Qwen/Qwen2.5-0.5B-Instruct +vendors: [Qwen] +clouds: [Civo] +accelerators: [CPU] +engines: [vLLM] +arch: Dense +precisions: ["BF16"] +size: 0.5B +ctx: "4,096" +servingModes: [Standalone] +engineImages: [vllm/vllm-openai-cpu:v0.32.0-x86_64] +--- + +A 0.5B dense chat model served on CPUs alone, on a Civo `g4p.kube.medium` +pool: one `Standalone` engine, no cache, weights pulled straight from Hugging +Face. A CPU pool is the cheapest way to exercise a Modelplane deployment end +to end - no GPU to find, no driver to wait for - and a 0.5B model is small +enough that CPU inference answers in sensible time. + +The cluster pins Kubernetes 1.36: the claim consumes the device's cores as +consumable capacity, which Kubernetes enables by default from 1.36, on an +older version the engine pod fails when the node prepares its claim. + +The class's provisioning block omits `accelerator`, which is what makes the +pool a CPU pool: the cluster composition builds it with a standard node image +and the `modelplane.ai/cpu` label and taint, and the serving stack installs +`dra-driver-cpu` - and no GPU machinery - because the class claims `dra.cpu`. +The claim pins the engine to its NUMA node's cpuset, and the engine's CPU +request is the one number that sizes everything: Kubernetes accounts the pod +by it, and Modelplane stamps it into the claim as the share of the device to +take. Five of the six claimable cores, because the driver refuses a claim +that would take them all - the node's shared containers keep the last one. The engine is vLLM's CPU +build, whose image serves the same OpenAI API as the CUDA one. The same class +without its provisioning block describes a bring-your-own CPU pool, on any +conformant cluster: label its nodes `modelplane.ai/pool=cpu-pool` and +`modelplane.ai/cpu=true` instead. + +## Validated deployments + +{{< validated-deployments >}} + +## Platform + +{{< manifests "recipes/qwen2.5-0.5b-cpu/inference-class.yaml" >}} + +{{< manifests "recipes/qwen2.5-0.5b-cpu/inference-cluster.yaml" >}} + +## Deployment + +{{< manifests "recipes/qwen2.5-0.5b-cpu/model-deployment.yaml" >}} + +{{< manifests "recipes/qwen2.5-0.5b-cpu/model-service.yaml" >}} + diff --git a/docs/content/recipes/qwen2.5-0.5b.md b/docs/content/recipes/qwen2.5-0.5b.md new file mode 100644 index 000000000..30a5bc234 --- /dev/null +++ b/docs/content/recipes/qwen2.5-0.5b.md @@ -0,0 +1,47 @@ +--- +title: Qwen2.5-0.5B on AMD +weight: 13 +description: A 0.5B dense chat model on one AMD Instinct MI325X on a bring-your-own cluster. +model: Qwen/Qwen2.5-0.5B-Instruct +vendors: [Qwen] +clouds: [Existing] +accelerators: [MI325X] +engines: [vLLM] +arch: Dense +precisions: ["BF16"] +size: 0.5B +ctx: "32,768" +servingModes: [Standalone] +engineImages: [rocm/vllm:rocm7.14.1_cdna_ubuntu24.04_py3.14_pytorch_2.11_vllm_0.23.0] +gpuNote: 1× of 8 per node +--- + +A 0.5B dense chat model on one AMD Instinct MI325X on a bring-your-own +cluster: one `Standalone` engine, no cache, weights pulled straight from +Hugging Face. The model is deliberately small - the point of this run is the +AMD serving path, not the model. The class claims the GPU through DRA +(`gpu.amd.com`); the serving stack installs the AMD GPU operator in DRA mode +when a class claims that driver, against the amdgpu kernel driver the node +image already carries. The engine is AMD's ROCm vLLM. + +`source: Existing` is the one cloud source that installs the AMD stack today. +Bring the cluster with the amdgpu driver on its GPU nodes, label the pool's +nodes `modelplane.ai/pool=gpu-mi325x`, and apply the platform side first, then +the ML side. + +## Validated deployments + +{{< validated-deployments >}} + +## Platform + +{{< manifests "recipes/qwen2.5-0.5b/inference-class.yaml" >}} + +{{< manifests "recipes/qwen2.5-0.5b/inference-cluster.yaml" >}} + +## Deployment + +{{< manifests "recipes/qwen2.5-0.5b/model-deployment.yaml" >}} + +{{< manifests "recipes/qwen2.5-0.5b/model-service.yaml" >}} + diff --git a/docs/data/recipes.yaml b/docs/data/recipes.yaml index 17fee9874..5cd2ac9e7 100644 --- a/docs/data/recipes.yaml +++ b/docs/data/recipes.yaml @@ -36,17 +36,32 @@ clouds: - name: Vultr term: Vultr icon: brand-vultr -gpus: + - name: Civo + term: Civo + icon: brand-civo + - name: Bring your own + term: Existing +accelerators: - name: A16 mem: 16G + icon: brand-nvidia - name: L4 mem: 24G + icon: brand-nvidia - name: A100 mem: 40/80G + icon: brand-nvidia - name: H100 mem: 80G + icon: brand-nvidia - name: H200 mem: 141G + icon: brand-nvidia + - name: MI325X + mem: 256G + icon: brand-amd + - name: CPU + icon: cpu servingModes: - Standalone - LeaderWorker diff --git a/docs/manifests/recipes/qwen2.5-0.5b-cpu/inference-class.yaml b/docs/manifests/recipes/qwen2.5-0.5b-cpu/inference-class.yaml new file mode 100644 index 000000000..7a3e5a149 --- /dev/null +++ b/docs/manifests/recipes/qwen2.5-0.5b-cpu/inference-class.yaml @@ -0,0 +1,29 @@ +# An InferenceClass describing a Civo CPU node pool - no GPU at all. +# +# The provisioning block has no accelerator, so the cluster composition +# builds the pool with the CPU role: a standard node image and the +# modelplane.ai/cpu label and taint, nothing GPU. The serving stack installs +# dra-driver-cpu when a class claims dra.cpu, which publishes each node's +# cores as claimable devices (one device per NUMA node) and registers the +# dra.cpu DeviceClass. +apiVersion: modelplane.ai/v1alpha1 +kind: InferenceClass +metadata: + name: civo-cpu-8vcpu +spec: + description: "Civo g4p.kube.medium, 8 vCPUs, no GPU" + provisioning: + provider: Civo + civo: + size: g4p.kube.medium + devices: + - name: cpu + claim: DRA + driver: dra.cpu + deviceClassName: dra.cpu + count: 1 + capacity: + # The node's 8 vCPUs less the two dra-driver-cpu reserves for the OS + # and kubelet (reservedCPUs 0-1), which is what the driver's + # ResourceSlices report as claimable. + cpu: { value: "6" } diff --git a/docs/manifests/recipes/qwen2.5-0.5b-cpu/inference-cluster.yaml b/docs/manifests/recipes/qwen2.5-0.5b-cpu/inference-cluster.yaml new file mode 100644 index 000000000..bba1ec564 --- /dev/null +++ b/docs/manifests/recipes/qwen2.5-0.5b-cpu/inference-cluster.yaml @@ -0,0 +1,25 @@ +# A Civo cluster in NYC1 with a single CPU node offering the class above. +# The composed ServingStack gets draDrivers: [dra.cpu], so dra-driver-cpu +# installs and the GPU device stacks stay out - a cluster with no GPU runs +# no GPU machinery. +apiVersion: modelplane.ai/v1alpha1 +kind: InferenceCluster +metadata: + name: civo-cpu + labels: + modelplane.ai/region: nyc1 +spec: + cluster: + source: Civo + civo: + region: NYC1 + # CPU cores are claimed as consumable capacity, which Kubernetes + # enables by default from 1.36 (DRAConsumableCapacity) - on an older + # version the engine pod fails at NodePrepareResources. Civo's + # default version may be older, so a CPU cluster pins it; list the + # exact strings with: civo kubernetes versions. + kubernetesVersion: 1.36.0-k3s1 + nodePools: + - name: cpu-pool + className: civo-cpu-8vcpu + nodeCount: 1 diff --git a/docs/manifests/recipes/qwen2.5-0.5b-cpu/model-deployment.yaml b/docs/manifests/recipes/qwen2.5-0.5b-cpu/model-deployment.yaml new file mode 100644 index 000000000..010fb69f1 --- /dev/null +++ b/docs/manifests/recipes/qwen2.5-0.5b-cpu/model-deployment.yaml @@ -0,0 +1,56 @@ +# A small model served on CPUs alone. The device selector matches against +# the claimable cores the cpu-8vcpu class declares, so >= 4 lands it on the +# CPU pool; the claim takes the whole NUMA-node device and pins the engine +# container to its cpuset. The engine is vLLM's CPU build - the default +# vllm/vllm-openai image is CUDA-only. --block-size=32 is the smallest +# block its CPU attention backend accepts (multiples of 32); Modelplane +# reads the flag and chunks the endpoint picker's prefix cache the same +# way, so routing stays aligned. VLLM_CPU_KVCACHE_SPACE sets the KV +# cache's share of the node's RAM in GiB. No ModelCache: the engine pulls +# weights straight from Hugging Face. +apiVersion: modelplane.ai/v1alpha1 +kind: ModelDeployment +metadata: + name: qwen-cpu-demo + namespace: ml-team +spec: + replicas: 1 + template: + spec: + engines: + - name: qwen + members: + - role: Standalone + nodeSelector: + devices: + - name: cpu + count: 1 + selectors: + - cel: | + device.capacity["dra.cpu"].cpu.compareTo(quantity("4")) >= 0 + template: + spec: + containers: + - name: engine + image: vllm/vllm-openai-cpu:v0.32.0-x86_64 + args: + - --model=Qwen/Qwen2.5-0.5B-Instruct + - --served-model-name=$(MODELPLANE_SERVED_MODEL_NAME) + - --max-model-len=4096 + - --block-size=32 + env: + - name: VLLM_CPU_KVCACHE_SPACE + value: "4" + # Sizes the pod's accounting and the DRA claim alike: + # compose-model-replica stamps this as the claim's capacity + # request, and the kubelet accounts CPUs only from requests + # until KEP-5517. It must stay below the device's six + # claimable cores - a whole-device claim exhausts the + # shared CPU pool and the driver refuses to prepare it + # while the node's DaemonSets need somewhere to run. Five + # leaves them one claimable core. + resources: + requests: + cpu: "5" + limits: + cpu: "5" diff --git a/docs/manifests/recipes/qwen2.5-0.5b-cpu/model-service.yaml b/docs/manifests/recipes/qwen2.5-0.5b-cpu/model-service.yaml new file mode 100644 index 000000000..eea7f0fe4 --- /dev/null +++ b/docs/manifests/recipes/qwen2.5-0.5b-cpu/model-service.yaml @@ -0,0 +1,15 @@ +# Publishes the deployment's endpoints as one model a caller can name: +# /, so ml-team/qwen-cpu here. Modelplane composes one +# ModelEndpoint per ModelReplica, labeled with the deployment name, so +# this selector reaches every replica. +apiVersion: modelplane.ai/v1alpha1 +kind: ModelService +metadata: + name: qwen-cpu + namespace: ml-team +spec: + endpoints: + - name: qwen-cpu-demo + selector: + matchLabels: + modelplane.ai/deployment: qwen-cpu-demo diff --git a/docs/manifests/recipes/qwen2.5-0.5b/inference-class.yaml b/docs/manifests/recipes/qwen2.5-0.5b/inference-class.yaml new file mode 100644 index 000000000..20eef725a --- /dev/null +++ b/docs/manifests/recipes/qwen2.5-0.5b/inference-class.yaml @@ -0,0 +1,29 @@ +# An InferenceClass describing a BYO 8x AMD Instinct MI325X node pool. +# +# No provisioning block: this class describes hardware that already exists on +# a bring-your-own cluster whose node images carry the amdgpu kernel driver. +# When a cluster references it, the serving stack installs the AMD GPU +# operator in DRA mode (driver management off), whose DRA driver publishes +# the GPUs as gpu.amd.com ResourceSlices. Label the pool's nodes +# modelplane.ai/pool=; the amd.com/gpu=true:NoSchedule taint is +# the convention that keeps system workloads off them, and engine pods +# tolerate it. +apiVersion: modelplane.ai/v1alpha1 +kind: InferenceClass +metadata: + name: amd-mi325x-8x +spec: + description: "BYO 8x AMD Instinct MI325X 256GB" + devices: + - name: gpu + claim: DRA + driver: gpu.amd.com + deviceClassName: gpu.amd.com + count: 8 + attributes: + productName: { string: AMD_Instinct_MI325_OAM } + capacity: + # The MI325X's usable HBM3e as the AMD DRA driver's ResourceSlices + # report it, not its nominal 256GB. A nodeSelector asking for + # >= 256Gi would never bind. + memory: { value: "262128Mi" } diff --git a/docs/manifests/recipes/qwen2.5-0.5b/inference-cluster.yaml b/docs/manifests/recipes/qwen2.5-0.5b/inference-cluster.yaml new file mode 100644 index 000000000..7523e172e --- /dev/null +++ b/docs/manifests/recipes/qwen2.5-0.5b/inference-cluster.yaml @@ -0,0 +1,20 @@ +# Registers the bring-your-own cluster and installs the serving stack onto +# it through the kubeconfig Secret. The composed ServingStack gets +# draDrivers: [gpu.amd.com], so the AMD GPU operator (DRA mode) installs and +# the NVIDIA and CPU device stacks stay out. Existing is the one cloud that +# installs the AMD stack today. +apiVersion: modelplane.ai/v1alpha1 +kind: InferenceCluster +metadata: + name: amd-byo +spec: + cluster: + source: Existing + existing: + secretRef: + name: byo-cluster-kubeconfig + key: kubeconfig + nodePools: + - name: gpu-mi325x + className: amd-mi325x-8x + nodeCount: 1 diff --git a/docs/manifests/recipes/qwen2.5-0.5b/model-deployment.yaml b/docs/manifests/recipes/qwen2.5-0.5b/model-deployment.yaml new file mode 100644 index 000000000..f22ef3fb6 --- /dev/null +++ b/docs/manifests/recipes/qwen2.5-0.5b/model-deployment.yaml @@ -0,0 +1,36 @@ +# A small model to prove placement and serving on the AMD pool. The device +# selector matches against the capacity the amd-mi325x-8x class declares +# (262128Mi), not the pod's resource requests, so >= 200Gi lands it on the +# MI325X pool; the claim takes 1 of the node's 8 GPUs. The engine is AMD's +# ROCm vLLM build (the CDNA variant covers MI325X/gfx942) - the upstream +# vllm/vllm-openai image is CUDA-only. Unlike that image, rocm/vllm's +# entrypoint is not the OpenAI server, so the command invokes it explicitly. +# No ModelCache: the engine pulls weights straight from Hugging Face. +apiVersion: modelplane.ai/v1alpha1 +kind: ModelDeployment +metadata: + name: qwen-demo + namespace: ml-team +spec: + replicas: 1 + template: + spec: + engines: + - name: qwen + members: + - role: Standalone + nodeSelector: + devices: + - name: gpu + count: 1 + selectors: + - cel: | + device.capacity["gpu.amd.com"].memory.compareTo(quantity("200Gi")) >= 0 + template: + spec: + containers: + - name: engine + image: rocm/vllm:rocm7.14.1_cdna_ubuntu24.04_py3.14_pytorch_2.11_vllm_0.23.0 + command: ["vllm", "serve", "Qwen/Qwen2.5-0.5B-Instruct"] + args: + - --served-model-name=$(MODELPLANE_SERVED_MODEL_NAME) diff --git a/docs/manifests/recipes/qwen2.5-0.5b/model-service.yaml b/docs/manifests/recipes/qwen2.5-0.5b/model-service.yaml new file mode 100644 index 000000000..41341ec24 --- /dev/null +++ b/docs/manifests/recipes/qwen2.5-0.5b/model-service.yaml @@ -0,0 +1,15 @@ +# Publishes the deployment's endpoints as one model a caller can name: +# /, so ml-team/qwen here. Modelplane composes one +# ModelEndpoint per ModelReplica, labeled with the deployment name, so +# this selector reaches every replica. +apiVersion: modelplane.ai/v1alpha1 +kind: ModelService +metadata: + name: qwen + namespace: ml-team +spec: + endpoints: + - name: qwen-demo + selector: + matchLabels: + modelplane.ai/deployment: qwen-demo From 0848772ec191370bda92215a42fff26771d1e951 Mon Sep 17 00:00:00 2001 From: Nic Cope Date: Thu, 29 Oct 2026 16:48:40 -0700 Subject: [PATCH 05/19] Observe only NVIDIA GPU pools with the cluster agent The cluster agent counts a node only if every NoSchedule taint on it is nvidia.com/gpu. CPU pools carry modelplane.ai/cpu and BYO AMD pools amd.com/gpu, so the agent reported their nodes as observed but left them out of its free-device histogram. The scheduler then saw those pools as full and never placed a replica on them. This composes a PoolObservation only for a pool whose class claims gpu.nvidia.com. Other pools go unobserved, and the scheduler falls back to whole-node accounting for them, as it did for every pool before the agent. Teaching the agent the other taints would let them use free-device accounting too, but needs an agent release. Signed-off-by: Nic Cope --- apis/inferenceclusters/definition.yaml | 2 + docs/content/architecture/scheduling.md | 7 +- docs/content/overview/how-it-works.md | 6 +- .../compose-inference-cluster/function/fn.py | 24 ++- .../tests/test_fn.py | 151 ++++++++++++++++++ 5 files changed, 180 insertions(+), 10 deletions(-) diff --git a/apis/inferenceclusters/definition.yaml b/apis/inferenceclusters/definition.yaml index 41e34647f..7788d473d 100644 --- a/apis/inferenceclusters/definition.yaml +++ b/apis/inferenceclusters/definition.yaml @@ -650,6 +650,8 @@ spec: The nodes this pool has now, ready or not, as the cluster agent observes them. Absent until the agent reports on the pool, and while it can't observe it. + Always absent on a pool that isn't an NVIDIA GPU + pool, which the agent doesn't observe. lastAllocationTime: type: string format: date-time diff --git a/docs/content/architecture/scheduling.md b/docs/content/architecture/scheduling.md index 2a7b4fa9e..adaec4b6e 100644 --- a/docs/content/architecture/scheduling.md +++ b/docs/content/architecture/scheduling.md @@ -95,8 +95,8 @@ The cluster's scheduler is free to bind a gang's pods to one node, so the scheduler lets them share a node like any other pods. A cluster agent, which the serving stack installs on every `InferenceCluster`, -reports how many of each pool's nodes have each number of free devices. The -`InferenceCluster` publishes that histogram in its status: +reports how many of each NVIDIA GPU pool's nodes have each number of free +devices. The `InferenceCluster` publishes that histogram in its status: ```yaml status: @@ -126,7 +126,8 @@ scheduled pods hold as taken, and the scheduler doesn't charge a pool for them twice. The scheduler counts a pool in whole nodes when the agent hasn't reported on it -or can't observe it. Such a pool publishes no `observedNodes`, and every node it +or can't observe it. The agent doesn't observe CPU or AMD GPU pools yet, so +they're always counted in whole nodes. Such a pool publishes no `observedNodes`, and every node it can have starts empty. A pod that claims any device takes a whole one. ## Pinning placement to a pool diff --git a/docs/content/overview/how-it-works.md b/docs/content/overview/how-it-works.md index faeb26455..d3698e0e6 100644 --- a/docs/content/overview/how-it-works.md +++ b/docs/content/overview/how-it-works.md @@ -119,9 +119,9 @@ For each replica, the scheduler picks a `(cluster, pool)` in two steps: "a GPU with at least 141Gi of memory." A pool fits when it has the devices the model asks for, free on enough of its nodes to fit a replica. -An agent on each cluster reports the free devices on each pool's nodes, and the -scheduler fits replicas against those, so replicas share a node when it has -room. Replicas are pinned to their cluster once placed and stay there across +An agent on each cluster reports the free devices on each NVIDIA GPU pool's +nodes, and the scheduler fits replicas against those, so replicas share a node +when it has room. Replicas are pinned to their cluster once placed and stay there across reconciles; if a cluster is deleted, the scheduler re-places its replicas elsewhere. [How it schedules]({{< ref "/architecture/scheduling.md" >}}) covers the placement rules and their limits in full. diff --git a/functions/compose-inference-cluster/function/fn.py b/functions/compose-inference-cluster/function/fn.py index 85368f352..72184af35 100644 --- a/functions/compose-inference-cluster/function/fn.py +++ b/functions/compose-inference-cluster/function/fn.py @@ -107,6 +107,10 @@ CLUSTER_SOURCE_CIVO: frozenset({"gpu.nvidia.com", "dra.cpu"}), } +# The only driver whose pools the cluster agent can observe (see +# compose_pool_observations). +_NVIDIA_GPU_DRIVER = "gpu.nvidia.com" + # Composed resource key for the backend XR. BACKEND_RESOURCE_KEY = "serving-stack" @@ -257,6 +261,17 @@ def _namespace(meta: metav1.ObjectMeta | None) -> str: return meta.namespace +def _agent_observable(cls: iclv1alpha1.InferenceClass) -> bool: + """Whether the cluster agent can observe a pool of this class. + + Only NVIDIA GPU pools. The agent counts a node only if every NoSchedule + taint on it is nvidia.com/gpu, so it would report a CPU or AMD pool as + having no free devices and the scheduler would never place a replica + there. Unobserved, they get whole-node accounting. + """ + return any(d.driver == _NVIDIA_GPU_DRIVER and (d.claim or "DRA") == "DRA" for d in cls.spec.devices or []) + + def _gateway_hostname(cluster_name: str) -> str: """The internal name an InferenceGateway addresses this cluster's gateway by. @@ -569,7 +584,7 @@ def compose_namespaces(self) -> None: self.rsp.desired.resources[key].ready = fnv1.READY_TRUE def compose_pool_observations(self) -> None: - """Compose a PoolObservation on the cluster for each of its GPU pools. + """Compose a PoolObservation on the cluster for each of its NVIDIA GPU pools. The cluster agent reports each pool's nodes and free devices in the status of its PoolObservation, which schedulable_pools publishes. The @@ -593,9 +608,8 @@ def compose_pool_observations(self) -> None: ) pc = resource.child_name(_name(self.xr.metadata), "cluster-kubeconfig") for pool in self.xr.spec.nodePools or []: - # Only the pools schedulable_pools publishes have devices to observe. cls = self.classes.get(pool.className) - if not cls or not cls.spec.devices: + if not cls or not _agent_observable(cls): continue key = f"pool-observation-{pool.name}" if not (backend_ready or key in self.req.observed.resources): @@ -2078,7 +2092,9 @@ def schedulable_pools(self) -> list[v1alpha1.SchedulablePool]: for d in cls.spec.devices ], ) - observed = self.observed_pool(pool.name) + # A pool that has stopped being observable may still have an + # observation until Crossplane deletes it. Don't publish it. + observed = self.observed_pool(pool.name) if _agent_observable(cls) else None if observed is not None: schedulable.observedNodes = observed.nodes.total schedulable.nodesByFreeDevices = [ diff --git a/functions/compose-inference-cluster/tests/test_fn.py b/functions/compose-inference-cluster/tests/test_fn.py index 52331afdd..e0632d70e 100644 --- a/functions/compose-inference-cluster/tests/test_fn.py +++ b/functions/compose-inference-cluster/tests/test_fn.py @@ -1141,6 +1141,157 @@ def _to_dict(msg: message.Message) -> dict: ], ), ), + ComposeCase( + name="ExistingCPUPoolUnobserved", + reason="Once its ServingStack is Ready, an Existing cluster composes a PoolObservation for its NVIDIA GPU pool but not its CPU pool, whose tainted nodes the agent can't count, and doesn't publish a leftover observation of the CPU pool.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_inference_cluster( + cluster=v1alpha1.Cluster( + source="Existing", + existing=v1alpha1.Existing(secretRef=v1alpha1.SecretRef(name="my-kubeconfig")), + ), + node_pools=[ + v1alpha1.NodePool(name="l4-pool", className="gpu-l4", nodeCount=2, maxNodeCount=4), + v1alpha1.NodePool(name="cpu-pool", className="cpu", nodeCount=1, maxNodeCount=2), + ], + ), + resources={ + "serving-stack": _observed_serving_stack(gateway={"address": "34.55.100.10"}), + # What the agent reports of a CPU pool: its node, which it + # leaves out of the histogram for its modelplane.ai/cpu taint. + "pool-observation-cpu-pool": _observed_pool_observation( + pool="cpu-pool", + status={ + "observedGeneration": 1, + "conditions": [ + { + "type": "Healthy", + "status": "True", + "reason": "PoolHealthy", + "message": "", + "lastTransitionTime": "2026-10-20T10:00:00Z", + "observedGeneration": 1, + }, + ], + "nodes": {"total": 1, "ready": 1, "schedulable": 0}, + "unschedulablePods": 0, + "devices": [], + "nodesByFreeDevices": [], + }, + ), + }, + ), + required_resources={ + "class-gpu-l4": fnv1.Resources( + items=[ + _inference_class( + name="gpu-l4", + count=1, + memory="24Gi", + provisioning={ + "provider": "GKE", + "gke": { + "machineType": "g2-standard-48", + "diskSizeGb": 100, + "accelerator": {"type": "nvidia-l4", "count": 1}, + }, + }, + ), + ], + ), + "class-cpu": fnv1.Resources( + items=[ + _cpu_inference_class( + name="cpu", + cpus="6", + provisioning={"provider": "Civo", "civo": {"size": "g4p.kube.medium"}}, + ), + ], + ), + }, + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_inference_cluster( + schedulable_pools=[ + { + "name": "l4-pool", + "nodes": 4, + "devices": [ + { + "name": "gpu", + "claim": "DRA", + "driver": "gpu.nvidia.com", + "deviceClassName": "gpu.nvidia.com", + "count": 1, + "capacity": {"memory": {"value": "24Gi"}}, + }, + ], + }, + { + "name": "cpu-pool", + "nodes": 2, + "devices": [ + { + "name": "cpu", + "claim": "DRA", + "driver": "dra.cpu", + "deviceClassName": "dra.cpu", + "count": 1, + "capacity": {"cpu": {"value": "6"}}, + }, + ], + }, + ], + cache=None, + gateway={"address": "34.55.100.10"}, + ), + resources={ + "cluster-provider-config-kubernetes": _cluster_provider_config( + kubeconfig="my-kubeconfig", identity=None + ), + "serving-stack": _serving_stack( + cloud="Existing", + dra_drivers=("dra.cpu", "gpu.nvidia.com"), + secrets=[{"type": "Kubeconfig", "name": "my-kubeconfig", "key": "kubeconfig"}], + client_cas=None, + gpu=None, + ready=fnv1.READY_TRUE, + ), + "pool-observation-l4-pool": _pool_observation(pool="l4-pool"), + }, + ), + context=structpb.Struct(), + requirements=fnv1.Requirements( + resources={ + "model-replicas": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", + kind="ModelReplica", + match_labels=fnv1.MatchLabels(labels={"modelplane.ai/cluster": "test-cluster"}), + ), + "model-routes": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", + kind="ModelRoute", + match_labels=fnv1.MatchLabels(labels={"modelplane.ai/cluster": "test-cluster"}), + ), + "model-caches": fnv1.ResourceSelector(api_version="modelplane.ai/v1alpha1", kind="ModelCache"), + "gateways": fnv1.ResourceSelector(api_version="modelplane.ai/v1alpha1", kind="InferenceGateway"), + "class-gpu-l4": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", kind="InferenceClass", match_name="gpu-l4" + ), + "class-cpu": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", kind="InferenceClass", match_name="cpu" + ), + }, + ), + conditions=[ + fnv1.Condition(type="ClusterReady", status=fnv1.STATUS_CONDITION_TRUE, reason="ClusterRunning"), + fnv1.Condition(type="BackendReady", status=fnv1.STATUS_CONDITION_TRUE, reason="BackendHealthy"), + ], + ), + ), # The PoolObservation status is what the agent reported for a pool of one # g6.12xlarge node with three of its four L4s allocated. ComposeCase( From 65e4a25722c197158b537669cd1405fbfaae7c15 Mon Sep 17 00:00:00 2001 From: Nic Cope Date: Thu, 29 Oct 2026 16:55:36 -0700 Subject: [PATCH 06/19] Replace MetricMapping's cumulative boolean with a fromType enum MetricMapping gained a cumulative boolean for a component that reports a running total as a gauge. A boolean leaves no room for the next type mismatch, and reads apart from fromUnit, which already describes the same kind of correction for units. This replaces it with fromType, an enum whose only value is Gauge, described the way fromUnit is: what the component reports, when that isn't what the name claims. Because Gauge converts the series to a counter, a rule requires the mapping's to name to end in _total. Signed-off-by: Nic Cope --- apis/metricmappings/definition.yaml | 20 ++++++++++++------- .../function/collector.py | 9 +++++++-- .../function/stacks/metrics.py | 11 +++++----- 3 files changed, 26 insertions(+), 14 deletions(-) diff --git a/apis/metricmappings/definition.yaml b/apis/metricmappings/definition.yaml index 467aed7e0..5635a6805 100644 --- a/apis/metricmappings/definition.yaml +++ b/apis/metricmappings/definition.yaml @@ -55,6 +55,9 @@ spec: items: type: object required: [from, to] + x-kubernetes-validations: + - rule: "!has(self.fromType) || self.to.endsWith('_total')" + message: fromType Gauge converts the metric to a counter, so to must name one, ending in _total. properties: from: type: string @@ -166,14 +169,17 @@ spec: Check rather than assume: vLLM publishes kv_cache_usage_perc and the value is a fraction, so a name is no guide. - cumulative: - type: boolean + fromType: + type: string + enum: [Gauge] description: >- - Set when the component reports a cumulative total as a - gauge. Modelplane converts the series to a monotonic - cumulative sum before renaming it, so the exported - type matches the measurement and a modelplane_* name - never carries two types across sources. + What type the component reports this as, when that + isn't the type the name claims. Gauge is for a + component that reports a running total, such as + accumulated energy, as a gauge where the name says a + counter. Modelplane converts it to a monotonic + cumulative sum, so a modelplane_* name carries one type + whichever component feeds it. status: type: object properties: diff --git a/functions/compose-serving-stack/function/collector.py b/functions/compose-serving-stack/function/collector.py index 639ee94b2..5a98a483d 100644 --- a/functions/compose-serving-stack/function/collector.py +++ b/functions/compose-serving-stack/function/collector.py @@ -427,6 +427,11 @@ def _scrape_configs() -> list[dict[str, Any]]: "Percent": "0.01", } +# The OTTL that converts each fromType to the counter a _total name claims. +_TYPE_CONVERSION = { + "Gauge": 'convert_gauge_to_sum("cumulative", true)', +} + def statements(mappings: list[mmv1alpha1.MetricMapping]) -> tuple[list[str], list[str], list[str], list[str]]: """Compile the mappings to OTTL, as (extract, scale, datapoint, metric) statements. @@ -455,11 +460,11 @@ def statements(mappings: list[mmv1alpha1.MetricMapping]) -> tuple[list[str], lis source = f"{source}{_PART_SUFFIX[m.part]}" if m.fromUnit: scale.append(f'scale_metric({_UNIT_FACTOR[m.fromUnit]}) where metric.name == "{source}"') - if m.cumulative: + if m.fromType: # In the conversion block with the unit scaling: both make # the series agree with the name it's about to get, and # both must run while it still answers to the source name. - scale.append(f'convert_gauge_to_sum("cumulative", true) where metric.name == "{source}"') + scale.append(f'{_TYPE_CONVERSION[m.fromType]} where metric.name == "{source}"') # Labels before the rename, while the series still answers to the # name this mapping selected on. After it, two folded mappings share # one name and a statement could no longer tell them apart. diff --git a/functions/compose-serving-stack/function/stacks/metrics.py b/functions/compose-serving-stack/function/stacks/metrics.py index f234eff0d..63223a971 100644 --- a/functions/compose-serving-stack/function/stacks/metrics.py +++ b/functions/compose-serving-stack/function/stacks/metrics.py @@ -153,7 +153,7 @@ # exporter reports # its accumulated energy as a gauge where DCGM's is a counter, and # modelplane_energy_joules_total is a counter, so the energy series is marked -# cumulative and the collector converts it to a monotonic sum - one name, +# fromType Gauge and the collector converts it to a monotonic sum - one name, # one type, whichever vendor feeds it. _AMD_GPU = { "gpu_used_vram": "modelplane_gpu_memory_used_bytes", @@ -175,7 +175,7 @@ "gpu_energy_consumed": "Microjoules", } -_AMD_GPU_CUMULATIVE = ("gpu_energy_consumed",) +_AMD_GPU_TYPES = {"gpu_energy_consumed": "Gauge"} # How each request vLLM finished ended, by vLLM's finished_reason renamed to @@ -237,13 +237,14 @@ def _mapping( units: dict[str, str] | None = None, tokens: tuple[str, str] | None = None, extra: tuple[dict[str, object], ...] = (), - cumulative: tuple[str, ...] = (), + types: dict[str, str] | None = None, ) -> v1alpha1.MetricMapping: units = units or {} + types = types or {} metrics: list[dict[str, object]] = [ {"from": src, "to": dst} | ({"fromUnit": units[src]} if src in units else {}) - | ({"cumulative": True} if src in cumulative else {}) + | ({"fromType": types[src]} if src in types else {}) for src, dst in pairs.items() ] if tokens: @@ -269,7 +270,7 @@ def mappings() -> list[v1alpha1.MetricMapping]: _mapping("modelplane-sglang", _SGLANG, tokens=_SGLANG_TOKENS, extra=(_SGLANG_RESPONSES,)), _mapping("modelplane-picker", _PICKER), _mapping("modelplane-gpu", _GPU, _GPU_UNITS), - _mapping("modelplane-gpu-amd", _AMD_GPU, _AMD_GPU_UNITS, cumulative=_AMD_GPU_CUMULATIVE), + _mapping("modelplane-gpu-amd", _AMD_GPU, _AMD_GPU_UNITS, types=_AMD_GPU_TYPES), ] From 1a0f54e7ac62e4be33f28a87f8e3bbcfe676aa42 Mon Sep 17 00:00:00 2001 From: Nic Cope Date: Thu, 29 Oct 2026 17:01:36 -0700 Subject: [PATCH 07/19] Note that CPU pools need Kubernetes 1.36 on GKE, AKS and Nebius A CPU pool's cores are claimed as consumable capacity, which Kubernetes enables by default only from 1.36. GKE defaults to 1.35, and AKS and Nebius to 1.34, so a CPU pool on their defaults fails when its engine pod's claim is prepared. Only Civo's version field said so. Signed-off-by: Nic Cope --- apis/inferenceclusters/definition.yaml | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/apis/inferenceclusters/definition.yaml b/apis/inferenceclusters/definition.yaml index 7788d473d..a28c3f873 100644 --- a/apis/inferenceclusters/definition.yaml +++ b/apis/inferenceclusters/definition.yaml @@ -141,6 +141,13 @@ spec: kubernetesVersion: type: string default: "1.35" + description: >- + GKE cluster Kubernetes version. Defaults to a + version where Dynamic Resource Allocation (how GPUs + bind to pods) is generally available. A CPU pool + needs 1.36 or newer: its cores are claimed as + consumable capacity, which Kubernetes enables by + default from 1.36 (DRAConsumableCapacity). credentials: type: object description: >- @@ -215,7 +222,10 @@ spec: description: >- AKS cluster Kubernetes version. Defaults to a version where Dynamic Resource Allocation (how GPUs - bind to pods) is generally available. + bind to pods) is generally available. A CPU pool + needs 1.36 or newer: its cores are claimed as + consumable capacity, which Kubernetes enables by + default from 1.36 (DRAConsumableCapacity). credentials: type: object description: >- @@ -250,7 +260,10 @@ spec: description: >- mk8s cluster Kubernetes version. Defaults to a version where Dynamic Resource Allocation (DRA) - is generally available. + is generally available. A CPU pool needs 1.36 or + newer: its cores are claimed as consumable + capacity, which Kubernetes enables by default from + 1.36 (DRAConsumableCapacity). credentials: type: object description: >- From d36e26c0ee5f4285dda59ba5d88c2868d41cdd64 Mon Sep 17 00:00:00 2001 From: Nic Cope Date: Thu, 29 Oct 2026 16:55:37 -0700 Subject: [PATCH 08/19] Regenerate schemas Signed-off-by: Nic Cope --- schemas/.lock.json | 2 +- .../models/ai/modelplane/inferencecluster/v1alpha1.py | 9 ++++++--- .../models/ai/modelplane/metricmapping/v1alpha1.py | 8 ++++---- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/schemas/.lock.json b/schemas/.lock.json index 26ea5efcb..a69bcded6 100644 --- a/schemas/.lock.json +++ b/schemas/.lock.json @@ -1,6 +1,6 @@ { "packages": { - "fs://apis": "172c9217209bee80bac15ff9cb1e98689fb66000e484303c81b2997811fdc8af", + "fs://apis": "1e153596108699430084d9760de985b57952cb019f048d373013dbb805c07fe8", "git://https://github.com/crossplane/crossplane/cluster/crds": "90d8b72ad8b829f0bcd7d7d5a98eaa0d579f244a", "git://https://github.com/modelplaneai/cluster-agent/cluster/charts/cluster-agent/files/crds": "ca7631d72ef8550a2d2f415f33db1ec52b20c65c", "xpkg://xpkg.upbound.io/upbound/provider-aws-ec2:v2.8.1": "sha256:ca2e9e3b2e3a8b6ca44a9700d5abf7abd733cfa388d1afe9bb2bf7c847cd39ef", diff --git a/schemas/python/models/ai/modelplane/inferencecluster/v1alpha1.py b/schemas/python/models/ai/modelplane/inferencecluster/v1alpha1.py index 822b29bb1..ca031af24 100644 --- a/schemas/python/models/ai/modelplane/inferencecluster/v1alpha1.py +++ b/schemas/python/models/ai/modelplane/inferencecluster/v1alpha1.py @@ -24,7 +24,7 @@ class Aks(BaseModel): """ kubernetesVersion: str | None = '1.34' """ - AKS cluster Kubernetes version. Defaults to a version where Dynamic Resource Allocation (how GPUs bind to pods) is generally available. + AKS cluster Kubernetes version. Defaults to a version where Dynamic Resource Allocation (how GPUs bind to pods) is generally available. A CPU pool needs 1.36 or newer: its cores are claimed as consumable capacity, which Kubernetes enables by default from 1.36 (DRAConsumableCapacity). """ location: constr(min_length=1, max_length=32) """ @@ -111,6 +111,9 @@ class Gke(BaseModel): GCP ProviderConfig or ClusterProviderConfig used to authenticate to the GCP API. Defaults to the ClusterProviderConfig named default. """ kubernetesVersion: str | None = '1.35' + """ + GKE cluster Kubernetes version. Defaults to a version where Dynamic Resource Allocation (how GPUs bind to pods) is generally available. A CPU pool needs 1.36 or newer: its cores are claimed as consumable capacity, which Kubernetes enables by default from 1.36 (DRAConsumableCapacity). + """ region: constr(min_length=1, max_length=32) @@ -121,7 +124,7 @@ class Nebius(BaseModel): """ kubernetesVersion: str | None = '1.34' """ - mk8s cluster Kubernetes version. Defaults to a version where Dynamic Resource Allocation (DRA) is generally available. + mk8s cluster Kubernetes version. Defaults to a version where Dynamic Resource Allocation (DRA) is generally available. A CPU pool needs 1.36 or newer: its cores are claimed as consumable capacity, which Kubernetes enables by default from 1.36 (DRAConsumableCapacity). """ @@ -401,7 +404,7 @@ class SchedulablePool(BaseModel): """ observedNodes: conint(ge=0) | None = None """ - The nodes this pool has now, ready or not, as the cluster agent observes them. Absent until the agent reports on the pool, and while it can't observe it. + The nodes this pool has now, ready or not, as the cluster agent observes them. Absent until the agent reports on the pool, and while it can't observe it. Always absent on a pool that isn't an NVIDIA GPU pool, which the agent doesn't observe. """ diff --git a/schemas/python/models/ai/modelplane/metricmapping/v1alpha1.py b/schemas/python/models/ai/modelplane/metricmapping/v1alpha1.py index 7ca6daef4..3e79ca1f6 100644 --- a/schemas/python/models/ai/modelplane/metricmapping/v1alpha1.py +++ b/schemas/python/models/ai/modelplane/metricmapping/v1alpha1.py @@ -65,10 +65,6 @@ class Label(BaseModel): class Metric(BaseModel): - cumulative: bool | None = None - """ - Set when the component reports a cumulative total as a gauge. Modelplane converts the series to a monotonic cumulative sum before renaming it, so the exported type matches the measurement and a modelplane_* name never carries two types across sources. - """ from_: constr(pattern=r'^[a-zA-Z_:][a-zA-Z0-9_:]*$', max_length=255) = Field( ..., alias='from' ) @@ -76,6 +72,10 @@ class Metric(BaseModel): The metric's name as the component exposes it, matched exactly wherever it appears in the fleet. A histogram is named by its base name, without the _count, _sum or _bucket a Prometheus query would use: the collector holds it as one metric, and `part` is what reaches into it. """ + fromType: Literal['Gauge'] | None = None + """ + What type the component reports this as, when that isn't the type the name claims. Gauge is for a component that reports a running total, such as accumulated energy, as a gauge where the name says a counter. Modelplane converts it to a monotonic cumulative sum, so a modelplane_* name carries one type whichever component feeds it. + """ fromUnit: ( Literal[ 'Millijoules', From 8de953544f7261912a102f10c006f75562056f4e Mon Sep 17 00:00:00 2001 From: Nic Cope Date: Thu, 29 Oct 2026 17:41:47 -0700 Subject: [PATCH 09/19] Rename the AMD Qwen2.5-0.5B recipe to qwen2.5-0.5b-amd The AMD recipe took the generic qwen2.5-0.5b slug while its CPU sibling is qwen2.5-0.5b-cpu, so an NVIDIA recipe for the same model would have had nowhere to go. Suffixing the AMD recipe matches the CPU one. Signed-off-by: Nic Cope --- .../recipes/{qwen2.5-0.5b.md => qwen2.5-0.5b-amd.md} | 8 ++++---- .../inference-class.yaml | 0 .../inference-cluster.yaml | 0 .../model-deployment.yaml | 0 .../{qwen2.5-0.5b => qwen2.5-0.5b-amd}/model-service.yaml | 0 5 files changed, 4 insertions(+), 4 deletions(-) rename docs/content/recipes/{qwen2.5-0.5b.md => qwen2.5-0.5b-amd.md} (83%) rename docs/manifests/recipes/{qwen2.5-0.5b => qwen2.5-0.5b-amd}/inference-class.yaml (100%) rename docs/manifests/recipes/{qwen2.5-0.5b => qwen2.5-0.5b-amd}/inference-cluster.yaml (100%) rename docs/manifests/recipes/{qwen2.5-0.5b => qwen2.5-0.5b-amd}/model-deployment.yaml (100%) rename docs/manifests/recipes/{qwen2.5-0.5b => qwen2.5-0.5b-amd}/model-service.yaml (100%) diff --git a/docs/content/recipes/qwen2.5-0.5b.md b/docs/content/recipes/qwen2.5-0.5b-amd.md similarity index 83% rename from docs/content/recipes/qwen2.5-0.5b.md rename to docs/content/recipes/qwen2.5-0.5b-amd.md index 30a5bc234..0d3fa7302 100644 --- a/docs/content/recipes/qwen2.5-0.5b.md +++ b/docs/content/recipes/qwen2.5-0.5b-amd.md @@ -35,13 +35,13 @@ the ML side. ## Platform -{{< manifests "recipes/qwen2.5-0.5b/inference-class.yaml" >}} +{{< manifests "recipes/qwen2.5-0.5b-amd/inference-class.yaml" >}} -{{< manifests "recipes/qwen2.5-0.5b/inference-cluster.yaml" >}} +{{< manifests "recipes/qwen2.5-0.5b-amd/inference-cluster.yaml" >}} ## Deployment -{{< manifests "recipes/qwen2.5-0.5b/model-deployment.yaml" >}} +{{< manifests "recipes/qwen2.5-0.5b-amd/model-deployment.yaml" >}} -{{< manifests "recipes/qwen2.5-0.5b/model-service.yaml" >}} +{{< manifests "recipes/qwen2.5-0.5b-amd/model-service.yaml" >}} diff --git a/docs/manifests/recipes/qwen2.5-0.5b/inference-class.yaml b/docs/manifests/recipes/qwen2.5-0.5b-amd/inference-class.yaml similarity index 100% rename from docs/manifests/recipes/qwen2.5-0.5b/inference-class.yaml rename to docs/manifests/recipes/qwen2.5-0.5b-amd/inference-class.yaml diff --git a/docs/manifests/recipes/qwen2.5-0.5b/inference-cluster.yaml b/docs/manifests/recipes/qwen2.5-0.5b-amd/inference-cluster.yaml similarity index 100% rename from docs/manifests/recipes/qwen2.5-0.5b/inference-cluster.yaml rename to docs/manifests/recipes/qwen2.5-0.5b-amd/inference-cluster.yaml diff --git a/docs/manifests/recipes/qwen2.5-0.5b/model-deployment.yaml b/docs/manifests/recipes/qwen2.5-0.5b-amd/model-deployment.yaml similarity index 100% rename from docs/manifests/recipes/qwen2.5-0.5b/model-deployment.yaml rename to docs/manifests/recipes/qwen2.5-0.5b-amd/model-deployment.yaml diff --git a/docs/manifests/recipes/qwen2.5-0.5b/model-service.yaml b/docs/manifests/recipes/qwen2.5-0.5b-amd/model-service.yaml similarity index 100% rename from docs/manifests/recipes/qwen2.5-0.5b/model-service.yaml rename to docs/manifests/recipes/qwen2.5-0.5b-amd/model-service.yaml From 42700fe05713b50f966aceb67db3c3037c905d39 Mon Sep 17 00:00:00 2001 From: Nic Cope Date: Thu, 29 Oct 2026 18:08:33 -0700 Subject: [PATCH 10/19] Check compose-inference-cluster's supported drivers against the stack's compose-inference-cluster rejects a class claiming a DRA driver its cloud can't serve, using a hand-kept copy of the serving stack's derived supported-drivers table. Nothing compared the two. A copy that admitted a driver the stack doesn't serve would skip the UnsupportedDevices condition, and the ServingStack's join would raise on every reconcile instead. This adds a supported-drivers-mirror flake check. It imports the stacks package from compose-serving-stack's venv and reads the copy from compose-inference-cluster's source, since neither function can import the other, and fails naming each cloud whose drivers differ. Signed-off-by: Nic Cope --- .../compose-inference-cluster/function/fn.py | 23 +++-- .../function/stacks/__init__.py | 9 +- hack/check_supported_drivers.py | 84 +++++++++++++++++++ nix/checks.nix | 22 ++++- pyproject.toml | 2 + 5 files changed, 121 insertions(+), 19 deletions(-) create mode 100644 hack/check_supported_drivers.py diff --git a/functions/compose-inference-cluster/function/fn.py b/functions/compose-inference-cluster/function/fn.py index 72184af35..f03903089 100644 --- a/functions/compose-inference-cluster/function/fn.py +++ b/functions/compose-inference-cluster/function/fn.py @@ -86,19 +86,16 @@ CONDITION_REASON_INVALID_NODE_POOL = "InvalidNodePool" CONDITION_REASON_UNSUPPORTED_DEVICES = "UnsupportedDevices" -# The DRA drivers each cloud's serving stack supports. A hand-kept -# mirror of supported_drivers() in compose-serving-stack's stacks -# package - functions are self-contained by design, so keep the two in -# sync by hand. A stale mirror fails closed: it rejects a newly -# supported driver until someone updates it, and never admits an -# unsupported one, because the serving stack join's raise backstops -# that direction. EKS's dra.net arrives outside the serving stack -# (compose_eks_cluster installs the EFA DRA driver when a pool asks for -# the fabric). Existing is deliberately absent and never gates: a -# claimed driver beyond the stack's own installs nothing and is never -# rejected - the rule, and why, is stated on the stacks package's -# join() in compose-serving-stack. -_SUPPORTED_DRA_DRIVERS: dict[str, frozenset[str]] = { +# The DRA drivers each cloud supports. A hand-kept mirror of +# supported_drivers() in compose-serving-stack's stacks package - +# functions are self-contained by design - which the +# supported-drivers-mirror flake check holds in sync. EKS's dra.net +# arrives outside the serving stack (compose_eks_cluster installs the +# EFA DRA driver when a pool asks for the fabric). Existing has no entry +# because it never gates: a claimed driver beyond the stack's own +# installs nothing and is never rejected - the rule, and why, is stated +# on the stacks package's join() in compose-serving-stack. +_SUPPORTED_DRA_DRIVERS: dict[Cloud, frozenset[str]] = { CLUSTER_SOURCE_GKE: frozenset({"gpu.nvidia.com", "dra.cpu"}), CLUSTER_SOURCE_EKS: frozenset({"gpu.nvidia.com", "dra.cpu", "dra.net"}), CLUSTER_SOURCE_AKS: frozenset({"gpu.nvidia.com", "dra.cpu"}), diff --git a/functions/compose-serving-stack/function/stacks/__init__.py b/functions/compose-serving-stack/function/stacks/__init__.py index f21adc26f..d2a238fe2 100644 --- a/functions/compose-serving-stack/function/stacks/__init__.py +++ b/functions/compose-serving-stack/function/stacks/__init__.py @@ -96,10 +96,11 @@ def supported_drivers(cloud: Cloud) -> frozenset[str]: (_EXTERNAL_DRIVERS), so the table can't drift from what the lists record. compose-inference-cluster mirrors this by hand in its _SUPPORTED_DRA_DRIVERS to reject unsupported pairings with a - condition before a ServingStack is ever composed - keep the two in - sync. Existing's tags are returned like any cloud's, but a driver - beyond them is not unsupported there: the user owns the cluster, so - join() never rejects on Existing. + condition before a ServingStack is ever composed, and the + supported-drivers-mirror flake check holds the two in sync. + Existing's tags are returned like any cloud's, but a driver beyond + them is not unsupported there: the user owns the cluster, so join() + never rejects on Existing. """ halves = [_CLOUDS[cloud], common.COMPONENTS, *_STACKS.values()] tagged = {driver for half in halves for c in half for driver in c.dra_drivers} diff --git a/hack/check_supported_drivers.py b/hack/check_supported_drivers.py new file mode 100644 index 000000000..c2e7ab8b4 --- /dev/null +++ b/hack/check_supported_drivers.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +# Copyright 2026 The Modelplane Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Check compose-inference-cluster's supported DRA drivers against the serving stack's. + +compose-inference-cluster rejects a class claiming a driver its cloud can't +serve before it composes a ServingStack, using _SUPPORTED_DRA_DRIVERS, a +hand-kept copy of compose-serving-stack's derived supported_drivers(). Each +function's package is named `function`, so neither can import the other. This +runs with compose-serving-stack installed and reads the copy from +compose-inference-cluster's source. +""" + +import ast +import pathlib +import sys + +from function import stacks # ty: ignore[unresolved-import] # compose-serving-stack, installed in the check's venv. + +REPO = pathlib.Path(__file__).resolve().parents[1] +MIRROR = REPO / "functions" / "compose-inference-cluster" / "function" / "fn.py" + +# Existing never gates, so it has no entry in the copy. +UNGATED = {"Existing"} + + +def mirror() -> dict[str, frozenset[str]]: + """Return compose-inference-cluster's _SUPPORTED_DRA_DRIVERS, read from its source.""" + tree = ast.parse(MIRROR.read_text()) + constants: dict[str, str] = {} + table: ast.Dict | None = None + for node in tree.body: + if isinstance(node, ast.Assign) and len(node.targets) == 1: + target, value = node.targets[0], node.value + elif isinstance(node, ast.AnnAssign) and node.value is not None: + target, value = node.target, node.value + else: + continue + if not isinstance(target, ast.Name): + continue + if isinstance(value, ast.Constant) and isinstance(value.value, str): + constants[target.id] = value.value + if target.id == "_SUPPORTED_DRA_DRIVERS" and isinstance(value, ast.Dict): + table = value + if table is None: + sys.exit(f"{MIRROR}: no _SUPPORTED_DRA_DRIVERS dict literal") + + # Each entry reads CLUSTER_SOURCE_: frozenset({"driver", ...}). + got: dict[str, frozenset[str]] = {} + for key, value in zip(table.keys, table.values, strict=True): + if not isinstance(key, ast.Name) or not isinstance(value, ast.Call) or len(value.args) != 1: + sys.exit(f"{MIRROR}:{table.lineno}: unexpected _SUPPORTED_DRA_DRIVERS entry shape") + got[constants[key.id]] = frozenset(ast.literal_eval(value.args[0])) + return got + + +def main() -> int: + """Report every cloud whose copied drivers differ from the serving stack's.""" + want = {cloud: stacks.supported_drivers(cloud) for cloud in stacks.clouds() if cloud not in UNGATED} + got = mirror() + drifted = [cloud for cloud in sorted(set(want) | set(got)) if want.get(cloud) != got.get(cloud)] + for cloud in drifted: + print( + f"{cloud}: compose-serving-stack supports {sorted(want.get(cloud, []))}, " + f"compose-inference-cluster's _SUPPORTED_DRA_DRIVERS has {sorted(got.get(cloud, []))}", + file=sys.stderr, + ) + return 1 if drifted else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/nix/checks.nix b/nix/checks.nix index b61bcc8ce..3f12c41b5 100644 --- a/nix/checks.nix +++ b/nix/checks.nix @@ -79,6 +79,23 @@ in touch $out/.e2e-legs-passed ''; + # compose-inference-cluster keeps a copy of the DRA drivers each cloud's + # serving stack supports, to reject an unsupported class before composing a + # ServingStack. The checker reads the copy from source and compares it with + # compose-serving-stack's, which it imports from that function's venv. + supported-drivers-mirror = + let + venv = pythonSet.mkVirtualEnv "supported-drivers-mirror-env" { + compose-serving-stack = [ ]; + }; + in + pkgs.runCommand "modelplane-supported-drivers-mirror" { } '' + cd ${self} + ${venv}/bin/python hack/check_supported_drivers.py + mkdir -p $out + touch $out/.supported-drivers-mirror-checked + ''; + # Check the function unit tests against the rules in CONTRIBUTING.md's Tests # section that an AST can decide, such as how a table and its test are laid # out and what a case's name and reason look like. The checker uses only the @@ -101,8 +118,9 @@ in # against a venv that provides the function's dependencies, pytest, which the # tests import, and the protobuf type stubs ty needs to resolve the SDK's # generated Struct and Duration. It then runs on e2e/, against the packages - # the e2e app runs the tests with (see apps.nix), and on hack/, whose checker - # uses only the standard library and so needs no venv. + # the e2e app runs the tests with (see apps.nix), and on hack/ with no venv: + # its checkers use the standard library, apart from the supported drivers + # checker's import of compose-serving-stack, which ty can't resolve there. # # Unlike mkFunctionTest, which runs the function module from the venv, ty # checks the source, so we copy it from the tree: each function's function/ diff --git a/pyproject.toml b/pyproject.toml index a0d46c5cc..f31055c71 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -91,6 +91,8 @@ allow-star-arg-any = true "docs/utils/validate/**" = ["T201"] # The function unit test checker is a CLI script; print is its output. "hack/check_function_tests.py" = ["T201"] +# The supported drivers checker is a CLI script; print is its output. +"hack/check_supported_drivers.py" = ["T201"] # Generated serving stack lists carry upstream chart values verbatim; long # string literals and unicode arrive from the recipes. "functions/*/function/stacks/clouds/generated/**" = ["E501", "RUF001", "RUF002", "RUF003"] From 934fb2ceb5d01149c08c2faf7ec425d7f644ccf4 Mon Sep 17 00:00:00 2001 From: Nic Cope Date: Thu, 29 Oct 2026 18:08:37 -0700 Subject: [PATCH 11/19] Name every unsupported class, and gate only Existing out check_devices_supported skipped any cloud missing from _SUPPORTED_DRA_DRIVERS, which is how Existing opted out, so a cloud added without an entry would never be gated. Its UnsupportedDevices message also stopped at the first offending class, and said the serving stack installs every supported driver, which on EKS includes dra.net, installed by compose-eks-cluster instead. This skips Existing explicitly and indexes the table for every other cloud, names every offending class in one message, and says which drivers the cloud supports rather than what its stack installs. It also replaces the "DRA" claim literals with a _CLAIM_DRA constant, as compose-model-deployment does. Signed-off-by: Nic Cope --- .../compose-inference-cluster/function/fn.py | 68 ++++++++++--------- .../tests/test_fn.py | 22 ++++-- 2 files changed, 51 insertions(+), 39 deletions(-) diff --git a/functions/compose-inference-cluster/function/fn.py b/functions/compose-inference-cluster/function/fn.py index f03903089..822c54424 100644 --- a/functions/compose-inference-cluster/function/fn.py +++ b/functions/compose-inference-cluster/function/fn.py @@ -108,6 +108,9 @@ # compose_pool_observations). _NVIDIA_GPU_DRIVER = "gpu.nvidia.com" +# InferenceClass device claim values. An unset claim means DRA. +_CLAIM_DRA = "DRA" + # Composed resource key for the backend XR. BACKEND_RESOURCE_KEY = "serving-stack" @@ -266,7 +269,7 @@ def _agent_observable(cls: iclv1alpha1.InferenceClass) -> bool: having no free devices and the scheduler would never place a replica there. Unobserved, they get whole-node accounting. """ - return any(d.driver == _NVIDIA_GPU_DRIVER and (d.claim or "DRA") == "DRA" for d in cls.spec.devices or []) + return any(d.driver == _NVIDIA_GPU_DRIVER and (d.claim or _CLAIM_DRA) == _CLAIM_DRA for d in cls.spec.devices or []) def _gateway_hostname(cluster_name: str) -> str: @@ -728,7 +731,7 @@ def dra_drivers(self) -> list[str]: drivers: set[str] = set() for cls in self.classes.values(): for device in cls.spec.devices or []: - if (device.claim or "DRA") == "DRA": + if (device.claim or _CLAIM_DRA) == _CLAIM_DRA: drivers.add(device.driver) return sorted(drivers) @@ -740,40 +743,41 @@ def check_devices_supported(self, source: Cloud) -> bool: not-yet-supported driver lands here too; unknown and unsupported are the same failure. Existing never gates (see _SUPPORTED_DRA_DRIVERS). Gating is a fatal result, so a cluster already running when a class - turns unsupported holds its existing resources. + turns unsupported holds its existing resources. Every offending class + is named, so fixing one doesn't just surface the next. """ - supported = _SUPPORTED_DRA_DRIVERS.get(source) - if supported is None: + if source == CLUSTER_SOURCE_EXISTING: return True + supported = _SUPPORTED_DRA_DRIVERS[source] + offending = [] for name in sorted(self.classes): - cls = self.classes[name] - claimed = {d.driver for d in (cls.spec.devices or []) if (d.claim or "DRA") == "DRA"} + claimed = { + d.driver for d in (self.classes[name].spec.devices or []) if (d.claim or _CLAIM_DRA) == _CLAIM_DRA + } unsupported = claimed - supported - if not unsupported: - continue - msg = ( - f"class {name} claims {', '.join(sorted(unsupported))}; " - f"the {source} serving stack installs {', '.join(sorted(supported))}" - ) - response.set_conditions( - self.rsp, - resource.Condition( - typ=CONDITION_TYPE_CLUSTER_READY, - status="False", - reason=CONDITION_REASON_UNSUPPORTED_DEVICES, - message=msg, - ), - ) - # A fatal result, not a bare return: everything after this gate - # composes the cluster, so returning an empty desired state would - # have Crossplane garbage-collect a running cluster - the cloud - # cluster XR, ServingStack and Usages - the moment a shared class - # starts claiming an unsupported driver. Crossplane stops before - # applying desired state or collecting anything on a fatal result, - # and still applies the condition above. - response.fatal(self.rsp, msg) - return False - return True + if unsupported: + offending.append(f"class {name} claims {', '.join(sorted(unsupported))}") + if not offending: + return True + msg = f"{'; '.join(offending)}; {source} clusters support {', '.join(sorted(supported))}" + response.set_conditions( + self.rsp, + resource.Condition( + typ=CONDITION_TYPE_CLUSTER_READY, + status="False", + reason=CONDITION_REASON_UNSUPPORTED_DEVICES, + message=msg, + ), + ) + # A fatal result, not a bare return: everything after this gate + # composes the cluster, so returning an empty desired state would + # have Crossplane garbage-collect a running cluster - the cloud + # cluster XR, ServingStack and Usages - the moment a shared class + # starts claiming an unsupported driver. Crossplane stops before + # applying desired state or collecting anything on a fatal result, + # and still applies the condition above. + response.fatal(self.rsp, msg) + return False def compose_gke(self, gke: v1alpha1.Gke | None) -> None: """Compose an InferenceCluster backed by a Modelplane-provisioned diff --git a/functions/compose-inference-cluster/tests/test_fn.py b/functions/compose-inference-cluster/tests/test_fn.py index e0632d70e..6a9e7104d 100644 --- a/functions/compose-inference-cluster/tests/test_fn.py +++ b/functions/compose-inference-cluster/tests/test_fn.py @@ -5852,7 +5852,7 @@ def _to_dict(msg: message.Message) -> dict: results=[ fnv1.Result( severity=fnv1.SEVERITY_FATAL, - message="class gpu-l4 claims gpu.amd.com; the GKE serving stack installs dra.cpu, gpu.nvidia.com", + message="class gpu-l4 claims gpu.amd.com; GKE clusters support dra.cpu, gpu.nvidia.com", ), ], context=structpb.Struct(), @@ -5880,7 +5880,7 @@ def _to_dict(msg: message.Message) -> dict: type="ClusterReady", status=fnv1.STATUS_CONDITION_FALSE, reason="UnsupportedDevices", - message="class gpu-l4 claims gpu.amd.com; the GKE serving stack installs dra.cpu, gpu.nvidia.com", + message="class gpu-l4 claims gpu.amd.com; GKE clusters support dra.cpu, gpu.nvidia.com", ), ], ), @@ -6369,9 +6369,9 @@ def test_dra_drivers_synthetic() -> None: def test_unsupported_devices() -> None: - """A class claiming a driver its cloud can't serve gates ClusterReady - with UnsupportedDevices and the design's message, and stops with a - fatal result so a running cluster holds its existing resources + """Classes claiming drivers their cloud can't serve gate ClusterReady + with UnsupportedDevices, naming every offending class, and stop with + a fatal result so a running cluster holds its existing resources rather than being garbage-collected.""" composer = _composer( source="Vultr", @@ -6379,6 +6379,12 @@ def test_unsupported_devices() -> None: "amd-mi355x-8x": [ {"name": "gpu", "claim": "DRA", "driver": "gpu.amd.com", "deviceClassName": "gpu.amd.com"}, ], + "nvidia-l40s": [ + {"name": "gpu", "claim": "DRA", "driver": "gpu.nvidia.com", "deviceClassName": "gpu.nvidia.com"}, + ], + "efa": [ + {"name": "efa", "claim": "DRA", "driver": "dra.net", "deviceClassName": "efa.networking.k8s.aws"}, + ], }, ) assert composer.check_devices_supported("Vultr") is False @@ -6386,12 +6392,14 @@ def test_unsupported_devices() -> None: assert conditions["ClusterReady"].status == fnv1.STATUS_CONDITION_FALSE assert conditions["ClusterReady"].reason == "UnsupportedDevices" assert conditions["ClusterReady"].message == ( - "class amd-mi355x-8x claims gpu.amd.com; the Vultr serving stack installs dra.cpu, gpu.nvidia.com" + "class amd-mi355x-8x claims gpu.amd.com; class efa claims dra.net; " + "Vultr clusters support dra.cpu, gpu.nvidia.com" ) assert [(r.severity, r.message) for r in composer.rsp.results] == [ ( fnv1.SEVERITY_FATAL, - "class amd-mi355x-8x claims gpu.amd.com; the Vultr serving stack installs dra.cpu, gpu.nvidia.com", + "class amd-mi355x-8x claims gpu.amd.com; class efa claims dra.net; " + "Vultr clusters support dra.cpu, gpu.nvidia.com", ), ] From 1664320b9133bb7e9668294d80edc3d5c4ef90d6 Mon Sep 17 00:00:00 2001 From: Nic Cope Date: Thu, 29 Oct 2026 18:08:43 -0700 Subject: [PATCH 12/19] Declare external DRA drivers in every cloud's module The generated cloud modules declare the drivers that arrive outside the serving stack as EXTERNAL_DRIVERS, while the stacks package hard-coded an empty set for each hand-written cloud. This declares EXTERNAL_DRIVERS in the hand-written modules too, so every cloud states its external drivers in one place. Signed-off-by: Nic Cope --- .../compose-serving-stack/function/stacks/__init__.py | 8 ++++---- .../compose-serving-stack/function/stacks/clouds/civo.py | 5 +++++ .../function/stacks/clouds/existing.py | 5 +++++ .../function/stacks/clouds/nebius.py | 5 +++++ .../compose-serving-stack/function/stacks/clouds/vultr.py | 5 +++++ 5 files changed, 24 insertions(+), 4 deletions(-) diff --git a/functions/compose-serving-stack/function/stacks/__init__.py b/functions/compose-serving-stack/function/stacks/__init__.py index d2a238fe2..8187499ab 100644 --- a/functions/compose-serving-stack/function/stacks/__init__.py +++ b/functions/compose-serving-stack/function/stacks/__init__.py @@ -70,10 +70,10 @@ "EKS": frozenset(eks.EXTERNAL_DRIVERS), "AKS": frozenset(aks.EXTERNAL_DRIVERS), "GKE": frozenset(gke.EXTERNAL_DRIVERS), - "Nebius": frozenset(), - "Vultr": frozenset(), - "Civo": frozenset(), - "Existing": frozenset(), + "Nebius": frozenset(nebius.EXTERNAL_DRIVERS), + "Vultr": frozenset(vultr.EXTERNAL_DRIVERS), + "Civo": frozenset(civo.EXTERNAL_DRIVERS), + "Existing": frozenset(existing.EXTERNAL_DRIVERS), } diff --git a/functions/compose-serving-stack/function/stacks/clouds/civo.py b/functions/compose-serving-stack/function/stacks/clouds/civo.py index 8c3d367db..58e5112f7 100644 --- a/functions/compose-serving-stack/function/stacks/clouds/civo.py +++ b/functions/compose-serving-stack/function/stacks/clouds/civo.py @@ -154,6 +154,11 @@ def transformed(c: Component) -> Component: return out +# DRA drivers this cloud supports that arrive outside the serving +# stack; the stacks package folds them into its derived +# supported-drivers table. +EXTERNAL_DRIVERS: list[str] = [] + COMPONENTS: list[Component] = [ Chart( key="cert-manager", diff --git a/functions/compose-serving-stack/function/stacks/clouds/existing.py b/functions/compose-serving-stack/function/stacks/clouds/existing.py index ecbebf30a..b86f318c9 100644 --- a/functions/compose-serving-stack/function/stacks/clouds/existing.py +++ b/functions/compose-serving-stack/function/stacks/clouds/existing.py @@ -32,6 +32,11 @@ from function.stacks.components import Chart, Component, Manifests +# DRA drivers this cloud supports that arrive outside the serving +# stack. None are declared: join() treats every driver this list +# doesn't tag as externally installed on Existing. +EXTERNAL_DRIVERS: list[str] = [] + COMPONENTS: list[Component] = [ Chart( key="cert-manager", diff --git a/functions/compose-serving-stack/function/stacks/clouds/nebius.py b/functions/compose-serving-stack/function/stacks/clouds/nebius.py index b3e40ff49..3014677c5 100644 --- a/functions/compose-serving-stack/function/stacks/clouds/nebius.py +++ b/functions/compose-serving-stack/function/stacks/clouds/nebius.py @@ -23,6 +23,11 @@ from function.stacks.components import Chart, Component +# DRA drivers this cloud supports that arrive outside the serving +# stack; the stacks package folds them into its derived +# supported-drivers table. +EXTERNAL_DRIVERS: list[str] = [] + COMPONENTS: list[Component] = [ Chart( key="cert-manager", diff --git a/functions/compose-serving-stack/function/stacks/clouds/vultr.py b/functions/compose-serving-stack/function/stacks/clouds/vultr.py index e5658c908..e186a9b5a 100644 --- a/functions/compose-serving-stack/function/stacks/clouds/vultr.py +++ b/functions/compose-serving-stack/function/stacks/clouds/vultr.py @@ -33,6 +33,11 @@ from function.stacks.components import Chart, Component +# DRA drivers this cloud supports that arrive outside the serving +# stack; the stacks package folds them into its derived +# supported-drivers table. +EXTERNAL_DRIVERS: list[str] = [] + COMPONENTS: list[Component] = [ Chart( key="cert-manager", From be957b9a6051edc2d4c0690389ddb6d744cab928 Mon Sep 17 00:00:00 2001 From: Nic Cope Date: Thu, 29 Oct 2026 18:08:47 -0700 Subject: [PATCH 13/19] Pin the AMD GPU operator's metrics exporter image The AMD GPU operator chart defaults the DeviceConfig's metrics exporter image to a floating latest tag. The DRA driver, node labeller and utility images were already pinned, but not the exporter, whose field names the modelplane-gpu-amd MetricMapping reads. This pins it to v1.5.1, the release that shipped with the v1.5.1 chart. Signed-off-by: Nic Cope --- .../function/stacks/clouds/existing.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/functions/compose-serving-stack/function/stacks/clouds/existing.py b/functions/compose-serving-stack/function/stacks/clouds/existing.py index b86f318c9..0e0217d46 100644 --- a/functions/compose-serving-stack/function/stacks/clouds/existing.py +++ b/functions/compose-serving-stack/function/stacks/clouds/existing.py @@ -222,10 +222,12 @@ # and leave the live DeviceConfig at its install-time spec. # # The chart pin pins the operator's own images but not the - # DeviceConfig's, whose DRA driver, node labeller and utility images - # default to floating tags - so each is pinned here. The DRA driver - # is the component the claims go through; on a floating tag any pod - # reschedule could change it under a running cluster. + # DeviceConfig's, whose DRA driver, node labeller, metrics exporter + # and utility images default to floating tags - so each is pinned + # here. The DRA driver is the component the claims go through; on a + # floating tag any pod reschedule could change it under a running + # cluster. The exporter's field names are what the modelplane-gpu-amd + # MetricMapping reads, so it is pinned to the operator's release. # # Driver management is off: a BYO node's image must already carry # the amdgpu kernel driver, and the chart's in-cluster driver build @@ -272,6 +274,7 @@ ], }, "metricsExporter": { + "image": "docker.io/rocm/device-metrics-exporter:v1.5.1", # The ConfigMap above: profiler metrics on, for # the tensor-activity mapping. "config": {"name": "mp-metrics-exporter-config"}, From f93d6ee6fb2842aecc3b2d4ff4f4674ee904b3cb Mon Sep 17 00:00:00 2001 From: Nic Cope Date: Thu, 29 Oct 2026 18:08:52 -0700 Subject: [PATCH 14/19] Reject provisioned classes that claim dra.cpu beside another driver dra-driver-cpu runs only on nodes labelled modelplane.ai/cpu, which the cluster compositions put only on CPU pools. A provisioned class claiming dra.cpu alongside a GPU driver gets a GPU pool, so its dra.cpu claim could never allocate, and nothing said why. This rejects that class at admission. A class without provisioning is unaffected, since its nodes are the user's to label. The descriptions of each cloud's machine shape still called the accelerator block informational, though whether it's set now decides whether a pool is provisioned as a GPU or a CPU pool. This corrects them, and the InferenceCluster nodePools description that called every pool a GPU pool. Signed-off-by: Nic Cope --- apis/inferenceclasses/definition.yaml | 94 ++++++++++++++++---------- apis/inferenceclusters/definition.yaml | 5 +- 2 files changed, 62 insertions(+), 37 deletions(-) diff --git a/apis/inferenceclasses/definition.yaml b/apis/inferenceclasses/definition.yaml index b1ead5206..c75f231c9 100644 --- a/apis/inferenceclasses/definition.yaml +++ b/apis/inferenceclasses/definition.yaml @@ -23,8 +23,9 @@ spec: type: object required: [devices] x-kubernetes-validations: - # The pool's role follows accelerator - compose--cluster - # composes a GPU pool when it's set and a CPU pool when it isn't - + # The pool's role follows accelerator - compose-inference-cluster + # gives a pool the GPU role when it's set and the CPU role when + # it isn't - # while the scheduler and ServingStack spec.draDrivers follow # spec.devices. These rules keep the two in step, one pair per # cloud because CEL can't address the provider's block @@ -116,6 +117,16 @@ spec: message: >- spec.provisioning.civo.accelerator must be omitted when no device claims a DRA driver other than dra.cpu. + # dra-driver-cpu runs only on CPU pools, so a provisioned GPU + # pool never publishes dra.cpu devices for a claim to allocate. + - rule: >- + !has(self.provisioning) || + !self.devices.exists(d, (!has(d.claim) || d.claim == 'DRA') && d.driver == 'dra.cpu') || + !self.devices.exists(d, (!has(d.claim) || d.claim == 'DRA') && d.driver != 'dra.cpu') + message: >- + A class with spec.provisioning can't claim dra.cpu alongside + another DRA driver: Modelplane runs dra-driver-cpu only on + CPU pools. properties: description: type: string @@ -164,9 +175,10 @@ spec: the GCP machine's GPU count and need not be restated in devices. Set this when and only when a device claims a DRA driver other than dra.cpu; the API - rejects a mismatch. Omitted, the pool has no GPU: - Modelplane provisions it with a standard node image - and the modelplane.ai/cpu label and taint. + rejects a mismatch. + Omitted, Modelplane provisions a CPU pool: a standard + node image, the modelplane.ai/cpu label and taint, + and no GPU stack. required: [type, count] properties: type: @@ -189,8 +201,10 @@ spec: description: >- EC2 instance type (e.g. g6.xlarge, p4d.24xlarge). The instance family determines - the GPU model; the accelerator block below is - informational. + the GPU model, if any. The accelerator block + below doesn't change the hardware, but whether + it's set decides whether the group is provisioned + as a GPU or a CPU pool. minLength: 1 diskSizeGb: type: integer @@ -203,10 +217,10 @@ spec: group. Provisioning input only: the scheduler matches against spec.devices, not this block. Set this when and only when a device claims a DRA driver other - than dra.cpu; the API rejects a mismatch. Omitted, - the group has no GPU: Modelplane provisions it with - a standard node image and the modelplane.ai/cpu - label and taint. + than dra.cpu; the API rejects a mismatch. + Omitted, Modelplane provisions a CPU group: a standard + node image, the modelplane.ai/cpu label and taint, + and no GPU stack. required: [type, count] properties: type: @@ -230,8 +244,10 @@ spec: description: >- Azure VM size (e.g. Standard_NC24ads_A100_v4, Standard_ND96isr_H100_v5). The VM size determines - the GPU model and count; the accelerator block - below is informational. + the GPU model and count, if any. The accelerator + block below doesn't change the hardware, but + whether it's set decides whether the pool is + provisioned as a GPU or a CPU pool. minLength: 1 diskSizeGb: type: integer @@ -244,10 +260,10 @@ spec: pool. Provisioning input only: the scheduler matches against spec.devices, not this block. Set this when and only when a device claims a DRA driver other - than dra.cpu; the API rejects a mismatch. Omitted, - the pool has no GPU: Modelplane provisions it with a - standard node image and the modelplane.ai/cpu label - and taint. + than dra.cpu; the API rejects a mismatch. + Omitted, Modelplane provisions a CPU pool: a standard + node image, the modelplane.ai/cpu label and taint, + and no GPU stack. required: [type, count] properties: type: @@ -271,8 +287,10 @@ spec: description: >- Nebius compute platform (e.g. gpu-h100-sxm, gpu-l40s-a). Together with preset this determines - the GPU model and count; the accelerator block - below is informational. + the GPU model and count, if any. The accelerator + block below doesn't change the hardware, but + whether it's set decides whether the group is + provisioned as a GPU or a CPU pool. minLength: 1 maxLength: 63 preset: @@ -307,10 +325,10 @@ spec: group. Provisioning input only: the scheduler matches against spec.devices, not this block. Set this when and only when a device claims a DRA driver other - than dra.cpu; the API rejects a mismatch. Omitted, - the group has no GPU: Modelplane provisions it with - a standard node image and the modelplane.ai/cpu - label and taint. + than dra.cpu; the API rejects a mismatch. + Omitted, Modelplane provisions a CPU group: a standard + node image, the modelplane.ai/cpu label and taint, + and no GPU stack. required: [type, count] properties: type: @@ -333,8 +351,11 @@ spec: type: string description: >- Vultr plan ID (e.g. vcg-l40s-16c-180g-48vram). - The plan determines the GPU model and count; the - accelerator block below is informational. + The plan determines the GPU model and count, if + any. The accelerator block below doesn't change + the hardware, but whether it's set decides + whether the pool is provisioned as a GPU or a CPU + pool. minLength: 1 maxLength: 63 accelerator: @@ -344,10 +365,10 @@ spec: pool. Provisioning input only: the scheduler matches against spec.devices, not this block. Set this when and only when a device claims a DRA driver other - than dra.cpu; the API rejects a mismatch. Omitted, - the pool has no GPU: Modelplane provisions it with a - standard node image and the modelplane.ai/cpu label - and taint. + than dra.cpu; the API rejects a mismatch. + Omitted, Modelplane provisions a CPU pool: a standard + node image, the modelplane.ai/cpu label and taint, + and no GPU stack. required: [type, count] properties: type: @@ -370,8 +391,11 @@ spec: type: string description: >- Civo Kubernetes size ID (e.g. an.g1.l40s.kube.x1). - The size determines the GPU model and count; the - accelerator block below is informational. + The size determines the GPU model and count, if + any. The accelerator block below doesn't change + the hardware, but whether it's set decides + whether the pool is provisioned as a GPU or a CPU + pool. minLength: 1 maxLength: 63 accelerator: @@ -381,10 +405,10 @@ spec: pool. Provisioning input only: the scheduler matches against spec.devices, not this block. Set this when and only when a device claims a DRA driver other - than dra.cpu; the API rejects a mismatch. Omitted, - the pool has no GPU: Modelplane provisions it with a - standard node image and the modelplane.ai/cpu label - and taint. + than dra.cpu; the API rejects a mismatch. + Omitted, Modelplane provisions a CPU pool: a standard + node image, the modelplane.ai/cpu label and taint, + and no GPU stack. required: [type, count] properties: type: diff --git a/apis/inferenceclusters/definition.yaml b/apis/inferenceclusters/definition.yaml index a28c3f873..0e0c8ca3c 100644 --- a/apis/inferenceclusters/definition.yaml +++ b/apis/inferenceclusters/definition.yaml @@ -447,7 +447,7 @@ spec: nodePools: type: array description: >- - GPU node pools available on this cluster. Each pool + Inference node pools available on this cluster. Each pool references an InferenceClass that describes the hardware shape and (for provisioned clusters) how to create the pool. System pools for control-plane @@ -467,7 +467,8 @@ spec: description: >- Name of the pool. Its nodes carry it as the value of the modelplane.ai/pool label, and the cluster agent - observes the pool through an object of this name. + observes an NVIDIA GPU pool through an object of + this name. className: type: string description: >- From 12bee5d26da2dc3ad718a36a1941508ecc1397f2 Mon Sep 17 00:00:00 2001 From: Nic Cope Date: Thu, 29 Oct 2026 18:08:52 -0700 Subject: [PATCH 15/19] Regenerate schemas Signed-off-by: Nic Cope --- schemas/.lock.json | 2 +- .../ai/modelplane/inferenceclass/v1alpha1.py | 22 +++++++++---------- .../modelplane/inferencecluster/v1alpha1.py | 4 ++-- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/schemas/.lock.json b/schemas/.lock.json index a69bcded6..13f3e5a47 100644 --- a/schemas/.lock.json +++ b/schemas/.lock.json @@ -1,6 +1,6 @@ { "packages": { - "fs://apis": "1e153596108699430084d9760de985b57952cb019f048d373013dbb805c07fe8", + "fs://apis": "a214ca0e4ca2772e256e3da42cb23ba4c79d25a648e3eee69acc5c87b9b3dab2", "git://https://github.com/crossplane/crossplane/cluster/crds": "90d8b72ad8b829f0bcd7d7d5a98eaa0d579f244a", "git://https://github.com/modelplaneai/cluster-agent/cluster/charts/cluster-agent/files/crds": "ca7631d72ef8550a2d2f415f33db1ec52b20c65c", "xpkg://xpkg.upbound.io/upbound/provider-aws-ec2:v2.8.1": "sha256:ca2e9e3b2e3a8b6ca44a9700d5abf7abd733cfa388d1afe9bb2bf7c847cd39ef", diff --git a/schemas/python/models/ai/modelplane/inferenceclass/v1alpha1.py b/schemas/python/models/ai/modelplane/inferenceclass/v1alpha1.py index 96d455664..22e16ec31 100644 --- a/schemas/python/models/ai/modelplane/inferenceclass/v1alpha1.py +++ b/schemas/python/models/ai/modelplane/inferenceclass/v1alpha1.py @@ -101,12 +101,12 @@ class Accelerator(BaseModel): class Aks(BaseModel): accelerator: Accelerator | None = None """ - GPU accelerator to attach when provisioning the node pool. Provisioning input only: the scheduler matches against spec.devices, not this block. Set this when and only when a device claims a DRA driver other than dra.cpu; the API rejects a mismatch. Omitted, the pool has no GPU: Modelplane provisions it with a standard node image and the modelplane.ai/cpu label and taint. + GPU accelerator to attach when provisioning the node pool. Provisioning input only: the scheduler matches against spec.devices, not this block. Set this when and only when a device claims a DRA driver other than dra.cpu; the API rejects a mismatch. Omitted, Modelplane provisions a CPU pool: a standard node image, the modelplane.ai/cpu label and taint, and no GPU stack. """ diskSizeGb: conint(ge=10) | None = 100 vmSize: constr(min_length=1) """ - Azure VM size (e.g. Standard_NC24ads_A100_v4, Standard_ND96isr_H100_v5). The VM size determines the GPU model and count; the accelerator block below is informational. + Azure VM size (e.g. Standard_NC24ads_A100_v4, Standard_ND96isr_H100_v5). The VM size determines the GPU model and count, if any. The accelerator block below doesn't change the hardware, but whether it's set decides whether the pool is provisioned as a GPU or a CPU pool. """ @@ -121,11 +121,11 @@ class AcceleratorModel(BaseModel): class Civo(BaseModel): accelerator: AcceleratorModel | None = None """ - GPU accelerator to attach when provisioning the node pool. Provisioning input only: the scheduler matches against spec.devices, not this block. Set this when and only when a device claims a DRA driver other than dra.cpu; the API rejects a mismatch. Omitted, the pool has no GPU: Modelplane provisions it with a standard node image and the modelplane.ai/cpu label and taint. + GPU accelerator to attach when provisioning the node pool. Provisioning input only: the scheduler matches against spec.devices, not this block. Set this when and only when a device claims a DRA driver other than dra.cpu; the API rejects a mismatch. Omitted, Modelplane provisions a CPU pool: a standard node image, the modelplane.ai/cpu label and taint, and no GPU stack. """ size: constr(min_length=1, max_length=63) """ - Civo Kubernetes size ID (e.g. an.g1.l40s.kube.x1). The size determines the GPU model and count; the accelerator block below is informational. + Civo Kubernetes size ID (e.g. an.g1.l40s.kube.x1). The size determines the GPU model and count, if any. The accelerator block below doesn't change the hardware, but whether it's set decides whether the pool is provisioned as a GPU or a CPU pool. """ @@ -140,12 +140,12 @@ class AcceleratorModel1(BaseModel): class Eks(BaseModel): accelerator: AcceleratorModel1 | None = None """ - GPU accelerator to attach when provisioning the node group. Provisioning input only: the scheduler matches against spec.devices, not this block. Set this when and only when a device claims a DRA driver other than dra.cpu; the API rejects a mismatch. Omitted, the group has no GPU: Modelplane provisions it with a standard node image and the modelplane.ai/cpu label and taint. + GPU accelerator to attach when provisioning the node group. Provisioning input only: the scheduler matches against spec.devices, not this block. Set this when and only when a device claims a DRA driver other than dra.cpu; the API rejects a mismatch. Omitted, Modelplane provisions a CPU group: a standard node image, the modelplane.ai/cpu label and taint, and no GPU stack. """ diskSizeGb: conint(ge=10) | None = 100 instanceType: constr(min_length=1) """ - EC2 instance type (e.g. g6.xlarge, p4d.24xlarge). The instance family determines the GPU model; the accelerator block below is informational. + EC2 instance type (e.g. g6.xlarge, p4d.24xlarge). The instance family determines the GPU model, if any. The accelerator block below doesn't change the hardware, but whether it's set decides whether the group is provisioned as a GPU or a CPU pool. """ @@ -160,7 +160,7 @@ class AcceleratorModel2(BaseModel): class Gke(BaseModel): accelerator: AcceleratorModel2 | None = None """ - GPU accelerator to attach when provisioning the node pool. Provisioning input only: the scheduler matches against spec.devices, not this block, so count here is the GCP machine's GPU count and need not be restated in devices. Set this when and only when a device claims a DRA driver other than dra.cpu; the API rejects a mismatch. Omitted, the pool has no GPU: Modelplane provisions it with a standard node image and the modelplane.ai/cpu label and taint. + GPU accelerator to attach when provisioning the node pool. Provisioning input only: the scheduler matches against spec.devices, not this block, so count here is the GCP machine's GPU count and need not be restated in devices. Set this when and only when a device claims a DRA driver other than dra.cpu; the API rejects a mismatch. Omitted, Modelplane provisions a CPU pool: a standard node image, the modelplane.ai/cpu label and taint, and no GPU stack. """ diskSizeGb: conint(ge=10) | None = 100 machineType: constr(min_length=1) @@ -177,7 +177,7 @@ class AcceleratorModel3(BaseModel): class Nebius(BaseModel): accelerator: AcceleratorModel3 | None = None """ - GPU accelerator to attach when provisioning the node group. Provisioning input only: the scheduler matches against spec.devices, not this block. Set this when and only when a device claims a DRA driver other than dra.cpu; the API rejects a mismatch. Omitted, the group has no GPU: Modelplane provisions it with a standard node image and the modelplane.ai/cpu label and taint. + GPU accelerator to attach when provisioning the node group. Provisioning input only: the scheduler matches against spec.devices, not this block. Set this when and only when a device claims a DRA driver other than dra.cpu; the API rejects a mismatch. Omitted, Modelplane provisions a CPU group: a standard node image, the modelplane.ai/cpu label and taint, and no GPU stack. """ diskSizeGb: conint(ge=10) | None = 100 driversPreset: constr(min_length=1, max_length=63) | None = 'cuda13.0' @@ -186,7 +186,7 @@ class Nebius(BaseModel): """ platform: constr(min_length=1, max_length=63) """ - Nebius compute platform (e.g. gpu-h100-sxm, gpu-l40s-a). Together with preset this determines the GPU model and count; the accelerator block below is informational. + Nebius compute platform (e.g. gpu-h100-sxm, gpu-l40s-a). Together with preset this determines the GPU model and count, if any. The accelerator block below doesn't change the hardware, but whether it's set decides whether the group is provisioned as a GPU or a CPU pool. """ preset: constr(min_length=1, max_length=63) """ @@ -205,11 +205,11 @@ class AcceleratorModel4(BaseModel): class Vultr(BaseModel): accelerator: AcceleratorModel4 | None = None """ - GPU accelerator to attach when provisioning the node pool. Provisioning input only: the scheduler matches against spec.devices, not this block. Set this when and only when a device claims a DRA driver other than dra.cpu; the API rejects a mismatch. Omitted, the pool has no GPU: Modelplane provisions it with a standard node image and the modelplane.ai/cpu label and taint. + GPU accelerator to attach when provisioning the node pool. Provisioning input only: the scheduler matches against spec.devices, not this block. Set this when and only when a device claims a DRA driver other than dra.cpu; the API rejects a mismatch. Omitted, Modelplane provisions a CPU pool: a standard node image, the modelplane.ai/cpu label and taint, and no GPU stack. """ plan: constr(min_length=1, max_length=63) """ - Vultr plan ID (e.g. vcg-l40s-16c-180g-48vram). The plan determines the GPU model and count; the accelerator block below is informational. + Vultr plan ID (e.g. vcg-l40s-16c-180g-48vram). The plan determines the GPU model and count, if any. The accelerator block below doesn't change the hardware, but whether it's set decides whether the pool is provisioned as a GPU or a CPU pool. """ diff --git a/schemas/python/models/ai/modelplane/inferencecluster/v1alpha1.py b/schemas/python/models/ai/modelplane/inferencecluster/v1alpha1.py index ca031af24..d6a4a53a6 100644 --- a/schemas/python/models/ai/modelplane/inferencecluster/v1alpha1.py +++ b/schemas/python/models/ai/modelplane/inferencecluster/v1alpha1.py @@ -257,7 +257,7 @@ class NodePool(BaseModel): minNodeCount: conint(ge=0) | None = None name: constr(pattern=r'^[a-z0-9]([-a-z0-9]*[a-z0-9])?$', max_length=40) """ - Name of the pool. Its nodes carry it as the value of the modelplane.ai/pool label, and the cluster agent observes the pool through an object of this name. + Name of the pool. Its nodes carry it as the value of the modelplane.ai/pool label, and the cluster agent observes an NVIDIA GPU pool through an object of this name. """ nodeCount: conint(ge=0) | None = 1 zones: list[str] | None = None @@ -295,7 +295,7 @@ class Spec(BaseModel): """ nodePools: list[NodePool] | None = Field(None, max_length=8, min_length=1) """ - GPU node pools available on this cluster. Each pool references an InferenceClass that describes the hardware shape and (for provisioned clusters) how to create the pool. System pools for control-plane components are provisioned automatically. + Inference node pools available on this cluster. Each pool references an InferenceClass that describes the hardware shape and (for provisioned clusters) how to create the pool. System pools for control-plane components are provisioned automatically. """ placement: Placement | None = None """ From d02d198706cc7003c94d82b6e02e77378adb0552 Mon Sep 17 00:00:00 2001 From: Nic Cope Date: Thu, 29 Oct 2026 18:08:58 -0700 Subject: [PATCH 16/19] Reject a dra.cpu claim that spans more than one device compose-model-replica sizes a dra.cpu claim by stamping the engine container's cpu request on the claim as device capacity. It stamped it on every device in the request, so a member claiming two CPU devices took the full request from each, pinning twice the cores Kubernetes accounts the pod for. This fails the replica on a member whose dra.cpu requests total more than one device, alongside the existing failure for a dra.cpu claim with no cpu request, which both now report through invalid_cpu_claim. Signed-off-by: Nic Cope --- .../function/backends/base.py | 57 ++++++++++------ .../compose-model-replica/function/fn.py | 17 ++--- .../tests/test_backends.py | 68 +++++++++++++++---- 3 files changed, 94 insertions(+), 48 deletions(-) diff --git a/functions/compose-model-replica/function/backends/base.py b/functions/compose-model-replica/function/backends/base.py index d68e2e31c..b08f67519 100644 --- a/functions/compose-model-replica/function/backends/base.py +++ b/functions/compose-model-replica/function/backends/base.py @@ -807,35 +807,48 @@ def engine_resources(container: v1alpha1.Container, *, claim: bool) -> dict: return resources -def cpu_claim_without_request(replica: v1alpha1.ModelReplica) -> str | None: - """Name the first engine/member claiming dra.cpu whose engine sets no cpu request. - - The claim's capacity is sized from that request (see - resource_claim_template), so its absence is a user error the function - stops on, rather than composing a whole-device claim the driver - refuses at pod start - every claimable core taken leaves the node's - shared containers nowhere to run. +# The DeviceClass dra-driver-cpu registers, and the qualified name its +# grouped devices hold their cores under. A member claiming through this +# class sizes its claim from the engine's cpu request (see +# resource_claim_template); the names are the driver's contract, kept in +# sync with the dra-driver-cpu pin in compose-serving-stack's +# stacks/common.py. +_CPU_DEVICE_CLASS = "dra.cpu" +_CPU_CAPACITY = "dra.cpu/cpu" + + +def invalid_cpu_claim(replica: v1alpha1.ModelReplica) -> str | None: + """Explain why a member's dra.cpu claim can't be sized, or None if every one can. + + A CPU claim takes the engine container's cpu request from one device + (see resource_claim_template), so the function stops on a claim it + can't size that way rather than compose one the driver refuses or + that over-claims. Without a request the claim would take the whole + device, which leaves the node's shared containers nowhere to run. + Across more than one device, each would take the full request. """ for engine in replica.spec.engines: for member in engine.members: - if not any(r.deviceClassName == CPU_DEVICE_CLASS for r in (member.deviceRequests or [])): + cpu = [r for r in (member.deviceRequests or []) if r.deviceClassName == _CPU_DEVICE_CLASS] + if not cpu: continue + name = f"{engine.name}/{member_role(member)}" + if sum(int(r.count or 1) for r in cpu) > 1: + return ( + f"{name} claims more than one dra.cpu device, but a dra.cpu claim takes the engine " + "container's cpu request from a single device: claim one" + ) container = engine_container(member) requests = (container.resources.requests if container.resources else None) or {} if "cpu" not in requests: - return f"{engine.name}/{member_role(member)}" + return ( + f"a dra.cpu claim is sized from the engine container's cpu request, and {name} sets none: " + "set resources requests and limits cpu to the cores the claim should take, below the " + "device's total so the node's shared containers keep a core" + ) return None -# The DeviceClass dra-driver-cpu registers, and the qualified name its -# grouped devices hold their cores under. A member claiming through this -# class sizes its claim from the engine's cpu request (see -# resource_claim_template); the names are the driver's contract, kept in -# sync with the dra-driver-cpu pin in compose-serving-stack's -# stacks/common.py. -CPU_DEVICE_CLASS = "dra.cpu" -CPU_CAPACITY = "dra.cpu/cpu" - # Taints inference node pools carry so unrelated pods don't land on them, one # per device stack: the cluster compositions taint NVIDIA GPU pools # nvidia.com/gpu and CPU pools modelplane.ai/cpu, and BYO AMD pools carry @@ -916,7 +929,7 @@ def resource_claim_template( selectors = [{"cel": {"expression": s.cel}} for s in (r.selectors or []) if s.cel] if selectors: exactly["selectors"] = selectors - if r.deviceClassName == CPU_DEVICE_CLASS: + if r.deviceClassName == _CPU_DEVICE_CLASS: # dra-driver-cpu's grouped device holds its cores as consumable # capacity, and a claim with no capacity request consumes the # whole device - which the driver refuses while the node's @@ -926,10 +939,10 @@ def resource_claim_template( # contract already requires there (the kubelet accounts CPUs # only from requests until KEP-5517). One number, stated # once, sizes the cgroup accounting and the cpuset alike; - # cpu_claim_without_request gates on it being present. + # invalid_cpu_claim gates on it being present. container = engine_container(member) assert container.resources is not None and container.resources.requests is not None - exactly["capacity"] = {"requests": {CPU_CAPACITY: str(container.resources.requests["cpu"])}} + exactly["capacity"] = {"requests": {_CPU_CAPACITY: str(container.resources.requests["cpu"])}} device_requests.append({"name": r.name, "exactly": exactly}) return wrap_object( diff --git a/functions/compose-model-replica/function/fn.py b/functions/compose-model-replica/function/fn.py index fd18e50dc..56be2da1d 100644 --- a/functions/compose-model-replica/function/fn.py +++ b/functions/compose-model-replica/function/fn.py @@ -183,19 +183,10 @@ def compose_model_serving(self) -> None: "PrefillDecode serving requires serving.router: EndpointPicker and serving.endpointPicker", ) return - if missing := base.cpu_claim_without_request(self.xr): - # Stop rather than compose a whole-device claim: a CPU claim is - # sized from the engine's cpu request (see - # resource_claim_template), and without one it would consume - # every claimable core, which dra-driver-cpu refuses to prepare - # while the node's shared containers are running. Fatal for the - # same reason as above: existing resources hold. - response.fatal( - self.rsp, - f"a dra.cpu claim is sized from the engine container's cpu request, and {missing} sets none: " - "set resources requests and limits cpu to the cores the claim should take, below the device's " - "total so the node's shared containers keep a core", - ) + if invalid := base.invalid_cpu_claim(self.xr): + # Stop rather than compose a CPU claim that can't be sized. Fatal + # for the same reason as above: existing resources hold. + response.fatal(self.rsp, invalid) return if routing.decode_without_sidecar(self.xr): response.warning( diff --git a/functions/compose-model-replica/tests/test_backends.py b/functions/compose-model-replica/tests/test_backends.py index 1c1668a13..3bc81c5e3 100644 --- a/functions/compose-model-replica/tests/test_backends.py +++ b/functions/compose-model-replica/tests/test_backends.py @@ -61,6 +61,16 @@ class SelectBackendCase: want: str +@dataclasses.dataclass +class InvalidCpuClaimCase: + """A test case for base.invalid_cpu_claim.""" + + name: str + reason: str + replica: v1alpha1.ModelReplica + want: str | None + + @dataclasses.dataclass class CacheMountsCase: """A test case for base.cache_mounts.""" @@ -5425,8 +5435,8 @@ def test_build(case: BuildCase) -> None: assert _sorted(objects=_to_dicts(objects=got)) == _sorted(objects=case.want), case.reason -def _cpu_member(*, cpu_request: str | None) -> v1alpha1.Member: - """A Standalone member claiming one dra.cpu device, its engine requesting cpu_request CPUs (None for none).""" +def _cpu_member(*, cpu_request: str | None, count: int) -> v1alpha1.Member: + """A Standalone member claiming count dra.cpu devices, its engine requesting cpu_request CPUs (None for none).""" resources = None if cpu_request is not None: resources = v1alpha1.Resources(requests={"cpu": cpu_request}, limits={"cpu": cpu_request}) @@ -5437,7 +5447,7 @@ def _cpu_member(*, cpu_request: str | None) -> v1alpha1.Member: v1alpha1.DeviceRequest( name="cpu", deviceClassName="dra.cpu", - count=1, + count=count, selectors=[v1alpha1.Selector(cel='device.capacity["dra.cpu"].cpu.compareTo(quantity("4")) >= 0')], ) ], @@ -5455,7 +5465,7 @@ def _cpu_member(*, cpu_request: str | None) -> v1alpha1.Member: ) -def _cpu_replica(*, cpu_request: str | None) -> v1alpha1.ModelReplica: +def _cpu_replica(*, cpu_request: str | None, count: int) -> v1alpha1.ModelReplica: """A one-engine replica of one _cpu_member.""" return _replica( name="r", @@ -5464,7 +5474,7 @@ def _cpu_replica(*, cpu_request: str | None) -> v1alpha1.ModelReplica: labels=None, model_cache_ref=None, serving=None, - engines=[v1alpha1.Engine(name="main", copies=1, members=[_cpu_member(cpu_request=cpu_request)])], + engines=[v1alpha1.Engine(name="main", copies=1, members=[_cpu_member(cpu_request=cpu_request, count=count)])], ) @@ -5520,7 +5530,7 @@ def test_cpu_claim_capacity() -> None: not the whole device: dra-driver-cpu refuses a claim that would exhaust the shared CPU pool, so the number the driver's contract already puts on the container sizes the claim too.""" - replica = _cpu_replica(cpu_request="5") + replica = _cpu_replica(cpu_request="5", count=1) engine = replica.spec.engines[0] obj = base.resource_claim_template(replica, engine, engine.members[0], "cluster-a-pc") got = obj.model_dump(exclude_unset=True, by_alias=True) @@ -5539,13 +5549,45 @@ def test_gpu_claim_uncapped() -> None: assert "capacity" not in requests[0]["exactly"] -def test_cpu_request_missing() -> None: - """cpu_claim_without_request names the member whose dra.cpu claim has no - cpu request to size itself from, and stays quiet when it has one or - when nothing claims CPUs.""" - assert base.cpu_claim_without_request(_cpu_replica(cpu_request=None)) == "main/standalone" - assert base.cpu_claim_without_request(_cpu_replica(cpu_request="5")) is None - assert base.cpu_claim_without_request(_gpu_replica()) is None +INVALID_CPU_CLAIM_CASES = [ + InvalidCpuClaimCase( + name="CpuRequestSet", + reason="A dra.cpu claim on one device whose engine requests cpu can be sized.", + replica=_cpu_replica(cpu_request="5", count=1), + want=None, + ), + InvalidCpuClaimCase( + name="CpuRequestMissing", + reason="A dra.cpu claim whose engine requests no cpu has nothing to size itself from.", + replica=_cpu_replica(cpu_request=None, count=1), + want=( + "a dra.cpu claim is sized from the engine container's cpu request, and main/standalone sets none: " + "set resources requests and limits cpu to the cores the claim should take, below the device's " + "total so the node's shared containers keep a core" + ), + ), + InvalidCpuClaimCase( + name="MoreThanOneDevice", + reason="A dra.cpu claim on two devices would take the engine's whole cpu request from each.", + replica=_cpu_replica(cpu_request="5", count=2), + want=( + "main/standalone claims more than one dra.cpu device, but a dra.cpu claim takes the engine " + "container's cpu request from a single device: claim one" + ), + ), + InvalidCpuClaimCase( + name="NoCpuClaim", + reason="A replica that claims no CPUs has no CPU claim to size.", + replica=_gpu_replica(), + want=None, + ), +] + + +@pytest.mark.parametrize("case", INVALID_CPU_CLAIM_CASES, ids=lambda case: case.name) +def test_invalid_cpu_claim(case: InvalidCpuClaimCase) -> None: + """invalid_cpu_claim explains why a member's dra.cpu claim can't be sized.""" + assert base.invalid_cpu_claim(case.replica) == case.want, case.reason SELECT_BACKEND_CASES = [ From 28aa28c1ea2f5b055f39260b957923a3ca3743f2 Mon Sep 17 00:00:00 2001 From: Nic Cope Date: Thu, 29 Oct 2026 18:08:58 -0700 Subject: [PATCH 17/19] Declare CPU pool taints like GPU pool taints The cluster compositions declare their GPU taint's key, value and effect as constants, but tainted CPU pools by reusing the CPU label constant as the key, an inline "true", and the GPU taint's effect. GKE inlined the effect too. This gives each composition _CPU_TAINT_KEY, _CPU_TAINT_VALUE and _CPU_TAINT_EFFECT, matching AKS's _CPU_TAINT. The e2e legs test now checks the CPU taint's value as well as its key, and the CPU label against each leg's fake nodes. Signed-off-by: Nic Cope --- e2e/test_legs.py | 26 +++++++++++++------ functions/compose-civo-cluster/function/fn.py | 9 ++++--- functions/compose-eks-cluster/function/fn.py | 9 ++++--- functions/compose-gke-cluster/function/fn.py | 5 +++- .../compose-nebius-cluster/function/fn.py | 9 ++++--- .../compose-vultr-cluster/function/fn.py | 9 ++++--- 6 files changed, 46 insertions(+), 21 deletions(-) diff --git a/e2e/test_legs.py b/e2e/test_legs.py index f28e80f14..4c435704e 100644 --- a/e2e/test_legs.py +++ b/e2e/test_legs.py @@ -92,15 +92,25 @@ def test_taint_matches_composition(leg: str) -> None: @pytest.mark.parametrize("leg", sorted(CPU_COMPOSITIONS)) def test_cpu_taint_matches(leg: str) -> None: - """The fake CPU pools' taint key is the one every cluster composition applies. + """The fake CPU pools' label and taint are the ones every cluster composition applies. - The compositions share one key (_LABEL_CPU doubles as the taint key and - the label dra-driver-cpu's nodeSelector reads), so the legs share one - CPU_TAINT rather than faking it per cloud - this pins both halves of - that assumption. + The compositions share one CPU taint, so the legs share one CPU_TAINT + rather than faking it per cloud - this pins that assumption. The label + key is the one dra-driver-cpu's nodeSelector reads. """ consts = constants(CPU_COMPOSITIONS[leg]) - assert consts["_LABEL_CPU"] == clouds.CPU_TAINT["key"], ( - f"the legs fake the CPU taint {clouds.CPU_TAINT['key']}, " - f"but {CPU_COMPOSITIONS[leg]} labels and taints CPU pools with {consts['_LABEL_CPU']}" + if "_CPU_TAINT_VALUE" in consts: + key, value = consts["_CPU_TAINT_KEY"], consts["_CPU_TAINT_VALUE"] + else: + # compose-aks-cluster declares the whole taint as one + # key=value:effect string. + key, rest = consts["_CPU_TAINT"].split("=", 1) + value = rest.split(":", 1)[0] + assert (key, value) == (clouds.CPU_TAINT["key"], clouds.CPU_TAINT["value"]), ( + f"the legs fake the CPU taint {clouds.CPU_TAINT['key']}={clouds.CPU_TAINT['value']}, " + f"but {CPU_COMPOSITIONS[leg]} taints CPU pools with {key}={value}" + ) + assert clouds.CLOUDS[leg].cpu_labels.get(consts["_LABEL_CPU"]) == "true", ( + f"{CPU_COMPOSITIONS[leg]} labels CPU pools {consts['_LABEL_CPU']}=true, " + f"but the {leg} leg's fake CPU nodes carry {clouds.CLOUDS[leg].cpu_labels}" ) diff --git a/functions/compose-civo-cluster/function/fn.py b/functions/compose-civo-cluster/function/fn.py index 081c931dc..5416d682d 100644 --- a/functions/compose-civo-cluster/function/fn.py +++ b/functions/compose-civo-cluster/function/fn.py @@ -101,6 +101,9 @@ # workloads off the pool as the GPU taint does on GPU pools, and engine # pods tolerate it (compose-model-replica). _LABEL_CPU = "modelplane.ai/cpu" +_CPU_TAINT_KEY = "modelplane.ai/cpu" +_CPU_TAINT_VALUE = "true" +_CPU_TAINT_EFFECT = "NoSchedule" # Civo's default marketplace applications include the Traefik ingress # controller, which would compete with the serving stack's Envoy Gateway @@ -462,9 +465,9 @@ def _node_pool(self, pool: v1alpha1.NodePool) -> nodepoolv1beta1.NodePool: elif pool.role == "CPU": fp.taint = [ nodepoolv1beta1.TaintItem( - key=_LABEL_CPU, - value="true", - effect=_GPU_TAINT_EFFECT, + key=_CPU_TAINT_KEY, + value=_CPU_TAINT_VALUE, + effect=_CPU_TAINT_EFFECT, ), ] diff --git a/functions/compose-eks-cluster/function/fn.py b/functions/compose-eks-cluster/function/fn.py index 72341c41c..14759acee 100644 --- a/functions/compose-eks-cluster/function/fn.py +++ b/functions/compose-eks-cluster/function/fn.py @@ -231,6 +231,9 @@ def _namespace(meta: metav1.ObjectMeta | None) -> str: # system workloads off the group as the GPU taint does on GPU groups, # and engine pods tolerate it (compose-model-replica). _LABEL_CPU = "modelplane.ai/cpu" +_CPU_TAINT_KEY = "modelplane.ai/cpu" +_CPU_TAINT_VALUE = "true" +_CPU_TAINT_EFFECT = "NO_SCHEDULE" # Kubernetes-API taint effect (as a toleration value, not the EC2 NO_SCHEDULE # form). The EFA DRA driver's DaemonSet must tolerate the GPU taint to run on @@ -923,9 +926,9 @@ def compose_node_groups(self) -> None: } fp.taint = [ ngv1beta1.TaintItem( - key=_LABEL_CPU, - value="true", - effect=_GPU_TAINT_EFFECT, + key=_CPU_TAINT_KEY, + value=_CPU_TAINT_VALUE, + effect=_CPU_TAINT_EFFECT, ), ] diff --git a/functions/compose-gke-cluster/function/fn.py b/functions/compose-gke-cluster/function/fn.py index e5a48445f..a45542bd0 100644 --- a/functions/compose-gke-cluster/function/fn.py +++ b/functions/compose-gke-cluster/function/fn.py @@ -72,6 +72,9 @@ # with nvidia.com/gpu but knows nothing of CPU inference pools, so the # taint is set explicitly here. _LABEL_CPU = "modelplane.ai/cpu" +_CPU_TAINT_KEY = "modelplane.ai/cpu" +_CPU_TAINT_VALUE = "true" +_CPU_TAINT_EFFECT = "NO_SCHEDULE" # GKE's cluster autoscaler uses this label to recognise a node pool as one whose # nodes run the NVIDIA GPU DRA driver, and so models the DRA ResourceSlices such @@ -404,7 +407,7 @@ def compose_node_pools(self) -> None: _LABEL_CPU: "true", } node_config.taint = [ - nodepoolv1beta1.TaintItem(key=_LABEL_CPU, value="true", effect="NO_SCHEDULE"), + nodepoolv1beta1.TaintItem(key=_CPU_TAINT_KEY, value=_CPU_TAINT_VALUE, effect=_CPU_TAINT_EFFECT), ] else: node_config.labels = { diff --git a/functions/compose-nebius-cluster/function/fn.py b/functions/compose-nebius-cluster/function/fn.py index 5ffb82493..0a32263be 100644 --- a/functions/compose-nebius-cluster/function/fn.py +++ b/functions/compose-nebius-cluster/function/fn.py @@ -107,6 +107,9 @@ # workloads off the group as the GPU taint does on GPU groups, and # engine pods tolerate it (compose-model-replica). _LABEL_CPU = "modelplane.ai/cpu" +_CPU_TAINT_KEY = "modelplane.ai/cpu" +_CPU_TAINT_VALUE = "true" +_CPU_TAINT_EFFECT = "NO_SCHEDULE" # Management policies that exclude Delete, used for resources installed on the # workload cluster (the RWX StorageClass Object and the CSI driver Helm @@ -596,9 +599,9 @@ def compose_node_groups(self) -> None: elif pool.role == "CPU": template.taints = [ nodegroupv1beta1.Taint( - key=_LABEL_CPU, - value="true", - effect=_GPU_TAINT_EFFECT, + key=_CPU_TAINT_KEY, + value=_CPU_TAINT_VALUE, + effect=_CPU_TAINT_EFFECT, ), ] diff --git a/functions/compose-vultr-cluster/function/fn.py b/functions/compose-vultr-cluster/function/fn.py index 8db939403..12670ba2d 100644 --- a/functions/compose-vultr-cluster/function/fn.py +++ b/functions/compose-vultr-cluster/function/fn.py @@ -90,6 +90,9 @@ # workloads off the pool as the GPU taint does on GPU pools, and engine # pods tolerate it (compose-model-replica). _LABEL_CPU = "modelplane.ai/cpu" +_CPU_TAINT_KEY = "modelplane.ai/cpu" +_CPU_TAINT_VALUE = "true" +_CPU_TAINT_EFFECT = "NoSchedule" # Vultr pre-installs the NVIDIA GPU Operator on every VKE GPU cluster. The # operator's validator DaemonSet runs a four-stage init sequence (driver → @@ -325,9 +328,9 @@ def _node_pool(self, pool: v1alpha1.NodePool) -> vkenodepoolv1beta1.KubernetesNo elif pool.role == "CPU": fp.taints = [ vkenodepoolv1beta1.Taint( - key=_LABEL_CPU, - value="true", - effect=_GPU_TAINT_EFFECT, + key=_CPU_TAINT_KEY, + value=_CPU_TAINT_VALUE, + effect=_CPU_TAINT_EFFECT, ), ] From 4dc4f9c7a040b97aed03223a53062e8f7cb966cc Mon Sep 17 00:00:00 2001 From: Nic Cope Date: Thu, 29 Oct 2026 18:09:04 -0700 Subject: [PATCH 18/19] Correct serving stack comments the device stacks made stale Several comments still described the NVIDIA-only stack. The GPU collector job's comment said Modelplane installs no GPU exporter, which is no longer true on Existing. generate.py's toleration comment left out the NODE class. On Existing, the NFD comment said the cluster compositions taint GPU nodes, and the AMD operator comment called DRA mode the chart's default when the values set it against the defaults. This corrects them and rewraps the comments left ragged by earlier edits. Signed-off-by: Nic Cope --- .../function/collector.py | 18 ++++++--------- .../function/stacks/clouds/civo.py | 6 ++--- .../function/stacks/clouds/existing.py | 23 ++++++++++--------- .../function/stacks/clouds/nebius.py | 6 ++--- .../function/stacks/clouds/vultr.py | 6 ++--- .../function/stacks/metrics.py | 11 ++++----- functions/compose-serving-stack/generate.py | 9 ++++---- 7 files changed, 38 insertions(+), 41 deletions(-) diff --git a/functions/compose-serving-stack/function/collector.py b/functions/compose-serving-stack/function/collector.py index 5a98a483d..13a18dd65 100644 --- a/functions/compose-serving-stack/function/collector.py +++ b/functions/compose-serving-stack/function/collector.py @@ -332,17 +332,13 @@ def _scrape_configs() -> list[dict[str, Any]]: "scrape_interval": _SCRAPE_INTERVAL, "kubernetes_sd_configs": [{"role": "pod"}], "relabel_configs": [ - # A job of its own because nobody annotates DCGM for scraping - # and Modelplane doesn't install it: GKE runs a managed one in - # gke-managed-system, the NVIDIA GPU operator installs its own - # elsewhere, and the substrate job sees neither. Without this - # every modelplane_gpu_* series is empty on a cloud that - # provides its own - which is every cloud. - # - # Matched on the name rather than an exact label, because the - # two spell it differently: gke-managed-dcgm-exporter and - # dcgm-exporter. Both labels are read, since which one carries - # the name depends on who packaged it. + # A job of its own because nobody annotates a GPU exporter + # for scraping: GKE runs a managed DCGM in + # gke-managed-system, the NVIDIA GPU operator installs its + # own elsewhere, the AMD GPU operator Modelplane installs on + # Existing runs AMD's device-metrics-exporter, and the + # substrate job sees none of them. Without this every + # modelplane_gpu_* series is empty. _gpu_exporter_selector("keep"), # The exporters name their metrics port differently: DCGM's # is metrics, the AMD operator's is exporter-port. diff --git a/functions/compose-serving-stack/function/stacks/clouds/civo.py b/functions/compose-serving-stack/function/stacks/clouds/civo.py index 58e5112f7..dbaed2bb6 100644 --- a/functions/compose-serving-stack/function/stacks/clouds/civo.py +++ b/functions/compose-serving-stack/function/stacks/clouds/civo.py @@ -228,9 +228,9 @@ def transformed(c: Component) -> Component: # The node-exporter chart's default toleration is keyless - it # tolerates every taint, landing the pod on tainted nodes by # accident rather than by intent. Scope it to the GPU and CPU pool - # taints, - # mirroring the generated clouds (TOLERATIONS in generate.py); as - # a DaemonSet it still reaches every untainted node. + # taints, mirroring the generated clouds (TOLERATIONS in + # generate.py); as a DaemonSet it still reaches every untainted + # node. "prometheus-node-exporter": { "tolerations": [ {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, diff --git a/functions/compose-serving-stack/function/stacks/clouds/existing.py b/functions/compose-serving-stack/function/stacks/clouds/existing.py index 0e0217d46..78225dbbc 100644 --- a/functions/compose-serving-stack/function/stacks/clouds/existing.py +++ b/functions/compose-serving-stack/function/stacks/clouds/existing.py @@ -23,8 +23,8 @@ Existing is also the open-ended cloud for devices: a claimed driver this list doesn't tag installs nothing and rejects nothing - join()'s -docstring states that rule. The list still pins device -stacks like any cloud's, each tagged with its driver so it installs +docstring states that rule. The list still pins device stacks like +any cloud's, each tagged with its driver so it installs only when a class claims it: NVIDIA's today, and AMD's - Existing is the one cloud that installs it, for BYO nodes whose images already carry the amdgpu kernel driver. @@ -106,9 +106,9 @@ # The node-exporter chart's default toleration is keyless - it # tolerates every taint, landing the pod on tainted nodes by # accident rather than by intent. Scope it to the inference pool - # taints - a BYO pool may carry any of the three - - # mirroring the generated clouds (TOLERATIONS in generate.py); as - # a DaemonSet it still reaches every untainted node. + # taints - a BYO pool may carry any of the three - mirroring the + # generated clouds (TOLERATIONS in generate.py); as a DaemonSet it + # still reaches every untainted node. "prometheus-node-exporter": { "tolerations": [ {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, @@ -137,12 +137,12 @@ # off clusters whose classes claim no GPUs. dra_drivers=["gpu.nvidia.com", "gpu.amd.com"], # The worker must run on the very nodes it is supposed to label: - # the DRA driver's kubelet plugin schedules only onto nodes + # each DRA driver's kubelet plugin schedules only onto nodes # carrying an NFD GPU label (feature.node.kubernetes.io/pci-10de - # and friends). The cluster compositions taint GPU nodes with + # and friends for NVIDIA). A BYO GPU node may be tainted # nvidia.com/gpu or amd.com/gpu, and the NFD chart's worker # tolerates nothing by default. Without these tolerations the - # chain breaks silently: no worker on the GPU node, no pci-10de + # chain breaks silently: no worker on the GPU node, no GPU # label, no kubelet plugin, no ResourceSlices, and every GPU # ResourceClaim stays unallocatable with all components looking # healthy. Exists matches every taint value. @@ -209,9 +209,10 @@ }, ], ), - # The AMD GPU operator, in DRA mode: the chart's default DeviceConfig - # (crds.defaultCR) enables the DRA driver and disables the device - # plugin - the two are mutually exclusive allocators - so the + # The AMD GPU operator, in DRA mode: the default DeviceConfig the + # chart renders (crds.defaultCR) is set below to enable the DRA + # driver and disable the device plugin, the reverse of the chart's + # defaults - the two are mutually exclusive allocators - so the # gpu.amd.com DeviceClass the chart registers is the sole path to the # GPUs. Existing is the one cloud that installs it today. # diff --git a/functions/compose-serving-stack/function/stacks/clouds/nebius.py b/functions/compose-serving-stack/function/stacks/clouds/nebius.py index 3014677c5..ded4548b1 100644 --- a/functions/compose-serving-stack/function/stacks/clouds/nebius.py +++ b/functions/compose-serving-stack/function/stacks/clouds/nebius.py @@ -97,9 +97,9 @@ # The node-exporter chart's default toleration is keyless - it # tolerates every taint, landing the pod on tainted nodes by # accident rather than by intent. Scope it to the GPU and CPU pool - # taints, - # mirroring the generated clouds (TOLERATIONS in generate.py); as - # a DaemonSet it still reaches every untainted node. + # taints, mirroring the generated clouds (TOLERATIONS in + # generate.py); as a DaemonSet it still reaches every untainted + # node. "prometheus-node-exporter": { "tolerations": [ {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, diff --git a/functions/compose-serving-stack/function/stacks/clouds/vultr.py b/functions/compose-serving-stack/function/stacks/clouds/vultr.py index e186a9b5a..3e887b945 100644 --- a/functions/compose-serving-stack/function/stacks/clouds/vultr.py +++ b/functions/compose-serving-stack/function/stacks/clouds/vultr.py @@ -107,9 +107,9 @@ # The node-exporter chart's default toleration is keyless - it # tolerates every taint, landing the pod on tainted nodes by # accident rather than by intent. Scope it to the GPU and CPU pool - # taints, - # mirroring the generated clouds (TOLERATIONS in generate.py); as - # a DaemonSet it still reaches every untainted node. + # taints, mirroring the generated clouds (TOLERATIONS in + # generate.py); as a DaemonSet it still reaches every untainted + # node. "prometheus-node-exporter": { "tolerations": [ {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, diff --git a/functions/compose-serving-stack/function/stacks/metrics.py b/functions/compose-serving-stack/function/stacks/metrics.py index 63223a971..ce2bfccdd 100644 --- a/functions/compose-serving-stack/function/stacks/metrics.py +++ b/functions/compose-serving-stack/function/stacks/metrics.py @@ -149,12 +149,11 @@ # activity) is a GPU_PROF_* field the exporter ships disabled; the stack's # exporter config turns ProfilerMetrics on to publish it (see the AMD # chart pin in clouds/existing.py) at the cost of the single profiler -# instance the hardware allows. The -# exporter reports -# its accumulated energy as a gauge where DCGM's is a counter, and -# modelplane_energy_joules_total is a counter, so the energy series is marked -# fromType Gauge and the collector converts it to a monotonic sum - one name, -# one type, whichever vendor feeds it. +# instance the hardware allows. The exporter reports its accumulated energy +# as a gauge where DCGM's is a counter, and modelplane_energy_joules_total is +# a counter, so the energy series is marked fromType Gauge and the collector +# converts it to a monotonic sum - one name, one type, whichever vendor feeds +# it. _AMD_GPU = { "gpu_used_vram": "modelplane_gpu_memory_used_bytes", "gpu_free_vram": "modelplane_gpu_memory_free_bytes", diff --git a/functions/compose-serving-stack/generate.py b/functions/compose-serving-stack/generate.py index 303e97f42..2f9b28922 100644 --- a/functions/compose-serving-stack/generate.py +++ b/functions/compose-serving-stack/generate.py @@ -476,10 +476,11 @@ # wildcard the bundle ships, per the table below: node-scoped workloads # that must reach GPU nodes tolerate exactly the GPU taint (GPU, # matching the hand-written stacks' NFD worker and the EKS -# composition's EFA driver), everything else tolerates nothing (SYSTEM) -# and schedules on the untainted system pool. A toleration the bundle -# already scoped (AKS's --accelerated-node-toleration form above) -# passes through untouched. The wildcard doesn't only arrive under keys +# composition's EFA driver), node-scoped workloads that must observe +# every inference node tolerate every inference pool taint (NODE), and +# everything else tolerates nothing (SYSTEM) and schedules on the +# untainted system pool. A toleration the bundle already scoped (AKS's +# --accelerated-node-toleration form above) passes through untouched. The wildcard doesn't only arrive under keys # named tolerations: nvsentinel's bundle puts one under # global.systemNodeTolerations, so the walkers match any *Tolerations # key too. From 4693b4b95eb661f8f2d081d43163b88e09be6a53 Mon Sep 17 00:00:00 2001 From: Nic Cope Date: Thu, 29 Oct 2026 18:09:11 -0700 Subject: [PATCH 19/19] Update the docs for per-driver device stacks, CPU pools and AMD Several docs still described an NVIDIA-only stack. The existing-cluster requirements said Modelplane always installs NVIDIA's DRA driver and tolerates only the nvidia.com/gpu taint, and said nothing of the AMD and CPU stacks' needs. The telemetry page said an existing cluster brings every GPU exporter and that dashboards need no change across vendors, though per-card labels differ, and didn't document fromType. The design doc and the CPU recipe had drifted from the implementation. This brings them in line with the code, and rewraps a few paragraphs left ragged by earlier edits. Signed-off-by: Nic Cope --- CONTRIBUTING.md | 2 +- design/multi-accelerator.md | 46 +++++++++--------- docs/content/architecture/scheduling.md | 5 +- docs/content/overview/how-it-works.md | 6 +-- docs/content/platform/inference-class.md | 9 ++-- docs/content/platform/inference-cluster.md | 48 +++++++++++++------ docs/content/platform/telemetry.md | 22 +++++---- docs/content/recipes/qwen2.5-0.5b-cpu.md | 10 ++-- .../qwen2.5-0.5b-cpu/model-deployment.yaml | 7 +-- e2e/README.md | 4 +- 10 files changed, 92 insertions(+), 67 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e2c8ca839..80098dcbb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -125,7 +125,7 @@ and it runs the pytest suite in `e2e/tests/`, exiting non-zero if a test fails. verify command is what the label-gated `E2E` workflow runs on CI (add the `test-e2e` label to a PR), so a green local `--verify` and a green CI run mean the same thing. `-- --cloud ` instead installs one cloud's serving -stack onto KWOK-faked GPU pools and audits placement, still with no +stack onto KWOK-faked device pools and audits placement, still with no credentials; its per-cloud CI matrix (the `E2E Clouds` workflow) runs on pull requests touching the serving stack or `e2e/`. See `e2e/README.md`. diff --git a/design/multi-accelerator.md b/design/multi-accelerator.md index f168fb566..59c391214 100644 --- a/design/multi-accelerator.md +++ b/design/multi-accelerator.md @@ -168,15 +168,14 @@ what is genuinely CPU-specific: the driver's reserved cores, the CPU is also why there is no opt-in path: a CPU class can run on any cloud, so every cloud filters and `spec.draDrivers` flows everywhere from the start. -One floor is CPU's alone: the driver's grouped mode publishes each NUMA -node's cores as one device whose CPUs are consumable capacity -(KEP-5075), which Kubernetes enables by default from 1.36 and managed -control planes can't enable earlier - on 1.34/1.35 the scheduler -allocates the device but strips the capacity accounting, and the engine -pod fails at NodePrepareResources. So a CPU pool needs Kubernetes 1.36 -on a managed cloud, while GPU pools keep the 1.34/1.35 floors; cluster -manifests for CPU pools pin the version rather than taking the cloud's -default. +One floor is CPU's alone: the driver's grouped mode publishes each NUMA node's +cores as one device whose CPUs are consumable capacity (KEP-5075), which +Kubernetes enables by default from 1.36 and managed control planes can't enable +earlier — on 1.34/1.35 the scheduler allocates the device but strips the +capacity accounting, and the engine pod fails at NodePrepareResources. So a CPU +pool needs Kubernetes 1.36 on a managed cloud, while GPU pools keep the +1.34/1.35 floors; cluster manifests for CPU pools pin the version rather than +taking the cloud's default. **Supported drivers per cloud.** A cloud supports the drivers its component list is tagged with, so in the stacks package the per-cloud table is derived @@ -192,20 +191,19 @@ supported, and the filter installs nothing for it. Existing (BYO) is the open-ended case: the user owns the cluster and may have installed any driver on it, which Modelplane can't enumerate, so on Existing every driver the list doesn't carry is treated as externally installed. Its -list still pins stacks like on any cloud — the NVIDIA DRA driver today, -dra-driver-cpu with #509 — but a claimed driver beyond them installs nothing -and is not rejected. The cost is deferred failure: a typo'd driver on a BYO -cluster surfaces when its ResourceClaim can't allocate, not as a condition. +list still pins stacks like on any cloud — NVIDIA's DRA driver and the AMD GPU +operator, beside the dra-driver-cpu every cloud pins — but a claimed driver +beyond them installs nothing and is not rejected. The cost is deferred +failure: a typo'd driver on a BYO cluster surfaces when its ResourceClaim +can't allocate, not as a condition. e2e runs on exactly this path, registering the kind cluster as Existing and claiming `gpu.example.com` from a driver the harness installs itself. The composition function that validates ahead of the join -(`compose-inference-cluster`) mirrors the derived table by hand with a -kept-in-sync comment pointing at the stacks package's copy; functions are -self-contained by design, so a shared library isn't on the table. A stale -mirror fails closed: it rejects a newly supported driver until someone -updates it, and never admits an unsupported one, because the join's raise -backstops that direction. +(`compose-inference-cluster`) mirrors the derived table by hand; functions are +self-contained by design, so a shared library isn't on the table. A flake +check reads the mirror from source and fails when it disagrees with the +stacks package's derived copy. **Device stacks follow classes.** Today the NVIDIA stack installs unconditionally; under the filter it installs when a class claims @@ -262,7 +260,7 @@ status: - type: ClusterReady status: "False" reason: UnsupportedDevices - message: "class amd-mi355x-8x claims gpu.amd.com; the Vultr serving stack installs gpu.nvidia.com, dra.cpu" + message: "class amd-mi355x-8x claims gpu.amd.com; Vultr clusters support dra.cpu, gpu.nvidia.com" ``` Accepting it is a data change, not a design change: pin the AMD operator and @@ -282,9 +280,9 @@ hand-written clouds' values and classified fail-closed in `generate.py`'s ## Future improvements -- **AMD on a first cloud**: pin the AMD operator and kernel-module manager on - its list, tagged `gpu.amd.com`. Vultr is the likely first: it already sells - AMD Instinct alongside NVIDIA. +- **AMD on a first provisioned cloud**: pin the AMD operator and kernel-module + manager on its list, tagged `gpu.amd.com`, as Existing's list already does. + Vultr is the likely first: it already sells AMD Instinct alongside NVIDIA. - **AMD on the generated clouds** (EKS, AKS, GKE): classify the AMD components in `ALLOW` and add `gpu.amd.com` to NFD's entry once one of them offers AMD shapes; the mechanism is already in place. @@ -293,7 +291,7 @@ hand-written clouds' values and classified fail-closed in `generate.py`'s from the class. - **Mirror generation**: if the per-cloud tables grow past a handful of drivers and clouds, generate `compose-inference-cluster`'s mirror from the - stacks package's derived copy instead of keeping it in sync by comment. + stacks package's derived copy instead of keeping it in sync by hand. ## Alternatives considered diff --git a/docs/content/architecture/scheduling.md b/docs/content/architecture/scheduling.md index adaec4b6e..8f4d12a03 100644 --- a/docs/content/architecture/scheduling.md +++ b/docs/content/architecture/scheduling.md @@ -127,8 +127,9 @@ twice. The scheduler counts a pool in whole nodes when the agent hasn't reported on it or can't observe it. The agent doesn't observe CPU or AMD GPU pools yet, so -they're always counted in whole nodes. Such a pool publishes no `observedNodes`, and every node it -can have starts empty. A pod that claims any device takes a whole one. +they're always counted in whole nodes. Such a pool publishes no +`observedNodes`, and every node it can have starts empty. A pod that claims any +device takes a whole one. ## Pinning placement to a pool diff --git a/docs/content/overview/how-it-works.md b/docs/content/overview/how-it-works.md index d3698e0e6..2ae50c473 100644 --- a/docs/content/overview/how-it-works.md +++ b/docs/content/overview/how-it-works.md @@ -121,9 +121,9 @@ For each replica, the scheduler picks a `(cluster, pool)` in two steps: An agent on each cluster reports the free devices on each NVIDIA GPU pool's nodes, and the scheduler fits replicas against those, so replicas share a node -when it has room. Replicas are pinned to their cluster once placed and stay there across -reconciles; if a cluster is deleted, the scheduler re-places its replicas -elsewhere. [How it schedules]({{< ref "/architecture/scheduling.md" >}}) +when it has room. Replicas are pinned to their cluster once placed and stay +there across reconciles; if a cluster is deleted, the scheduler re-places its +replicas elsewhere. [How it schedules]({{< ref "/architecture/scheduling.md" >}}) covers the placement rules and their limits in full. ## Deploying a model diff --git a/docs/content/platform/inference-class.md b/docs/content/platform/inference-class.md index 7ebc63308..6d8a7acd8 100644 --- a/docs/content/platform/inference-class.md +++ b/docs/content/platform/inference-class.md @@ -1,12 +1,12 @@ --- title: Define Hardware Classes weight: 20 -description: Hardware recipe defining GPU type, count, and provisioning for a node pool. +description: Hardware recipe defining devices and provisioning for a node pool. --- **API:** [`modelplane.ai/v1alpha1` · InferenceClass]({{< ref "/reference/inferenceclasses" >}}) -An `InferenceClass` is a tested recipe for a GPU node pool. It bundles: +An `InferenceClass` is a tested recipe for a GPU or CPU node pool. It bundles: - **Devices**: the node's hardware as a list of Dynamic Resource Allocation (DRA) @@ -39,8 +39,9 @@ architecture, and compute capability, and use the same keys the driver reports. Each device sets a `claim` discriminator: -- **`DRA`** (the default) is hardware a real DRA driver exposes, today GPUs. - Modelplane both schedules against it and binds it to pods. +- **`DRA`** (the default) is hardware a real DRA driver exposes, such as + NVIDIA GPUs (`gpu.nvidia.com`), AMD GPUs (`gpu.amd.com`), or CPU cores + (`dra.cpu`). Modelplane both schedules against it and binds it to pods. - **`Synthetic`** is described for scheduling only, never claimed. Use it for hardware that matters for placement but has no DRA driver yet, like an InfiniBand fabric. diff --git a/docs/content/platform/inference-cluster.md b/docs/content/platform/inference-cluster.md index 3f640e130..55f2eef83 100644 --- a/docs/content/platform/inference-cluster.md +++ b/docs/content/platform/inference-cluster.md @@ -53,25 +53,43 @@ The `cluster.source` discriminator picks one of two models: ## Requirements for an existing cluster -An existing cluster must meet what Modelplane would otherwise set up for you: - -- **Kubernetes 1.34.2 or newer.** Modelplane binds GPUs with the generally - available Dynamic Resource Allocation API (`resource.k8s.io/v1`), first served - in Kubernetes 1.34, and the DRA driver it installs wants 1.34.2. Prefer a - [maintained release](https://kubernetes.io/releases/). -- **GPU nodes that already run the NVIDIA driver.** Modelplane installs NVIDIA's - DRA driver, but not the GPU driver itself. Its +Modelplane installs a device stack for each DRA driver a pool's +`InferenceClass` claims: NVIDIA's DRA driver for `gpu.nvidia.com`, the AMD GPU +operator in DRA mode for `gpu.amd.com`, and `dra-driver-cpu` for `dra.cpu`. The +AMD stack installs only on an existing cluster. Beyond that, an existing cluster +must meet what Modelplane would otherwise set up for you: + +- **Kubernetes 1.34.2 or newer, or 1.36 for a CPU pool.** Modelplane binds + devices with the generally available Dynamic Resource Allocation API + (`resource.k8s.io/v1`), first served in Kubernetes 1.34, and NVIDIA's DRA + driver wants 1.34.2. A CPU pool's cores are claimed as consumable capacity, + which Kubernetes enables by default from 1.36. Prefer a [maintained + release](https://kubernetes.io/releases/). +- **NVIDIA GPU nodes that already run the NVIDIA driver.** Modelplane installs + NVIDIA's DRA driver, but not the GPU driver itself. Its [prerequisites](https://dra-driver-nvidia-gpu.sigs.k8s.io/docs/prerequisites/) call for NVIDIA GPU driver v565 or newer and NVIDIA Container Toolkit v1.18.0 or newer, which provides the CDI the driver uses to expose GPUs to the container runtime. A provisioned cluster's node image includes this. +- **AMD GPU nodes that already run the amdgpu kernel driver.** The AMD GPU + operator Modelplane installs has its driver management turned off. +- **A `modelplane.ai/cpu=true` label on CPU pool nodes.** `dra-driver-cpu` runs + only on nodes with that label, so without it a CPU pool doesn't publish its + cores. - **A `modelplane.ai/pool=` label on each pool's nodes**, matching the pool's `name`. Modelplane provisions no nodes here, so you apply the label yourself. The [scheduler pins each pool's pods to it]({{< ref "/architecture/scheduling.md#pinning-placement-to-a-pool" >}}), so worker pods stay Pending without it. -- **The `nvidia.com/gpu` taint key, if you taint GPU nodes.** Modelplane's GPU - workloads tolerate that key. A different taint keeps them off the nodes. +- **The `nvidia.com/gpu`, `amd.com/gpu` or `modelplane.ai/cpu` taint key, if you + taint pool nodes.** Modelplane's engine pods tolerate those keys, and a + different taint keeps them off the nodes. The AMD operator's per-node + components tolerate only `amd.com/gpu` and `dra-driver-cpu` only + `modelplane.ai/cpu`, so taint AMD and CPU nodes with their own key or not at + all. +- **No AMD GPU operator or `dra-driver-cpu` of your own** for a driver a class + claims. Modelplane installs its own, which would run a second copy beside + yours. - **A load balancer.** Modelplane exposes the cluster's serving gateway through a `LoadBalancer` Service. The cluster needs a load balancer that assigns the Service an external address. @@ -82,11 +100,11 @@ An existing cluster must meet what Modelplane would otherwise set up for you: - **Any multi-node fabric you need.** For multi-node serving you provide and configure the RDMA or InfiniBand fabric and its drivers. Modelplane installs those only on the clouds it provisions. -- **A DCGM exporter, if you want GPU metrics.** Modelplane doesn't install - one here because your cluster may already run one - a GPU Operator install - does by default - and a second exporter would collect every GPU twice. - Without one, engine and gateway series arrive and every `modelplane_gpu_*` - series is empty. [GPUs]({{< ref "/platform/telemetry.md#gpus" >}}) +- **A DCGM exporter, if you want NVIDIA GPU metrics.** Modelplane doesn't + install one here because your cluster may already run one, as a GPU Operator + install does by default, and a second exporter would collect every GPU twice. + Without one, NVIDIA pools' `modelplane_gpu_*` series are empty. The AMD stack + brings its own exporter. [GPUs]({{< ref "/platform/telemetry.md#gpus" >}}) describes how the collector discovers the exporter, the DCGM fields it needs, and how to check what yours publishes. diff --git a/docs/content/platform/telemetry.md b/docs/content/platform/telemetry.md index 6d08d1f2e..830a33695 100644 --- a/docs/content/platform/telemetry.md +++ b/docs/content/platform/telemetry.md @@ -11,8 +11,8 @@ description: Collect normalized metrics across the fleet and send them to any ba Modelplane runs an OpenTelemetry collector on every inference cluster. It collects from every component Modelplane installs. This includes the inference server engine, inference gateway and Envoy proxy, router, and the cluster's -GPU exporter - on a provisioned cluster Modelplane or the cloud installs one, -and on an existing cluster you provide it, as +GPU exporter. On a provisioned cluster Modelplane or the cloud installs one. On +an existing cluster the AMD stack brings AMD's and you provide NVIDIA's, as [Requirements for an existing cluster]({{< ref "/platform/inference-cluster.md#requirements-for-an-existing-cluster" >}}) describes. It renames each component's series to a common `modelplane_*` vocabulary and exports them to wherever you say - any backend the @@ -407,6 +407,10 @@ claims, and Modelplane converts to the base one. Skipping this is the expensive a series named `_seconds` that holds milliseconds reads a thousand times fast, and nothing downstream can tell. +Say `fromType: Gauge` when the engine reports a running total, such as accumulated +energy, as a gauge where the name claims a counter. Modelplane converts it to a counter, +so `to` must end in `_total`. + Rename only where the measurements agree. If two engines' histograms share a name and their buckets disagree, a quantile over them is wrong rather than approximate. @@ -474,11 +478,13 @@ Where the exporter comes from depends on the cluster: - **A cluster Modelplane provisions** runs one already: the serving stack installs an exporter wherever the cloud doesn't provide its own. -- **An existing cluster provides its own**, as [Requirements for an existing +- **An existing cluster provides its own DCGM exporter**, as + [Requirements for an existing cluster]({{< ref "/platform/inference-cluster.md#requirements-for-an-existing-cluster" >}}) - describes. A GPU Operator install runs one by default; a cluster without - one needs NVIDIA's dcgm-exporter chart, tolerating your GPU taint. The - chart's pod already has the label and port the collector matches on. + describes. The AMD stack still brings AMD's. A GPU Operator install runs a + DCGM exporter by default; a cluster without one needs NVIDIA's dcgm-exporter + chart, tolerating your GPU taint. The chart's pod already has the label and + port the collector matches on. Don't run two exporters: the collector scrapes every matching pod and would collect each GPU twice. @@ -509,8 +515,8 @@ curl -s localhost:9400/metrics | grep -c -E \ ``` On an AMD pool the same `modelplane_*` names come from the -device-metrics-exporter's fields instead. A dashboard built on them needs no -change when a deployment moves between vendors' pools. +device-metrics-exporter's fields instead, in the same units. Per-card labels +differ: AMD's exporter identifies a card by `gpu_id`, DCGM by `gpu` and `UUID`. | AMD field | Becomes | Unit | | --- | --- | --- | diff --git a/docs/content/recipes/qwen2.5-0.5b-cpu.md b/docs/content/recipes/qwen2.5-0.5b-cpu.md index 6f819d243..5e96aec8e 100644 --- a/docs/content/recipes/qwen2.5-0.5b-cpu.md +++ b/docs/content/recipes/qwen2.5-0.5b-cpu.md @@ -33,11 +33,11 @@ The claim pins the engine to its NUMA node's cpuset, and the engine's CPU request is the one number that sizes everything: Kubernetes accounts the pod by it, and Modelplane stamps it into the claim as the share of the device to take. Five of the six claimable cores, because the driver refuses a claim -that would take them all - the node's shared containers keep the last one. The engine is vLLM's CPU -build, whose image serves the same OpenAI API as the CUDA one. The same class -without its provisioning block describes a bring-your-own CPU pool, on any -conformant cluster: label its nodes `modelplane.ai/pool=cpu-pool` and -`modelplane.ai/cpu=true` instead. +that would take them all - the node's shared containers keep the last one. +The engine is vLLM's CPU build, whose image serves the same OpenAI API as the +CUDA one. The same class without its provisioning block describes a +bring-your-own CPU pool, on any conformant cluster: label its nodes +`modelplane.ai/pool=cpu-pool` and `modelplane.ai/cpu=true` instead. ## Validated deployments diff --git a/docs/manifests/recipes/qwen2.5-0.5b-cpu/model-deployment.yaml b/docs/manifests/recipes/qwen2.5-0.5b-cpu/model-deployment.yaml index 010fb69f1..85e8e7c4b 100644 --- a/docs/manifests/recipes/qwen2.5-0.5b-cpu/model-deployment.yaml +++ b/docs/manifests/recipes/qwen2.5-0.5b-cpu/model-deployment.yaml @@ -1,7 +1,8 @@ # A small model served on CPUs alone. The device selector matches against -# the claimable cores the cpu-8vcpu class declares, so >= 4 lands it on the -# CPU pool; the claim takes the whole NUMA-node device and pins the engine -# container to its cpuset. The engine is vLLM's CPU build - the default +# the claimable cores the civo-cpu-8vcpu class declares, so >= 4 lands it on +# the CPU pool; the claim takes the engine's CPU request, five cores, as +# capacity from the NUMA-node device and pins the engine container to a +# cpuset. The engine is vLLM's CPU build - the default # vllm/vllm-openai image is CUDA-only. --block-size=32 is the smallest # block its CPU attention backend accepts (multiples of 32); Modelplane # reads the flag and chunks the endpoint picker's prefix cache the same diff --git a/e2e/README.md b/e2e/README.md index ecefa1891..446e5458c 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -185,7 +185,7 @@ against. The control-plane cluster needs no DRA. nix run .#e2e # bring up both clusters + deploy the mock model nix run .#e2e -- --verify # same, then run the tests nix run .#e2e -- --test # run the tests against clusters already up -nix run .#e2e -- --cloud eks # one cloud's serving stack on KWOK-faked GPU pools (see below) +nix run .#e2e -- --cloud eks # one cloud's serving stack on KWOK-faked device pools (see below) nix run .#e2e -- --clean # tear both clusters down ``` @@ -269,7 +269,7 @@ e2e/ test_metering.py # it logs a usage record per request test_cluster_gateway.py # the cluster gateway requires a client certificate test_telemetry.py # the engine's series reach the collector's sink - clouds/ # --cloud mode: cloud stacks on KWOK-faked GPU pools + clouds/ # --cloud mode: cloud stacks on KWOK-faked device pools __init__.py # the matrix: per-cloud node shape, taint, expected DaemonSets conftest.py # fixtures: the clusters, the ServingStack Ready wait test_placement.py # DaemonSet fan-out + placement/toleration audits