diff --git a/apis/inferenceclusters/definition.yaml b/apis/inferenceclusters/definition.yaml index a3b6d897f..74c18fab3 100644 --- a/apis/inferenceclusters/definition.yaml +++ b/apis/inferenceclusters/definition.yaml @@ -446,6 +446,11 @@ spec: name: type: string maxLength: 40 + pattern: '^[a-z0-9]([-a-z0-9]*[a-z0-9])?$' + description: >- + Name of the pool. Its nodes carry it as the value of + the modelplane.ai/pool label, and the cluster agent + observes the pool through an object of this name. className: type: string description: >- @@ -612,8 +617,9 @@ spec: type: array description: >- Schedulable GPU node pools on this cluster, derived from the - referenced classes and the per-pool node counts. - ModelDeployment scheduling matches against these. + referenced classes, the per-pool node counts, and what the + cluster agent observes of each pool. ModelDeployment + scheduling matches against these. maxItems: 8 x-kubernetes-list-type: map x-kubernetes-list-map-keys: [name] @@ -630,8 +636,53 @@ spec: nodes: type: integer description: >- - Number of nodes in this pool. Derived from + The most nodes this pool can have. Derived from maxNodeCount (if autoscaling) or nodeCount. + observedNodes: + type: integer + minimum: 0 + description: >- + The nodes this pool has now, ready or not, as the + cluster agent observes them. Absent until the agent + reports on the pool, and while it can't observe it. + lastAllocationTime: + type: string + format: date-time + description: >- + The latest time the cluster agent first saw a + ResourceClaim allocated devices on this pool's nodes. + nodesByFreeDevices accounts for the devices of a + ModelReplica whose lastAllocationTime for this pool is + no later than this. Published with observedNodes, once + a claim has allocated devices on the pool. + nodesByFreeDevices: + type: array + description: >- + A histogram of this pool's schedulable nodes by how + many devices of each DeviceClass are free on them, as + the cluster agent observes them. Nodes with the same + number of free devices of every class share a bucket. + Published with observedNodes. + maxItems: 256 + x-kubernetes-list-type: atomic + items: + type: object + required: [nodes] + properties: + freeDevices: + type: object + description: >- + Each node's free devices, keyed by DeviceClass. + A class the nodes publish no devices of is + absent. + maxProperties: 32 + additionalProperties: + type: integer + minimum: 0 + nodes: + type: integer + minimum: 1 + description: Nodes with these free devices. devices: type: array description: >- diff --git a/apis/modelreplicas/definition.yaml b/apis/modelreplicas/definition.yaml index 25342be5b..5f070a13d 100644 --- a/apis/modelreplicas/definition.yaml +++ b/apis/modelreplicas/definition.yaml @@ -607,3 +607,30 @@ spec: type: string status: type: object + properties: + pools: + type: array + description: >- + The pools whose nodes this replica's devices are attached + to, as the cluster agent observes them. + maxItems: 16 + x-kubernetes-list-type: map + x-kubernetes-list-map-keys: [name] + items: + type: object + required: [name, lastAllocationTime] + properties: + name: + type: string + description: >- + Node pool name, matching the InferenceCluster's + spec.nodePools[].name. + lastAllocationTime: + type: string + format: date-time + description: >- + The latest time the cluster agent first saw one of + this replica's ResourceClaims allocated devices on the + pool. The pool's nodesByFreeDevices accounts for them + once its own lastAllocationTime is no earlier than + this. diff --git a/crossplane-project.yaml b/crossplane-project.yaml index 95fbb9a9c..14f8b2cd2 100644 --- a/crossplane-project.yaml +++ b/crossplane-project.yaml @@ -97,6 +97,13 @@ spec: repository: https://github.com/crossplane/crossplane ref: v2.3.2 path: cluster/crds + # The serving stack installs the cluster agent, at the version + # compose-serving-stack pins, on every InferenceCluster. + - type: crd + git: + repository: https://github.com/modelplaneai/cluster-agent + ref: v0.1.0 + path: cluster/charts/cluster-agent/files/crds - type: xpkg xpkg: apiVersion: pkg.crossplane.io/v1 diff --git a/design/free-gpu-scheduling.md b/design/free-gpu-scheduling.md new file mode 100644 index 000000000..ebd13a40d --- /dev/null +++ b/design/free-gpu-scheduling.md @@ -0,0 +1,251 @@ +# Scheduling by free GPUs + +**Status:** Draft +**Date:** October 2026 +**Author:** Nic Cope + +This document proposes that the fleet scheduler count free GPUs per node rather +than free nodes, using what a new agent on each cluster reports, so replicas can +share a node. It builds on the base design in [design.md](./design.md). + +## Background + +The fleet scheduler places a ModelReplica on a cluster when one of the cluster's +pools has room for it, and it measures room in nodes. The InferenceCluster +publishes the most nodes each pool can have. Every pod that claims a GPU charges +a whole node against that, however many of the node's GPUs it claims. + +This leaves GPUs idle wherever a pod claims less than a whole node. Qwen3-8B +fits on one NVIDIA L4. A pool of two g6.12xlarge nodes, which have four L4s +each, has room for eight Qwen3-8B replicas, but the scheduler places only two. +The cluster's scheduler might even put both on one node. A replica that serves +a large model from one 8-GPU node, with prefill on four GPUs and decode on the +other four, charges two nodes, so a pool of one node can't host it. + +The accounting is deliberately coarse. It leaves contention for devices to DRA +on the workload cluster. Finer accounting needs to know how a pool's free GPUs +spread across its nodes: four free GPUs on one node fit a pod claiming two, and +one free GPU on each of four nodes doesn't. The control plane can't see that +today. + +## Goals + +GPUs are scarce and expensive, so the fleet should use every one it can. This +design should: + +- Let replicas that claim less than a node share one. +- Choose only a cluster and a pool, and leave the choice of node to the + cluster's scheduler. +- Keep the scheduler a pure function of what clusters publish. +- Keep what each cluster publishes bounded, however many nodes its pools have. + +It's not yet a goal to recover a replica placed on a pool that can't fit it. +This can happen in cases where the cluster's scheduler binds pods differently +than the fleet scheduler predicted, leaving some without room. + +## Proposal + +I propose a cluster agent that runs on every InferenceCluster and reports a +histogram of each pool's nodes by free devices. One bucket might count the nodes +with four free GPUs, and another the nodes with none. Where a pool's nodes also +have other kinds of DRA devices, such as EFA adapters, a bucket counts the nodes +with the same number free of each kind. The fleet scheduler fits replicas +against these histograms instead of against node counts. + +Modelplane installs the agent as part of the serving stack, including on +clusters it didn't provision. The agent needs cluster-wide read access to nodes, +pods, events, and DRA's ResourceSlices, ResourceClaims and DeviceClasses. It +reports in the status of two custom resources: + +- `PoolObservation`. The InferenceCluster creates one for each of its pools. +- `ReplicaObservation`. Each ModelReplica creates one, which selects the + replica's engine pods by their deployment and replica index labels. + +Provider-kubernetes watches both resources, and copies their status to the +control plane as it changes rather than on a poll. The InferenceCluster and +ModelReplica then publish what the agent reported in their own status. + +For a pool of one g6.12xlarge node on a test EKS cluster, with three of its four +L4s allocated, the InferenceCluster and a ModelReplica with one of the L4s +publish: + +```yaml +# The InferenceCluster. +status: + gpuPools: + - name: gpu-l4x4-a + nodes: 1 # The most nodes the pool can have. + observedNodes: 1 # New. The nodes the pool has now, ready or not. + lastAllocationTime: "2026-10-20T10:05:40.318204Z" # New. See below. + devices: + - name: gpu + claim: DRA + driver: gpu.nvidia.com + deviceClassName: gpu.nvidia.com + count: 4 + nodesByFreeDevices: # New. Schedulable nodes by free devices. + - nodes: 1 + freeDevices: + gpu.nvidia.com: 1 + # Omitted for brevity. +--- +# The ModelReplica. +status: + conditions: + - type: ModelPodsScheduled # New. See below. + status: "True" + reason: AllPodsScheduled + # Omitted for brevity. + pools: # New. See below. + - name: gpu-l4x4-a + lastAllocationTime: "2026-10-20T10:02:13.482915Z" +``` + +`ModelPodsScheduled` is true once the cluster has bound every pod the replica +runs to a node, including a gang's workers and pods that claim nothing. The +agent must also have seen each of those pods' claims allocated. While it's +false, the condition gives the agent's reason, with the cluster scheduler's +message from when the reason last changed. A Pending replica's status then +changes only when its reason does. + +The fleet scheduler still chooses only a cluster and a pool, as it does today. +The histogram shows which of a pool's nodes could fit a pod, but the cluster's +scheduler chooses which one it binds to. To check that a replica plausibly +fits, the fleet scheduler fits its pods against the histogram one at a time, +guessing that each binds to the node with the most of its devices free. Before +it places a new replica, the scheduler: + +1. Starts from the published histogram. The agent leaves out cordoned and + unready nodes, and nodes with taints Modelplane's GPU pods don't tolerate. + It treats devices that other workloads claim as taken. +2. Adds the nodes the pool can still grow by, `nodes` minus `observedNodes`, + with all of their devices free. A pool that a surge upgrade has taken past + `nodes` can't grow. +3. Places the pods of earlier replicas that the histogram doesn't count yet. + +A replica claiming one L4 fits the pool above, and one claiming two doesn't. The +replica from the background, with prefill on four GPUs and decode on four more, +fits one empty 8-GPU node. A pod claiming GPUs and EFA adapters fits only a node +with enough of both free. A node that's still booting counts toward +`observedNodes` but not the histogram, which leaves the pool looking one node +smaller until the node is ready. A pool with no published histogram gets +whole-node accounting, as every pool does today. So does a pool whose nodes span +more than 256 combinations of free devices, the histogram's limit. + +The agent records when it first saw each claim allocated, to the microsecond +and later than any time it has already published. It reports the latest of +those times as `lastAllocationTime`, for each pool and for each pool a replica +is on. It computes a pool's histogram and time from one read of its cache, and +that read includes every claim still allocated that the agent saw before the +latest one in it. So a pool time no earlier than a replica's means the +histogram counts the replica's claims on that pool. The scheduler places a +replica's pods on a pool in the third step until the replica's +`ModelPodsScheduled` condition is true and its time on that pool is no later +than the pool's. + +If the agent stops reporting, the InferenceCluster goes on publishing its last +observations, and pools tend to look fuller than they are. New replicas can't +count as observed against a frozen pool. The scheduler keeps charging them, and +GPUs that replicas free still count as taken. A pool can look emptier than it +is only after a change whole-node accounting misses too, such as a node failing. + +Because it guesses which node each pod binds to, the fleet scheduler can +mispredict. When the pods it places on a pool all claim identical devices, +which node each binds to doesn't change how many fit. When they claim different +devices, the cluster's scheduler can place them differently than the fleet +scheduler predicted. Deployments that place replicas on a pool at the same +moment, or an autoscaled pool that fails to grow, can overcommit it too, as +they can today. A mispredicted replica's pods stay Pending until room frees up, +which on a full pool that can't grow may be never. Its ModelDeployment reports +it not ready, and its `ModelPodsScheduled` condition gives the cluster +scheduler's reason. + +## Future improvements + +If mispredictions turn out to be common, I'd revisit how the fleet scheduler +places replicas rather than move them after the fact. Each cluster could decide +whether a replica fits, the way MultiKueue offers a workload to every worker +cluster and keeps the first that admits it. Mispredictions would probably become +common under GPU scarcity, when more workloads wait than there are GPUs to run +them. Many replicas would then compete for the same few freed devices at once. + +Modelplane doesn't request CPU or memory for engine pods unless a deployment +sets them, so replicas that share a node would compete for both, and the kubelet +evicts them first under memory pressure. Requesting a share of the node's CPU +and memory in proportion to the pod's share of its GPUs would let the cluster's +scheduler account for them. + +The scheduler could also take Prometheus metrics, like those the +[telemetry design](./telemetry.md) proposes collecting, into account when it +places replicas. The agent could then export its histogram as a metric rather +than in a PoolObservation's status. + +## Alternatives considered + +### Count GPUs against each pool's total + +The scheduler could count a pool's GPUs as its nodes times GPUs per node, and +subtract what placed replicas claim. That needs nothing new on the cluster: it's +a change to arithmetic the scheduler already does, on state it already reads. It +can't see fragmentation though. It would mispredict whenever no node has enough +free GPUs for a pod, so it would need to move mispredicted replicas from the +start. It can't see nodes that are down either. + +### Publish every node's free devices + +The InferenceCluster could publish each node's free devices, which would tell +the scheduler exactly where each pod could go. A pool's status would then grow +with its node count. Whether a pod fits depends on how many devices each node +has free, not on which node it is. The histogram gives the same fit at a size +bounded by devices per node. + +### Pin pods to nodes + +The fleet scheduler could choose a node for each pod and pin the pod to it. Its +predictions would then be exact. Pods of different sizes couldn't strand a +replica, and no gang could end up with only some of its pods started. + +The cluster's scheduler would then have nothing left to decide, and Modelplane +would become the node-level scheduler for every cluster in the fleet. It would +take on the gang scheduling, topology, priority and preemption that clusters +run kube-scheduler, Volcano, KAI or Kueue for, and it would have to do that +across a fleet potentially far larger than the largest cluster. Splitting the +decision lets each level scale. The fleet scheduler chooses among pools, and +each cluster's scheduler chooses among its own nodes from live state. Pinning +also couldn't use nodes an autoscaled pool hasn't created yet. + +### Fall back on stale observations + +The agent could refresh a time on each PoolObservation every minute, and the +scheduler could fall back to whole-node accounting for a pool whose time is a +few minutes old. That would stop the scheduler trusting a crashed agent's view. +Stale observations err toward pools looking full though, and +falling back would trade that for forgetting what the agent last saw, such as +GPUs that other workloads hold. Each refresh would also change the +InferenceCluster's status, which reconciles the XRs that require it. + +### Publish capacity on its own kind + +The InferenceCluster could compose an XR for each pool to publish its free +devices. Only the scheduler reads capacity, but a change to an +InferenceCluster's status reconciles the XRs that require it, including every +InferenceGateway, MetricMapping and ModelRoute. Capacity changes only when +allocations or nodes do though, which is rare outside a rollout. Because only +the scheduler reads it, moving it to its own kind later wouldn't change anyone's +manifests. + +### Charge replicas for a while after they schedule + +The scheduler could keep charging a replica for a fixed time, say a minute, +after its `ModelPodsScheduled` condition turns true, to give the pool's +observation time to catch up. The agent wouldn't need to record allocation +times. The time would be a guess though. If the observation took longer, the +scheduler would stop charging GPUs the histogram doesn't count yet, and if it +took less, it would count them twice. The scheduler would also depend on the +clock, and on a periodic reconcile to notice the time passing. + +### Keep whole-node accounting + +Whole-node accounting is simple, and it costs nothing when every pod claims a +whole node. It leaves GPUs idle whenever a pod claims less, and it charges two +nodes for prefill and decode on one node. diff --git a/docs/content/architecture/scheduling.md b/docs/content/architecture/scheduling.md index 55ce26cbd..57718b2b0 100644 --- a/docs/content/architecture/scheduling.md +++ b/docs/content/architecture/scheduling.md @@ -78,30 +78,56 @@ graph TD ``` A member with no `nodeSelector` doesn't claim any devices. It matches the -engine's pool at no node cost, and the cluster's own scheduler packs its pods -onto the gang's nodes. +engine's pool without taking any of its capacity, and the cluster's own +scheduler places its pods on any of the pool's nodes. -## Counting capacity in nodes +## Counting free devices -Capacity is counted in **nodes**, not individual GPUs. The only number the -scheduler reads from a member is its node cost: +The scheduler places a replica only where each of its pods fits on a node with +enough of the devices it claims free. A member runs this many pods, each +claiming the devices its `nodeSelector` resolves: ```text -nodes = pods × copies -pods = 1 for a Standalone or Leader, or worker.nodes for a Worker +pods = copies × (1 for a Standalone or Leader, or worker.nodes for a Worker) ``` -A member that resolves no `claim: DRA` device, because it has no `nodeSelector` -or matches only synthetic devices, costs zero nodes. The scheduler sums the cost -of a replica's members and places the replica only where every engine's pool has -enough free nodes, tracking a running ledger so it never overcommits a cluster. +The cluster's scheduler is free to bind a gang's pods to one node, so the +scheduler lets them share a node like any other pods. + +A cluster agent, which the serving stack installs on every `InferenceCluster`, +reports how many of each pool's nodes have each number of free devices. The +`InferenceCluster` publishes that histogram in its status: + +```yaml +status: + gpuPools: + - name: gpu-l4x4-a + nodes: 3 # The most nodes the pool can have. + observedNodes: 1 # The nodes the pool has now, ready or not. + nodesByFreeDevices: + - freeDevices: + gpu.nvidia.com: 1 + nodes: 1 + devices: + - name: gpu + claim: DRA + driver: gpu.nvidia.com + deviceClassName: gpu.nvidia.com + count: 4 +``` + +A replica claiming one GPU fits this pool's node, as do two replicas claiming +four GPUs each, on the two nodes the pool can still grow by. Pods share a node +whenever it has the devices for them, so four one-GPU replicas fit one four-GPU +node. A pod that claims GPUs and network devices fits only a node with enough +of both free. The histogram leaves out cordoned and unready nodes, and nodes +with taints Modelplane's GPU pods don't tolerate. It counts the devices +scheduled pods hold as taken, and the scheduler doesn't charge a pool for them +twice. -This accounting is deliberately coarse. The control-plane scheduler answers -"could this cluster plausibly host this replica," not "exactly which GPU does -each pod get." Device-level contention between deployments is left to DRA -admission on the workload cluster, which is authoritative: it rejects a pod whose -`ResourceClaim` can't be satisfied, and the next reconcile sees the updated -state. +The scheduler counts a pool in whole nodes when the agent hasn't reported on it +or can't observe it. Such a pool publishes no `observedNodes`, and every node it +can have starts empty. A pod that claims any device takes a whole one. ## Pinning placement to a pool @@ -139,19 +165,29 @@ treats a pod whose node is gone. ## Known limitations -The scheduler is built to be conservative and predictable rather than optimal. -The limits below follow from that, and each is tracked for future work: +The scheduler is built to be conservative and predictable rather than optimal, +and has these limits: -- **A whole node is charged per pod** - ([#172](https://github.com/modelplaneai/modelplane/issues/172)). A pod that - claims one GPU of an eight-GPU node still charges the whole node in the - scheduler's accounting. This is safe, it can only under-count a pool's - capacity, never overcommit it, but it can strand GPUs on deployments of - sub-node engines. - **An engine can't span pools, even on one fabric** ([#149](https://github.com/modelplaneai/modelplane/issues/149)). Because the scheduler has no concept of fabric, it refuses to split a gang across pools at all. That forecloses a legitimate case, GPU workers on one pool and a no-GPU coordinator on another within the same fabric, until fabric-aware placement lands. +- **A wrong prediction can leave a replica Pending.** The scheduler assumes each + pod binds to the node with the most of its claimed devices free, and the + cluster's scheduler may bind pods of different sizes differently. A replica + left without room for its pods waits in Pending until devices free up, and its + `ModelPodsScheduled` condition says why. The scheduler doesn't move it to + another cluster. +- **A pool looks fuller than it is while its agent is down.** If the cluster agent stops + reporting, the `InferenceCluster` keeps publishing what it last saw. The + scheduler goes on charging every replica it places on the pool, and GPUs that + replicas free still count as taken, until the agent reports again. +- **Modelplane doesn't request CPU or memory for engines.** Unless a deployment + sets requests on its engine containers, replicas that share a node compete for + both, and Kubernetes evicts them first under memory pressure. +- **A rollout can need room for a new pod.** An engine that runs as a Deployment + of fewer than four copies starts each new pod before it stops an old one. On a + full pool that can't grow, the rollout stalls with the old pods still serving. diff --git a/docs/content/overview/how-it-works.md b/docs/content/overview/how-it-works.md index ef59546cf..faeb26455 100644 --- a/docs/content/overview/how-it-works.md +++ b/docs/content/overview/how-it-works.md @@ -117,12 +117,13 @@ For each replica, the scheduler picks a `(cluster, pool)` in two steps: `nodeSelector.devices` against the pool's `InferenceClass`. A request is based on DRA: a `count` and CEL selectors over a device's attributes and capacity, like "a GPU with at least 141Gi of memory." A pool fits when it has the devices the - model asks for and enough free nodes to hold a replica. + model asks for, free on enough of its nodes to fit a replica. -Capacity is accounted at the node level across the fleet, so Modelplane never -overcommits a pool. Replicas are pinned to their cluster once placed and stay -there across reconciles; if a cluster is deleted, the scheduler re-places its -replicas elsewhere. [How it schedules]({{< ref "/architecture/scheduling.md" >}}) +An agent on each cluster reports the free devices on each pool's nodes, and the +scheduler fits replicas against those, so replicas share a node when it has +room. Replicas are pinned to their cluster once placed and stay there across +reconciles; if a cluster is deleted, the scheduler re-places its replicas +elsewhere. [How it schedules]({{< ref "/architecture/scheduling.md" >}}) covers the placement rules and their limits in full. ## Deploying a model diff --git a/functions/compose-inference-cluster/function/fn.py b/functions/compose-inference-cluster/function/fn.py index e5d1dd4ee..87d0fd35a 100644 --- a/functions/compose-inference-cluster/function/fn.py +++ b/functions/compose-inference-cluster/function/fn.py @@ -33,9 +33,11 @@ from typing import Final, Literal import grpc +import pydantic from crossplane.function import logging, request, resource, response from crossplane.function.proto.v1 import run_function_pb2 as fnv1 from crossplane.function.proto.v1 import run_function_pb2_grpc as grpcv1 +from models.ai.modelplane.agent.poolobservation import v1alpha1 as pov1alpha1 from models.ai.modelplane.inferenceclass import v1alpha1 as iclv1alpha1 from models.ai.modelplane.inferencecluster import v1alpha1 from models.ai.modelplane.inferencegateway import v1alpha1 as igv1alpha1 @@ -123,6 +125,21 @@ # ClusterProviderConfig secretRefs, and status.namespace. _NAMESPACE_SYSTEM = "modelplane-system" +# The cluster agent, which the serving stack installs, reports a pool's nodes +# and free devices in the status of a PoolObservation. It reports its Healthy +# condition Unknown, with this reason, when it can't observe the pool. +_AGENT_CONDITION_HEALTHY = "Healthy" +_AGENT_REASON_OBSERVATION_FAILED = "ObservationFailed" + +# provider-kubernetes stops receiving watch events once the token in a +# cluster's kubeconfig expires, which on EKS is after 15 minutes. A watched +# PoolObservation then only reaches this XR when it's polled, every 10 minutes +# by default, so it's polled more often. +# TODO(negz): Remove once provider-kubernetes fixes this. +# https://github.com/crossplane-contrib/provider-kubernetes/issues/598 +_ANNOTATION_POLL_INTERVAL = "crossplane.io/poll-interval" +_POOL_OBSERVATION_POLL_INTERVAL = "30s" + # Accelerator type naming an NVIDIA H100 in an InferenceClass. On Civo, # a pool of single-H100 nodes needs the serving stack to load that # pool's GPU driver with NVLink disabled (see civo_nvlink_disabled_pools). @@ -317,6 +334,9 @@ def compose(self) -> None: self.compose_existing(cluster.existing) else: response.warning(self.rsp, f"unsupported cluster source: {source}") + return + + self.compose_pool_observations() def compose_activation(self, kinds: tuple[str, ...]) -> None: """Activate the cloud managed resource kinds the cluster XR composes. @@ -522,6 +542,92 @@ def compose_namespaces(self) -> None: resource.update(self.rsp.desired.resources[key], obj) self.rsp.desired.resources[key].ready = fnv1.READY_TRUE + def compose_pool_observations(self) -> None: + """Compose a PoolObservation on the cluster for each of its GPU pools. + + The cluster agent reports each pool's nodes and free devices in the + status of its PoolObservation, which gpu_pools publishes. The agent and + the PoolObservation CRD arrive with the serving stack, so they're first + composed once the stack is ready, and kept after. + + provider-kubernetes watches each one, and polls it often in case the + watch has stopped (see _POOL_OBSERVATION_POLL_INTERVAL), so the agent's + reports reach this XR soon after they change. Each is + marked ready unconditionally: the scheduler falls back to whole-node + accounting for a pool it has no observation of, so a missing one mustn't + hold up the cluster. + + Each is composed no-Delete. Nothing holds the cluster until they're + gone, so deleting one could race the cluster's own deletion and never + finish. A PoolObservation left on a cluster, or for a pool that's since + been removed, has no effect on scheduling. + """ + backend_ready = ( + resource.get_condition(self.req.observed.resources.get(BACKEND_RESOURCE_KEY), "Ready").status == "True" + ) + pc = resource.child_name(_name(self.xr.metadata), "cluster-kubeconfig") + for pool in self.xr.spec.nodePools or []: + # Only the pools gpu_pools publishes have devices to observe. + cls = self.classes.get(pool.className) + if not cls or not cls.spec.devices: + continue + key = f"pool-observation-{pool.name}" + if not (backend_ready or key in self.req.observed.resources): + continue + obj = k8sobjv1alpha1.Object( + metadata=metav1.ObjectMeta( + namespace=_NAMESPACE_SYSTEM, + annotations={_ANNOTATION_POLL_INTERVAL: _POOL_OBSERVATION_POLL_INTERVAL}, + ), + spec=k8sobjv1alpha1.Spec( + providerConfigRef=k8sobjv1alpha1.ProviderConfigRef(kind="ClusterProviderConfig", name=pc), + readiness=k8sobjv1alpha1.Readiness(policy="SuccessfulCreate"), + watch=True, + forProvider=k8sobjv1alpha1.ForProvider( + manifest=pov1alpha1.PoolObservation( + apiVersion="agent.modelplane.ai/v1alpha1", + kind="PoolObservation", + metadata=metav1.ObjectMeta(name=pool.name), + spec=pov1alpha1.Spec(poolName=pool.name), + ).model_dump(exclude_unset=True), + ), + ), + ) + obj.spec.managementPolicies = ["Observe", "Create", "Update"] + resource.update(self.rsp.desired.resources[key], obj) + self.rsp.desired.resources[key].ready = fnv1.READY_TRUE + + def observed_pool(self, pool_name: str) -> pov1alpha1.Status | None: + """The status of a pool's PoolObservation, or None until the agent has observed it. + + The agent reports its conditions Unknown, and no nodes or devices, when + it can't observe a pool. That's no observation at all, rather than an + observation of an empty pool. So is a status that doesn't match the + agent's schema, which an agent other than the version Modelplane + installs could report. Raising would fail the whole cluster. + """ + observed = self.req.observed.resources.get(f"pool-observation-{pool_name}") + if not observed: + return None + try: + obj = k8sobjv1alpha1.Object.model_validate(resource.struct_to_dict(observed.resource)) + if not obj.status or not obj.status.atProvider or not obj.status.atProvider.manifest: + return None + po = pov1alpha1.PoolObservation.model_validate(obj.status.atProvider.manifest) + except pydantic.ValidationError as err: + response.warning( + self.rsp, + f"Ignoring the cluster agent's observation of pool {pool_name}: it does not match the agent's " + f"schema ({err.error_count()} problems), so the pool is counted in whole nodes.", + ) + return None + if po.status is None: + return None + healthy = next((c for c in po.status.conditions or [] if c.type == _AGENT_CONDITION_HEALTHY), None) + if healthy is None or healthy.reason == _AGENT_REASON_OBSERVATION_FAILED: + return None + return po.status + def resolve_gateway_cas(self) -> None: """Collect the client CA of every InferenceGateway in the fleet. @@ -945,14 +1051,14 @@ def compose_cluster_provider_config( ) self.rsp.desired.resources["cluster-provider-config-kubernetes"].ready = fnv1.READY_TRUE - def write_status(self, gpu_pools: list[dict[str, object]]) -> None: + def write_status(self, gpu_pools: list[v1alpha1.GpuPool]) -> None: """Write the InferenceCluster status.""" status = v1alpha1.Status( providerConfigRef=v1alpha1.ProviderConfigRef( name=resource.child_name(_name(self.xr.metadata), "cluster-kubeconfig"), ), namespace=_NAMESPACE_SYSTEM, - gpuPools=gpu_pools, # gpu_pools builds the by-alias wire form; pydantic coerces it to GpuPool + gpuPools=gpu_pools, ) cache_storage_class = self.observed_cache_storage_class() if cache_storage_class: @@ -986,7 +1092,12 @@ def write_status(self, gpu_pools: list[dict[str, object]]) -> None: # every request routed to it would be stranded. if ca and self.gateway_cas: status.gateway.hostname = _gateway_hostname(_name(self.xr.metadata)) - resource.update_status(self.rsp.desired.composite, status) + # The SDK dumps a model in Python mode, leaving a pool's + # lastAllocationTime a datetime a protobuf Struct can't hold. + resource.update_status( + self.rsp.desired.composite, + status.model_dump(mode="json", exclude_unset=True, by_alias=True, warnings=False), + ) def derive_conditions(self, *, cluster_ready: bool) -> None: """Derive ClusterReady and BackendReady conditions.""" @@ -1820,32 +1931,46 @@ def resolve_gke_backend_secrets(self, *, gke_ready: bool, backend_exists: bool) return None - def gpu_pools(self) -> list[dict[str, object]]: - """Derive status.gpuPools from each node pool's class. + def gpu_pools(self) -> list[v1alpha1.GpuPool]: + """Derive status.gpuPools from each node pool's class and observation. The class declares the node's devices (DRA-style); the pool declares how - many nodes. We copy the class's devices verbatim so + many nodes it can have. We copy the class's devices verbatim so ModelDeployment.nodeSelector can match against them, and record the node - count for the scheduler's available-node gate. + count. Once the cluster agent has observed the pool, we also record the + nodes it has now and how many devices are free on them, which the + scheduler fits replicas against. """ gpu_pools = [] for pool in self.xr.spec.nodePools or []: cls = self.classes.get(pool.className) if not cls or not cls.spec.devices: continue - # Copy the class's devices verbatim. model_dump drops None fields, - # keeping the typed attribute value objects - # (string/version/bool/int) one-of clean. by_alias keeps DRA's wire - # names (bool/int) rather than the generated bool_/int_ attributes, - # so the published status matches the InferenceClass schema. - devices = [d.model_dump(by_alias=True, exclude_none=True) for d in cls.spec.devices] - gpu_pools.append( - { - "name": pool.name, - "nodes": pool.maxNodeCount or pool.nodeCount, - "devices": devices, - } + # Copy the class's devices verbatim, from the InferenceClass's model + # to the InferenceCluster's. model_dump drops None fields, keeping + # the typed attribute value objects (string/version/bool/int) one-of + # clean. by_alias keeps DRA's wire names (bool/int) rather than the + # generated bool_/int_ attributes. + gpu_pool = v1alpha1.GpuPool( + name=pool.name, + nodes=pool.maxNodeCount or pool.nodeCount, + devices=[ + v1alpha1.Device.model_validate(d.model_dump(by_alias=True, exclude_none=True)) + for d in cls.spec.devices + ], ) + observed = self.observed_pool(pool.name) + if observed is not None: + gpu_pool.observedNodes = observed.nodes.total + gpu_pool.nodesByFreeDevices = [ + v1alpha1.NodesByFreeDevice.model_validate(b.model_dump(exclude_none=True)) + for b in observed.nodesByFreeDevices or [] + ] + # The agent reports a time once a claim has allocated devices + # on the pool. + if observed.lastAllocationTime is not None: + gpu_pool.lastAllocationTime = observed.lastAllocationTime + gpu_pools.append(gpu_pool) return gpu_pools def observed_gke_secrets(self) -> list[gkev1alpha1.Secret] | None: diff --git a/functions/compose-inference-cluster/tests/test_fn.py b/functions/compose-inference-cluster/tests/test_fn.py index dfbcb7600..1da0a9baa 100644 --- a/functions/compose-inference-cluster/tests/test_fn.py +++ b/functions/compose-inference-cluster/tests/test_fn.py @@ -409,6 +409,81 @@ def _namespace_object(*, team: str, name: str) -> fnv1.Resource: ) +def _pool_observation(*, pool: str) -> fnv1.Resource: + """The composed Object that creates a PoolObservation of pool on the cluster, watched and polled, without deleting it, and marked ready.""" + return fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "kubernetes.m.crossplane.io/v1alpha1", + "kind": "Object", + "metadata": { + "namespace": "modelplane-system", + "annotations": {"crossplane.io/poll-interval": "30s"}, + }, + "spec": { + "providerConfigRef": { + "kind": "ClusterProviderConfig", + "name": "test-cluster-cluster-kubeconfig-d0f89", + }, + "readiness": {"policy": "SuccessfulCreate"}, + "watch": True, + "managementPolicies": ["Observe", "Create", "Update"], + "forProvider": { + "manifest": { + "apiVersion": "agent.modelplane.ai/v1alpha1", + "kind": "PoolObservation", + "metadata": {"name": pool}, + "spec": {"poolName": pool}, + }, + }, + }, + } + ), + ready=fnv1.READY_TRUE, + ) + + +def _observed_pool_observation(*, pool: str, status: dict) -> fnv1.Resource: + """The observed Object for pool's PoolObservation, whose remote manifest carries the status the agent reported.""" + return fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "kubernetes.m.crossplane.io/v1alpha1", + "kind": "Object", + "metadata": {"namespace": "modelplane-system"}, + "spec": { + "providerConfigRef": { + "kind": "ClusterProviderConfig", + "name": "test-cluster-cluster-kubeconfig-d0f89", + }, + "readiness": {"policy": "SuccessfulCreate"}, + "watch": True, + "managementPolicies": ["Observe", "Create", "Update"], + "forProvider": { + "manifest": { + "apiVersion": "agent.modelplane.ai/v1alpha1", + "kind": "PoolObservation", + "metadata": {"name": pool}, + "spec": {"poolName": pool}, + }, + }, + }, + "status": { + "atProvider": { + "manifest": { + "apiVersion": "agent.modelplane.ai/v1alpha1", + "kind": "PoolObservation", + "metadata": {"name": pool}, + "spec": {"poolName": pool}, + "status": status, + }, + }, + }, + } + ), + ) + + def _to_dict(msg: message.Message) -> dict: """msg as a dict with sorted keys, so pytest's diff of two lines them up.""" return json.loads(json_format.MessageToJson(msg, sort_keys=True)) @@ -893,7 +968,7 @@ def _to_dict(msg: message.Message) -> dict: ), ComposeCase( name="ExistingBackendReady", - reason="Once its ServingStack is observed Ready with an address, an Existing cluster relays the address and reports its backend healthy.", + reason="Once its ServingStack is observed Ready with an address, an Existing cluster relays the address, reports its backend healthy, and composes a PoolObservation for each pool.", req=fnv1.RunFunctionRequest( observed=fnv1.State( composite=_inference_cluster( @@ -963,6 +1038,7 @@ def _to_dict(msg: message.Message) -> dict: gpu=None, ready=fnv1.READY_TRUE, ), + "pool-observation-l4-pool": _pool_observation(pool="l4-pool"), }, ), context=structpb.Struct(), @@ -991,6 +1067,695 @@ def _to_dict(msg: message.Message) -> dict: ], ), ), + # The PoolObservation status is what the agent reported for a pool of one + # g6.12xlarge node with three of its four L4s allocated. + ComposeCase( + name="ExistingPoolObserved", + reason="Once the agent reports on a pool, the cluster publishes the nodes the pool has, how many devices are free on them, and when the agent last saw a claim allocated on them.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_inference_cluster( + cluster=v1alpha1.Cluster( + source="Existing", + existing=v1alpha1.Existing(secretRef=v1alpha1.SecretRef(name="my-kubeconfig")), + ), + node_pools=[ + v1alpha1.NodePool(name="gpu-l4x4-a", className="gpu-l4x4", nodeCount=1, maxNodeCount=3), + ], + ), + resources={ + "serving-stack": _observed_serving_stack(gateway={"address": "34.55.100.10"}), + "pool-observation-gpu-l4x4-a": _observed_pool_observation( + pool="gpu-l4x4-a", + status={ + "observedGeneration": 1, + "conditions": [ + { + "type": "Healthy", + "status": "True", + "reason": "PoolHealthy", + "message": "", + "lastTransitionTime": "2026-10-20T10:00:00Z", + "observedGeneration": 1, + }, + { + "type": "ScaleUpBlocked", + "status": "False", + "reason": "NoUnschedulablePods", + "message": "", + "lastTransitionTime": "2026-10-20T10:00:00Z", + "observedGeneration": 1, + }, + ], + "nodes": {"total": 1, "ready": 1, "schedulable": 1}, + "unschedulablePods": 0, + "devices": [ + {"deviceClassName": "gpu.nvidia.com", "nodes": 1, "allocatable": 4, "allocated": 3}, + ], + "nodesByFreeDevices": [{"freeDevices": {"gpu.nvidia.com": 1}, "nodes": 1}], + "lastAllocationTime": "2026-10-20T10:05:40.318204Z", + }, + ), + }, + ), + required_resources={ + "class-gpu-l4x4": fnv1.Resources( + items=[ + _inference_class( + name="gpu-l4x4", + count=4, + memory="23034Mi", + provisioning={ + "provider": "EKS", + "eks": { + "instanceType": "g6.12xlarge", + "diskSizeGb": 100, + "accelerator": {"type": "nvidia-l4", "count": 4}, + }, + }, + ), + ], + ), + }, + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_inference_cluster( + gpu_pools=[ + { + "name": "gpu-l4x4-a", + "nodes": 3, + "observedNodes": 1, + "nodesByFreeDevices": [{"freeDevices": {"gpu.nvidia.com": 1}, "nodes": 1}], + "lastAllocationTime": "2026-10-20T10:05:40.318204Z", + "devices": [ + { + "name": "gpu", + "claim": "DRA", + "driver": "gpu.nvidia.com", + "deviceClassName": "gpu.nvidia.com", + "count": 4, + "capacity": {"memory": {"value": "23034Mi"}}, + }, + ], + }, + ], + cache=None, + gateway={"address": "34.55.100.10"}, + ), + resources={ + "cluster-provider-config-kubernetes": _cluster_provider_config( + kubeconfig="my-kubeconfig", identity=None + ), + "serving-stack": _serving_stack( + cloud="Existing", + secrets=[{"type": "Kubeconfig", "name": "my-kubeconfig", "key": "kubeconfig"}], + client_cas=None, + gpu=None, + ready=fnv1.READY_TRUE, + ), + "pool-observation-gpu-l4x4-a": _pool_observation(pool="gpu-l4x4-a"), + }, + ), + context=structpb.Struct(), + requirements=fnv1.Requirements( + resources={ + "model-replicas": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", + kind="ModelReplica", + match_labels=fnv1.MatchLabels(labels={"modelplane.ai/cluster": "test-cluster"}), + ), + "model-routes": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", + kind="ModelRoute", + match_labels=fnv1.MatchLabels(labels={"modelplane.ai/cluster": "test-cluster"}), + ), + "model-caches": fnv1.ResourceSelector(api_version="modelplane.ai/v1alpha1", kind="ModelCache"), + "gateways": fnv1.ResourceSelector(api_version="modelplane.ai/v1alpha1", kind="InferenceGateway"), + "class-gpu-l4x4": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", kind="InferenceClass", match_name="gpu-l4x4" + ), + }, + ), + conditions=[ + fnv1.Condition(type="ClusterReady", status=fnv1.STATUS_CONDITION_TRUE, reason="ClusterRunning"), + fnv1.Condition(type="BackendReady", status=fnv1.STATUS_CONDITION_TRUE, reason="BackendHealthy"), + ], + ), + ), + ComposeCase( + name="ExistingPoolNotObservable", + reason="A pool the agent reports it can't observe publishes no observation, rather than an observation of an empty pool.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_inference_cluster( + cluster=v1alpha1.Cluster( + source="Existing", + existing=v1alpha1.Existing(secretRef=v1alpha1.SecretRef(name="my-kubeconfig")), + ), + node_pools=[ + v1alpha1.NodePool(name="gpu-l4x4-a", className="gpu-l4x4", nodeCount=1, maxNodeCount=3), + ], + ), + resources={ + "serving-stack": _observed_serving_stack(gateway={"address": "34.55.100.10"}), + "pool-observation-gpu-l4x4-a": _observed_pool_observation( + pool="gpu-l4x4-a", + status={ + "observedGeneration": 1, + "conditions": [ + { + "type": "Healthy", + "status": "Unknown", + "reason": "ObservationFailed", + "message": "cannot observe pool: boom", + "lastTransitionTime": "2026-10-20T10:00:00Z", + "observedGeneration": 1, + }, + { + "type": "ScaleUpBlocked", + "status": "Unknown", + "reason": "ObservationFailed", + "message": "cannot observe pool: boom", + "lastTransitionTime": "2026-10-20T10:00:00Z", + "observedGeneration": 1, + }, + ], + "nodes": {"total": 0, "ready": 0, "schedulable": 0}, + "unschedulablePods": 0, + "lastAllocationTime": "2026-10-20T10:05:40.318204Z", + }, + ), + }, + ), + required_resources={ + "class-gpu-l4x4": fnv1.Resources( + items=[ + _inference_class( + name="gpu-l4x4", + count=4, + memory="23034Mi", + provisioning={ + "provider": "EKS", + "eks": { + "instanceType": "g6.12xlarge", + "diskSizeGb": 100, + "accelerator": {"type": "nvidia-l4", "count": 4}, + }, + }, + ), + ], + ), + }, + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_inference_cluster( + gpu_pools=[ + { + "name": "gpu-l4x4-a", + "nodes": 3, + "devices": [ + { + "name": "gpu", + "claim": "DRA", + "driver": "gpu.nvidia.com", + "deviceClassName": "gpu.nvidia.com", + "count": 4, + "capacity": {"memory": {"value": "23034Mi"}}, + }, + ], + }, + ], + cache=None, + gateway={"address": "34.55.100.10"}, + ), + resources={ + "cluster-provider-config-kubernetes": _cluster_provider_config( + kubeconfig="my-kubeconfig", identity=None + ), + "serving-stack": _serving_stack( + cloud="Existing", + secrets=[{"type": "Kubeconfig", "name": "my-kubeconfig", "key": "kubeconfig"}], + client_cas=None, + gpu=None, + ready=fnv1.READY_TRUE, + ), + "pool-observation-gpu-l4x4-a": _pool_observation(pool="gpu-l4x4-a"), + }, + ), + context=structpb.Struct(), + requirements=fnv1.Requirements( + resources={ + "model-replicas": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", + kind="ModelReplica", + match_labels=fnv1.MatchLabels(labels={"modelplane.ai/cluster": "test-cluster"}), + ), + "model-routes": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", + kind="ModelRoute", + match_labels=fnv1.MatchLabels(labels={"modelplane.ai/cluster": "test-cluster"}), + ), + "model-caches": fnv1.ResourceSelector(api_version="modelplane.ai/v1alpha1", kind="ModelCache"), + "gateways": fnv1.ResourceSelector(api_version="modelplane.ai/v1alpha1", kind="InferenceGateway"), + "class-gpu-l4x4": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", kind="InferenceClass", match_name="gpu-l4x4" + ), + }, + ), + conditions=[ + fnv1.Condition(type="ClusterReady", status=fnv1.STATUS_CONDITION_TRUE, reason="ClusterRunning"), + fnv1.Condition(type="BackendReady", status=fnv1.STATUS_CONDITION_TRUE, reason="BackendHealthy"), + ], + ), + ), + # One of the pool's two nodes is cordoned, so the histogram counts only + # the other. + ComposeCase( + name="ExistingPoolUnhealthy", + reason="A pool the agent observes as unhealthy, but not unobservable, still publishes its observation.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_inference_cluster( + cluster=v1alpha1.Cluster( + source="Existing", + existing=v1alpha1.Existing(secretRef=v1alpha1.SecretRef(name="my-kubeconfig")), + ), + node_pools=[ + v1alpha1.NodePool(name="gpu-l4x4-a", className="gpu-l4x4", nodeCount=2, maxNodeCount=3), + ], + ), + resources={ + "serving-stack": _observed_serving_stack(gateway={"address": "34.55.100.10"}), + "pool-observation-gpu-l4x4-a": _observed_pool_observation( + pool="gpu-l4x4-a", + status={ + "observedGeneration": 1, + "conditions": [ + { + "type": "Healthy", + "status": "False", + "reason": "NodesCordoned", + "message": "1 of 2 nodes cordoned", + "lastTransitionTime": "2026-10-20T10:00:00Z", + "observedGeneration": 1, + }, + { + "type": "ScaleUpBlocked", + "status": "False", + "reason": "NoUnschedulablePods", + "message": "", + "lastTransitionTime": "2026-10-20T10:00:00Z", + "observedGeneration": 1, + }, + ], + "nodes": {"total": 2, "ready": 2, "schedulable": 1}, + "unschedulablePods": 0, + "devices": [ + {"deviceClassName": "gpu.nvidia.com", "nodes": 2, "allocatable": 4, "allocated": 3}, + ], + "nodesByFreeDevices": [{"freeDevices": {"gpu.nvidia.com": 1}, "nodes": 1}], + "lastAllocationTime": "2026-10-20T10:05:40.318204Z", + }, + ), + }, + ), + required_resources={ + "class-gpu-l4x4": fnv1.Resources( + items=[ + _inference_class( + name="gpu-l4x4", + count=4, + memory="23034Mi", + provisioning={ + "provider": "EKS", + "eks": { + "instanceType": "g6.12xlarge", + "diskSizeGb": 100, + "accelerator": {"type": "nvidia-l4", "count": 4}, + }, + }, + ), + ], + ), + }, + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_inference_cluster( + gpu_pools=[ + { + "name": "gpu-l4x4-a", + "nodes": 3, + "observedNodes": 2, + "nodesByFreeDevices": [{"freeDevices": {"gpu.nvidia.com": 1}, "nodes": 1}], + "lastAllocationTime": "2026-10-20T10:05:40.318204Z", + "devices": [ + { + "name": "gpu", + "claim": "DRA", + "driver": "gpu.nvidia.com", + "deviceClassName": "gpu.nvidia.com", + "count": 4, + "capacity": {"memory": {"value": "23034Mi"}}, + }, + ], + }, + ], + cache=None, + gateway={"address": "34.55.100.10"}, + ), + resources={ + "cluster-provider-config-kubernetes": _cluster_provider_config( + kubeconfig="my-kubeconfig", identity=None + ), + "serving-stack": _serving_stack( + cloud="Existing", + secrets=[{"type": "Kubeconfig", "name": "my-kubeconfig", "key": "kubeconfig"}], + client_cas=None, + gpu=None, + ready=fnv1.READY_TRUE, + ), + "pool-observation-gpu-l4x4-a": _pool_observation(pool="gpu-l4x4-a"), + }, + ), + context=structpb.Struct(), + requirements=fnv1.Requirements( + resources={ + "model-replicas": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", + kind="ModelReplica", + match_labels=fnv1.MatchLabels(labels={"modelplane.ai/cluster": "test-cluster"}), + ), + "model-routes": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", + kind="ModelRoute", + match_labels=fnv1.MatchLabels(labels={"modelplane.ai/cluster": "test-cluster"}), + ), + "model-caches": fnv1.ResourceSelector(api_version="modelplane.ai/v1alpha1", kind="ModelCache"), + "gateways": fnv1.ResourceSelector(api_version="modelplane.ai/v1alpha1", kind="InferenceGateway"), + "class-gpu-l4x4": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", kind="InferenceClass", match_name="gpu-l4x4" + ), + }, + ), + conditions=[ + fnv1.Condition(type="ClusterReady", status=fnv1.STATUS_CONDITION_TRUE, reason="ClusterRunning"), + fnv1.Condition(type="BackendReady", status=fnv1.STATUS_CONDITION_TRUE, reason="BackendHealthy"), + ], + ), + ), + # The agent's schema requires each bucket to count at least one node. + ComposeCase( + name="ExistingPoolObservationInvalid", + reason="A pool whose observation doesn't match the agent's schema publishes no observation, rather than failing the whole cluster.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_inference_cluster( + cluster=v1alpha1.Cluster( + source="Existing", + existing=v1alpha1.Existing(secretRef=v1alpha1.SecretRef(name="my-kubeconfig")), + ), + node_pools=[ + v1alpha1.NodePool(name="gpu-l4x4-a", className="gpu-l4x4", nodeCount=1, maxNodeCount=3), + ], + ), + resources={ + "serving-stack": _observed_serving_stack(gateway={"address": "34.55.100.10"}), + "pool-observation-gpu-l4x4-a": _observed_pool_observation( + pool="gpu-l4x4-a", + status={ + "observedGeneration": 1, + "conditions": [ + { + "type": "Healthy", + "status": "True", + "reason": "PoolHealthy", + "message": "", + "lastTransitionTime": "2026-10-20T10:00:00Z", + "observedGeneration": 1, + }, + { + "type": "ScaleUpBlocked", + "status": "False", + "reason": "NoUnschedulablePods", + "message": "", + "lastTransitionTime": "2026-10-20T10:00:00Z", + "observedGeneration": 1, + }, + ], + "nodes": {"total": 1, "ready": 1, "schedulable": 1}, + "unschedulablePods": 0, + "devices": [ + {"deviceClassName": "gpu.nvidia.com", "nodes": 1, "allocatable": 4, "allocated": 3}, + ], + "nodesByFreeDevices": [{"freeDevices": {"gpu.nvidia.com": 1}, "nodes": 0}], + "lastAllocationTime": "2026-10-20T10:05:40.318204Z", + }, + ), + }, + ), + required_resources={ + "class-gpu-l4x4": fnv1.Resources( + items=[ + _inference_class( + name="gpu-l4x4", + count=4, + memory="23034Mi", + provisioning={ + "provider": "EKS", + "eks": { + "instanceType": "g6.12xlarge", + "diskSizeGb": 100, + "accelerator": {"type": "nvidia-l4", "count": 4}, + }, + }, + ), + ], + ), + }, + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_inference_cluster( + gpu_pools=[ + { + "name": "gpu-l4x4-a", + "nodes": 3, + "devices": [ + { + "name": "gpu", + "claim": "DRA", + "driver": "gpu.nvidia.com", + "deviceClassName": "gpu.nvidia.com", + "count": 4, + "capacity": {"memory": {"value": "23034Mi"}}, + }, + ], + }, + ], + cache=None, + gateway={"address": "34.55.100.10"}, + ), + resources={ + "cluster-provider-config-kubernetes": _cluster_provider_config( + kubeconfig="my-kubeconfig", identity=None + ), + "serving-stack": _serving_stack( + cloud="Existing", + secrets=[{"type": "Kubeconfig", "name": "my-kubeconfig", "key": "kubeconfig"}], + client_cas=None, + gpu=None, + ready=fnv1.READY_TRUE, + ), + "pool-observation-gpu-l4x4-a": _pool_observation(pool="gpu-l4x4-a"), + }, + ), + results=[ + fnv1.Result( + severity=fnv1.SEVERITY_WARNING, + message="Ignoring the cluster agent's observation of pool gpu-l4x4-a: it does not match the " + "agent's schema (1 problems), so the pool is counted in whole nodes.", + ), + ], + context=structpb.Struct(), + requirements=fnv1.Requirements( + resources={ + "model-replicas": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", + kind="ModelReplica", + match_labels=fnv1.MatchLabels(labels={"modelplane.ai/cluster": "test-cluster"}), + ), + "model-routes": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", + kind="ModelRoute", + match_labels=fnv1.MatchLabels(labels={"modelplane.ai/cluster": "test-cluster"}), + ), + "model-caches": fnv1.ResourceSelector(api_version="modelplane.ai/v1alpha1", kind="ModelCache"), + "gateways": fnv1.ResourceSelector(api_version="modelplane.ai/v1alpha1", kind="InferenceGateway"), + "class-gpu-l4x4": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", kind="InferenceClass", match_name="gpu-l4x4" + ), + }, + ), + conditions=[ + fnv1.Condition(type="ClusterReady", status=fnv1.STATUS_CONDITION_TRUE, reason="ClusterRunning"), + fnv1.Condition(type="BackendReady", status=fnv1.STATUS_CONDITION_TRUE, reason="BackendHealthy"), + ], + ), + ), + # The agent hasn't seen a claim allocated on the pool, so it reports no + # allocation time. + ComposeCase( + name="ExistingPoolObservationKept", + reason="A PoolObservation that already exists stays composed when the ServingStack stops being Ready, so the agent's reports keep flowing.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_inference_cluster( + cluster=v1alpha1.Cluster( + source="Existing", + existing=v1alpha1.Existing(secretRef=v1alpha1.SecretRef(name="my-kubeconfig")), + ), + node_pools=[ + v1alpha1.NodePool(name="gpu-l4x4-a", className="gpu-l4x4", nodeCount=1, maxNodeCount=3), + ], + ), + resources={ + "serving-stack": fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "infrastructure.modelplane.ai/v1alpha1", + "kind": "ServingStack", + "metadata": {"name": "test-cluster-serving-stack-fd00b"}, + "status": { + "conditions": [{"type": "Ready", "status": "False"}], + "gateway": {"address": "34.55.100.10"}, + }, + } + ), + ), + "pool-observation-gpu-l4x4-a": _observed_pool_observation( + pool="gpu-l4x4-a", + status={ + "observedGeneration": 1, + "conditions": [ + { + "type": "Healthy", + "status": "True", + "reason": "PoolHealthy", + "message": "", + "lastTransitionTime": "2026-10-20T10:00:00Z", + "observedGeneration": 1, + }, + { + "type": "ScaleUpBlocked", + "status": "False", + "reason": "NoUnschedulablePods", + "message": "", + "lastTransitionTime": "2026-10-20T10:00:00Z", + "observedGeneration": 1, + }, + ], + "nodes": {"total": 1, "ready": 1, "schedulable": 1}, + "unschedulablePods": 0, + "devices": [ + {"deviceClassName": "gpu.nvidia.com", "nodes": 1, "allocatable": 4, "allocated": 0}, + ], + "nodesByFreeDevices": [{"freeDevices": {"gpu.nvidia.com": 4}, "nodes": 1}], + }, + ), + }, + ), + required_resources={ + "class-gpu-l4x4": fnv1.Resources( + items=[ + _inference_class( + name="gpu-l4x4", + count=4, + memory="23034Mi", + provisioning={ + "provider": "EKS", + "eks": { + "instanceType": "g6.12xlarge", + "diskSizeGb": 100, + "accelerator": {"type": "nvidia-l4", "count": 4}, + }, + }, + ), + ], + ), + }, + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_inference_cluster( + gpu_pools=[ + { + "name": "gpu-l4x4-a", + "nodes": 3, + "observedNodes": 1, + "nodesByFreeDevices": [{"freeDevices": {"gpu.nvidia.com": 4}, "nodes": 1}], + "devices": [ + { + "name": "gpu", + "claim": "DRA", + "driver": "gpu.nvidia.com", + "deviceClassName": "gpu.nvidia.com", + "count": 4, + "capacity": {"memory": {"value": "23034Mi"}}, + }, + ], + }, + ], + cache=None, + gateway={"address": "34.55.100.10"}, + ), + resources={ + "cluster-provider-config-kubernetes": _cluster_provider_config( + kubeconfig="my-kubeconfig", identity=None + ), + "serving-stack": _serving_stack( + cloud="Existing", + secrets=[{"type": "Kubeconfig", "name": "my-kubeconfig", "key": "kubeconfig"}], + client_cas=None, + gpu=None, + ready=fnv1.READY_UNSPECIFIED, + ), + "pool-observation-gpu-l4x4-a": _pool_observation(pool="gpu-l4x4-a"), + }, + ), + context=structpb.Struct(), + requirements=fnv1.Requirements( + resources={ + "model-replicas": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", + kind="ModelReplica", + match_labels=fnv1.MatchLabels(labels={"modelplane.ai/cluster": "test-cluster"}), + ), + "model-routes": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", + kind="ModelRoute", + match_labels=fnv1.MatchLabels(labels={"modelplane.ai/cluster": "test-cluster"}), + ), + "model-caches": fnv1.ResourceSelector(api_version="modelplane.ai/v1alpha1", kind="ModelCache"), + "gateways": fnv1.ResourceSelector(api_version="modelplane.ai/v1alpha1", kind="InferenceGateway"), + "class-gpu-l4x4": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", kind="InferenceClass", match_name="gpu-l4x4" + ), + }, + ), + conditions=[ + fnv1.Condition(type="ClusterReady", status=fnv1.STATUS_CONDITION_TRUE, reason="ClusterRunning"), + fnv1.Condition(type="BackendReady", status=fnv1.STATUS_CONDITION_FALSE, reason="Installing"), + ], + ), + ), ComposeCase( name="EKSFirstPass", reason="With no EKSCluster observed yet, an EKS cluster composes only the activation policy and the EKSCluster.", diff --git a/functions/compose-model-deployment/function/scheduling.py b/functions/compose-model-deployment/function/scheduling.py index c46388b2d..9d3a06d1b 100644 --- a/functions/compose-model-deployment/function/scheduling.py +++ b/functions/compose-model-deployment/function/scheduling.py @@ -38,9 +38,9 @@ fabrics and silently hang. An engine that no single pool satisfies is not scheduled on that cluster (see #149 for the same-fabric multi-pool case). A member with no nodeSelector claims no devices - it matches the engine's pool at -zero node cost, packed onto the gang's nodes by the cluster's scheduler. The +no cost, and the cluster's scheduler places it on any of the pool's nodes. The scheduler therefore asks, for each candidate cluster, whether every engine can -be assigned a single pool with enough free nodes, all on that one cluster. +be assigned a single pool with room for its pods, all on that one cluster. Scheduling runs in two phases: @@ -57,21 +57,30 @@ shortfall one at a time. Each new replica goes to the eligible cluster hosting the fewest of this deployment's replicas (spread first, pack only when every eligible cluster already has its share), against a running ledger - of free node capacity so we never overcommit a cluster. If the deployment - wants fewer, drop the highest-index replicas first, consolidating off the - clusters we packed onto last. - -Capacity is gated on nodes, not on individual DRA devices. The per-node device -count is a device request's count; the only number the scheduler reads from a -member is its node cost, which it gates against a pool's available nodes. A -member's pods occupy nodes only when they claim devices, so a member that -resolves no claim: DRA device - it carried no nodeSelector, or matched only -synthetic devices - costs zero nodes. Device-count contention BETWEEN -deployments is left to DRA admission on the workload cluster, which is -authoritative: it rejects a Pod whose ResourceClaim can't be satisfied, and the -next reconcile sees the updated observed state. The control-plane scheduler -stays deliberately coarse - "could this cluster plausibly host this replica" - -rather than duplicating the real DRA scheduler. + of each pool's free capacity. If the deployment wants fewer, drop the + highest-index replicas first, consolidating off the clusters we packed onto + last. + +A pool's free capacity is how many of its nodes have each set of free devices, +and a replica fits when each of its pods can be placed on a node with enough of +the devices it claims free. Any pods can share a node, including a gang's, +because nothing stops the cluster's scheduler binding them to one. A pod that +claims no claim: DRA device - its member carried no nodeSelector, or matched +only synthetic devices - takes no capacity. + +Where the cluster agent reports on a pool, its capacity is the agent's +histogram of the pool's nodes by free devices, plus the nodes the pool can +still grow by with every device free. The histogram counts the devices +scheduled pods hold as taken, so to avoid counting them twice the ledger +charges a replica's pods on the pool only until the histogram accounts for +them (see _settled). Where the agent doesn't +report on a pool, every one of the nodes it can have starts with every device +free, and a pod that claims any device takes a whole node. + +The scheduler predicts and the workload cluster's scheduler decides. It can't +know which node each pod will bind to, so it guesses, and a wrong guess leaves a +pod Pending until devices free up. The next reconcile sees the updated observed +state. """ from dataclasses import dataclass, field @@ -117,6 +126,10 @@ def _labels(meta: metav1.ObjectMeta | None) -> dict[str, str]: _ROLE_STANDALONE = "Standalone" _ROLE_WORKER = "Worker" +# The condition compose-model-replica sets True once every one of a replica's +# pods has scheduled. +_CONDITION_MODEL_PODS_SCHEDULED = "ModelPodsScheduled" + def _published_count(value: int | None) -> int: """A cluster-published device/node count, floored at zero. @@ -129,7 +142,7 @@ def _published_count(value: int | None) -> int: device count of 0: it differs from a deployment-side count, which the XRD floors at 1, so `x or 1` is a safe default-for-None there but would wrongly turn a published 0 into 1 here. Flooring None and negatives to 0 is defensive - hygiene - both already fail the capacity gate as a non-positive count. + hygiene - both already read as no capacity. """ if value is None or value < 0: return 0 @@ -231,15 +244,14 @@ class _CompiledRequest: class _CompiledMember: """A member reduced to what the scheduler needs. - nodes is the member's total node cost across the engine's copies (its pod - count x copies), charged only when the member resolves a claimable device. - requests carries the member's compiled nodeSelector requests, used to match - a pool; empty when the member carries no nodeSelector and so matches every - pool. + pods is the member's pod count in one copy of its engine, each of which + claims the devices the member resolves. requests carries the member's + compiled nodeSelector requests, used to match a pool; empty when the member + carries no nodeSelector and so matches every pool. """ role: str - nodes: int + pods: int requests: list[_CompiledRequest] @@ -247,11 +259,13 @@ class _CompiledMember: class _CompiledEngine: """An engine reduced to what the scheduler needs. - name identifies the engine; members are its compiled members in deployment - order. Every member of an engine is placed on a single shared pool. + name identifies the engine; copies is how many copies of it each replica + runs; members are its compiled members in deployment order. Every member of + an engine is placed on a single shared pool. """ name: str + copies: int members: list[_CompiledMember] @@ -259,7 +273,9 @@ def _member_pods(member: _Member) -> int: """Pods a single member contributes: a Worker's node span, else 1. A Standalone or a Leader is always exactly one pod; only a Worker fans out - to worker.nodes follower pods (one per node). + to worker.nodes follower pods. compose-model-replica's _expected_pods counts + a replica's pods the same way to tell when they all exist, which is when + _settled stops charging them, so the two must agree. """ if (member.role or _ROLE_STANDALONE) == _ROLE_WORKER and member.worker: return int(member.worker.nodes) @@ -296,16 +312,15 @@ def compile_engines(deployment: mdv1alpha1.ModelDeployment) -> list[_CompiledEng """ engines = [] for engine in deployment.spec.template.spec.engines: - copies = int(engine.copies or 1) members = [ _CompiledMember( role=member.role or _ROLE_STANDALONE, - nodes=_member_pods(member) * copies, + pods=_member_pods(member), requests=_compile_requests(member.nodeSelector) if member.nodeSelector else [], ) for member in engine.members ] - engines.append(_CompiledEngine(name=engine.name, members=members)) + engines.append(_CompiledEngine(name=engine.name, copies=int(engine.copies or 1), members=members)) return engines @@ -386,8 +401,8 @@ def _match_member(pool: icv1alpha1.GpuPool, member: _CompiledMember) -> list[Dev Returns the resolved claim: DRA DeviceRequests when the pool satisfies every request, or None when the member fails any request. The requests describe what ONE of the member's pods needs from its node; every pod of - the member runs on its own node of the matched pool and claims the same - devices. A member with no requests matches every pool, resolving nothing. + the member claims the same devices on a node of the matched pool. A member + with no requests matches every pool, resolving nothing. The resolved list may also be empty when every matched device is claim: Synthetic (matched for fleet scheduling but never claimed); the caller rejects an engine NONE of whose members resolve a claimable device, @@ -398,8 +413,8 @@ def _match_member(pool: icv1alpha1.GpuPool, member: _CompiledMember) -> list[Dev when requests don't overlap on a device (the practical shape: one GPU request, maybe a NIC request); when they do overlap it can only false-reject, never overcommit, so it fails safe. Members are matched - independently of each other - their pods run on different nodes, so two - members' requests never contend for one node's devices. + independently of each other, against what one node of the pool has. Whether + their pods fit the nodes the pool has free is up to its capacity. """ devices = pool.devices or [] remaining = [_published_count(d.count) for d in devices] @@ -428,28 +443,39 @@ def _match_member(pool: icv1alpha1.GpuPool, member: _CompiledMember) -> list[Dev return resolved -def _member_cost(member: _CompiledMember, resolved: list[DeviceRequest]) -> int: - """The nodes a member consumes on its pool, given its resolved requests. - - A member that claims devices occupies its nodes exclusively (the coarse - pod-per-node model). One that claims nothing - no nodeSelector, or only - synthetic devices matched - shares the pool's nodes with the pods that do - claim, so it costs zero NODES. The same rule charges observed replicas in - the ledger (keyed on their stamped deviceRequests), so fill-time and - rebuild-time accounting agree. - - "Zero nodes" is the fleet scheduler's whole-node accounting, not zero - resources: a claimless member (e.g. a gang's coordinator-only leader) is - still pinned to its gang's GPU pool and its pod still consumes CPU and memory - on whatever GPU node the cluster's scheduler packs it onto. We model only - nodes here and leave that to the cluster scheduler, on the assumption a GPU - node has slack to host a coordinator beside its worker pods. If it doesn't - the pod stays Pending - real-resource contention the fleet scheduler doesn't - see. Pinning to the gang's pool is deliberate even so: a coordinator on - another pool could be in a different zone or interconnect fabric from the - workers it coordinates. +# The devices one pod claims, keyed by DeviceClass. +_Claim = dict[str, int] + +# A node's free devices as sorted (DeviceClass, count) pairs, so nodes with the +# same free devices share a key. +_Free = tuple[tuple[str, int], ...] + + +def _claim(requests: list[DeviceRequest]) -> _Claim: + """The devices each of a member's pods claims, given the requests it resolved. + + A member that resolves no claim: DRA device - no nodeSelector, or only + synthetic devices matched - claims nothing, and takes no capacity. Its pods + still use CPU and memory on whatever node of its gang's pool the cluster's + scheduler packs them onto, which the scheduler doesn't model. """ - return member.nodes if resolved else 0 + claim: _Claim = {} + for r in requests: + claim[r.device_class_name] = claim.get(r.device_class_name, 0) + r.count + return claim + + +def _observed_claim(member: mrv1alpha1.Member) -> _Claim: + """The devices each of an observed replica member's pods claims, from its stamped deviceRequests. + + This reads what the member was STAMPED with, not what it would resolve to + now. For a retained member whose pinned pool drifted, the two can disagree + for one reconcile, until the re-resolved spec is observed. + """ + claim: _Claim = {} + for r in member.deviceRequests or []: + claim[r.deviceClassName] = claim.get(r.deviceClassName, 0) + int(r.count or 1) + return claim def _gpu_pools(cluster: icv1alpha1.InferenceCluster) -> list[icv1alpha1.GpuPool]: @@ -489,43 +515,118 @@ def _replica_index(replica: mrv1alpha1.ModelReplica) -> int: @dataclass -class _Ledger: - """Free node capacity per (cluster, pool). +class _Pool: + """A pool's free capacity: how many of its nodes have each set of free devices. - Built by _build_ledger from published capacity minus the replicas already - committed to each pool (see there for exactly which replicas count). The - fill phase then decrements it via consume() as it places each new replica's - members, which is what stops a single scheduling pass overcommitting a pool. + whole_node is set for a pool the cluster agent hasn't reported on. Each of + its nodes starts with every device free, and a pod that claims any device + takes a whole node, however few of its devices it claims. """ - free: dict[tuple[str, str], int] + nodes: dict[_Free, int] + whole_node: bool + + def copy(self) -> "_Pool": + """A copy to place pods on without changing this one.""" + return _Pool(nodes=dict(self.nodes), whole_node=self.whole_node) + + def place(self, pods: list[_Claim]) -> bool: + """Place one copy of an engine's pods, each claiming its devices, or return False if they don't all fit. + + Pods that claim nothing take no capacity, so they always fit. Pods that + don't fit leave the others placed. In a whole_node pool any free node + fits a pod, whatever it claims, so a replica stamped with a claim its + pool's class no longer declares still takes nodes. + + The cluster's scheduler decides which node a pod binds to, and the + scheduler can't know which. It assumes the node with the most of the + claimed devices free. It places the smallest pods first, for the reason + _place_engines gives. A gang's pods can share a node, because nothing + stops the cluster's scheduler binding them to one. + """ + fit = True + for claim in sorted(pods, key=lambda c: sum(c.values())): + if not claim: + continue + fits = [f for f in self.nodes if self.whole_node or all(dict(f).get(c, 0) >= n for c, n in claim.items())] + if not fits: + fit = False + continue + node = max(fits, key=lambda f: (sum(dict(f).get(c, 0) for c in claim), f)) + self.nodes[node] -= 1 + if self.nodes[node] == 0: + del self.nodes[node] + if self.whole_node: + continue + free = dict(node) + for c, n in claim.items(): + free[c] -= n + left = tuple(sorted(free.items())) + self.nodes[left] = self.nodes.get(left, 0) + 1 + return fit - def available(self, cluster: str, pool: str) -> int: - return self.free.get((cluster, pool), 0) - def consume(self, cluster: str, pool: str, nodes: int) -> None: - self.free[(cluster, pool)] = self.available(cluster, pool) - nodes +def _pool_capacity(pool: icv1alpha1.GpuPool) -> _Pool: + """A pool's free capacity, before any replica is charged to it. + A node the pool hasn't got yet will have every device its class declares + free. Where the cluster agent reports on the pool, its capacity is the + agent's histogram plus the nodes the pool can still grow by. Where it + doesn't, it's every node the pool can have, taken whole. + """ + # Keyed as _match_member keys the requests it resolves. + full: dict[str, int] = {} + for d in pool.devices or []: + if (d.claim or _CLAIM_DRA) == _CLAIM_DRA: + key = d.deviceClassName or "" + full[key] = full.get(key, 0) + _published_count(d.count) + empty = tuple(sorted(full.items())) + nodes = _published_count(pool.nodes) + + if pool.observedNodes is None: + return _Pool(nodes={empty: nodes} if nodes else {}, whole_node=True) + + capacity = _Pool(nodes={}, whole_node=False) + for bucket in pool.nodesByFreeDevices or []: + free = tuple(sorted((bucket.freeDevices or {}).items())) + capacity.nodes[free] = capacity.nodes.get(free, 0) + bucket.nodes + growth = nodes - pool.observedNodes + if growth > 0: + capacity.nodes[empty] = capacity.nodes.get(empty, 0) + growth + return capacity -def _observed_member_cost(engine: mrv1alpha1.Engine, member: mrv1alpha1.Member) -> int: - """An observed replica member's node cost, from its stamped deviceRequests. - Mirrors _member_cost: a member that claims devices occupies its nodes; one - with no deviceRequests shares its gang's nodes and costs nothing. +@dataclass +class _Ledger: + """Free capacity per (cluster, pool). - This reads the cost the member was STAMPED with, not what it would resolve - to now. For a retained member whose pinned pool drifted across the - claimless/claimable boundary (e.g. a synthetic-only match that now resolves a - claim: DRA device, or vice versa) the stamped cost and the re-resolved cost - disagree for one reconcile: the ledger charges the stamped cost here while - the composed ModelReplica carries the re-resolved one. It reconverges on the - next reconcile once the new spec is observed, and never durably overcommits - (DRA admission on the workload cluster is the backstop), so we accept the - one-pass skew rather than couple ledger accounting to retain re-resolution. + Built by _build_ledger from each pool's published capacity, less the + replicas already committed to it (see there for exactly which). The fill + phase then places each new replica's pods against it, which is what stops a + single scheduling pass overcommitting a pool. """ - if not member.deviceRequests: - return 0 - return _member_pods(member) * int(engine.copies or 1) + + pools: dict[tuple[str, str], _Pool] + + +def _settled(replica: mrv1alpha1.ModelReplica, pool: icv1alpha1.GpuPool) -> bool: + """Whether a pool's histogram accounts for the devices a replica's pods hold on it. + + The cluster agent records when it first saw each claim allocated. It + observes a pool from a view of the cluster that includes every claim it saw + allocated no later than the time it publishes for the pool. So once all of a + replica's pods have scheduled, the pool's histogram accounts for their + devices if the replica's time for the pool is no later than the pool's. A + replica with no time for the pool hasn't been seen holding devices on it, and + a pool the agent doesn't report on has no time. + """ + status = replica.status + published = pool.lastAllocationTime + if status is None or published is None: + return False + if not any(c.type == _CONDITION_MODEL_PODS_SCHEDULED and c.status == "True" for c in status.conditions or []): + return False + return any(p.name == pool.name and p.lastAllocationTime <= published for p in status.pools or []) def _build_ledger( @@ -534,42 +635,35 @@ def _build_ledger( retained: list[Candidate], all_replicas: list[mrv1alpha1.ModelReplica], ) -> _Ledger: - """Compute free node capacity per (cluster, pool). + """Compute free capacity per (cluster, pool). - Starts from each pool's published node count and subtracts the nodes already - committed to it. A replica counts when it is either: + Starts from each pool's published capacity and charges it the pods of the + replicas committed to it. A replica counts when it is either: * another deployment's replica - capacity we don't control, or * one of THIS deployment's RETAINED replicas - a placement we're keeping. - It deliberately does NOT subtract this deployment's observed replicas that + It deliberately does NOT charge this deployment's observed replicas that were dropped from the retained set (cluster gone, or a pinned pool no longer - matches the nodeSelectors). Those are being deleted, so their nodes are + matches the nodeSelectors). Those are being deleted, so their devices are freeing up and must be available to the fill phase that re-places them - - otherwise re-placement (delete-old + create-new) could never converge. - - Every counted replica is charged per member at its OWN observed node cost - (derived from its spec.engines), to the pool that member is pinned to. A - member pinned to a pool the cluster still publishes is subtracted from that - pool. One pinned to a pool that's since been deleted is charged nothing - its - pods are unschedulable (Pending) and occupy no node (see `charge`). + otherwise re-placement (delete-old + create-new) could never converge. A + pool the cluster agent reports on still counts their devices until their + pods are gone, so re-placement onto it waits for them. + + A pool the agent reports on counts the devices of scheduled pods as taken, + so to avoid counting them twice it's charged only a replica's pods that its + histogram doesn't account for yet (see _settled). Every counted replica is charged one copy of each + engine at a time, each member at its OWN stamped claim, on the pool that + member is pinned to. A member pinned to a pool that's since been deleted is + charged nothing: its pods are hard-pinned (modelplane.ai/pool nodeSelector) + to a node label no node carries, so they hold nothing. Neither does a pod + that doesn't fit - it's Pending. """ - free: dict[tuple[str, str], int] = {} - for cluster in clusters: - name = _name(cluster.metadata) - for pool in _gpu_pools(cluster): - free[(name, pool.name or "")] = _published_count(pool.nodes) - - def charge(cluster_name: str, pool_name: str, nodes: int) -> None: - # Charge the member's nodes to the pool it's pinned to. A member whose - # pool isn't among the cluster's published pools is charged nothing: its - # pods are hard-pinned (modelplane.ai/pool nodeSelector) to a node label - # no node carries, so they're unschedulable and occupy no node. (A - # member's nodePoolName is XRD-required and always names a pool that was - # published when the replica was placed, so this is the deleted-pool - # case, not a missing pin.) - if (cluster_name, pool_name) in free: - free[(cluster_name, pool_name)] -= nodes + published = { + (_name(cluster.metadata), pool.name or ""): pool for cluster in clusters for pool in _gpu_pools(cluster) + } + pools = {key: _pool_capacity(pool) for key, pool in published.items()} # Identities (cluster, index) of the replicas we're keeping. retained_ids = {(c.name, c.index) for c in retained} @@ -579,16 +673,30 @@ def charge(cluster_name: str, pool_name: str, nodes: int) -> None: continue ours = _is_ours(r, deployment) # Skip our own replicas that aren't being retained: dropped (re-placed) - # ones are freeing their nodes, and scaled-down ones are going away. + # ones are freeing their devices, and scaled-down ones are going away. if ours and (r.spec.clusterName, _replica_index(r)) not in retained_ids: continue - for engine in r.spec.engines: + # Smallest pod first, as _place_engines places them. + engines = sorted( + r.spec.engines, + key=lambda e: max((sum(_observed_claim(m).values()) for m in e.members or []), default=0), + ) + for engine in engines: + # The pods of one copy of the engine, by the pool they're pinned to. + by_pool: dict[str, list[_Claim]] = {} for member in engine.members or []: - nodes = _observed_member_cost(engine, member) - if nodes: - charge(r.spec.clusterName, member.nodePoolName or "", nodes) + by_pool.setdefault(member.nodePoolName or "", []).extend( + [_observed_claim(member)] * _member_pods(member) + ) + for pool_name, pods in by_pool.items(): + key = (r.spec.clusterName, pool_name) + pool = pools.get(key) + if pool is None or _settled(r, published[key]): + continue + for _ in range(int(engine.copies or 1)): + pool.place(pods) - return _Ledger(free=free) + return _Ledger(pools=pools) def _retain( @@ -740,46 +848,53 @@ def _place_engines( cluster: icv1alpha1.InferenceCluster, engines: list[_CompiledEngine], ledger: _Ledger, -) -> list[EnginePlacement] | None: +) -> tuple[list[EnginePlacement], dict[str, _Pool]] | None: """Assign every member of one replica's engines to a pool on this cluster. Members are placed against a TRIAL copy of this cluster's free capacity so two members of the same replica don't double-book one pool. Returns one - EnginePlacement per engine (in deployment order) when every member fits, or - None when any member has no eligible pool - the replica can't be - co-scheduled here. - - Engines are placed in deployment order, each greedily taking the first - eligible pool(s). Like device matching within a pool, this is greedy without - backtracking, so it is not complete: when an earlier engine's selector - overlaps a pool a later engine also needs and capacity is tight, greedy can - consume that pool early and fail to place the later engine even though a - different assignment would have fit them both. It stays greedy because the - failure is safe - a false reject surfaces as InsufficientCapacity, never an - overcommit - and the common shapes don't hit it: a gang's members repeat one - nodeSelector (one pool), and disaggregated phases target disjoint hardware - (disjoint pools). Overlapping selectors across engines of one replica are the - case that can false-reject. + EnginePlacement per engine (in deployment order) and the trial capacity + with the replica's pods placed, when every member fits, or None when any + member has no eligible pool - the replica can't be co-scheduled here. + + Engines are placed smallest pod first, each greedily taking the first + eligible pool(s). Small pods then take the emptiest nodes, approximating the + worst order the cluster's scheduler could bind them in. A replica of + mixed-size pods placed on a better guess could get stuck partly scheduled, + holding GPUs it can't serve with. + + TODO(negz): Place largest pod first once we can recover a replica from a + bad placement. + + Like device matching within a pool, this is greedy without backtracking, so + it is not complete: when an earlier engine's selector overlaps a pool a later + engine also needs and capacity is tight, greedy can consume that pool early + and fail to place the later engine even though a different assignment would + have fit them both. A search wouldn't help much, because the cluster's + scheduler doesn't search either: it binds one pod at a time. """ cluster_name = _name(cluster.metadata) - # Trial free counts for this cluster's pools, decremented as we place each - # member so a later member sees capacity an earlier one took. Discarded - # wholesale when any member fails to place, so partial placement never - # leaks into the real ledger. - trial = {(pool.name or ""): ledger.available(cluster_name, pool.name or "") for pool in _gpu_pools(cluster)} - placements: list[EnginePlacement] = [] - for engine in engines: + # Discarded wholesale when any member fails to place, so partial placement + # never leaks into the real ledger. + trial = {(pool.name or ""): ledger.pools[(cluster_name, pool.name or "")].copy() for pool in _gpu_pools(cluster)} + placed: dict[str, EnginePlacement] = {} + for engine in sorted(engines, key=_largest_pod): members = _place_engine(cluster, engine, trial) if members is None: return None - placements.append(EnginePlacement(name=engine.name, members=members)) - return placements + placed[engine.name] = EnginePlacement(name=engine.name, members=members) + return [placed[e.name] for e in engines], trial + + +def _largest_pod(engine: _CompiledEngine) -> int: + """How many devices the largest of an engine's pods requests, on any pool.""" + return max((sum(r.count for r in m.requests) for m in engine.members), default=0) def _place_engine( cluster: icv1alpha1.InferenceCluster, engine: _CompiledEngine, - trial: dict[str, int], + trial: dict[str, _Pool], ) -> list[MemberPlacement] | None: """Place a whole engine on a single pool of this cluster, or reject it. @@ -799,14 +914,15 @@ def _place_engine( A pool hosts the engine when every member matches it, no member that could claim a real device elsewhere is stranded on a synthetic-only match, the engine has at least one member that resolves a claim: DRA device (or its - pods would have no ResourceClaim to bind GPUs through), and the pool has free - nodes for the engine's whole cost. A member with no nodeSelector claims - nothing, matches every pool, and rides along on the engine's pool at zero - node cost. Returns one MemberPlacement per member, or None when no single - pool can host the engine. - - Decrements trial as it places; on failure the caller discards the whole - trial, so partial decrements don't leak. + pods would have no ResourceClaim to bind GPUs through), and the pool has + room for every one of the engine's pods. A member with no nodeSelector + claims nothing, matches every pool, and rides along on the engine's pool, + because a coordinator on another pool could be in a different zone or fabric + from the workers it coordinates. Returns one MemberPlacement per member, or + None when no single pool can host the engine. + + Places pods on trial as it goes; on failure the caller discards the whole + trial, so partial placement doesn't leak. """ pools = _gpu_pools(cluster) # Whether each member's requests resolve a claimable device on any pool of @@ -823,11 +939,11 @@ def _place_engine( members = _match_members(pool, engine, claimable) if members is None: continue - cost = sum(_member_cost(m, p.device_requests) for m, p in zip(engine.members, members, strict=True)) - if trial[pool_name] < cost: - continue - trial[pool_name] -= cost - return members + capacity = trial[pool_name].copy() + pods = [_claim(p.device_requests) for m, p in zip(engine.members, members, strict=True) for _ in range(m.pods)] + if all(capacity.place(pods) for _ in range(engine.copies)): + trial[pool_name] = capacity + return members return None @@ -864,11 +980,10 @@ def _fill( replicas so far (spread), breaking ties by cluster name for determinism. A second replica lands on a cluster only once every other eligible cluster already has its share; when capacity forces it, replicas pack onto fewer - clusters. Each placement decrements the ledger (per member, on the pools the - members took) and takes the lowest free index on its chosen cluster, so the - next iteration sees the updated load. Stops early (placing fewer than n) - when no cluster can host another replica - the caller surfaces that as - InsufficientCapacity. + clusters. Each placement commits its pods to the ledger and takes the lowest + free index on its chosen cluster, so the next iteration sees the updated + load. Stops early (placing fewer than n) when no cluster can host another + replica - the caller surfaces that as InsufficientCapacity. """ # Per-cluster load and used indices seeded from retained replicas, so spread # accounts for what's already there and new indices don't collide. @@ -883,7 +998,7 @@ def _fill( choice = _pick_cluster(engines, clusters, load, ledger) if choice is None: break - cluster, placements = choice + cluster, placements, trial = choice name = _name(cluster.metadata) index = _lowest_free_index(used_indices.setdefault(name, set())) @@ -899,46 +1014,40 @@ def _fill( load[name] = load.get(name, 0) + 1 used_indices[name].add(index) - for ep in placements: - engine = _engine_by_name(engines, ep.name) - for member, mp in zip(engine.members, ep.members, strict=True): - ledger.consume(name, mp.pool, _member_cost(member, mp.device_requests)) + for pool_name, capacity in trial.items(): + ledger.pools[(name, pool_name)] = capacity return placed -def _engine_by_name(engines: list[_CompiledEngine], name: str) -> _CompiledEngine: - """The compiled engine with this name. Always present for a placement we built.""" - return next(g for g in engines if g.name == name) - - def _pick_cluster( engines: list[_CompiledEngine], clusters: list[icv1alpha1.InferenceCluster], load: dict[str, int], ledger: _Ledger, -) -> tuple[icv1alpha1.InferenceCluster, list[EnginePlacement]] | None: +) -> tuple[icv1alpha1.InferenceCluster, list[EnginePlacement], dict[str, _Pool]] | None: """Pick the eligible cluster hosting the fewest of this deployment's replicas. Eligible means Ready and able to co-schedule every member on its pools given the ledger. The chosen key is (load on the cluster, cluster name): fewest replicas first for spread, name for a deterministic tiebreak. load already counts only this deployment's replicas (seeded from retained plus those - placed earlier in the pass). Returns (cluster, engine placements) or None - when no cluster is eligible. + placed earlier in the pass). Returns (cluster, engine placements, the + cluster's capacity with the replica placed) or None when no cluster is + eligible. """ best = None best_key = None for cluster in clusters: if not _cluster_ready(cluster): continue - placements = _place_engines(cluster, engines, ledger) - if placements is None: + placed = _place_engines(cluster, engines, ledger) + if placed is None: continue key = (load.get(_name(cluster.metadata), 0), _name(cluster.metadata)) if best_key is None or key < best_key: best_key = key - best = (cluster, placements) + best = (cluster, *placed) return best @@ -1043,8 +1152,8 @@ def schedule( if fill and len(retained) < desired: # Build the ledger AFTER retain and scale-down: it charges the replicas # in the final `retained` set (plus other deployments' replicas), and - # must not charge our dropped or scaled-down replicas, whose nodes are - # freeing up. Fill then decrements it only as it places NEW replicas. + # must not charge our dropped or scaled-down replicas, whose devices + # are freeing up. Fill then charges it only as it places NEW replicas. ledger = _build_ledger(deployment, clusters, retained, all_replicas) # New replicas avoid clusters carrying an untolerated taint (NoSchedule # or NoExecute). The ledger still spans every cluster, so a tainted diff --git a/functions/compose-model-deployment/tests/test_scheduling.py b/functions/compose-model-deployment/tests/test_scheduling.py index 560dee16c..44d54fe17 100644 --- a/functions/compose-model-deployment/tests/test_scheduling.py +++ b/functions/compose-model-deployment/tests/test_scheduling.py @@ -19,8 +19,10 @@ logic without the protobuf/gRPC ceremony of the fn tests. Pool selection is driven by nodeSelector device requests (DRA CEL matched -against a pool's devices) plus the available-node gate. Per-node GPU count is -expressed as a device request's count, not derived from topology. +against a pool's devices) plus the pool's free capacity. Per-node GPU count is +expressed as a device request's count, not derived from topology. A pool with +no observedNodes is one the cluster agent doesn't report on, so its capacity is +counted in whole nodes. """ import dataclasses @@ -42,6 +44,8 @@ _MEM_200 = 'device.capacity["gpu.nvidia.com"].memory.compareTo(quantity("200Gi")) >= 0' _MEM_LT_200 = 'device.capacity["gpu.nvidia.com"].memory.compareTo(quantity("200Gi")) < 0' _IB = 'device.attributes["nic.nvidia.com"].linkType == "infiniband"' +_MEM_20 = 'device.capacity["gpu.nvidia.com"].memory.compareTo(quantity("20Gi")) >= 0' +_EFA = 'device.driver == "dra.net"' _Role = Literal["Standalone", "Leader", "Worker"] @@ -74,17 +78,18 @@ def _member(*, role: _Role, worker_nodes: int | None, devices: list[mdv1alpha1.D def _deployment( *, replicas: int, + copies: int, members: list[mdv1alpha1.Member], tolerations: list[mdv1alpha1.Toleration] | None, ) -> mdv1alpha1.ModelDeployment: - """The ModelDeployment my-model, whose one engine, main, has the given members.""" + """The ModelDeployment my-model, whose one engine, main, runs copies of the given members.""" return mdv1alpha1.ModelDeployment( metadata=metav1.ObjectMeta(name="my-model", namespace="ml-team"), spec=mdv1alpha1.SpecModel2( replicas=replicas, template=mdv1alpha1.TemplateModel1( spec=mdv1alpha1.SpecModel1( - engines=[mdv1alpha1.Engine(name="main", copies=1, members=members)], + engines=[mdv1alpha1.Engine(name="main", copies=copies, members=members)], tolerations=tolerations, ), ), @@ -177,9 +182,22 @@ def _replica_member( def _replica( - *, name: str, deployment: str, cluster: str, index: int, members: list[mrv1alpha1.Member] + *, + name: str, + deployment: str, + cluster: str, + index: int, + copies: int, + pods_scheduled: bool, + pools: list[mrv1alpha1.Pool] | None, + members: list[mrv1alpha1.Member], ) -> mrv1alpha1.ModelReplica: - """An observed ModelReplica of deployment, pinned to (cluster, index), whose one engine, main, has members.""" + """An observed ModelReplica of deployment, pinned to (cluster, index), whose one engine, main, has members. + + Its pods have all scheduled if pods_scheduled, and the cluster agent saw + their claims allocated on pools. It has no status if neither, like a replica + that's just been created. + """ return mrv1alpha1.ModelReplica( metadata=metav1.ObjectMeta( name=name, @@ -191,7 +209,22 @@ def _replica( }, ), spec=mrv1alpha1.SpecModel1( - clusterName=cluster, engines=[mrv1alpha1.Engine(name="main", copies=1, members=members)] + clusterName=cluster, engines=[mrv1alpha1.Engine(name="main", copies=copies, members=members)] + ), + status=( + mrv1alpha1.Status( + conditions=[ + mrv1alpha1.Condition( + type="ModelPodsScheduled", + status="True" if pods_scheduled else "False", + reason="AllPodsScheduled" if pods_scheduled else "SchedulingPending", + lastTransitionTime=datetime.datetime(2026, 10, 1, 12, 0, tzinfo=datetime.UTC), + ) + ], + pools=pools, + ) + if pods_scheduled or pools is not None + else None ), ) @@ -223,6 +256,7 @@ def _candidate( reason="With no clusters, schedule places nothing.", deployment=_deployment( replicas=1, + copies=1, members=[ _member( role="Standalone", @@ -242,6 +276,7 @@ def _candidate( reason="A new replica lands on the one ready cluster, with no placement labels because the cluster declares none.", deployment=_deployment( replicas=1, + copies=1, members=[ _member( role="Standalone", @@ -295,6 +330,7 @@ def _candidate( reason="A cluster that isn't Ready gets no new replica.", deployment=_deployment( replicas=1, + copies=1, members=[ _member( role="Standalone", @@ -327,6 +363,7 @@ def _candidate( reason="A cluster without a gateway hostname gets no new replica.", deployment=_deployment( replicas=1, + copies=1, members=[ _member( role="Standalone", @@ -359,6 +396,7 @@ def _candidate( reason="A Leader and a three-node Worker need four nodes, so a two-node pool hosts no replica.", deployment=_deployment( replicas=1, + copies=1, members=[ _member( role="Leader", @@ -396,6 +434,7 @@ def _candidate( reason="An existing replica on cluster-a stays there with its resolved pool and requests, where fresh placement would also put it as the lower name.", deployment=_deployment( replicas=1, + copies=1, members=[ _member( role="Standalone", @@ -437,6 +476,9 @@ def _candidate( deployment="my-model", cluster="cluster-a", index=0, + copies=1, + pods_scheduled=False, + pools=None, members=[ _replica_member( role="Standalone", @@ -483,6 +525,7 @@ def _candidate( reason="A replica stays on a cluster that isn't Ready and has no gateway hostname, and its candidate's gateway hostname is empty.", deployment=_deployment( replicas=1, + copies=1, members=[ _member( role="Standalone", @@ -512,6 +555,9 @@ def _candidate( deployment="my-model", cluster="cluster-a", index=0, + copies=1, + pods_scheduled=False, + pools=None, members=[ _replica_member( role="Standalone", @@ -558,6 +604,7 @@ def _candidate( reason="A replica whose cluster no longer exists is re-placed on another cluster.", deployment=_deployment( replicas=1, + copies=1, members=[ _member( role="Standalone", @@ -587,6 +634,9 @@ def _candidate( deployment="my-model", cluster="cluster-a", index=0, + copies=1, + pods_scheduled=False, + pools=None, members=[ _replica_member( role="Standalone", @@ -633,6 +683,7 @@ def _candidate( reason="Scaling from one replica to two places the new one on the cluster that has none.", deployment=_deployment( replicas=2, + copies=1, members=[ _member( role="Standalone", @@ -674,6 +725,9 @@ def _candidate( deployment="my-model", cluster="cluster-a", index=0, + copies=1, + pods_scheduled=False, + pools=None, members=[ _replica_member( role="Standalone", @@ -740,6 +794,7 @@ def _candidate( reason="Scaling up when the retained replica fills the only one-node pool places no new replica.", deployment=_deployment( replicas=2, + copies=1, members=[ _member( role="Standalone", @@ -769,6 +824,9 @@ def _candidate( deployment="my-model", cluster="cluster-a", index=0, + copies=1, + pods_scheduled=False, + pools=None, members=[ _replica_member( role="Standalone", @@ -815,6 +873,7 @@ def _candidate( reason="With one cluster to place on, two replicas pack onto it at indices 0 and 1.", deployment=_deployment( replicas=2, + copies=1, members=[ _member( role="Standalone", @@ -888,6 +947,7 @@ def _candidate( reason="Two replicas spread one to each of two clusters, though either cluster could hold both.", deployment=_deployment( replicas=2, + copies=1, members=[ _member( role="Standalone", @@ -973,6 +1033,7 @@ def _candidate( reason="Three replicas spread across two clusters, then the third lands on cluster-a, the lower name at equal load.", deployment=_deployment( replicas=3, + copies=1, members=[ _member( role="Standalone", @@ -1078,6 +1139,7 @@ def _candidate( reason="Three replicas spread one to each cluster, filling cluster-b's one-node pool, and the third lands on cluster-a, which the name tiebreak picks whether or not cluster-b is full.", deployment=_deployment( replicas=3, + copies=1, members=[ _member( role="Standalone", @@ -1183,6 +1245,7 @@ def _candidate( reason="A new replica goes to empty cluster-b rather than doubling up on cluster-a, which already hosts one.", deployment=_deployment( replicas=2, + copies=1, members=[ _member( role="Standalone", @@ -1224,6 +1287,9 @@ def _candidate( deployment="my-model", cluster="cluster-a", index=0, + copies=1, + pods_scheduled=False, + pools=None, members=[ _replica_member( role="Standalone", @@ -1290,6 +1356,7 @@ def _candidate( reason="A new replica on a cluster that hosts indices 0 and 2 takes the free index 1.", deployment=_deployment( replicas=3, + copies=1, members=[ _member( role="Standalone", @@ -1319,6 +1386,9 @@ def _candidate( deployment="my-model", cluster="cluster-a", index=0, + copies=1, + pods_scheduled=False, + pools=None, members=[ _replica_member( role="Standalone", @@ -1340,6 +1410,9 @@ def _candidate( deployment="my-model", cluster="cluster-a", index=2, + copies=1, + pods_scheduled=False, + pools=None, members=[ _replica_member( role="Standalone", @@ -1426,6 +1499,7 @@ def _candidate( reason="Scaling three replicas to two drops cluster-a's index 1, keeping one replica on each cluster.", deployment=_deployment( replicas=2, + copies=1, members=[ _member( role="Standalone", @@ -1467,6 +1541,9 @@ def _candidate( deployment="my-model", cluster="cluster-a", index=0, + copies=1, + pods_scheduled=False, + pools=None, members=[ _replica_member( role="Standalone", @@ -1488,6 +1565,9 @@ def _candidate( deployment="my-model", cluster="cluster-a", index=1, + copies=1, + pods_scheduled=False, + pools=None, members=[ _replica_member( role="Standalone", @@ -1509,6 +1589,9 @@ def _candidate( deployment="my-model", cluster="cluster-b", index=0, + copies=1, + pods_scheduled=False, + pools=None, members=[ _replica_member( role="Standalone", @@ -1579,6 +1662,7 @@ def _candidate( reason="A retained replica costs the two nodes it was placed with, not the deployment's new four, so a second replica fits the six-node pool.", deployment=_deployment( replicas=2, + copies=1, members=[ _member( role="Leader", @@ -1613,6 +1697,9 @@ def _candidate( deployment="my-model", cluster="cluster-a", index=0, + copies=1, + pods_scheduled=False, + pools=None, members=[ _replica_member( role="Leader", @@ -1720,6 +1807,7 @@ def _candidate( reason="Scaling three replicas to two drops cluster-a's second replica rather than cluster-b's only one.", deployment=_deployment( replicas=2, + copies=1, members=[ _member( role="Standalone", @@ -1761,6 +1849,9 @@ def _candidate( deployment="my-model", cluster="cluster-a", index=0, + copies=1, + pods_scheduled=False, + pools=None, members=[ _replica_member( role="Standalone", @@ -1782,6 +1873,9 @@ def _candidate( deployment="my-model", cluster="cluster-a", index=1, + copies=1, + pods_scheduled=False, + pools=None, members=[ _replica_member( role="Standalone", @@ -1803,6 +1897,9 @@ def _candidate( deployment="my-model", cluster="cluster-b", index=3, + copies=1, + pods_scheduled=False, + pools=None, members=[ _replica_member( role="Standalone", @@ -1869,6 +1966,7 @@ def _candidate( reason="Two existing replicas on cluster-a, the only cluster, stay at a/0 and a/1, where fresh placement would also put them.", deployment=_deployment( replicas=2, + copies=1, members=[ _member( role="Standalone", @@ -1898,6 +1996,9 @@ def _candidate( deployment="my-model", cluster="cluster-a", index=0, + copies=1, + pods_scheduled=False, + pools=None, members=[ _replica_member( role="Standalone", @@ -1919,6 +2020,9 @@ def _candidate( deployment="my-model", cluster="cluster-a", index=1, + copies=1, + pods_scheduled=False, + pools=None, members=[ _replica_member( role="Standalone", @@ -1985,6 +2089,7 @@ def _candidate( reason="Scaling two replicas at index 0 down to one keeps cluster-a's, the lower name.", deployment=_deployment( replicas=1, + copies=1, members=[ _member( role="Standalone", @@ -2026,6 +2131,9 @@ def _candidate( deployment="my-model", cluster="cluster-b", index=0, + copies=1, + pods_scheduled=False, + pools=None, members=[ _replica_member( role="Standalone", @@ -2047,6 +2155,9 @@ def _candidate( deployment="my-model", cluster="cluster-a", index=0, + copies=1, + pods_scheduled=False, + pools=None, members=[ _replica_member( role="Standalone", @@ -2093,6 +2204,7 @@ def _candidate( reason="New replicas go to clusters in name order, whatever order the clusters arrive in.", deployment=_deployment( replicas=2, + copies=1, members=[ _member( role="Standalone", @@ -2190,6 +2302,7 @@ def _candidate( reason="Another deployment's replica fills the only node, leaving no room for a new replica.", deployment=_deployment( replicas=1, + copies=1, members=[ _member( role="Standalone", @@ -2219,6 +2332,9 @@ def _candidate( deployment="other-model", cluster="cluster-a", index=0, + copies=1, + pods_scheduled=False, + pools=None, members=[ _replica_member( role="Standalone", @@ -2244,6 +2360,7 @@ def _candidate( reason="An existing replica filling cluster-a's one-node pool stays there, and with the deployment already at its one replica, schedule builds no capacity ledger that could charge it.", deployment=_deployment( replicas=1, + copies=1, members=[ _member( role="Standalone", @@ -2273,6 +2390,9 @@ def _candidate( deployment="my-model", cluster="cluster-a", index=0, + copies=1, + pods_scheduled=False, + pools=None, members=[ _replica_member( role="Standalone", @@ -2322,6 +2442,7 @@ def _candidate( reason="Another deployment's replica pinned to a pool the cluster no longer publishes uses no node, so a new replica takes the free one.", deployment=_deployment( replicas=1, + copies=1, members=[ _member( role="Standalone", @@ -2353,6 +2474,9 @@ def _candidate( deployment="other-model", cluster="cluster-a", index=0, + copies=1, + pods_scheduled=False, + pools=None, members=[ _replica_member( role="Standalone", @@ -2402,6 +2526,7 @@ def _candidate( reason="Of two replicas on the same cluster and index, the one whose name sorts first is retained, whatever their input order.", deployment=_deployment( replicas=1, + copies=1, members=[ _member( role="Standalone", @@ -2434,6 +2559,9 @@ def _candidate( deployment="my-model", cluster="cluster-a", index=0, + copies=1, + pods_scheduled=False, + pools=None, members=[ _replica_member( role="Standalone", @@ -2455,6 +2583,9 @@ def _candidate( deployment="my-model", cluster="cluster-a", index=0, + copies=1, + pods_scheduled=False, + pools=None, members=[ _replica_member( role="Standalone", @@ -2505,6 +2636,7 @@ def _candidate( reason="Without fill and with no existing replicas, nothing is placed.", deployment=_deployment( replicas=1, + copies=1, members=[ _member( role="Standalone", @@ -2537,6 +2669,7 @@ def _candidate( reason="Without fill, an existing replica is still retained.", deployment=_deployment( replicas=1, + copies=1, members=[ _member( role="Standalone", @@ -2566,6 +2699,9 @@ def _candidate( deployment="my-model", cluster="cluster-a", index=0, + copies=1, + pods_scheduled=False, + pools=None, members=[ _replica_member( role="Standalone", @@ -2612,6 +2748,7 @@ def _candidate( reason="Without fill, scaling to three replicas keeps the existing one and places no more.", deployment=_deployment( replicas=3, + copies=1, members=[ _member( role="Standalone", @@ -2653,6 +2790,9 @@ def _candidate( deployment="my-model", cluster="cluster-a", index=0, + copies=1, + pods_scheduled=False, + pools=None, members=[ _replica_member( role="Standalone", @@ -2700,6 +2840,7 @@ def _candidate( reason="A device request that matches a pool's GPU places the replica in that pool.", deployment=_deployment( replicas=1, + copies=1, members=[ _member( role="Standalone", @@ -2755,6 +2896,7 @@ def _candidate( reason="A request for a 200Gi GPU matches no 141Gi GPU, so nothing is placed.", deployment=_deployment( replicas=1, + copies=1, members=[ _member( role="Standalone", @@ -2789,6 +2931,7 @@ def _candidate( reason="A request for 8 GPUs doesn't fit a device with only 4, so nothing is placed.", deployment=_deployment( replicas=1, + copies=1, members=[ _member( role="Standalone", @@ -2826,6 +2969,7 @@ def _candidate( reason="A device published with count 0 satisfies no request, so nothing is placed.", deployment=_deployment( replicas=1, + copies=1, members=[ _member( role="Standalone", @@ -2860,6 +3004,7 @@ def _candidate( reason="A pool with a matching GPU but no nodes, say one autoscaled to zero, hosts no replica.", deployment=_deployment( replicas=1, + copies=1, members=[ _member( role="Standalone", @@ -2894,6 +3039,7 @@ def _candidate( reason="A synthetic NIC counts toward matching a pool but, unlike the DRA GPU, isn't among the resolved requests.", deployment=_deployment( replicas=1, + copies=1, members=[ _member( role="Standalone", @@ -2955,6 +3101,7 @@ def _candidate( reason="A pool without the NIC a request selects hosts no replica.", deployment=_deployment( replicas=1, + copies=1, members=[ _member( role="Standalone", @@ -2993,6 +3140,7 @@ def _candidate( reason="Two requests can't both claim one GPU device of count 1, so the pool doesn't match.", deployment=_deployment( replicas=1, + copies=1, members=[ _member( role="Standalone", @@ -3032,6 +3180,7 @@ def _candidate( reason="Two requests for 5 GPUs each need 10, more than the device's 8, so the pool doesn't match.", deployment=_deployment( replicas=1, + copies=1, members=[ _member( role="Standalone", @@ -3069,6 +3218,7 @@ def _candidate( reason="Two requests for 4 GPUs each fit a device of 8, and both resolve.", deployment=_deployment( replicas=1, + copies=1, members=[ _member( role="Standalone", @@ -3133,6 +3283,7 @@ def _candidate( reason="Of two pools with a GPU, only the one with an infiniband NIC matches, so it's picked though it's listed second.", deployment=_deployment( replicas=1, + copies=1, members=[ _member( role="Standalone", @@ -3204,6 +3355,7 @@ def _candidate( reason="A request that matches only a synthetic NIC leaves nothing to claim, so nothing is placed.", deployment=_deployment( replicas=1, + copies=1, members=[ _member( role="Standalone", @@ -3241,6 +3393,7 @@ def _candidate( reason="A replica pinned to the cluster's only pool stays on it, where fresh placement would also put it.", deployment=_deployment( replicas=1, + copies=1, members=[ _member( role="Standalone", @@ -3272,6 +3425,9 @@ def _candidate( deployment="my-model", cluster="cluster-a", index=0, + copies=1, + pods_scheduled=False, + pools=None, members=[ _replica_member( role="Standalone", @@ -3320,6 +3476,7 @@ def _candidate( reason="A replica pinned to a pool its selector no longer matches is re-placed onto the pool that matches.", deployment=_deployment( replicas=1, + copies=1, members=[ _member( role="Standalone", @@ -3365,6 +3522,9 @@ def _candidate( deployment="my-model", cluster="cluster-a", index=0, + copies=1, + pods_scheduled=False, + pools=None, members=[ _replica_member( role="Standalone", @@ -3411,6 +3571,7 @@ def _candidate( reason="A replica stays on pool a while that pool still matches, where fresh placement would also put it as the first of two matching pools.", deployment=_deployment( replicas=1, + copies=1, members=[ _member( role="Standalone", @@ -3456,6 +3617,9 @@ def _candidate( deployment="my-model", cluster="cluster-a", index=0, + copies=1, + pods_scheduled=False, + pools=None, members=[ _replica_member( role="Standalone", @@ -3502,6 +3666,7 @@ def _candidate( reason="A replica whose pool no longer matches is dropped when no other pool matches either.", deployment=_deployment( replicas=1, + copies=1, members=[ _member( role="Standalone", @@ -3539,6 +3704,9 @@ def _candidate( deployment="my-model", cluster="cluster-a", index=0, + copies=1, + pods_scheduled=False, + pools=None, members=[ _replica_member( role="Standalone", @@ -3564,6 +3732,7 @@ def _candidate( reason="A replica pinned to a pool the cluster no longer publishes is re-placed onto one that matches.", deployment=_deployment( replicas=1, + copies=1, members=[ _member( role="Standalone", @@ -3601,6 +3770,9 @@ def _candidate( deployment="my-model", cluster="cluster-a", index=0, + copies=1, + pods_scheduled=False, + pools=None, members=[ _replica_member( role="Standalone", @@ -3651,6 +3823,7 @@ def _candidate( reason="A replica dropped for its unpublished pool is re-placed at index 1 on frontier, taking the node a/0 leaves free in the two-node pool.", deployment=_deployment( replicas=2, + copies=1, members=[ _member( role="Standalone", @@ -3682,6 +3855,9 @@ def _candidate( deployment="my-model", cluster="cluster-a", index=0, + copies=1, + pods_scheduled=False, + pools=None, members=[ _replica_member( role="Standalone", @@ -3703,6 +3879,9 @@ def _candidate( deployment="my-model", cluster="cluster-a", index=1, + copies=1, + pods_scheduled=False, + pools=None, members=[ _replica_member( role="Standalone", @@ -3769,6 +3948,7 @@ def _candidate( reason="A request for 8 GPUs pins to pool b, which has 8, rather than counting pool a's 4 toward a cluster-wide sum.", deployment=_deployment( replicas=1, + copies=1, members=[ _member( role="Standalone", @@ -3827,6 +4007,7 @@ def _candidate( reason="A leader that matches both pools and a worker that matches only big both go on big.", deployment=_deployment( replicas=1, + copies=1, members=[ _member( role="Leader", @@ -3902,6 +4083,7 @@ def _candidate( reason="A leader that fits only big and a worker that fits only small leave the replica unplaced.", deployment=_deployment( replicas=1, + copies=1, members=[ _member( role="Leader", @@ -3942,6 +4124,7 @@ def _candidate( reason="A two-node gang whose only matching pool has one node goes unplaced.", deployment=_deployment( replicas=1, + copies=1, members=[ _member( role="Leader", @@ -3982,6 +4165,7 @@ def _candidate( reason="A two-node gang too big for cluster-a's matching pool lands whole on cluster-b.", deployment=_deployment( replicas=1, + copies=1, members=[ _member( role="Leader", @@ -4069,6 +4253,7 @@ def _candidate( reason="A leader that matches only a synthetic device in pool a goes with its gang to pool b, where it can claim a GPU.", deployment=_deployment( replicas=1, + copies=1, members=[ _member( role="Leader", @@ -4149,6 +4334,7 @@ def _candidate( reason="A leader that matches only the pool's synthetic NIC places claimless alongside its claiming worker.", deployment=_deployment( replicas=1, + copies=1, members=[ _member( role="Leader", @@ -4217,6 +4403,7 @@ def _candidate( reason="A worker that matches no pool leaves the whole replica unplaced.", deployment=_deployment( replicas=1, + copies=1, members=[ _member( role="Leader", @@ -4251,9 +4438,10 @@ def _candidate( ), Case( name="ClaimlessLeader", - reason="A leader with no nodeSelector follows the worker's pool at no node cost, so the gang fits a one-node pool.", + reason="A leader with no nodeSelector follows the worker's pool without taking capacity, so the gang fits a one-node pool.", deployment=_deployment( replicas=1, + copies=1, members=[ _member(role="Leader", worker_nodes=None, devices=None), _member( @@ -4315,6 +4503,7 @@ def _candidate( reason="A replica pinned to the cluster's only pool, claimless leader and worker alike, stays there, where fresh placement would also put both members.", deployment=_deployment( replicas=1, + copies=1, members=[ _member(role="Leader", worker_nodes=None, devices=None), _member( @@ -4347,6 +4536,9 @@ def _candidate( deployment="my-model", cluster="cluster-a", index=0, + copies=1, + pods_scheduled=False, + pools=None, members=[ _replica_member(role="Leader", worker_nodes=None, pool="frontier", device_requests=None), _replica_member( @@ -4401,6 +4593,7 @@ def _candidate( reason="Another deployment's claimless leader uses no node, leaving one of two free for a new replica.", deployment=_deployment( replicas=1, + copies=1, members=[ _member( role="Standalone", @@ -4430,6 +4623,9 @@ def _candidate( deployment="other-model", cluster="cluster-a", index=0, + copies=1, + pods_scheduled=False, + pools=None, members=[ _replica_member(role="Leader", worker_nodes=None, pool="default", device_requests=None), _replica_member( @@ -4477,6 +4673,7 @@ def _candidate( reason="A replica placed as one Standalone member is re-placed when the deployment grows a Worker.", deployment=_deployment( replicas=1, + copies=1, members=[ _member( role="Leader", @@ -4511,6 +4708,9 @@ def _candidate( deployment="my-model", cluster="cluster-a", index=0, + copies=1, + pods_scheduled=False, + pools=None, members=[ _replica_member( role="Standalone", @@ -4573,6 +4773,7 @@ def _candidate( reason="A NoSchedule taint keeps a new replica off cluster-a, so it lands on cluster-b.", deployment=_deployment( replicas=1, + copies=1, members=[ _member( role="Standalone", @@ -4638,6 +4839,7 @@ def _candidate( reason="A NoSchedule taint leaves an existing replica on its cluster.", deployment=_deployment( replicas=1, + copies=1, members=[ _member( role="Standalone", @@ -4667,6 +4869,9 @@ def _candidate( deployment="my-model", cluster="cluster-a", index=0, + copies=1, + pods_scheduled=False, + pools=None, members=[ _replica_member( role="Standalone", @@ -4713,6 +4918,7 @@ def _candidate( reason="A toleration for a cluster's taint lets a new replica land there.", deployment=_deployment( replicas=1, + copies=1, members=[ _member( role="Standalone", @@ -4766,6 +4972,7 @@ def _candidate( reason="A NoExecute taint drains an existing replica, which is rescheduled onto an untainted cluster.", deployment=_deployment( replicas=1, + copies=1, members=[ _member( role="Standalone", @@ -4807,6 +5014,9 @@ def _candidate( deployment="my-model", cluster="cluster-a", index=0, + copies=1, + pods_scheduled=False, + pools=None, members=[ _replica_member( role="Standalone", @@ -4853,6 +5063,7 @@ def _candidate( reason="A toleration for cluster-a's NoExecute taint leaves the replica at cluster-a/0, as it would be even if retain drained it, since fill honours the toleration too.", deployment=_deployment( replicas=1, + copies=1, members=[ _member( role="Standalone", @@ -4882,6 +5093,9 @@ def _candidate( deployment="my-model", cluster="cluster-a", index=0, + copies=1, + pods_scheduled=False, + pools=None, members=[ _replica_member( role="Standalone", @@ -4929,6 +5143,7 @@ def _candidate( reason="A NoExecute taint drains the only replica, and with no other cluster nothing replaces it.", deployment=_deployment( replicas=1, + copies=1, members=[ _member( role="Standalone", @@ -4958,6 +5173,9 @@ def _candidate( deployment="my-model", cluster="cluster-a", index=0, + copies=1, + pods_scheduled=False, + pools=None, members=[ _replica_member( role="Standalone", @@ -4983,6 +5201,7 @@ def _candidate( reason="A NoSchedule toleration doesn't cover a NoExecute taint with the same key, so the replica is drained to cluster-b.", deployment=_deployment( replicas=1, + copies=1, members=[ _member( role="Standalone", @@ -5026,6 +5245,9 @@ def _candidate( deployment="my-model", cluster="cluster-a", index=0, + copies=1, + pods_scheduled=False, + pools=None, members=[ _replica_member( role="Standalone", @@ -5072,6 +5294,7 @@ def _candidate( reason="Tolerating one of a cluster's two taints isn't enough, so a new replica lands on cluster-b.", deployment=_deployment( replicas=1, + copies=1, members=[ _member( role="Standalone", @@ -5140,6 +5363,7 @@ def _candidate( reason="An Equal toleration whose key and value match the taint's lets a new replica land.", deployment=_deployment( replicas=1, + copies=1, members=[ _member( role="Standalone", @@ -5193,6 +5417,7 @@ def _candidate( reason="An Equal toleration whose value differs from the taint's keeps a new replica off.", deployment=_deployment( replicas=1, + copies=1, members=[ _member( role="Standalone", @@ -5225,6 +5450,7 @@ def _candidate( reason="An Exists toleration with no key tolerates both of a cluster's taints.", deployment=_deployment( replicas=1, + copies=1, members=[ _member( role="Standalone", @@ -5286,6 +5512,7 @@ def _candidate( reason="A cluster's placement labels reach the candidate placed there.", deployment=_deployment( replicas=1, + copies=1, members=[ _member( role="Standalone", @@ -5334,6 +5561,1435 @@ def _candidate( ), ], ), + # Pools the cluster agent reports on. The scheduler fits each pod against + # the agent's histogram of the pool's nodes by free devices, rather than + # charging it a whole node. + # + # gpu-l4x4-a is a pool of one g6.12xlarge node, four L4s, three of them + # allocated, as the agent reported it. + Case( + name="ObservedPoolFits", + reason="A replica claiming one GPU fits a node of an observed pool with one GPU free.", + deployment=_deployment( + replicas=1, + copies=1, + members=[ + _member( + role="Standalone", + worker_nodes=None, + devices=[mdv1alpha1.Device(name="gpu", count=1, selectors=[mdv1alpha1.Selector(cel=_MEM_20)])], + ) + ], + tolerations=None, + ), + clusters=[ + _cluster( + name="cluster-a", + gateway_hostname="cluster-a.clusters.example.com", + ready=True, + pools=[ + icv1alpha1.GpuPool( + name="gpu-l4x4-a", + nodes=1, + observedNodes=1, + nodesByFreeDevices=[ + icv1alpha1.NodesByFreeDevice(freeDevices={"gpu.nvidia.com": 1}, nodes=1), + ], + devices=[_gpu_device(name="gpu", claim="DRA", count=4, memory="23034Mi")], + ) + ], + taints=None, + placement_labels=None, + ) + ], + all_replicas=[], + fill=True, + want=[ + _candidate( + name="cluster-a", + index=0, + gateway_hostname="cluster-a.clusters.example.com", + placement_labels={}, + members=[ + scheduling.MemberPlacement( + role="Standalone", + pool="gpu-l4x4-a", + device_requests=[ + scheduling.DeviceRequest( + name="gpu", device_class_name="gpu.nvidia.com", count=1, cel_selectors=[_MEM_20] + ) + ], + ), + ], + ), + ], + ), + Case( + name="ObservedPoolTooFew", + reason="A replica claiming two GPUs doesn't fit an observed pool whose only node has one GPU free.", + deployment=_deployment( + replicas=1, + copies=1, + members=[ + _member( + role="Standalone", + worker_nodes=None, + devices=[mdv1alpha1.Device(name="gpu", count=2, selectors=[mdv1alpha1.Selector(cel=_MEM_20)])], + ) + ], + tolerations=None, + ), + clusters=[ + _cluster( + name="cluster-a", + gateway_hostname="cluster-a.clusters.example.com", + ready=True, + pools=[ + icv1alpha1.GpuPool( + name="gpu-l4x4-a", + nodes=1, + observedNodes=1, + nodesByFreeDevices=[ + icv1alpha1.NodesByFreeDevice(freeDevices={"gpu.nvidia.com": 1}, nodes=1), + ], + devices=[_gpu_device(name="gpu", claim="DRA", count=4, memory="23034Mi")], + ) + ], + taints=None, + placement_labels=None, + ) + ], + all_replicas=[], + fill=True, + want=[], + ), + Case( + name="ObservedPoolSharesNode", + reason="Four replicas claiming one GPU each share an observed pool's one node with four GPUs free.", + deployment=_deployment( + replicas=4, + copies=1, + members=[ + _member( + role="Standalone", + worker_nodes=None, + devices=[mdv1alpha1.Device(name="gpu", count=1, selectors=[mdv1alpha1.Selector(cel=_MEM_20)])], + ) + ], + tolerations=None, + ), + clusters=[ + _cluster( + name="cluster-a", + gateway_hostname="cluster-a.clusters.example.com", + ready=True, + pools=[ + icv1alpha1.GpuPool( + name="gpu-l4x4-a", + nodes=1, + observedNodes=1, + nodesByFreeDevices=[ + icv1alpha1.NodesByFreeDevice(freeDevices={"gpu.nvidia.com": 4}, nodes=1), + ], + devices=[_gpu_device(name="gpu", claim="DRA", count=4, memory="23034Mi")], + ) + ], + taints=None, + placement_labels=None, + ) + ], + all_replicas=[], + fill=True, + want=[ + _candidate( + name="cluster-a", + index=0, + gateway_hostname="cluster-a.clusters.example.com", + placement_labels={}, + members=[ + scheduling.MemberPlacement( + role="Standalone", + pool="gpu-l4x4-a", + device_requests=[ + scheduling.DeviceRequest( + name="gpu", device_class_name="gpu.nvidia.com", count=1, cel_selectors=[_MEM_20] + ) + ], + ), + ], + ), + _candidate( + name="cluster-a", + index=1, + gateway_hostname="cluster-a.clusters.example.com", + placement_labels={}, + members=[ + scheduling.MemberPlacement( + role="Standalone", + pool="gpu-l4x4-a", + device_requests=[ + scheduling.DeviceRequest( + name="gpu", device_class_name="gpu.nvidia.com", count=1, cel_selectors=[_MEM_20] + ) + ], + ), + ], + ), + _candidate( + name="cluster-a", + index=2, + gateway_hostname="cluster-a.clusters.example.com", + placement_labels={}, + members=[ + scheduling.MemberPlacement( + role="Standalone", + pool="gpu-l4x4-a", + device_requests=[ + scheduling.DeviceRequest( + name="gpu", device_class_name="gpu.nvidia.com", count=1, cel_selectors=[_MEM_20] + ) + ], + ), + ], + ), + _candidate( + name="cluster-a", + index=3, + gateway_hostname="cluster-a.clusters.example.com", + placement_labels={}, + members=[ + scheduling.MemberPlacement( + role="Standalone", + pool="gpu-l4x4-a", + device_requests=[ + scheduling.DeviceRequest( + name="gpu", device_class_name="gpu.nvidia.com", count=1, cel_selectors=[_MEM_20] + ) + ], + ), + ], + ), + ], + ), + Case( + name="ObservedPoolGrows", + reason="An observed pool that can grow by two nodes fits two replicas claiming a whole node each, but not a third.", + deployment=_deployment( + replicas=3, + copies=1, + members=[ + _member( + role="Standalone", + worker_nodes=None, + devices=[mdv1alpha1.Device(name="gpu", count=4, selectors=[mdv1alpha1.Selector(cel=_MEM_20)])], + ) + ], + tolerations=None, + ), + clusters=[ + _cluster( + name="cluster-a", + gateway_hostname="cluster-a.clusters.example.com", + ready=True, + pools=[ + icv1alpha1.GpuPool( + name="gpu-l4x4-a", + nodes=3, + observedNodes=1, + nodesByFreeDevices=[ + icv1alpha1.NodesByFreeDevice(freeDevices={"gpu.nvidia.com": 0}, nodes=1), + ], + devices=[_gpu_device(name="gpu", claim="DRA", count=4, memory="23034Mi")], + ) + ], + taints=None, + placement_labels=None, + ) + ], + all_replicas=[], + fill=True, + want=[ + _candidate( + name="cluster-a", + index=0, + gateway_hostname="cluster-a.clusters.example.com", + placement_labels={}, + members=[ + scheduling.MemberPlacement( + role="Standalone", + pool="gpu-l4x4-a", + device_requests=[ + scheduling.DeviceRequest( + name="gpu", device_class_name="gpu.nvidia.com", count=4, cel_selectors=[_MEM_20] + ) + ], + ), + ], + ), + _candidate( + name="cluster-a", + index=1, + gateway_hostname="cluster-a.clusters.example.com", + placement_labels={}, + members=[ + scheduling.MemberPlacement( + role="Standalone", + pool="gpu-l4x4-a", + device_requests=[ + scheduling.DeviceRequest( + name="gpu", device_class_name="gpu.nvidia.com", count=4, cel_selectors=[_MEM_20] + ) + ], + ), + ], + ), + ], + ), + # A disaggregated deployment's phases are two engines. Each must fit on + # its own, but they can share a node. + Case( + name="PrefillDecodeShareNode", + reason="Prefill and decode engines claiming four GPUs each share an observed pool's one node with eight GPUs free.", + deployment=mdv1alpha1.ModelDeployment( + metadata=metav1.ObjectMeta(name="my-model", namespace="ml-team"), + spec=mdv1alpha1.SpecModel2( + replicas=1, + template=mdv1alpha1.TemplateModel1( + spec=mdv1alpha1.SpecModel1( + engines=[ + mdv1alpha1.Engine( + name="prefill", + copies=1, + members=[ + _member( + role="Standalone", + worker_nodes=None, + devices=[ + mdv1alpha1.Device( + name="gpu", count=4, selectors=[mdv1alpha1.Selector(cel=_MEM_141)] + ) + ], + ) + ], + ), + mdv1alpha1.Engine( + name="decode", + copies=1, + members=[ + _member( + role="Standalone", + worker_nodes=None, + devices=[ + mdv1alpha1.Device( + name="gpu", count=4, selectors=[mdv1alpha1.Selector(cel=_MEM_141)] + ) + ], + ) + ], + ), + ], + ), + ), + ), + ), + clusters=[ + _cluster( + name="cluster-a", + gateway_hostname="cluster-a.clusters.example.com", + ready=True, + pools=[ + icv1alpha1.GpuPool( + name="default", + nodes=1, + observedNodes=1, + nodesByFreeDevices=[ + icv1alpha1.NodesByFreeDevice(freeDevices={"gpu.nvidia.com": 8}, nodes=1), + ], + devices=[_gpu_device(name="gpu", claim="DRA", count=8, memory="141Gi")], + ) + ], + taints=None, + placement_labels=None, + ) + ], + all_replicas=[], + fill=True, + want=[ + scheduling.Candidate( + name="cluster-a", + index=0, + gateway_hostname="cluster-a.clusters.example.com", + placement_labels={}, + engines=[ + scheduling.EnginePlacement( + name="prefill", + members=[ + scheduling.MemberPlacement( + role="Standalone", + pool="default", + device_requests=[ + scheduling.DeviceRequest( + name="gpu", + device_class_name="gpu.nvidia.com", + count=4, + cel_selectors=[_MEM_141], + ) + ], + ), + ], + ), + scheduling.EnginePlacement( + name="decode", + members=[ + scheduling.MemberPlacement( + role="Standalone", + pool="default", + device_requests=[ + scheduling.DeviceRequest( + name="gpu", + device_class_name="gpu.nvidia.com", + count=4, + cel_selectors=[_MEM_141], + ) + ], + ), + ], + ), + ], + ), + ], + ), + Case( + name="SmallestEngineFirst", + reason="A replica's engines are placed smallest pod first, so a one-GPU decode takes the node with four GPUs free and the four-GPU prefill declared before it doesn't fit.", + deployment=mdv1alpha1.ModelDeployment( + metadata=metav1.ObjectMeta(name="my-model", namespace="ml-team"), + spec=mdv1alpha1.SpecModel2( + replicas=1, + template=mdv1alpha1.TemplateModel1( + spec=mdv1alpha1.SpecModel1( + engines=[ + mdv1alpha1.Engine( + name="prefill", + copies=1, + members=[ + _member( + role="Standalone", + worker_nodes=None, + devices=[ + mdv1alpha1.Device( + name="gpu", count=4, selectors=[mdv1alpha1.Selector(cel=_MEM_141)] + ) + ], + ) + ], + ), + mdv1alpha1.Engine( + name="decode", + copies=1, + members=[ + _member( + role="Standalone", + worker_nodes=None, + devices=[ + mdv1alpha1.Device( + name="gpu", count=1, selectors=[mdv1alpha1.Selector(cel=_MEM_141)] + ) + ], + ) + ], + ), + ], + ), + ), + ), + ), + clusters=[ + _cluster( + name="cluster-a", + gateway_hostname="cluster-a.clusters.example.com", + ready=True, + pools=[ + icv1alpha1.GpuPool( + name="default", + nodes=2, + observedNodes=2, + nodesByFreeDevices=[ + icv1alpha1.NodesByFreeDevice(freeDevices={"gpu.nvidia.com": 4}, nodes=1), + icv1alpha1.NodesByFreeDevice(freeDevices={"gpu.nvidia.com": 1}, nodes=1), + ], + devices=[_gpu_device(name="gpu", claim="DRA", count=8, memory="141Gi")], + ) + ], + taints=None, + placement_labels=None, + ) + ], + all_replicas=[], + fill=True, + want=[], + ), + Case( + name="ObservedPoolUnscheduledReplica", + reason="Another deployment's replica whose pods haven't scheduled is charged against an observed pool, because its histogram can't count them yet.", + deployment=_deployment( + replicas=1, + copies=1, + members=[ + _member( + role="Standalone", + worker_nodes=None, + devices=[mdv1alpha1.Device(name="gpu", count=1, selectors=[mdv1alpha1.Selector(cel=_MEM_141)])], + ) + ], + tolerations=None, + ), + clusters=[ + _cluster( + name="cluster-a", + gateway_hostname="cluster-a.clusters.example.com", + ready=True, + pools=[ + icv1alpha1.GpuPool( + name="default", + nodes=1, + observedNodes=1, + nodesByFreeDevices=[ + icv1alpha1.NodesByFreeDevice(freeDevices={"gpu.nvidia.com": 8}, nodes=1), + ], + devices=[_gpu_device(name="gpu", claim="DRA", count=8, memory="141Gi")], + ) + ], + taints=None, + placement_labels=None, + ) + ], + all_replicas=[ + _replica( + name="other-model-cluster-a-0", + deployment="other-model", + cluster="cluster-a", + index=0, + copies=1, + pods_scheduled=False, + pools=None, + members=[ + _replica_member( + role="Standalone", + worker_nodes=None, + pool="default", + device_requests=[ + mrv1alpha1.DeviceRequest( + name="gpu", + deviceClassName="gpu.nvidia.com", + count=8, + selectors=[mrv1alpha1.Selector(cel=_MEM_141)], + ) + ], + ) + ], + ) + ], + fill=True, + want=[], + ), + # other-model's replica holds four of the node's eight GPUs. They were the + # last the agent saw allocated on the pool when it last observed it, so the + # histogram counts them. + Case( + name="ObservedPoolSettledReplica", + reason="Another deployment's replica whose claims the agent saw allocated no later than it last observed the pool isn't charged against it, because its histogram counts them.", + deployment=_deployment( + replicas=1, + copies=1, + members=[ + _member( + role="Standalone", + worker_nodes=None, + devices=[mdv1alpha1.Device(name="gpu", count=4, selectors=[mdv1alpha1.Selector(cel=_MEM_141)])], + ) + ], + tolerations=None, + ), + clusters=[ + _cluster( + name="cluster-a", + gateway_hostname="cluster-a.clusters.example.com", + ready=True, + pools=[ + icv1alpha1.GpuPool( + name="default", + nodes=1, + observedNodes=1, + lastAllocationTime=datetime.datetime(2026, 10, 1, 11, 58, tzinfo=datetime.UTC), + nodesByFreeDevices=[ + icv1alpha1.NodesByFreeDevice(freeDevices={"gpu.nvidia.com": 4}, nodes=1), + ], + devices=[_gpu_device(name="gpu", claim="DRA", count=8, memory="141Gi")], + ) + ], + taints=None, + placement_labels=None, + ) + ], + all_replicas=[ + _replica( + name="other-model-cluster-a-0", + deployment="other-model", + cluster="cluster-a", + index=0, + copies=1, + pods_scheduled=True, + pools=[ + mrv1alpha1.Pool( + name="default", lastAllocationTime=datetime.datetime(2026, 10, 1, 11, 58, tzinfo=datetime.UTC) + ) + ], + members=[ + _replica_member( + role="Standalone", + worker_nodes=None, + pool="default", + device_requests=[ + mrv1alpha1.DeviceRequest( + name="gpu", + deviceClassName="gpu.nvidia.com", + count=4, + selectors=[mrv1alpha1.Selector(cel=_MEM_141)], + ) + ], + ) + ], + ) + ], + fill=True, + want=[ + _candidate( + name="cluster-a", + index=0, + gateway_hostname="cluster-a.clusters.example.com", + placement_labels={}, + members=[ + scheduling.MemberPlacement( + role="Standalone", + pool="default", + device_requests=[ + scheduling.DeviceRequest( + name="gpu", device_class_name="gpu.nvidia.com", count=4, cel_selectors=[_MEM_141] + ) + ], + ), + ], + ), + ], + ), + Case( + name="UnobservedReplicaCharged", + reason="Another deployment's replica whose claims the agent saw allocated after it last observed the pool is still charged against it, because its histogram doesn't count them yet.", + deployment=_deployment( + replicas=1, + copies=1, + members=[ + _member( + role="Standalone", + worker_nodes=None, + devices=[mdv1alpha1.Device(name="gpu", count=4, selectors=[mdv1alpha1.Selector(cel=_MEM_141)])], + ) + ], + tolerations=None, + ), + clusters=[ + _cluster( + name="cluster-a", + gateway_hostname="cluster-a.clusters.example.com", + ready=True, + pools=[ + icv1alpha1.GpuPool( + name="default", + nodes=1, + observedNodes=1, + lastAllocationTime=datetime.datetime(2026, 10, 1, 11, 58, tzinfo=datetime.UTC), + nodesByFreeDevices=[ + icv1alpha1.NodesByFreeDevice(freeDevices={"gpu.nvidia.com": 4}, nodes=1), + ], + devices=[_gpu_device(name="gpu", claim="DRA", count=8, memory="141Gi")], + ) + ], + taints=None, + placement_labels=None, + ) + ], + all_replicas=[ + _replica( + name="other-model-cluster-a-0", + deployment="other-model", + cluster="cluster-a", + index=0, + copies=1, + pods_scheduled=True, + pools=[ + mrv1alpha1.Pool( + name="default", lastAllocationTime=datetime.datetime(2026, 10, 1, 11, 59, tzinfo=datetime.UTC) + ) + ], + members=[ + _replica_member( + role="Standalone", + worker_nodes=None, + pool="default", + device_requests=[ + mrv1alpha1.DeviceRequest( + name="gpu", + deviceClassName="gpu.nvidia.com", + count=4, + selectors=[mrv1alpha1.Selector(cel=_MEM_141)], + ) + ], + ) + ], + ) + ], + fill=True, + want=[], + ), + Case( + name="DevicesOnDifferentNodes", + reason="A replica claiming GPUs and EFA adapters doesn't fit an observed pool whose free GPUs and free adapters are on different nodes.", + deployment=_deployment( + replicas=1, + copies=1, + members=[ + _member( + role="Standalone", + worker_nodes=None, + devices=[ + mdv1alpha1.Device(name="gpu", count=4, selectors=[mdv1alpha1.Selector(cel=_MEM_141)]), + mdv1alpha1.Device(name="efa", count=8, selectors=[mdv1alpha1.Selector(cel=_EFA)]), + ], + ) + ], + tolerations=None, + ), + clusters=[ + _cluster( + name="cluster-a", + gateway_hostname="cluster-a.clusters.example.com", + ready=True, + pools=[ + icv1alpha1.GpuPool( + name="default", + nodes=2, + observedNodes=2, + nodesByFreeDevices=[ + icv1alpha1.NodesByFreeDevice( + freeDevices={"efa.networking.k8s.aws": 0, "gpu.nvidia.com": 8}, nodes=1 + ), + icv1alpha1.NodesByFreeDevice( + freeDevices={"efa.networking.k8s.aws": 16, "gpu.nvidia.com": 0}, nodes=1 + ), + ], + devices=[ + _gpu_device(name="gpu", claim="DRA", count=8, memory="141Gi"), + icv1alpha1.Device( + name="efa", + claim="DRA", + driver="dra.net", + deviceClassName="efa.networking.k8s.aws", + count=16, + ), + ], + ) + ], + taints=None, + placement_labels=None, + ) + ], + all_replicas=[], + fill=True, + want=[], + ), + # The agent saw other-model's claims allocated before it last observed the + # pool, but one of its pods hasn't scheduled. + Case( + name="UnscheduledReplicaCharged", + reason="Another deployment's replica whose pods haven't all scheduled is still charged against an observed pool.", + deployment=_deployment( + replicas=1, + copies=1, + members=[ + _member( + role="Standalone", + worker_nodes=None, + devices=[mdv1alpha1.Device(name="gpu", count=4, selectors=[mdv1alpha1.Selector(cel=_MEM_141)])], + ) + ], + tolerations=None, + ), + clusters=[ + _cluster( + name="cluster-a", + gateway_hostname="cluster-a.clusters.example.com", + ready=True, + pools=[ + icv1alpha1.GpuPool( + name="default", + nodes=1, + observedNodes=1, + lastAllocationTime=datetime.datetime(2026, 10, 1, 11, 59, tzinfo=datetime.UTC), + nodesByFreeDevices=[ + icv1alpha1.NodesByFreeDevice(freeDevices={"gpu.nvidia.com": 4}, nodes=1), + ], + devices=[_gpu_device(name="gpu", claim="DRA", count=8, memory="141Gi")], + ) + ], + taints=None, + placement_labels=None, + ) + ], + all_replicas=[ + _replica( + name="other-model-cluster-a-0", + deployment="other-model", + cluster="cluster-a", + index=0, + copies=1, + pods_scheduled=False, + pools=[ + mrv1alpha1.Pool( + name="default", lastAllocationTime=datetime.datetime(2026, 10, 1, 11, 58, tzinfo=datetime.UTC) + ) + ], + members=[ + _replica_member( + role="Standalone", + worker_nodes=None, + pool="default", + device_requests=[ + mrv1alpha1.DeviceRequest( + name="gpu", + deviceClassName="gpu.nvidia.com", + count=4, + selectors=[mrv1alpha1.Selector(cel=_MEM_141)], + ) + ], + ) + ], + ) + ], + fill=True, + want=[], + ), + Case( + name="NoPoolTimeCharged", + reason="Another deployment's replica the agent hasn't seen holding devices on an observed pool is still charged against it.", + deployment=_deployment( + replicas=1, + copies=1, + members=[ + _member( + role="Standalone", + worker_nodes=None, + devices=[mdv1alpha1.Device(name="gpu", count=4, selectors=[mdv1alpha1.Selector(cel=_MEM_141)])], + ) + ], + tolerations=None, + ), + clusters=[ + _cluster( + name="cluster-a", + gateway_hostname="cluster-a.clusters.example.com", + ready=True, + pools=[ + icv1alpha1.GpuPool( + name="default", + nodes=1, + observedNodes=1, + lastAllocationTime=datetime.datetime(2026, 10, 1, 11, 59, tzinfo=datetime.UTC), + nodesByFreeDevices=[ + icv1alpha1.NodesByFreeDevice(freeDevices={"gpu.nvidia.com": 4}, nodes=1), + ], + devices=[_gpu_device(name="gpu", claim="DRA", count=8, memory="141Gi")], + ) + ], + taints=None, + placement_labels=None, + ) + ], + all_replicas=[ + _replica( + name="other-model-cluster-a-0", + deployment="other-model", + cluster="cluster-a", + index=0, + copies=1, + pods_scheduled=True, + pools=[ + mrv1alpha1.Pool( + name="other", lastAllocationTime=datetime.datetime(2026, 10, 1, 11, 58, tzinfo=datetime.UTC) + ) + ], + members=[ + _replica_member( + role="Standalone", + worker_nodes=None, + pool="default", + device_requests=[ + mrv1alpha1.DeviceRequest( + name="gpu", + deviceClassName="gpu.nvidia.com", + count=4, + selectors=[mrv1alpha1.Selector(cel=_MEM_141)], + ) + ], + ) + ], + ) + ], + fill=True, + want=[], + ), + Case( + name="GangSharesNode", + reason="A gang whose leader and worker claim one GPU each fits an observed pool of one node with four free.", + deployment=_deployment( + replicas=1, + copies=1, + members=[ + _member( + role="Leader", + worker_nodes=None, + devices=[mdv1alpha1.Device(name="gpu", count=1, selectors=[mdv1alpha1.Selector(cel=_MEM_141)])], + ), + _member( + role="Worker", + worker_nodes=1, + devices=[mdv1alpha1.Device(name="gpu", count=1, selectors=[mdv1alpha1.Selector(cel=_MEM_141)])], + ), + ], + tolerations=None, + ), + clusters=[ + _cluster( + name="cluster-a", + gateway_hostname="cluster-a.clusters.example.com", + ready=True, + pools=[ + icv1alpha1.GpuPool( + name="default", + nodes=1, + observedNodes=1, + nodesByFreeDevices=[ + icv1alpha1.NodesByFreeDevice(freeDevices={"gpu.nvidia.com": 4}, nodes=1), + ], + devices=[_gpu_device(name="gpu", claim="DRA", count=4, memory="141Gi")], + ) + ], + taints=None, + placement_labels=None, + ) + ], + all_replicas=[], + fill=True, + want=[ + _candidate( + name="cluster-a", + index=0, + gateway_hostname="cluster-a.clusters.example.com", + placement_labels={}, + members=[ + scheduling.MemberPlacement( + role="Leader", + pool="default", + device_requests=[ + scheduling.DeviceRequest( + name="gpu", device_class_name="gpu.nvidia.com", count=1, cel_selectors=[_MEM_141] + ) + ], + ), + scheduling.MemberPlacement( + role="Worker", + pool="default", + device_requests=[ + scheduling.DeviceRequest( + name="gpu", device_class_name="gpu.nvidia.com", count=1, cel_selectors=[_MEM_141] + ) + ], + ), + ], + ), + ], + ), + # other-model's gang hasn't scheduled, so the histogram doesn't count it. + # Its leader and worker each take two of the GPUs free on the node with + # eight, leaving neither node with six free for the new replica. + Case( + name="GangChargedSharedNode", + reason="Another deployment's gang is charged against an observed pool with its pods sharing a node.", + deployment=_deployment( + replicas=1, + copies=1, + members=[ + _member( + role="Standalone", + worker_nodes=None, + devices=[mdv1alpha1.Device(name="gpu", count=6, selectors=[mdv1alpha1.Selector(cel=_MEM_141)])], + ) + ], + tolerations=None, + ), + clusters=[ + _cluster( + name="cluster-a", + gateway_hostname="cluster-a.clusters.example.com", + ready=True, + pools=[ + icv1alpha1.GpuPool( + name="default", + nodes=2, + observedNodes=2, + lastAllocationTime=datetime.datetime(2026, 10, 1, 11, 58, tzinfo=datetime.UTC), + nodesByFreeDevices=[ + icv1alpha1.NodesByFreeDevice(freeDevices={"gpu.nvidia.com": 4}, nodes=1), + icv1alpha1.NodesByFreeDevice(freeDevices={"gpu.nvidia.com": 8}, nodes=1), + ], + devices=[_gpu_device(name="gpu", claim="DRA", count=8, memory="141Gi")], + ) + ], + taints=None, + placement_labels=None, + ) + ], + all_replicas=[ + _replica( + name="other-model-cluster-a-0", + deployment="other-model", + cluster="cluster-a", + index=0, + copies=1, + pods_scheduled=False, + pools=None, + members=[ + _replica_member( + role="Leader", + worker_nodes=None, + pool="default", + device_requests=[ + mrv1alpha1.DeviceRequest( + name="gpu", + deviceClassName="gpu.nvidia.com", + count=2, + selectors=[mrv1alpha1.Selector(cel=_MEM_141)], + ) + ], + ), + _replica_member( + role="Worker", + worker_nodes=1, + pool="default", + device_requests=[ + mrv1alpha1.DeviceRequest( + name="gpu", + deviceClassName="gpu.nvidia.com", + count=2, + selectors=[mrv1alpha1.Selector(cel=_MEM_141)], + ) + ], + ), + ], + ) + ], + fill=True, + want=[], + ), + # Standalone copies may share a node, but the node needs room for both. + Case( + name="TwoCopiesDontFit", + reason="Two copies of an engine claiming four GPUs each don't fit an observed pool of one node with four free.", + deployment=_deployment( + replicas=1, + copies=2, + members=[ + _member( + role="Standalone", + worker_nodes=None, + devices=[mdv1alpha1.Device(name="gpu", count=4, selectors=[mdv1alpha1.Selector(cel=_MEM_141)])], + ) + ], + tolerations=None, + ), + clusters=[ + _cluster( + name="cluster-a", + gateway_hostname="cluster-a.clusters.example.com", + ready=True, + pools=[ + icv1alpha1.GpuPool( + name="default", + nodes=1, + observedNodes=1, + nodesByFreeDevices=[ + icv1alpha1.NodesByFreeDevice(freeDevices={"gpu.nvidia.com": 4}, nodes=1), + ], + devices=[_gpu_device(name="gpu", claim="DRA", count=4, memory="141Gi")], + ) + ], + taints=None, + placement_labels=None, + ) + ], + all_replicas=[], + fill=True, + want=[], + ), + # other-model's two copies haven't scheduled, so the histogram doesn't count + # them. They take all four GPUs between them. + Case( + name="CopiesCharged", + reason="Another deployment's replica is charged against an observed pool for every copy of its engine.", + deployment=_deployment( + replicas=1, + copies=1, + members=[ + _member( + role="Standalone", + worker_nodes=None, + devices=[mdv1alpha1.Device(name="gpu", count=1, selectors=[mdv1alpha1.Selector(cel=_MEM_141)])], + ) + ], + tolerations=None, + ), + clusters=[ + _cluster( + name="cluster-a", + gateway_hostname="cluster-a.clusters.example.com", + ready=True, + pools=[ + icv1alpha1.GpuPool( + name="default", + nodes=1, + observedNodes=1, + nodesByFreeDevices=[ + icv1alpha1.NodesByFreeDevice(freeDevices={"gpu.nvidia.com": 4}, nodes=1), + ], + devices=[_gpu_device(name="gpu", claim="DRA", count=4, memory="141Gi")], + ) + ], + taints=None, + placement_labels=None, + ) + ], + all_replicas=[ + _replica( + name="other-model-cluster-a-0", + deployment="other-model", + cluster="cluster-a", + index=0, + copies=2, + pods_scheduled=False, + pools=None, + members=[ + _replica_member( + role="Standalone", + worker_nodes=None, + pool="default", + device_requests=[ + mrv1alpha1.DeviceRequest( + name="gpu", + deviceClassName="gpu.nvidia.com", + count=2, + selectors=[mrv1alpha1.Selector(cel=_MEM_141)], + ) + ], + ) + ], + ) + ], + fill=True, + want=[], + ), + # other-model's claims are the first allocated on the pool, and the agent + # hasn't observed the pool since. + Case( + name="PoolWithoutTimeCharged", + reason="Another deployment's scheduled replica is still charged against an observed pool that publishes no allocation time.", + deployment=_deployment( + replicas=1, + copies=1, + members=[ + _member( + role="Standalone", + worker_nodes=None, + devices=[mdv1alpha1.Device(name="gpu", count=4, selectors=[mdv1alpha1.Selector(cel=_MEM_141)])], + ) + ], + tolerations=None, + ), + clusters=[ + _cluster( + name="cluster-a", + gateway_hostname="cluster-a.clusters.example.com", + ready=True, + pools=[ + icv1alpha1.GpuPool( + name="default", + nodes=1, + observedNodes=1, + nodesByFreeDevices=[ + icv1alpha1.NodesByFreeDevice(freeDevices={"gpu.nvidia.com": 4}, nodes=1), + ], + devices=[_gpu_device(name="gpu", claim="DRA", count=4, memory="141Gi")], + ) + ], + taints=None, + placement_labels=None, + ) + ], + all_replicas=[ + _replica( + name="other-model-cluster-a-0", + deployment="other-model", + cluster="cluster-a", + index=0, + copies=1, + pods_scheduled=True, + pools=[ + mrv1alpha1.Pool( + name="default", lastAllocationTime=datetime.datetime(2026, 10, 1, 11, 58, tzinfo=datetime.UTC) + ) + ], + members=[ + _replica_member( + role="Standalone", + worker_nodes=None, + pool="default", + device_requests=[ + mrv1alpha1.DeviceRequest( + name="gpu", + deviceClassName="gpu.nvidia.com", + count=4, + selectors=[mrv1alpha1.Selector(cel=_MEM_141)], + ) + ], + ) + ], + ) + ], + fill=True, + want=[], + ), + # other-model's leader and worker are pinned to different pools, as they can + # be for one reconcile after its deployment's nodeSelectors change. Each + # takes the one node of its pool. + Case( + name="SplitGangCharged", + reason="Another deployment's gang whose pods are pinned to two observed pools is charged against each.", + deployment=_deployment( + replicas=1, + copies=1, + members=[ + _member( + role="Standalone", + worker_nodes=None, + devices=[mdv1alpha1.Device(name="gpu", count=4, selectors=[mdv1alpha1.Selector(cel=_MEM_141)])], + ) + ], + tolerations=None, + ), + clusters=[ + _cluster( + name="cluster-a", + gateway_hostname="cluster-a.clusters.example.com", + ready=True, + pools=[ + icv1alpha1.GpuPool( + name="pool-a", + nodes=1, + observedNodes=1, + nodesByFreeDevices=[ + icv1alpha1.NodesByFreeDevice(freeDevices={"gpu.nvidia.com": 4}, nodes=1), + ], + devices=[_gpu_device(name="gpu", claim="DRA", count=4, memory="141Gi")], + ), + icv1alpha1.GpuPool( + name="pool-b", + nodes=1, + observedNodes=1, + nodesByFreeDevices=[ + icv1alpha1.NodesByFreeDevice(freeDevices={"gpu.nvidia.com": 4}, nodes=1), + ], + devices=[_gpu_device(name="gpu", claim="DRA", count=4, memory="141Gi")], + ), + ], + taints=None, + placement_labels=None, + ) + ], + all_replicas=[ + _replica( + name="other-model-cluster-a-0", + deployment="other-model", + cluster="cluster-a", + index=0, + copies=1, + pods_scheduled=False, + pools=None, + members=[ + _replica_member( + role="Leader", + worker_nodes=None, + pool="pool-a", + device_requests=[ + mrv1alpha1.DeviceRequest( + name="gpu", + deviceClassName="gpu.nvidia.com", + count=4, + selectors=[mrv1alpha1.Selector(cel=_MEM_141)], + ) + ], + ), + _replica_member( + role="Worker", + worker_nodes=1, + pool="pool-b", + device_requests=[ + mrv1alpha1.DeviceRequest( + name="gpu", + deviceClassName="gpu.nvidia.com", + count=4, + selectors=[mrv1alpha1.Selector(cel=_MEM_141)], + ) + ], + ), + ], + ) + ], + fill=True, + want=[], + ), + # Pools the cluster agent doesn't report on count whole nodes whether or not + # a replica's pods have scheduled, because nothing else counts them. + Case( + name="WholeNodeSettledReplica", + reason="Another deployment's replica whose pods have all scheduled still takes a whole node of a pool the agent doesn't report on.", + deployment=_deployment( + replicas=1, + copies=1, + members=[ + _member( + role="Standalone", + worker_nodes=None, + devices=[mdv1alpha1.Device(name="gpu", count=1, selectors=[mdv1alpha1.Selector(cel=_MEM_141)])], + ) + ], + tolerations=None, + ), + clusters=[ + _cluster( + name="cluster-a", + gateway_hostname="cluster-a.clusters.example.com", + ready=True, + pools=[ + icv1alpha1.GpuPool( + name="default", nodes=1, devices=[_gpu_device(name="gpu", claim="DRA", count=8, memory="141Gi")] + ) + ], + taints=None, + placement_labels=None, + ) + ], + all_replicas=[ + _replica( + name="other-model-cluster-a-0", + deployment="other-model", + cluster="cluster-a", + index=0, + copies=1, + pods_scheduled=True, + pools=[ + mrv1alpha1.Pool( + name="default", lastAllocationTime=datetime.datetime(2026, 10, 1, 11, 58, tzinfo=datetime.UTC) + ) + ], + members=[ + _replica_member( + role="Standalone", + worker_nodes=None, + pool="default", + device_requests=[ + mrv1alpha1.DeviceRequest( + name="gpu", + deviceClassName="gpu.nvidia.com", + count=1, + selectors=[mrv1alpha1.Selector(cel=_MEM_141)], + ) + ], + ) + ], + ) + ], + fill=True, + want=[], + ), + # other-model's replica was stamped when the pool's class declared eight + # GPUs a node. It now declares four, but the replica's pods still run. + Case( + name="WholeNodeStaleClaim", + reason="Another deployment's replica claiming more GPUs than its pool's class now declares still takes a whole node of a pool the agent doesn't report on.", + deployment=_deployment( + replicas=1, + copies=1, + members=[ + _member( + role="Standalone", + worker_nodes=None, + devices=[mdv1alpha1.Device(name="gpu", count=1, selectors=[mdv1alpha1.Selector(cel=_MEM_141)])], + ) + ], + tolerations=None, + ), + clusters=[ + _cluster( + name="cluster-a", + gateway_hostname="cluster-a.clusters.example.com", + ready=True, + pools=[ + icv1alpha1.GpuPool( + name="default", nodes=1, devices=[_gpu_device(name="gpu", claim="DRA", count=4, memory="141Gi")] + ) + ], + taints=None, + placement_labels=None, + ) + ], + all_replicas=[ + _replica( + name="other-model-cluster-a-0", + deployment="other-model", + cluster="cluster-a", + index=0, + copies=1, + pods_scheduled=False, + pools=None, + members=[ + _replica_member( + role="Standalone", + worker_nodes=None, + pool="default", + device_requests=[ + mrv1alpha1.DeviceRequest( + name="gpu", + deviceClassName="gpu.nvidia.com", + count=8, + selectors=[mrv1alpha1.Selector(cel=_MEM_141)], + ) + ], + ) + ], + ) + ], + fill=True, + want=[], + ), ] @@ -5352,6 +7008,7 @@ def test_schedule_error() -> None: scheduling.schedule( _deployment( replicas=1, + copies=1, members=[ _member( role="Standalone", diff --git a/functions/compose-model-replica/function/backends/base.py b/functions/compose-model-replica/function/backends/base.py index f962652b5..1463afd5d 100644 --- a/functions/compose-model-replica/function/backends/base.py +++ b/functions/compose-model-replica/function/backends/base.py @@ -258,8 +258,9 @@ def remote_namespace(replica: v1alpha1.ModelReplica) -> str: # Pod labels carrying the identity every metric off this pod is attributed to. # The collector reads them off the pod during service discovery and stamps them # on each series, so a MetricMapping needs no labels of its own and a series -# merged across replicas keeps the identity all of them share. Metrics are the -# only reason these exist; nothing selects on them. +# merged across replicas keeps the identity all of them share. A replica's +# ReplicaObservation also selects its pods by deployment, replica and engine (see +# observation.compose), so renaming those breaks the scheduler's accounting. LABEL_DEPLOYMENT = "modelplane.ai/deployment" LABEL_REPLICA = "modelplane.ai/replica" LABEL_ENGINE = "modelplane.ai/engine" @@ -268,7 +269,7 @@ def remote_namespace(replica: v1alpha1.ModelReplica) -> str: # Set on the ModelReplica by the composite that scheduled it. -_LABEL_REPLICA_INDEX = "modelplane.ai/replica-index" +LABEL_REPLICA_INDEX = "modelplane.ai/replica-index" def telemetry_labels( @@ -292,7 +293,7 @@ def telemetry_labels( # one of them is simply lost. The index rather than the pod: it is bounded # by the replica count, and it survives a restart and a rolling update, # where a pod name is minted afresh each time. - if index := own.get(_LABEL_REPLICA_INDEX): + if index := own.get(LABEL_REPLICA_INDEX): labels[LABEL_REPLICA] = index if engine.name: labels[LABEL_ENGINE] = engine.name @@ -319,7 +320,7 @@ def fleet_labels(replica: v1alpha1.ModelReplica, role: str) -> dict[str, str]: own = (replica.metadata.labels if replica.metadata else None) or {} if name := own.get(LABEL_DEPLOYMENT): labels[LABEL_DEPLOYMENT] = name - if index := own.get(_LABEL_REPLICA_INDEX): + if index := own.get(LABEL_REPLICA_INDEX): labels[LABEL_REPLICA] = index return labels diff --git a/functions/compose-model-replica/function/fn.py b/functions/compose-model-replica/function/fn.py index 6ed0814c1..c63984149 100644 --- a/functions/compose-model-replica/function/fn.py +++ b/functions/compose-model-replica/function/fn.py @@ -28,6 +28,7 @@ """ import grpc +import pydantic from crossplane.function import logging, request, resource, response from crossplane.function.proto.v1 import run_function_pb2 as fnv1 from crossplane.function.proto.v1 import run_function_pb2_grpc as grpcv1 @@ -35,7 +36,7 @@ from models.ai.modelplane.modelreplica import v1alpha1 from models.io.crossplane.m.kubernetes.object import v1alpha1 as k8sobjv1alpha1 -from function import routing +from function import observation, routing from function.backends import base, grove, llmd, native # Condition types and reasons for the ModelReplica XR. @@ -202,6 +203,18 @@ def compose_model_serving(self) -> None: image = base.engine_container(self.xr.spec.engines[0].members[0]).image response.normal(self.rsp, f"Composing {image} on {self.xr.spec.clusterName}") + # The observation is first composed once the agent reports on one of + # the replica's pools, and kept after. It's optional - the scheduler + # counts a replica it can't see scheduled against the cluster's + # capacity - so it's marked ready unconditionally rather than holding + # the replica up. + if observation.KEY not in self.req.observed.resources and not observation.agent_reports(self.ic, self.xr): + return + obs = observation.compose(self.xr, pc) + if obs is not None: + resource.update(self.rsp.desired.resources[observation.KEY], obs) + self.rsp.desired.resources[observation.KEY].ready = fnv1.READY_TRUE + def derive_conditions(self) -> None: """Derive ModelAccepted, ModelServing and ModelReady. @@ -211,7 +224,9 @@ def derive_conditions(self) -> None: copy of every engine is ready and the routing in front of them is in place, so the replica can take traffic. ModelReady is True once every copy is ready, so the replica is at full capacity; the replica's Ready - mirrors it. + mirrors it. Where the cluster agent observes the replica, it also + relays whether the replica's pods have scheduled, and the pools their + devices are on. """ engines = self.xr.spec.engines observed = self.req.observed.resources @@ -221,8 +236,10 @@ def ready(key: str) -> bool: # The PrefillDecode routing check stops composing with a fatal result, # which leaves nothing desired. Crossplane keeps the existing resources - # then, so judge the replica on what it observes. - keys = set(self.rsp.desired.resources) or set(observed) + # then, so judge the replica on what it observes. The observation is + # composed partway through a serving replica's life, and isn't what + # serves it, so it's left out. + keys = (set(self.rsp.desired.resources) or set(observed)) - {observation.KEY} workloads = {base.workload_key(e) for e in engines} | {base.scaling_group_key(e) for e in engines} # The endpoint picker, the routing in front of it, and the engines' # claim templates. @@ -274,6 +291,30 @@ def ready(key: str) -> bool: ), ) + if observation.KEY in self.rsp.desired.resources: + # A status that doesn't match the agent's schema, which an agent + # other than the version Modelplane installs could report, is no + # observation. Raising would fail the whole replica. + try: + status = observation.status(observed.get(observation.KEY)) + except pydantic.ValidationError as err: + response.warning( + self.rsp, + f"Ignoring the cluster agent's observation of this replica: it does not match the agent's " + f"schema ({err.error_count()} problems).", + ) + status = None + response.set_conditions(self.rsp, observation.pods_scheduled(self.xr, status)) + if pools := observation.pools(status): + # The SDK dumps a model in Python mode, leaving each pool's + # lastAllocationTime a datetime a protobuf Struct can't hold. + resource.update_status( + self.rsp.desired.composite, + v1alpha1.Status(pools=pools).model_dump( + mode="json", exclude_unset=True, by_alias=True, warnings=False + ), + ) + # Per-resource readiness. Crossplane gates the XR's Ready on every # composed resource being ready, but doesn't read a composed resource's # own Ready condition, so the function relays it. Each Object's diff --git a/functions/compose-model-replica/function/observation.py b/functions/compose-model-replica/function/observation.py new file mode 100644 index 000000000..d6740109b --- /dev/null +++ b/functions/compose-model-replica/function/observation.py @@ -0,0 +1,187 @@ +# Copyright 2026 The Modelplane Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Observe whether a replica's pods have scheduled, and where. + +The cluster agent, which the serving stack installs, reports on the pods a +ReplicaObservation's selector matches. The replica relays whether every one of +its pods has scheduled as its ModelPodsScheduled condition, and when the agent +first saw its claims allocated on each pool as status.pools. +compose-model-deployment reads both to tell which replicas a cluster's histogram +of free devices accounts for. +""" + +from crossplane.function import resource +from crossplane.function.proto.v1 import run_function_pb2 as fnv1 +from models.ai.modelplane.agent.replicaobservation import v1alpha1 as rov1alpha1 +from models.ai.modelplane.inferencecluster import v1alpha1 as icv1alpha1 +from models.ai.modelplane.modelreplica import v1alpha1 +from models.io.crossplane.m.kubernetes.object import v1alpha1 as k8sobjv1alpha1 +from models.io.k8s.apimachinery.pkg.apis.meta import v1 as metav1 + +from function.backends import base + +# The composed resource key of the replica's ReplicaObservation. +KEY = "replica-observation" + +# The replica's ModelPodsScheduled condition. Its type and True status are a +# contract with compose-model-deployment. A False status copies the agent's +# reason where it has one. +CONDITION_TYPE = "ModelPodsScheduled" +REASON_ALL_PODS_SCHEDULED = "AllPodsScheduled" +REASON_WAITING_FOR_OBSERVATION = "WaitingForObservation" +REASON_WAITING_FOR_PODS = "WaitingForPods" + +# The agent's condition reporting whether a ReplicaObservation's pods are bound +# to nodes. +_AGENT_CONDITION_SCHEDULED = "Scheduled" + +# provider-kubernetes stops receiving watch events once the token in a +# cluster's kubeconfig expires, which on EKS is after 15 minutes. A watched +# ReplicaObservation then only reaches the replica when it's polled, every 10 +# minutes by default, so it's polled more often. +# TODO(negz): Remove once provider-kubernetes fixes this. +# https://github.com/crossplane-contrib/provider-kubernetes/issues/598 +_ANNOTATION_POLL_INTERVAL = "crossplane.io/poll-interval" +_POLL_INTERVAL = "30s" + + +def compose(replica: v1alpha1.ModelReplica, provider_config: str) -> k8sobjv1alpha1.Object | None: + """The Object creating the replica's ReplicaObservation on its cluster. + + It selects the replica's engine pods by the deployment and replica index + they carry, which together identify a replica within its namespace. The + endpoint picker carries both too, but no engine. The agent ignores a pod + without a replica label, so a replica its deployment didn't label gets no + observation. + + provider-kubernetes watches the ReplicaObservation, and polls it often in + case the watch has stopped (see _POLL_INTERVAL), so the agent's reports + reach the replica soon after they change. + """ + meta = replica.metadata + if meta is None or meta.name is None: + raise ValueError("metadata.name is unexpectedly absent") + labels = meta.labels or {} + deployment = labels.get(base.LABEL_DEPLOYMENT) + index = labels.get(base.LABEL_REPLICA_INDEX) + if not deployment or not index: + return None + obj = base.wrap_object( + provider_config, + rov1alpha1.ReplicaObservation( + apiVersion="agent.modelplane.ai/v1alpha1", + kind="ReplicaObservation", + metadata=metav1.ObjectMeta(name=meta.name, namespace=base.remote_namespace(replica)), + spec=rov1alpha1.Spec( + podSelector=rov1alpha1.PodSelector( + matchLabels={base.LABEL_DEPLOYMENT: deployment, base.LABEL_REPLICA: index}, + matchExpressions=[rov1alpha1.MatchExpression(key=base.LABEL_ENGINE, operator="Exists")], + ), + ), + ).model_dump(exclude_unset=True), + watch=True, + ) + obj.metadata = metav1.ObjectMeta(annotations={_ANNOTATION_POLL_INTERVAL: _POLL_INTERVAL}) + return obj + + +def agent_reports(cluster: icv1alpha1.InferenceCluster, replica: v1alpha1.ModelReplica) -> bool: + """Whether the cluster agent reports on any of the pools the replica's members are pinned to. + + The agent installs the ReplicaObservation CRD. An Object of a kind the + cluster doesn't serve can't be created, and can't be deleted either, so it + would hold up the replica's deletion. A pool the cluster publishes + observedNodes for is one the agent is reporting on. + """ + pinned = {m.nodePoolName for e in replica.spec.engines for m in e.members} + gpu_pools = (cluster.status.gpuPools if cluster.status else None) or [] + return any(p.name in pinned and p.observedNodes is not None for p in gpu_pools) + + +def pods_scheduled(replica: v1alpha1.ModelReplica, status: rov1alpha1.Status | None) -> resource.Condition: + """The replica's ModelPodsScheduled condition, given the status the agent reported for it. + + The agent's Scheduled condition is True once every pod it can see is bound + to a node with its claims allocated, which a gang's workers can make true + before they've all been created. So the replica's pods count as scheduled + only once the agent sees as many as the replica runs. + + While the agent's reason is unchanged, the condition keeps the message it + relayed when the reason last changed. Every change to a replica's status + reconciles every ModelDeployment, and the cluster scheduler's message can + change each time it retries a pod. + """ + if status is None: + return resource.Condition(typ=CONDITION_TYPE, status="False", reason=REASON_WAITING_FOR_OBSERVATION) + + scheduled = next((c for c in status.conditions or [] if c.type == _AGENT_CONDITION_SCHEDULED), None) + if scheduled is None: + return resource.Condition(typ=CONDITION_TYPE, status="False", reason=REASON_WAITING_FOR_OBSERVATION) + if scheduled.status != "True": + message = scheduled.message + for c in (replica.status.conditions if replica.status else None) or []: + if c.type == CONDITION_TYPE and c.status == "False" and c.reason == scheduled.reason: + message = c.message + return resource.Condition(typ=CONDITION_TYPE, status="False", reason=scheduled.reason, message=message) + + total = status.pods.total + expected = _expected_pods(replica) + if total < expected: + return resource.Condition( + typ=CONDITION_TYPE, + status="False", + reason=REASON_WAITING_FOR_PODS, + message=f"{total} of {expected} pods exist", + ) + return resource.Condition(typ=CONDITION_TYPE, status="True", reason=REASON_ALL_PODS_SCHEDULED) + + +def pools(status: rov1alpha1.Status | None) -> list[v1alpha1.Pool]: + """The pools the replica's devices are on, given the status the agent reported for it. + + Each carries the latest time the agent first saw one of the replica's claims + allocated devices on it, which compose-model-deployment compares with the + time its InferenceCluster publishes for the pool. + """ + if status is None: + return [] + return [v1alpha1.Pool(name=p.name, lastAllocationTime=p.lastAllocationTime) for p in status.pools or []] + + +def status(observed: fnv1.Resource | None) -> rov1alpha1.Status | None: + """The status the agent reported in the observed ReplicaObservation, or None until it reports. + + Raises pydantic.ValidationError if the status doesn't match the agent's + schema. + """ + if observed is None: + return None + obj = k8sobjv1alpha1.Object.model_validate(resource.struct_to_dict(observed.resource)) + if not obj.status or not obj.status.atProvider or not obj.status.atProvider.manifest: + return None + return rov1alpha1.ReplicaObservation.model_validate(obj.status.atProvider.manifest).status + + +def _expected_pods(replica: v1alpha1.ModelReplica) -> int: + """How many engine pods the replica runs: a Worker's nodes, otherwise one, for each copy of each engine. + + compose-model-deployment's scheduler charges a replica's pods by counting + them the same way, in _member_pods, so the two must agree. + """ + return sum( + int(engine.copies or 1) + * sum(int(m.worker.nodes) if m.role == base.ROLE_WORKER and m.worker else 1 for m in engine.members) + for engine in replica.spec.engines + ) diff --git a/functions/compose-model-replica/tests/test_fn.py b/functions/compose-model-replica/tests/test_fn.py index c2fd025c7..f37a42ab9 100644 --- a/functions/compose-model-replica/tests/test_fn.py +++ b/functions/compose-model-replica/tests/test_fn.py @@ -44,7 +44,7 @@ class Case: def _model_replica(*, copies: int) -> fnv1.Resource: - """The ModelReplica XR test-replica in ml-team, with one Standalone engine. + """The ModelReplica XR test-replica in ml-team, replica 0 of my-deployment, with one Standalone engine. The device request's CEL selector is as compose-model-deployment stamps it. """ @@ -55,6 +55,7 @@ def _model_replica(*, copies: int) -> fnv1.Resource: labels={ "modelplane.ai/deployment": "my-deployment", "modelplane.ai/cluster": "cluster-a", + "modelplane.ai/replica-index": "0", }, ), spec=v1alpha1.SpecModel1( @@ -100,7 +101,7 @@ def _model_replica(*, copies: int) -> fnv1.Resource: def _prefill_decode_replica(*, serving: v1alpha1.Serving) -> fnv1.Resource: - """The ModelReplica XR test-replica in ml-team, serving PrefillDecode from a prefill and a decode engine. + """The ModelReplica XR test-replica in ml-team, replica 0 of my-deployment, serving PrefillDecode from a prefill and a decode engine. Each engine's pod runs only its engine container, so the decode pod has no sidecar. @@ -112,6 +113,7 @@ def _prefill_decode_replica(*, serving: v1alpha1.Serving) -> fnv1.Resource: labels={ "modelplane.ai/deployment": "my-deployment", "modelplane.ai/cluster": "cluster-a", + "modelplane.ai/replica-index": "0", }, ), spec=v1alpha1.SpecModel1( @@ -192,8 +194,12 @@ def _prefill_decode_replica(*, serving: v1alpha1.Serving) -> fnv1.Resource: return fnv1.Resource(resource=resource.dict_to_struct(xr.model_dump(exclude_none=True, mode="json", by_alias=True))) -def _cluster(*, provider_config_ref: str | None) -> fnv1.Resource: - """The InferenceCluster cluster-a, with no status until it reports a providerConfigRef.""" +def _cluster(*, provider_config_ref: str | None, observed: bool) -> fnv1.Resource: + """The InferenceCluster cluster-a, with no status until it reports a providerConfigRef. + + Its pool frontier, of one node with eight GPUs free, is one the cluster agent + reports on if observed is True. + """ cluster = { "apiVersion": "modelplane.ai/v1alpha1", "kind": "InferenceCluster", @@ -202,10 +208,15 @@ def _cluster(*, provider_config_ref: str | None) -> fnv1.Resource: "cluster": {"source": "Existing", "existing": {"secretRef": {"name": "k"}}}, }, } + pool: dict = {"name": "frontier", "nodes": 1} + if observed: + pool["observedNodes"] = 1 + pool["nodesByFreeDevices"] = [{"freeDevices": {"gpu.nvidia.com": 8}, "nodes": 1}] if provider_config_ref is not None: cluster["status"] = { "providerConfigRef": {"name": provider_config_ref}, "gateway": {"address": "10.0.0.1"}, + "gpuPools": [pool], } return fnv1.Resource(resource=resource.dict_to_struct(cluster)) @@ -214,7 +225,7 @@ def _deployment(*, ready: fnv1.Ready, name: str, claim_template_name: str, label """The Object composing an engine's Deployment. Its pod carries the labels the collector attributes the engine's metrics by: - the deployment, the engine and the member's role. + the deployment, the replica, the engine and the member's role. """ return fnv1.Resource( resource=resource.dict_to_struct( @@ -422,8 +433,8 @@ def _epp(*, ready: fnv1.Ready, config_checksum: str) -> fnv1.Resource: """The Object composing the endpoint picker's Deployment, running llm-d's picker with a config. Its pod carries the labels the collector attributes the picker's metrics by: - the deployment and the picker role. The picker's command line takes the pool - and the config's path from the environment Modelplane sets. + the deployment, the replica and the picker role. The picker's command line + takes the pool and the config's path from the environment Modelplane sets. """ return fnv1.Resource( resource=resource.dict_to_struct( @@ -454,6 +465,7 @@ def _epp(*, ready: fnv1.Ready, config_checksum: str) -> fnv1.Resource: "app": "test-replica-epp", "modelplane.ai/role": "picker", "modelplane.ai/deployment": "my-deployment", + "modelplane.ai/replica": "0", }, "annotations": {"modelplane.ai/epp-config-checksum": config_checksum}, }, @@ -695,6 +707,41 @@ def _epp_service(*, ready: fnv1.Ready) -> fnv1.Resource: ) +def _replica_observation() -> fnv1.Resource: + """The watched and polled Object creating the replica's ReplicaObservation, selecting its engine pods, and marked ready.""" + return fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "kubernetes.m.crossplane.io/v1alpha1", + "kind": "Object", + "metadata": {"annotations": {"crossplane.io/poll-interval": "30s"}}, + "spec": { + "providerConfigRef": {"kind": "ClusterProviderConfig", "name": "cluster-a-pc"}, + "readiness": {"policy": "SuccessfulCreate"}, + "watch": True, + "forProvider": { + "manifest": { + "apiVersion": "agent.modelplane.ai/v1alpha1", + "kind": "ReplicaObservation", + "metadata": {"name": "test-replica", "namespace": "mp-ml-team-51733"}, + "spec": { + "podSelector": { + "matchLabels": { + "modelplane.ai/deployment": "my-deployment", + "modelplane.ai/replica": "0", + }, + "matchExpressions": [{"key": "modelplane.ai/engine", "operator": "Exists"}], + }, + }, + } + }, + }, + } + ), + ready=fnv1.READY_TRUE, + ) + + def _observed_deployment(*, ready: bool, ready_replicas: int) -> fnv1.Resource: """The engine's Deployment Object as observed back. @@ -775,7 +822,9 @@ def _to_dict(msg: message.Message) -> dict: "model-serving-decode": _observed_deployment(ready=True, ready_replicas=1), }, ), - required_resources={"cluster": fnv1.Resources(items=[_cluster(provider_config_ref="cluster-a-pc")])}, + required_resources={ + "cluster": fnv1.Resources(items=[_cluster(provider_config_ref="cluster-a-pc", observed=False)]) + }, ), want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), @@ -822,7 +871,9 @@ def _to_dict(msg: message.Message) -> dict: "model-serving-decode": _observed_deployment(ready=False, ready_replicas=0), }, ), - required_resources={"cluster": fnv1.Resources(items=[_cluster(provider_config_ref="cluster-a-pc")])}, + required_resources={ + "cluster": fnv1.Resources(items=[_cluster(provider_config_ref="cluster-a-pc", observed=False)]) + }, ), want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), @@ -867,7 +918,9 @@ def _to_dict(msg: message.Message) -> dict: serving=v1alpha1.Serving(mode="PrefillDecode", router="EndpointPicker") ) ), - required_resources={"cluster": fnv1.Resources(items=[_cluster(provider_config_ref="cluster-a-pc")])}, + required_resources={ + "cluster": fnv1.Resources(items=[_cluster(provider_config_ref="cluster-a-pc", observed=False)]) + }, ), want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), @@ -941,7 +994,9 @@ def _to_dict(msg: message.Message) -> dict: ) ) ), - required_resources={"cluster": fnv1.Resources(items=[_cluster(provider_config_ref="cluster-a-pc")])}, + required_resources={ + "cluster": fnv1.Resources(items=[_cluster(provider_config_ref="cluster-a-pc", observed=False)]) + }, ), want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), @@ -955,6 +1010,7 @@ def _to_dict(msg: message.Message) -> dict: labels={ "modelplane.ai/deployment": "my-deployment", "modelplane.ai/engine": "prefill", + "modelplane.ai/replica": "0", "modelplane.ai/phase": "Prefill", "modelplane.ai/role": "Standalone", "modelplane.ai/serving": "test-replica", @@ -973,6 +1029,7 @@ def _to_dict(msg: message.Message) -> dict: labels={ "modelplane.ai/deployment": "my-deployment", "modelplane.ai/engine": "decode", + "modelplane.ai/replica": "0", "modelplane.ai/phase": "Decode", "modelplane.ai/role": "Standalone", "modelplane.ai/serving": "test-replica", @@ -1068,11 +1125,270 @@ def _to_dict(msg: message.Message) -> dict: ), req=fnv1.RunFunctionRequest( observed=fnv1.State(composite=_model_replica(copies=1)), - required_resources={"cluster": fnv1.Resources(items=[_cluster(provider_config_ref="cluster-a-pc")])}, + required_resources={ + "cluster": fnv1.Resources(items=[_cluster(provider_config_ref="cluster-a-pc", observed=True)]) + }, + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + resources={ + "model-serving-main": _deployment( + copies=1, + ready=fnv1.READY_UNSPECIFIED, + name="test-replica-main-6b608", + claim_template_name="test-replica-main-standalone-devices-e609d", + labels={ + "modelplane.ai/deployment": "my-deployment", + "modelplane.ai/engine": "main", + "modelplane.ai/replica": "0", + "modelplane.ai/role": "Standalone", + "modelplane.ai/serving": "test-replica", + "modelplane.ai/workload": "test-replica-main-6b608", + }, + ), + "model-route": _route(ready=fnv1.READY_UNSPECIFIED), + "resource-claim-main-standalone": _claim_template( + ready=fnv1.READY_UNSPECIFIED, name="test-replica-main-standalone-devices-e609d" + ), + "inference-pool": _inference_pool(ready=fnv1.READY_UNSPECIFIED), + "epp": _epp( + ready=fnv1.READY_UNSPECIFIED, + config_checksum="c4f1cefa4a1fe67069eaa6e9dde3e5b9bc7e79d3f807ecd076dd021259b2ab39", + ), + "epp-config": _epp_config(ready=fnv1.READY_UNSPECIFIED), + "epp-role": _epp_role(ready=fnv1.READY_UNSPECIFIED), + "epp-rolebinding": _epp_role_binding(ready=fnv1.READY_UNSPECIFIED), + "epp-serviceaccount": _epp_service_account(ready=fnv1.READY_UNSPECIFIED), + "epp-service": _epp_service(ready=fnv1.READY_UNSPECIFIED), + "replica-observation": _replica_observation(), + }, + ), + results=[ + fnv1.Result( + severity=fnv1.SEVERITY_NORMAL, + message="Composing vllm/vllm-openai:latest on cluster-a", + ), + ], + context=structpb.Struct(), + requirements=fnv1.Requirements( + resources={ + "cluster": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", + kind="InferenceCluster", + match_name="cluster-a", + ), + }, + ), + conditions=[ + fnv1.Condition( + type="ModelAccepted", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Deploying", + ), + fnv1.Condition( + type="ModelServing", + status=fnv1.STATUS_CONDITION_FALSE, + reason="WaitingForModel", + ), + fnv1.Condition( + type="ModelReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="WaitingForModel", + message="0 of 1 engine copies ready", + ), + fnv1.Condition( + type="ModelPodsScheduled", status=fnv1.STATUS_CONDITION_FALSE, reason="WaitingForObservation" + ), + ], + ), + ), + Case( + name="ObservationKept", + reason="A replica keeps its ReplicaObservation once composed, even when its cluster stops publishing an observation of its pool.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_model_replica(copies=1), + resources={ + "replica-observation": fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "kubernetes.m.crossplane.io/v1alpha1", + "kind": "Object", + "spec": {"forProvider": {"manifest": {"kind": "ReplicaObservation"}}}, + } + ) + ), + }, + ), + required_resources={ + "cluster": fnv1.Resources(items=[_cluster(provider_config_ref="cluster-a-pc", observed=False)]) + }, + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + resources={ + "model-serving-main": _deployment( + copies=1, + ready=fnv1.READY_UNSPECIFIED, + name="test-replica-main-6b608", + claim_template_name="test-replica-main-standalone-devices-e609d", + labels={ + "modelplane.ai/deployment": "my-deployment", + "modelplane.ai/engine": "main", + "modelplane.ai/replica": "0", + "modelplane.ai/role": "Standalone", + "modelplane.ai/serving": "test-replica", + "modelplane.ai/workload": "test-replica-main-6b608", + }, + ), + "model-route": _route(ready=fnv1.READY_UNSPECIFIED), + "resource-claim-main-standalone": _claim_template( + ready=fnv1.READY_UNSPECIFIED, name="test-replica-main-standalone-devices-e609d" + ), + "inference-pool": _inference_pool(ready=fnv1.READY_UNSPECIFIED), + "epp": _epp( + ready=fnv1.READY_UNSPECIFIED, + config_checksum="c4f1cefa4a1fe67069eaa6e9dde3e5b9bc7e79d3f807ecd076dd021259b2ab39", + ), + "epp-config": _epp_config(ready=fnv1.READY_UNSPECIFIED), + "epp-role": _epp_role(ready=fnv1.READY_UNSPECIFIED), + "epp-rolebinding": _epp_role_binding(ready=fnv1.READY_UNSPECIFIED), + "epp-serviceaccount": _epp_service_account(ready=fnv1.READY_UNSPECIFIED), + "epp-service": _epp_service(ready=fnv1.READY_UNSPECIFIED), + "replica-observation": _replica_observation(), + }, + ), + results=[ + fnv1.Result( + severity=fnv1.SEVERITY_NORMAL, + message="Composing vllm/vllm-openai:latest on cluster-a", + ), + ], + context=structpb.Struct(), + requirements=fnv1.Requirements( + resources={ + "cluster": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", + kind="InferenceCluster", + match_name="cluster-a", + ), + }, + ), + conditions=[ + fnv1.Condition( + type="ModelAccepted", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Deploying", + ), + fnv1.Condition( + type="ModelServing", + status=fnv1.STATUS_CONDITION_FALSE, + reason="WaitingForModel", + ), + fnv1.Condition( + type="ModelReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="WaitingForModel", + message="0 of 1 engine copies ready", + ), + fnv1.Condition( + type="ModelPodsScheduled", status=fnv1.STATUS_CONDITION_FALSE, reason="WaitingForObservation" + ), + ], + ), + ), + Case( + name="ObservationRelayed", + reason="A replica relays that the agent has seen its pods scheduled, and when it first saw their claims allocated on each pool.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_model_replica(copies=1), + resources={ + "replica-observation": fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "kubernetes.m.crossplane.io/v1alpha1", + "kind": "Object", + "spec": {"forProvider": {"manifest": {"kind": "ReplicaObservation"}}}, + "status": { + "atProvider": { + "manifest": { + "apiVersion": "agent.modelplane.ai/v1alpha1", + "kind": "ReplicaObservation", + "metadata": {"name": "test-replica", "namespace": "mp-ml-team-51733"}, + "spec": { + "podSelector": { + "matchLabels": { + "modelplane.ai/deployment": "my-deployment", + "modelplane.ai/replica": "0", + }, + "matchExpressions": [ + {"key": "modelplane.ai/engine", "operator": "Exists"} + ], + }, + }, + "status": { + "observedGeneration": 1, + "conditions": [ + { + "type": "Scheduled", + "status": "True", + "reason": "PodsScheduled", + "message": "", + "lastTransitionTime": "2026-10-20T10:00:00Z", + "observedGeneration": 1, + }, + { + "type": "Ready", + "status": "False", + "reason": "Starting", + "message": "0 of 1 pods ready", + "lastTransitionTime": "2026-10-20T10:00:00Z", + "observedGeneration": 1, + }, + ], + "pods": {"total": 1, "scheduled": 1, "ready": 0}, + "devices": [ + { + "driver": "gpu.nvidia.com", + "pool": "ip-10-0-54-161.ec2.internal", + "device": "gpu-2", + "nodeName": "ip-10-0-54-161.ec2.internal", + }, + ], + "pools": [ + { + "name": "frontier", + "lastAllocationTime": "2026-10-20T10:02:13.482915Z", + }, + ], + }, + }, + }, + }, + } + ) + ), + }, + ), + required_resources={ + "cluster": fnv1.Resources(items=[_cluster(provider_config_ref="cluster-a-pc", observed=True)]) + }, ), want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( + composite=fnv1.Resource( + resource=resource.dict_to_struct( + { + "status": { + "pools": [{"name": "frontier", "lastAllocationTime": "2026-10-20T10:02:13.482915Z"}] + } + } + ) + ), resources={ "model-serving-main": _deployment( copies=1, @@ -1082,6 +1398,241 @@ def _to_dict(msg: message.Message) -> dict: labels={ "modelplane.ai/deployment": "my-deployment", "modelplane.ai/engine": "main", + "modelplane.ai/replica": "0", + "modelplane.ai/role": "Standalone", + "modelplane.ai/serving": "test-replica", + "modelplane.ai/workload": "test-replica-main-6b608", + }, + ), + "model-route": _route(ready=fnv1.READY_UNSPECIFIED), + "resource-claim-main-standalone": _claim_template( + ready=fnv1.READY_UNSPECIFIED, name="test-replica-main-standalone-devices-e609d" + ), + "inference-pool": _inference_pool(ready=fnv1.READY_UNSPECIFIED), + "epp": _epp( + ready=fnv1.READY_UNSPECIFIED, + config_checksum="c4f1cefa4a1fe67069eaa6e9dde3e5b9bc7e79d3f807ecd076dd021259b2ab39", + ), + "epp-config": _epp_config(ready=fnv1.READY_UNSPECIFIED), + "epp-role": _epp_role(ready=fnv1.READY_UNSPECIFIED), + "epp-rolebinding": _epp_role_binding(ready=fnv1.READY_UNSPECIFIED), + "epp-serviceaccount": _epp_service_account(ready=fnv1.READY_UNSPECIFIED), + "epp-service": _epp_service(ready=fnv1.READY_UNSPECIFIED), + "replica-observation": _replica_observation(), + }, + ), + results=[ + fnv1.Result( + severity=fnv1.SEVERITY_NORMAL, + message="Composing vllm/vllm-openai:latest on cluster-a", + ), + ], + context=structpb.Struct(), + requirements=fnv1.Requirements( + resources={ + "cluster": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", + kind="InferenceCluster", + match_name="cluster-a", + ), + }, + ), + conditions=[ + fnv1.Condition( + type="ModelAccepted", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Deploying", + ), + fnv1.Condition( + type="ModelServing", + status=fnv1.STATUS_CONDITION_FALSE, + reason="WaitingForModel", + ), + fnv1.Condition( + type="ModelReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="WaitingForModel", + message="0 of 1 engine copies ready", + ), + fnv1.Condition(type="ModelPodsScheduled", status=fnv1.STATUS_CONDITION_TRUE, reason="AllPodsScheduled"), + ], + ), + ), + # The agent's schema requires a pool's allocation time. + Case( + name="ObservationInvalid", + reason="A replica whose observation doesn't match the agent's schema waits for an observation, rather than failing.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_model_replica(copies=1), + resources={ + "replica-observation": fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "kubernetes.m.crossplane.io/v1alpha1", + "kind": "Object", + "spec": {"forProvider": {"manifest": {"kind": "ReplicaObservation"}}}, + "status": { + "atProvider": { + "manifest": { + "apiVersion": "agent.modelplane.ai/v1alpha1", + "kind": "ReplicaObservation", + "metadata": {"name": "test-replica", "namespace": "mp-ml-team-51733"}, + "spec": { + "podSelector": { + "matchLabels": { + "modelplane.ai/deployment": "my-deployment", + "modelplane.ai/replica": "0", + }, + "matchExpressions": [ + {"key": "modelplane.ai/engine", "operator": "Exists"} + ], + }, + }, + "status": { + "observedGeneration": 1, + "conditions": [ + { + "type": "Scheduled", + "status": "True", + "reason": "PodsScheduled", + "message": "", + "lastTransitionTime": "2026-10-20T10:00:00Z", + "observedGeneration": 1, + }, + { + "type": "Ready", + "status": "False", + "reason": "Starting", + "message": "0 of 1 pods ready", + "lastTransitionTime": "2026-10-20T10:00:00Z", + "observedGeneration": 1, + }, + ], + "pods": {"total": 1, "scheduled": 1, "ready": 0}, + "devices": [ + { + "driver": "gpu.nvidia.com", + "pool": "ip-10-0-54-161.ec2.internal", + "device": "gpu-2", + "nodeName": "ip-10-0-54-161.ec2.internal", + }, + ], + "pools": [{"name": "frontier"}], + }, + }, + }, + }, + } + ) + ), + }, + ), + required_resources={ + "cluster": fnv1.Resources(items=[_cluster(provider_config_ref="cluster-a-pc", observed=True)]) + }, + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + resources={ + "model-serving-main": _deployment( + copies=1, + ready=fnv1.READY_UNSPECIFIED, + name="test-replica-main-6b608", + claim_template_name="test-replica-main-standalone-devices-e609d", + labels={ + "modelplane.ai/deployment": "my-deployment", + "modelplane.ai/engine": "main", + "modelplane.ai/replica": "0", + "modelplane.ai/role": "Standalone", + "modelplane.ai/serving": "test-replica", + "modelplane.ai/workload": "test-replica-main-6b608", + }, + ), + "model-route": _route(ready=fnv1.READY_UNSPECIFIED), + "resource-claim-main-standalone": _claim_template( + ready=fnv1.READY_UNSPECIFIED, name="test-replica-main-standalone-devices-e609d" + ), + "inference-pool": _inference_pool(ready=fnv1.READY_UNSPECIFIED), + "epp": _epp( + ready=fnv1.READY_UNSPECIFIED, + config_checksum="c4f1cefa4a1fe67069eaa6e9dde3e5b9bc7e79d3f807ecd076dd021259b2ab39", + ), + "epp-config": _epp_config(ready=fnv1.READY_UNSPECIFIED), + "epp-role": _epp_role(ready=fnv1.READY_UNSPECIFIED), + "epp-rolebinding": _epp_role_binding(ready=fnv1.READY_UNSPECIFIED), + "epp-serviceaccount": _epp_service_account(ready=fnv1.READY_UNSPECIFIED), + "epp-service": _epp_service(ready=fnv1.READY_UNSPECIFIED), + "replica-observation": _replica_observation(), + }, + ), + results=[ + fnv1.Result( + severity=fnv1.SEVERITY_NORMAL, + message="Composing vllm/vllm-openai:latest on cluster-a", + ), + fnv1.Result( + severity=fnv1.SEVERITY_WARNING, + message="Ignoring the cluster agent's observation of this replica: it does not match the agent's " + "schema (1 problems).", + ), + ], + context=structpb.Struct(), + requirements=fnv1.Requirements( + resources={ + "cluster": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", + kind="InferenceCluster", + match_name="cluster-a", + ), + }, + ), + conditions=[ + fnv1.Condition( + type="ModelAccepted", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Deploying", + ), + fnv1.Condition( + type="ModelServing", + status=fnv1.STATUS_CONDITION_FALSE, + reason="WaitingForModel", + ), + fnv1.Condition( + type="ModelReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="WaitingForModel", + message="0 of 1 engine copies ready", + ), + fnv1.Condition( + type="ModelPodsScheduled", status=fnv1.STATUS_CONDITION_FALSE, reason="WaitingForObservation" + ), + ], + ), + ), + Case( + name="AgentNotReporting", + reason="A replica on a cluster whose agent reports on none of its pools composes no ReplicaObservation and reports no ModelPodsScheduled condition.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State(composite=_model_replica(copies=1)), + required_resources={ + "cluster": fnv1.Resources(items=[_cluster(provider_config_ref="cluster-a-pc", observed=False)]) + }, + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + resources={ + "model-serving-main": _deployment( + copies=1, + ready=fnv1.READY_UNSPECIFIED, + name="test-replica-main-6b608", + claim_template_name="test-replica-main-standalone-devices-e609d", + labels={ + "modelplane.ai/deployment": "my-deployment", + "modelplane.ai/engine": "main", + "modelplane.ai/replica": "0", "modelplane.ai/role": "Standalone", "modelplane.ai/serving": "test-replica", "modelplane.ai/workload": "test-replica-main-6b608", @@ -1173,7 +1724,7 @@ def _to_dict(msg: message.Message) -> dict: ), req=fnv1.RunFunctionRequest( observed=fnv1.State(composite=_model_replica(copies=1)), - required_resources={"cluster": fnv1.Resources(items=[_cluster(provider_config_ref=None)])}, + required_resources={"cluster": fnv1.Resources(items=[_cluster(provider_config_ref=None, observed=False)])}, ), want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), @@ -1220,7 +1771,9 @@ def _to_dict(msg: message.Message) -> dict: "resource-claim-main-standalone": _observed_object(ready=True), }, ), - required_resources={"cluster": fnv1.Resources(items=[_cluster(provider_config_ref="cluster-a-pc")])}, + required_resources={ + "cluster": fnv1.Resources(items=[_cluster(provider_config_ref="cluster-a-pc", observed=True)]) + }, ), want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), @@ -1234,6 +1787,7 @@ def _to_dict(msg: message.Message) -> dict: labels={ "modelplane.ai/deployment": "my-deployment", "modelplane.ai/engine": "main", + "modelplane.ai/replica": "0", "modelplane.ai/role": "Standalone", "modelplane.ai/serving": "test-replica", "modelplane.ai/workload": "test-replica-main-6b608", @@ -1253,6 +1807,7 @@ def _to_dict(msg: message.Message) -> dict: "epp-rolebinding": _epp_role_binding(ready=fnv1.READY_UNSPECIFIED), "epp-serviceaccount": _epp_service_account(ready=fnv1.READY_UNSPECIFIED), "epp-service": _epp_service(ready=fnv1.READY_UNSPECIFIED), + "replica-observation": _replica_observation(), }, ), context=structpb.Struct(), @@ -1274,6 +1829,9 @@ def _to_dict(msg: message.Message) -> dict: reason="WaitingForRouting", message="1 of 1 engine copies ready", ), + fnv1.Condition( + type="ModelPodsScheduled", status=fnv1.STATUS_CONDITION_FALSE, reason="WaitingForObservation" + ), ], ), ), @@ -1302,7 +1860,9 @@ def _to_dict(msg: message.Message) -> dict: "epp-service": _observed_object(ready=False), }, ), - required_resources={"cluster": fnv1.Resources(items=[_cluster(provider_config_ref="cluster-a-pc")])}, + required_resources={ + "cluster": fnv1.Resources(items=[_cluster(provider_config_ref="cluster-a-pc", observed=True)]) + }, ), want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), @@ -1316,6 +1876,7 @@ def _to_dict(msg: message.Message) -> dict: labels={ "modelplane.ai/deployment": "my-deployment", "modelplane.ai/engine": "main", + "modelplane.ai/replica": "0", "modelplane.ai/role": "Standalone", "modelplane.ai/serving": "test-replica", "modelplane.ai/workload": "test-replica-main-6b608", @@ -1335,6 +1896,7 @@ def _to_dict(msg: message.Message) -> dict: "epp-rolebinding": _epp_role_binding(ready=fnv1.READY_TRUE), "epp-serviceaccount": _epp_service_account(ready=fnv1.READY_TRUE), "epp-service": _epp_service(ready=fnv1.READY_UNSPECIFIED), + "replica-observation": _replica_observation(), }, ), context=structpb.Struct(), @@ -1356,6 +1918,9 @@ def _to_dict(msg: message.Message) -> dict: reason="WaitingForRouting", message="1 of 1 engine copies ready", ), + fnv1.Condition( + type="ModelPodsScheduled", status=fnv1.STATUS_CONDITION_FALSE, reason="WaitingForObservation" + ), ], ), ), @@ -1384,7 +1949,9 @@ def _to_dict(msg: message.Message) -> dict: "epp-service": _observed_object(ready=True), }, ), - required_resources={"cluster": fnv1.Resources(items=[_cluster(provider_config_ref="cluster-a-pc")])}, + required_resources={ + "cluster": fnv1.Resources(items=[_cluster(provider_config_ref="cluster-a-pc", observed=True)]) + }, ), want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), @@ -1398,6 +1965,7 @@ def _to_dict(msg: message.Message) -> dict: labels={ "modelplane.ai/deployment": "my-deployment", "modelplane.ai/engine": "main", + "modelplane.ai/replica": "0", "modelplane.ai/role": "Standalone", "modelplane.ai/serving": "test-replica", "modelplane.ai/workload": "test-replica-main-6b608", @@ -1417,6 +1985,7 @@ def _to_dict(msg: message.Message) -> dict: "epp-rolebinding": _epp_role_binding(ready=fnv1.READY_TRUE), "epp-serviceaccount": _epp_service_account(ready=fnv1.READY_TRUE), "epp-service": _epp_service(ready=fnv1.READY_TRUE), + "replica-observation": _replica_observation(), }, ), context=structpb.Struct(), @@ -1438,6 +2007,9 @@ def _to_dict(msg: message.Message) -> dict: reason="WaitingForRouting", message="1 of 1 engine copies ready", ), + fnv1.Condition( + type="ModelPodsScheduled", status=fnv1.STATUS_CONDITION_FALSE, reason="WaitingForObservation" + ), ], ), ), @@ -1463,7 +2035,9 @@ def _to_dict(msg: message.Message) -> dict: "epp-service": _observed_object(ready=True), }, ), - required_resources={"cluster": fnv1.Resources(items=[_cluster(provider_config_ref="cluster-a-pc")])}, + required_resources={ + "cluster": fnv1.Resources(items=[_cluster(provider_config_ref="cluster-a-pc", observed=False)]) + }, ), want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), @@ -1477,6 +2051,7 @@ def _to_dict(msg: message.Message) -> dict: labels={ "modelplane.ai/deployment": "my-deployment", "modelplane.ai/engine": "main", + "modelplane.ai/replica": "0", "modelplane.ai/role": "Standalone", "modelplane.ai/serving": "test-replica", "modelplane.ai/workload": "test-replica-main-6b608", @@ -1539,7 +2114,9 @@ def _to_dict(msg: message.Message) -> dict: "epp-service": _observed_object(ready=True), }, ), - required_resources={"cluster": fnv1.Resources(items=[_cluster(provider_config_ref="cluster-a-pc")])}, + required_resources={ + "cluster": fnv1.Resources(items=[_cluster(provider_config_ref="cluster-a-pc", observed=False)]) + }, ), want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), @@ -1553,6 +2130,7 @@ def _to_dict(msg: message.Message) -> dict: labels={ "modelplane.ai/deployment": "my-deployment", "modelplane.ai/engine": "main", + "modelplane.ai/replica": "0", "modelplane.ai/role": "Standalone", "modelplane.ai/serving": "test-replica", "modelplane.ai/workload": "test-replica-main-6b608", @@ -1618,7 +2196,9 @@ def _to_dict(msg: message.Message) -> dict: "epp-service": _observed_object(ready=True), }, ), - required_resources={"cluster": fnv1.Resources(items=[_cluster(provider_config_ref="cluster-a-pc")])}, + required_resources={ + "cluster": fnv1.Resources(items=[_cluster(provider_config_ref="cluster-a-pc", observed=False)]) + }, ), want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), @@ -1632,6 +2212,7 @@ def _to_dict(msg: message.Message) -> dict: labels={ "modelplane.ai/deployment": "my-deployment", "modelplane.ai/engine": "main", + "modelplane.ai/replica": "0", "modelplane.ai/role": "Standalone", "modelplane.ai/serving": "test-replica", "modelplane.ai/workload": "test-replica-main-6b608", @@ -1694,7 +2275,9 @@ def _to_dict(msg: message.Message) -> dict: "epp-service": _observed_object(ready=True), }, ), - required_resources={"cluster": fnv1.Resources(items=[_cluster(provider_config_ref="cluster-a-pc")])}, + required_resources={ + "cluster": fnv1.Resources(items=[_cluster(provider_config_ref="cluster-a-pc", observed=False)]) + }, ), want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), @@ -1708,6 +2291,7 @@ def _to_dict(msg: message.Message) -> dict: labels={ "modelplane.ai/deployment": "my-deployment", "modelplane.ai/engine": "main", + "modelplane.ai/replica": "0", "modelplane.ai/role": "Standalone", "modelplane.ai/serving": "test-replica", "modelplane.ai/workload": "test-replica-main-6b608", @@ -1751,6 +2335,92 @@ def _to_dict(msg: message.Message) -> dict: ], ), ), + Case( + name="ObservationComposedWhileServing", + reason=( + "A serving replica keeps serving at full capacity when the agent starts reporting on its pool, while the " + "ReplicaObservation it then composes isn't yet observed." + ), + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_model_replica(copies=2), + resources={ + "model-serving-main": _observed_deployment(ready=True, ready_replicas=2), + "model-route": _observed_object(ready=True), + "resource-claim-main-standalone": _observed_object(ready=True), + "inference-pool": _observed_object(ready=True), + "epp": _observed_object(ready=True), + "epp-config": _observed_object(ready=True), + "epp-role": _observed_object(ready=True), + "epp-rolebinding": _observed_object(ready=True), + "epp-serviceaccount": _observed_object(ready=True), + "epp-service": _observed_object(ready=True), + }, + ), + required_resources={ + "cluster": fnv1.Resources(items=[_cluster(provider_config_ref="cluster-a-pc", observed=True)]) + }, + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + resources={ + "model-serving-main": _deployment( + ready=fnv1.READY_TRUE, + copies=2, + name="test-replica-main-6b608", + claim_template_name="test-replica-main-standalone-devices-e609d", + labels={ + "modelplane.ai/deployment": "my-deployment", + "modelplane.ai/engine": "main", + "modelplane.ai/replica": "0", + "modelplane.ai/role": "Standalone", + "modelplane.ai/serving": "test-replica", + "modelplane.ai/workload": "test-replica-main-6b608", + }, + ), + "model-route": _route(ready=fnv1.READY_TRUE), + "resource-claim-main-standalone": _claim_template( + ready=fnv1.READY_TRUE, name="test-replica-main-standalone-devices-e609d" + ), + "inference-pool": _inference_pool(ready=fnv1.READY_TRUE), + "epp": _epp( + ready=fnv1.READY_TRUE, + config_checksum="c4f1cefa4a1fe67069eaa6e9dde3e5b9bc7e79d3f807ecd076dd021259b2ab39", + ), + "epp-config": _epp_config(ready=fnv1.READY_TRUE), + "epp-role": _epp_role(ready=fnv1.READY_TRUE), + "epp-rolebinding": _epp_role_binding(ready=fnv1.READY_TRUE), + "epp-serviceaccount": _epp_service_account(ready=fnv1.READY_TRUE), + "epp-service": _epp_service(ready=fnv1.READY_TRUE), + "replica-observation": _replica_observation(), + }, + ), + context=structpb.Struct(), + requirements=fnv1.Requirements( + resources={ + "cluster": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", + kind="InferenceCluster", + match_name="cluster-a", + ), + }, + ), + conditions=[ + fnv1.Condition(type="ModelAccepted", status=fnv1.STATUS_CONDITION_TRUE, reason="Accepted"), + fnv1.Condition(type="ModelServing", status=fnv1.STATUS_CONDITION_TRUE, reason="Serving"), + fnv1.Condition( + type="ModelReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="AllCopiesReady", + message="2 of 2 engine copies ready", + ), + fnv1.Condition( + type="ModelPodsScheduled", status=fnv1.STATUS_CONDITION_FALSE, reason="WaitingForObservation" + ), + ], + ), + ), ] diff --git a/functions/compose-model-replica/tests/test_observation.py b/functions/compose-model-replica/tests/test_observation.py new file mode 100644 index 000000000..75d67191f --- /dev/null +++ b/functions/compose-model-replica/tests/test_observation.py @@ -0,0 +1,702 @@ +# Copyright 2026 The Modelplane Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for compose-model-replica's observation of whether a replica's pods have scheduled. + +The observed ReplicaObservation statuses are what the cluster agent reports. +""" + +import dataclasses +import datetime +from typing import Literal + +import pydantic +import pytest +from crossplane.function import resource +from crossplane.function.proto.v1 import run_function_pb2 as fnv1 +from function import observation +from models.ai.modelplane.agent.replicaobservation import v1alpha1 as rov1alpha1 +from models.ai.modelplane.modelreplica import v1alpha1 +from models.io.k8s.apimachinery.pkg.apis.meta import v1 as metav1 + + +@dataclasses.dataclass +class ComposeCase: + """A test case for observation.compose.""" + + name: str + reason: str + replica: v1alpha1.ModelReplica + provider_config: str + want: dict | None + + +@dataclasses.dataclass +class PodsScheduledCase: + """A test case for observation.pods_scheduled.""" + + name: str + reason: str + replica: v1alpha1.ModelReplica + status: rov1alpha1.Status | None + want: resource.Condition + + +def _member(*, role: Literal["Standalone", "Leader", "Worker"], worker_nodes: int | None) -> v1alpha1.Member: + """A member of role pinned to pool frontier, claiming one GPU, with a Worker's node span unless it's None.""" + return v1alpha1.Member( + role=role, + worker=v1alpha1.Worker(nodes=worker_nodes) if worker_nodes is not None else None, + nodePoolName="frontier", + deviceRequests=[v1alpha1.DeviceRequest(name="gpu", deviceClassName="gpu.nvidia.com", count=1)], + template=v1alpha1.Template( + spec=v1alpha1.Spec(containers=[v1alpha1.Container(name="engine", image="vllm/vllm-openai:latest")]), + ), + ) + + +def _replica( + *, labels: dict[str, str], members: list[v1alpha1.Member], conditions: list[v1alpha1.Condition] | None +) -> v1alpha1.ModelReplica: + """The ModelReplica test-replica in ml-team, whose one engine, main, has members, with conditions unless they're None.""" + return v1alpha1.ModelReplica( + metadata=metav1.ObjectMeta(name="test-replica", namespace="ml-team", labels=labels), + spec=v1alpha1.SpecModel1( + clusterName="cluster-a", + engines=[v1alpha1.Engine(name="main", copies=1, members=members)], + ), + status=v1alpha1.Status(conditions=conditions) if conditions is not None else None, + ) + + +def _observed(*, status: dict) -> fnv1.Resource: + """The observed ReplicaObservation Object, whose remote manifest carries the status the agent reported.""" + return fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "kubernetes.m.crossplane.io/v1alpha1", + "kind": "Object", + "spec": {"forProvider": {"manifest": {"kind": "ReplicaObservation"}}}, + "status": { + "atProvider": { + "manifest": { + "apiVersion": "agent.modelplane.ai/v1alpha1", + "kind": "ReplicaObservation", + "metadata": {"name": "test-replica", "namespace": "mp-ml-team-51733"}, + "spec": { + "podSelector": { + "matchLabels": { + "modelplane.ai/deployment": "my-deployment", + "modelplane.ai/replica": "0", + }, + "matchExpressions": [{"key": "modelplane.ai/engine", "operator": "Exists"}], + }, + }, + "status": status, + }, + }, + }, + } + ), + ) + + +@dataclasses.dataclass +class StatusCase: + """A test case for observation.status.""" + + name: str + reason: str + observed: fnv1.Resource | None + want: rov1alpha1.Status | None + + +STATUS_CASES = [ + StatusCase( + name="NotObserved", + reason="Before its ReplicaObservation is observed, a replica has no status from the agent.", + observed=None, + want=None, + ), + StatusCase( + name="NotReported", + reason="Before the agent reports on its pods, a replica has no status from the agent.", + observed=fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "kubernetes.m.crossplane.io/v1alpha1", + "kind": "Object", + "spec": {"forProvider": {"manifest": {"kind": "ReplicaObservation"}}}, + } + ) + ), + want=None, + ), + StatusCase( + name="NoStatus", + reason="Before the agent writes its status, a ReplicaObservation that exists gives the replica no status from the agent.", + observed=fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "kubernetes.m.crossplane.io/v1alpha1", + "kind": "Object", + "spec": {"forProvider": {"manifest": {"kind": "ReplicaObservation"}}}, + "status": { + "atProvider": { + "manifest": { + "apiVersion": "agent.modelplane.ai/v1alpha1", + "kind": "ReplicaObservation", + "metadata": {"name": "test-replica", "namespace": "mp-ml-team-51733"}, + "spec": { + "podSelector": { + "matchLabels": { + "modelplane.ai/deployment": "my-deployment", + "modelplane.ai/replica": "0", + }, + "matchExpressions": [{"key": "modelplane.ai/engine", "operator": "Exists"}], + }, + }, + }, + }, + }, + } + ) + ), + want=None, + ), + StatusCase( + name="Reported", + reason="A replica reads the status the agent reported.", + observed=_observed( + status={ + "observedGeneration": 1, + "conditions": [ + { + "type": "Scheduled", + "status": "True", + "reason": "PodsScheduled", + "message": "", + "lastTransitionTime": "2026-10-20T10:00:00Z", + "observedGeneration": 1, + }, + { + "type": "Ready", + "status": "True", + "reason": "PodsReady", + "message": "", + "lastTransitionTime": "2026-10-20T10:00:00Z", + "observedGeneration": 1, + }, + ], + "pods": {"total": 1, "scheduled": 1, "ready": 1}, + "devices": [ + { + "driver": "gpu.nvidia.com", + "pool": "ip-10-0-54-161.ec2.internal", + "device": "gpu-2", + "nodeName": "ip-10-0-54-161.ec2.internal", + }, + ], + "pools": [{"name": "frontier", "lastAllocationTime": "2026-10-20T10:02:13.482915Z"}], + } + ), + want=rov1alpha1.Status( + observedGeneration=1, + conditions=[ + rov1alpha1.Condition( + type="Scheduled", + status="True", + reason="PodsScheduled", + message="", + lastTransitionTime=datetime.datetime(2026, 10, 20, 10, 0, tzinfo=datetime.UTC), + observedGeneration=1, + ), + rov1alpha1.Condition( + type="Ready", + status="True", + reason="PodsReady", + message="", + lastTransitionTime=datetime.datetime(2026, 10, 20, 10, 0, tzinfo=datetime.UTC), + observedGeneration=1, + ), + ], + pods=rov1alpha1.Pods(total=1, scheduled=1, ready=1), + devices=[ + rov1alpha1.Device( + driver="gpu.nvidia.com", + pool="ip-10-0-54-161.ec2.internal", + device="gpu-2", + nodeName="ip-10-0-54-161.ec2.internal", + ), + ], + pools=[ + rov1alpha1.Pool( + name="frontier", + lastAllocationTime=datetime.datetime(2026, 10, 20, 10, 2, 13, 482915, tzinfo=datetime.UTC), + ), + ], + ), + ), +] + + +@pytest.mark.parametrize("case", STATUS_CASES, ids=lambda case: case.name) +def test_status(case: StatusCase) -> None: + """status reads the status the agent reported in a replica's ReplicaObservation.""" + got = observation.status(case.observed) + assert got == case.want, case.reason + + +def test_status_invalid() -> None: + """status raises when the agent reports a status that doesn't match its schema.""" + # The agent's schema requires a pool's allocation time. + observed = _observed( + status={ + "observedGeneration": 1, + "pods": {"total": 1, "scheduled": 1, "ready": 1}, + "devices": [ + { + "driver": "gpu.nvidia.com", + "pool": "ip-10-0-54-161.ec2.internal", + "device": "gpu-2", + "nodeName": "ip-10-0-54-161.ec2.internal", + }, + ], + "pools": [{"name": "frontier"}], + } + ) + with pytest.raises(pydantic.ValidationError, match=r"pools\.0\.lastAllocationTime"): + observation.status(observed) + + +COMPOSE_CASES = [ + ComposeCase( + name="Unlabelled", + reason="A replica without a replica index composes no observation, because its pods carry no replica label for the agent to see.", + replica=_replica( + labels={"modelplane.ai/deployment": "my-deployment"}, + members=[_member(role="Standalone", worker_nodes=None)], + conditions=None, + ), + provider_config="cluster-a-pc", + want=None, + ), + ComposeCase( + name="Labelled", + reason="A replica of a deployment observes its engine pods by the deployment and replica index they carry.", + replica=_replica( + labels={"modelplane.ai/deployment": "my-deployment", "modelplane.ai/replica-index": "2"}, + members=[_member(role="Standalone", worker_nodes=None)], + conditions=None, + ), + provider_config="cluster-a-pc", + want={ + "metadata": {"annotations": {"crossplane.io/poll-interval": "30s"}}, + "spec": { + "providerConfigRef": {"kind": "ClusterProviderConfig", "name": "cluster-a-pc"}, + "readiness": {"policy": "SuccessfulCreate"}, + "watch": True, + "forProvider": { + "manifest": { + "apiVersion": "agent.modelplane.ai/v1alpha1", + "kind": "ReplicaObservation", + "metadata": {"name": "test-replica", "namespace": "mp-ml-team-51733"}, + "spec": { + "podSelector": { + "matchLabels": { + "modelplane.ai/deployment": "my-deployment", + "modelplane.ai/replica": "2", + }, + "matchExpressions": [{"key": "modelplane.ai/engine", "operator": "Exists"}], + }, + }, + }, + }, + }, + }, + ), +] + + +@pytest.mark.parametrize("case", COMPOSE_CASES, ids=lambda case: case.name) +def test_compose(case: ComposeCase) -> None: + """compose builds the Object that creates a replica's ReplicaObservation.""" + got = observation.compose(case.replica, case.provider_config) + got_dict = got.model_dump(exclude_unset=True, by_alias=True) if got is not None else None + assert got_dict == case.want, case.reason + + +PODS_SCHEDULED_CASES = [ + PodsScheduledCase( + name="NotReported", + reason="Before the agent reports on its pods, a replica waits for the observation.", + replica=_replica(labels={}, members=[_member(role="Standalone", worker_nodes=None)], conditions=None), + status=None, + want=resource.Condition(typ="ModelPodsScheduled", status="False", reason="WaitingForObservation"), + ), + PodsScheduledCase( + name="Unschedulable", + reason="A replica whose pod the scheduler can't place relays the agent's reason and message.", + replica=_replica(labels={}, members=[_member(role="Standalone", worker_nodes=None)], conditions=None), + status=rov1alpha1.Status( + observedGeneration=1, + conditions=[ + rov1alpha1.Condition( + type="Scheduled", + status="False", + reason="Unschedulable", + message="pod too-big-5859cf75cf-jzm8n: 0/2 nodes are available: 1 cannot allocate all claims, 1 node(s) didn't match Pod's node affinity/selector.", + lastTransitionTime=datetime.datetime(2026, 10, 20, 10, 0, tzinfo=datetime.UTC), + observedGeneration=1, + ), + rov1alpha1.Condition( + type="Ready", + status="False", + reason="NotScheduled", + message="0 of 1 pods scheduled", + lastTransitionTime=datetime.datetime(2026, 10, 20, 10, 0, tzinfo=datetime.UTC), + observedGeneration=1, + ), + ], + pods=rov1alpha1.Pods(total=1, scheduled=0, ready=0), + ), + want=resource.Condition( + typ="ModelPodsScheduled", + status="False", + reason="Unschedulable", + message="pod too-big-5859cf75cf-jzm8n: 0/2 nodes are available: 1 cannot allocate all claims, 1 node(s) didn't match Pod's node affinity/selector.", + ), + ), + PodsScheduledCase( + name="WorkersNotCreated", + reason="A gang whose workers haven't all been created hasn't scheduled, even when every pod the agent sees has.", + replica=_replica( + labels={}, + members=[_member(role="Leader", worker_nodes=None), _member(role="Worker", worker_nodes=2)], + conditions=None, + ), + status=rov1alpha1.Status( + observedGeneration=1, + conditions=[ + rov1alpha1.Condition( + type="Scheduled", + status="True", + reason="PodsScheduled", + message="", + lastTransitionTime=datetime.datetime(2026, 10, 20, 10, 0, tzinfo=datetime.UTC), + observedGeneration=1, + ), + rov1alpha1.Condition( + type="Ready", + status="False", + reason="Starting", + message="0 of 2 pods ready", + lastTransitionTime=datetime.datetime(2026, 10, 20, 10, 0, tzinfo=datetime.UTC), + observedGeneration=1, + ), + ], + pods=rov1alpha1.Pods(total=2, scheduled=2, ready=0), + ), + want=resource.Condition( + typ="ModelPodsScheduled", status="False", reason="WaitingForPods", message="2 of 3 pods exist" + ), + ), + PodsScheduledCase( + name="Scheduled", + reason="A gang whose every pod exists and is bound to a node has scheduled.", + replica=_replica( + labels={}, + members=[_member(role="Leader", worker_nodes=None), _member(role="Worker", worker_nodes=2)], + conditions=None, + ), + status=rov1alpha1.Status( + observedGeneration=1, + conditions=[ + rov1alpha1.Condition( + type="Scheduled", + status="True", + reason="PodsScheduled", + message="", + lastTransitionTime=datetime.datetime(2026, 10, 20, 10, 0, tzinfo=datetime.UTC), + observedGeneration=1, + ), + rov1alpha1.Condition( + type="Ready", + status="False", + reason="Starting", + message="0 of 3 pods ready", + lastTransitionTime=datetime.datetime(2026, 10, 20, 10, 0, tzinfo=datetime.UTC), + observedGeneration=1, + ), + ], + pods=rov1alpha1.Pods(total=3, scheduled=3, ready=0), + ), + want=resource.Condition(typ="ModelPodsScheduled", status="True", reason="AllPodsScheduled"), + ), + PodsScheduledCase( + name="ReasonUnchanged", + reason="A replica whose pod is still unschedulable keeps the message it relayed when the pod became unschedulable.", + replica=_replica( + labels={}, + members=[_member(role="Standalone", worker_nodes=None)], + conditions=[ + v1alpha1.Condition( + type="ModelPodsScheduled", + status="False", + reason="Unschedulable", + message="pod llama-0: 0/2 nodes are available: 2 cannot allocate all claims.", + lastTransitionTime=datetime.datetime(2026, 10, 20, 10, 0, tzinfo=datetime.UTC), + observedGeneration=1, + ), + ], + ), + status=rov1alpha1.Status( + observedGeneration=1, + conditions=[ + rov1alpha1.Condition( + type="Scheduled", + status="False", + reason="Unschedulable", + message="pod llama-0: 0/3 nodes are available: 3 cannot allocate all claims.", + lastTransitionTime=datetime.datetime(2026, 10, 20, 10, 0, tzinfo=datetime.UTC), + observedGeneration=1, + ), + rov1alpha1.Condition( + type="Ready", + status="False", + reason="NotScheduled", + message="0 of 1 pods scheduled", + lastTransitionTime=datetime.datetime(2026, 10, 20, 10, 0, tzinfo=datetime.UTC), + observedGeneration=1, + ), + ], + pods=rov1alpha1.Pods(total=1, scheduled=0, ready=0), + ), + want=resource.Condition( + typ="ModelPodsScheduled", + status="False", + reason="Unschedulable", + message="pod llama-0: 0/2 nodes are available: 2 cannot allocate all claims.", + ), + ), + PodsScheduledCase( + name="PodCountChanged", + reason="A gang still waiting for its workers reports how many of its pods exist now, since that's Modelplane's count rather than the agent's message.", + replica=_replica( + labels={}, + members=[_member(role="Leader", worker_nodes=None), _member(role="Worker", worker_nodes=2)], + conditions=[ + v1alpha1.Condition( + type="ModelPodsScheduled", + status="False", + reason="WaitingForPods", + message="1 of 3 pods exist", + lastTransitionTime=datetime.datetime(2026, 10, 20, 10, 0, tzinfo=datetime.UTC), + observedGeneration=1, + ), + ], + ), + status=rov1alpha1.Status( + observedGeneration=1, + conditions=[ + rov1alpha1.Condition( + type="Scheduled", + status="True", + reason="PodsScheduled", + message="", + lastTransitionTime=datetime.datetime(2026, 10, 20, 10, 0, tzinfo=datetime.UTC), + observedGeneration=1, + ), + rov1alpha1.Condition( + type="Ready", + status="False", + reason="Starting", + message="0 of 2 pods ready", + lastTransitionTime=datetime.datetime(2026, 10, 20, 10, 0, tzinfo=datetime.UTC), + observedGeneration=1, + ), + ], + pods=rov1alpha1.Pods(total=2, scheduled=2, ready=0), + ), + want=resource.Condition( + typ="ModelPodsScheduled", status="False", reason="WaitingForPods", message="2 of 3 pods exist" + ), + ), + PodsScheduledCase( + name="ReasonChanged", + reason="A replica whose pod has a new reason not to be scheduled relays the agent's new message.", + replica=_replica( + labels={}, + members=[_member(role="Standalone", worker_nodes=None)], + conditions=[ + v1alpha1.Condition( + type="ModelPodsScheduled", + status="False", + reason="SchedulingPending", + message="pod llama-0", + lastTransitionTime=datetime.datetime(2026, 10, 20, 10, 0, tzinfo=datetime.UTC), + observedGeneration=1, + ), + ], + ), + status=rov1alpha1.Status( + observedGeneration=1, + conditions=[ + rov1alpha1.Condition( + type="Scheduled", + status="False", + reason="Unschedulable", + message="pod llama-0: 0/3 nodes are available: 3 cannot allocate all claims.", + lastTransitionTime=datetime.datetime(2026, 10, 20, 10, 0, tzinfo=datetime.UTC), + observedGeneration=1, + ), + rov1alpha1.Condition( + type="Ready", + status="False", + reason="NotScheduled", + message="0 of 1 pods scheduled", + lastTransitionTime=datetime.datetime(2026, 10, 20, 10, 0, tzinfo=datetime.UTC), + observedGeneration=1, + ), + ], + pods=rov1alpha1.Pods(total=1, scheduled=0, ready=0), + ), + want=resource.Condition( + typ="ModelPodsScheduled", + status="False", + reason="Unschedulable", + message="pod llama-0: 0/3 nodes are available: 3 cannot allocate all claims.", + ), + ), +] + + +@pytest.mark.parametrize("case", PODS_SCHEDULED_CASES, ids=lambda case: case.name) +def test_pods_scheduled(case: PodsScheduledCase) -> None: + """pods_scheduled relays whether every one of a replica's pods has scheduled.""" + got = observation.pods_scheduled(case.replica, case.status) + assert got == case.want, case.reason + + +@dataclasses.dataclass +class PoolsCase: + """A test case for observation.pools.""" + + name: str + reason: str + status: rov1alpha1.Status | None + want: list[v1alpha1.Pool] + + +POOLS_CASES = [ + PoolsCase( + name="NotReported", + reason="Before the agent reports on its pods, a replica is on no pools.", + status=None, + want=[], + ), + PoolsCase( + name="NoDevices", + reason="A replica the agent has seen no devices allocated to is on no pools.", + status=rov1alpha1.Status( + observedGeneration=1, + conditions=[ + rov1alpha1.Condition( + type="Scheduled", + status="False", + reason="SchedulingPending", + message="pod llama-0", + lastTransitionTime=datetime.datetime(2026, 10, 20, 10, 0, tzinfo=datetime.UTC), + observedGeneration=1, + ), + rov1alpha1.Condition( + type="Ready", + status="False", + reason="NotScheduled", + message="0 of 1 pods scheduled", + lastTransitionTime=datetime.datetime(2026, 10, 20, 10, 0, tzinfo=datetime.UTC), + observedGeneration=1, + ), + ], + pods=rov1alpha1.Pods(total=1, scheduled=0, ready=0), + ), + want=[], + ), + PoolsCase( + name="Pools", + reason="A replica is on the pools the agent reports, with the latest time it first saw the replica's claims allocated on each.", + status=rov1alpha1.Status( + observedGeneration=1, + conditions=[ + rov1alpha1.Condition( + type="Scheduled", + status="True", + reason="PodsScheduled", + message="", + lastTransitionTime=datetime.datetime(2026, 10, 20, 10, 0, tzinfo=datetime.UTC), + observedGeneration=1, + ), + rov1alpha1.Condition( + type="Ready", + status="True", + reason="PodsReady", + message="", + lastTransitionTime=datetime.datetime(2026, 10, 20, 10, 0, tzinfo=datetime.UTC), + observedGeneration=1, + ), + ], + pods=rov1alpha1.Pods(total=2, scheduled=2, ready=2), + devices=[ + rov1alpha1.Device( + driver="gpu.nvidia.com", + pool="ip-10-0-54-161.ec2.internal", + device="gpu-2", + nodeName="ip-10-0-54-161.ec2.internal", + ), + rov1alpha1.Device( + driver="gpu.nvidia.com", + pool="ip-10-0-61-20.ec2.internal", + device="gpu-0", + nodeName="ip-10-0-61-20.ec2.internal", + ), + ], + pools=[ + rov1alpha1.Pool( + name="decode", + lastAllocationTime=datetime.datetime(2026, 10, 20, 10, 2, 13, 482915, tzinfo=datetime.UTC), + ), + rov1alpha1.Pool( + name="prefill", + lastAllocationTime=datetime.datetime(2026, 10, 20, 10, 2, 14, 1, tzinfo=datetime.UTC), + ), + ], + ), + want=[ + v1alpha1.Pool( + name="decode", + lastAllocationTime=datetime.datetime(2026, 10, 20, 10, 2, 13, 482915, tzinfo=datetime.UTC), + ), + v1alpha1.Pool( + name="prefill", lastAllocationTime=datetime.datetime(2026, 10, 20, 10, 2, 14, 1, tzinfo=datetime.UTC) + ), + ], + ), +] + + +@pytest.mark.parametrize("case", POOLS_CASES, ids=lambda case: case.name) +def test_pools(case: PoolsCase) -> None: + """pools relays the pools the agent reports a replica's devices are on.""" + got = observation.pools(case.status) + assert got == case.want, case.reason diff --git a/functions/compose-serving-stack/function/stacks/common.py b/functions/compose-serving-stack/function/stacks/common.py index 10c5ec3ee..96f68940a 100644 --- a/functions/compose-serving-stack/function/stacks/common.py +++ b/functions/compose-serving-stack/function/stacks/common.py @@ -18,7 +18,8 @@ Dynamo stacks don't differ: the gateway path (Envoy Gateway, Envoy AI Gateway and its CRDs, the Gateway API Inference Extension CRDs, the gateway namespace and EnvoyProxy), the gateway's PKI (a self-signed -Issuer and trust-manager) and the DRA critical-pods quota. +Issuer and trust-manager), the DRA critical-pods quota and the cluster +agent. The GatewayClass and Gateway are deliberately absent: they read spec.gateway (className, hostname), which is per-cluster configuration, @@ -68,6 +69,13 @@ # where the kubelet plugin runs. _DRA_DRIVER_NAMESPACE = "nvidia-dra-driver" +# The cluster agent is Modelplane's own. Its chart and image share one +# repository, tagged 0.1.0 and v0.1.0 respectively, and the chart's default +# image tag is its appVersion. The schema dependency on the agent's CRDs in +# crossplane-project.yaml pins the same version. +_CLUSTER_AGENT_REPO = "oci://xpkg.upbound.io/modelplane" +_CLUSTER_AGENT_VERSION = "0.1.0" + _CRDS_DIR = pathlib.Path(__file__).parent / "crds" @@ -304,4 +312,18 @@ def _crds(filename: str) -> list[dict[str, Any]]: }, ], ), + # The cluster agent reports the free devices on each GPU pool's nodes, + # and whether each replica's pods have scheduled, in the status of the + # PoolObservations compose-inference-cluster creates and the + # ReplicaObservations compose-model-replica creates. The fleet scheduler + # places replicas by what it reports. + Chart( + key="cluster-agent", + release="mp-cluster-agent", + namespace="modelplane-system", + chart="cluster-agent", + repository=_CLUSTER_AGENT_REPO, + version=_CLUSTER_AGENT_VERSION, + depends_on=["gateway-namespace"], + ), ] diff --git a/functions/compose-serving-stack/tests/test_fn.py b/functions/compose-serving-stack/tests/test_fn.py index 0dde1c289..ae1e46f40 100644 --- a/functions/compose-serving-stack/tests/test_fn.py +++ b/functions/compose-serving-stack/tests/test_fn.py @@ -571,6 +571,34 @@ def _trust_manager(*, ready: fnv1.Ready) -> fnv1.Resource: ) +def _cluster_agent(*, ready: fnv1.Ready) -> fnv1.Resource: + """The composed cluster agent Release.""" + return fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "helm.m.crossplane.io/v1beta1", + "kind": "Release", + "metadata": { + "annotations": {"crossplane.io/external-name": "mp-cluster-agent"}, + "labels": {"modelplane.ai/resource": "cluster-agent"}, + }, + "spec": { + "providerConfigRef": {"kind": "ProviderConfig", "name": "test-backend-cluster-63fde"}, + "forProvider": { + "chart": { + "name": "cluster-agent", + "repository": "oci://xpkg.upbound.io/modelplane", + "version": "0.1.0", + }, + "namespace": "modelplane-system", + }, + }, + } + ), + ready=ready, + ) + + def _dra_driver_critical_pods_quota(*, ready: fnv1.Ready) -> fnv1.Resource: """The composed ResourceQuota admitting the DRA driver's critical pods.""" return fnv1.Resource( @@ -1901,6 +1929,14 @@ def _to_dict(msg: message.Message) -> dict: by_kind="Object", by_key="gateway-proxy", ), + "usage-gateway-namespace-by-cluster-agent": _usage( + of_api_version="kubernetes.m.crossplane.io/v1alpha1", + of_kind="Object", + of_key="gateway-namespace", + by_api_version="helm.m.crossplane.io/v1beta1", + by_kind="Release", + by_key="cluster-agent", + ), "usage-cert-manager-by-gateway-selfsigned-issuer": _usage( of_api_version="helm.m.crossplane.io/v1beta1", of_kind="Release", @@ -2325,6 +2361,14 @@ def _to_dict(msg: message.Message) -> dict: by_kind="Object", by_key="gateway-proxy", ), + "usage-gateway-namespace-by-cluster-agent": _usage( + of_api_version="kubernetes.m.crossplane.io/v1alpha1", + of_kind="Object", + of_key="gateway-namespace", + by_api_version="helm.m.crossplane.io/v1beta1", + by_kind="Release", + by_key="cluster-agent", + ), "usage-cert-manager-by-gateway-selfsigned-issuer": _usage( of_api_version="helm.m.crossplane.io/v1beta1", of_kind="Release", @@ -2444,6 +2488,7 @@ def _to_dict(msg: message.Message) -> dict: "nvidia-dra-driver-gpu": _observed_ready(), "envoy-gateway": _observed_ready(), "ai-gateway-crds": _observed_ready(), + "cluster-agent": _observed_ready(), "ai-gateway": _observed_ready(), "gaie-crds-inferenceobjectives.inference.networking.x-k8s.io": _observed_ready(), "gaie-crds-inferencepools.inference.networking.k8s.io": _observed_ready(), @@ -2648,6 +2693,7 @@ def _to_dict(msg: message.Message) -> dict: ), "gateway-selfsigned-issuer": _gateway_selfsigned_issuer(), "trust-manager": _trust_manager(ready=fnv1.READY_TRUE), + "cluster-agent": _cluster_agent(ready=fnv1.READY_TRUE), "dra-driver-critical-pods-quota": _dra_driver_critical_pods_quota(ready=fnv1.READY_TRUE), "grove": fnv1.Resource( resource=resource.dict_to_struct( @@ -3060,6 +3106,14 @@ def _to_dict(msg: message.Message) -> dict: by_kind="Object", by_key="gateway-proxy", ), + "usage-gateway-namespace-by-cluster-agent": _usage( + of_api_version="kubernetes.m.crossplane.io/v1alpha1", + of_kind="Object", + of_key="gateway-namespace", + by_api_version="helm.m.crossplane.io/v1beta1", + by_kind="Release", + by_key="cluster-agent", + ), "usage-cert-manager-by-gateway-selfsigned-issuer": _usage( of_api_version="helm.m.crossplane.io/v1beta1", of_kind="Release", @@ -3214,6 +3268,14 @@ def _to_dict(msg: message.Message) -> dict: by_kind="Object", by_key="gateway-proxy", ), + "usage-gateway-namespace-by-cluster-agent": _usage( + of_api_version="kubernetes.m.crossplane.io/v1alpha1", + of_kind="Object", + of_key="gateway-namespace", + by_api_version="helm.m.crossplane.io/v1beta1", + by_kind="Release", + by_key="cluster-agent", + ), "usage-cert-manager-by-gateway-selfsigned-issuer": _usage( of_api_version="helm.m.crossplane.io/v1beta1", of_kind="Release", @@ -3374,6 +3436,14 @@ def _to_dict(msg: message.Message) -> dict: by_kind="Object", by_key="gateway-proxy", ), + "usage-gateway-namespace-by-cluster-agent": _usage( + of_api_version="kubernetes.m.crossplane.io/v1alpha1", + of_kind="Object", + of_key="gateway-namespace", + by_api_version="helm.m.crossplane.io/v1beta1", + by_kind="Release", + by_key="cluster-agent", + ), "usage-cert-manager-by-gateway-selfsigned-issuer": _usage( of_api_version="helm.m.crossplane.io/v1beta1", of_kind="Release", @@ -3540,6 +3610,14 @@ def _to_dict(msg: message.Message) -> dict: by_kind="Object", by_key="gateway-proxy", ), + "usage-gateway-namespace-by-cluster-agent": _usage( + of_api_version="kubernetes.m.crossplane.io/v1alpha1", + of_kind="Object", + of_key="gateway-namespace", + by_api_version="helm.m.crossplane.io/v1beta1", + by_kind="Release", + by_key="cluster-agent", + ), "usage-cert-manager-by-gateway-selfsigned-issuer": _usage( of_api_version="helm.m.crossplane.io/v1beta1", of_kind="Release", @@ -3673,6 +3751,14 @@ def _to_dict(msg: message.Message) -> dict: by_kind="Object", by_key="gateway-proxy", ), + "usage-gateway-namespace-by-cluster-agent": _usage( + of_api_version="kubernetes.m.crossplane.io/v1alpha1", + of_kind="Object", + of_key="gateway-namespace", + by_api_version="helm.m.crossplane.io/v1beta1", + by_kind="Release", + by_key="cluster-agent", + ), "usage-cert-manager-by-gateway-selfsigned-issuer": _usage( of_api_version="helm.m.crossplane.io/v1beta1", of_kind="Release", @@ -4074,6 +4160,14 @@ def _to_dict(msg: message.Message) -> dict: by_kind="Object", by_key="gateway-proxy", ), + "usage-gateway-namespace-by-cluster-agent": _usage( + of_api_version="kubernetes.m.crossplane.io/v1alpha1", + of_kind="Object", + of_key="gateway-namespace", + by_api_version="helm.m.crossplane.io/v1beta1", + by_kind="Release", + by_key="cluster-agent", + ), "usage-cert-manager-by-gateway-selfsigned-issuer": _usage( of_api_version="helm.m.crossplane.io/v1beta1", of_kind="Release", @@ -4365,6 +4459,14 @@ def _to_dict(msg: message.Message) -> dict: by_kind="Object", by_key="gateway-proxy", ), + "usage-gateway-namespace-by-cluster-agent": _usage( + of_api_version="kubernetes.m.crossplane.io/v1alpha1", + of_kind="Object", + of_key="gateway-namespace", + by_api_version="helm.m.crossplane.io/v1beta1", + by_kind="Release", + by_key="cluster-agent", + ), "usage-cert-manager-by-gateway-selfsigned-issuer": _usage( of_api_version="helm.m.crossplane.io/v1beta1", of_kind="Release", @@ -4662,6 +4764,14 @@ def _to_dict(msg: message.Message) -> dict: by_kind="Object", by_key="gateway-proxy", ), + "usage-gateway-namespace-by-cluster-agent": _usage( + of_api_version="kubernetes.m.crossplane.io/v1alpha1", + of_kind="Object", + of_key="gateway-namespace", + by_api_version="helm.m.crossplane.io/v1beta1", + by_kind="Release", + by_key="cluster-agent", + ), "usage-cert-manager-by-gateway-selfsigned-issuer": _usage( of_api_version="helm.m.crossplane.io/v1beta1", of_kind="Release", @@ -5029,6 +5139,14 @@ def _to_dict(msg: message.Message) -> dict: by_kind="Object", by_key="gateway-proxy", ), + "usage-gateway-namespace-by-cluster-agent": _usage( + of_api_version="kubernetes.m.crossplane.io/v1alpha1", + of_kind="Object", + of_key="gateway-namespace", + by_api_version="helm.m.crossplane.io/v1beta1", + by_kind="Release", + by_key="cluster-agent", + ), "usage-cert-manager-by-gateway-selfsigned-issuer": _usage( of_api_version="helm.m.crossplane.io/v1beta1", of_kind="Release", @@ -5344,6 +5462,14 @@ def _to_dict(msg: message.Message) -> dict: by_kind="Object", by_key="gateway-proxy", ), + "usage-gateway-namespace-by-cluster-agent": _usage( + of_api_version="kubernetes.m.crossplane.io/v1alpha1", + of_kind="Object", + of_key="gateway-namespace", + by_api_version="helm.m.crossplane.io/v1beta1", + by_kind="Release", + by_key="cluster-agent", + ), "usage-cert-manager-by-gateway-selfsigned-issuer": _usage( of_api_version="helm.m.crossplane.io/v1beta1", of_kind="Release", @@ -5645,6 +5771,14 @@ def _to_dict(msg: message.Message) -> dict: by_kind="Object", by_key="gateway-proxy", ), + "usage-gateway-namespace-by-cluster-agent": _usage( + of_api_version="kubernetes.m.crossplane.io/v1alpha1", + of_kind="Object", + of_key="gateway-namespace", + by_api_version="helm.m.crossplane.io/v1beta1", + by_kind="Release", + by_key="cluster-agent", + ), "usage-cert-manager-by-gateway-selfsigned-issuer": _usage( of_api_version="helm.m.crossplane.io/v1beta1", of_kind="Release", @@ -5974,6 +6108,14 @@ def _to_dict(msg: message.Message) -> dict: by_kind="Object", by_key="gateway-proxy", ), + "usage-gateway-namespace-by-cluster-agent": _usage( + of_api_version="kubernetes.m.crossplane.io/v1alpha1", + of_kind="Object", + of_key="gateway-namespace", + by_api_version="helm.m.crossplane.io/v1beta1", + by_kind="Release", + by_key="cluster-agent", + ), "usage-cert-manager-by-gateway-selfsigned-issuer": _usage( of_api_version="helm.m.crossplane.io/v1beta1", of_kind="Release", @@ -6102,6 +6244,7 @@ def test_compose(case: ComposeCase) -> None: "usage-envoy-gateway-by-gateway-class": _observed_ready(), "ai-gateway": _observed_ready(), "ai-gateway-crds": _observed_ready(), + "cluster-agent": _observed_ready(), "dra-driver-critical-pods-quota": _observed_ready(), "envoy-gateway": _observed_ready(), "gaie-crds-inferenceobjectives.inference.networking.x-k8s.io": _observed_ready(), @@ -6115,6 +6258,7 @@ def test_compose(case: ComposeCase) -> None: "usage-cert-manager-by-envoy-gateway": _observed_ready(), "usage-cert-manager-by-gateway-selfsigned-issuer": _observed_ready(), "usage-gateway-namespace-by-gateway-proxy": _observed_ready(), + "usage-gateway-namespace-by-cluster-agent": _observed_ready(), "usage-gateway-namespace-by-gateway-selfsigned-issuer": _observed_ready(), "usage-gateway-selfsigned-issuer-by-trust-manager": _observed_ready(), "cert-manager": _observed_ready(), @@ -6160,6 +6304,7 @@ def test_compose(case: ComposeCase) -> None: # The common components. "ai-gateway", "ai-gateway-crds", + "cluster-agent", "dra-driver-critical-pods-quota", "envoy-gateway", "gaie-crds-inferenceobjectives.inference.networking.x-k8s.io", @@ -6173,6 +6318,7 @@ def test_compose(case: ComposeCase) -> None: "usage-cert-manager-by-envoy-gateway", "usage-cert-manager-by-gateway-selfsigned-issuer", "usage-gateway-namespace-by-gateway-proxy", + "usage-gateway-namespace-by-cluster-agent", "usage-gateway-namespace-by-gateway-selfsigned-issuer", "usage-gateway-selfsigned-issuer-by-trust-manager", # The EKS half, generated from aicr. @@ -6243,6 +6389,7 @@ def test_compose(case: ComposeCase) -> None: "usage-envoy-gateway-by-gateway-class": _observed_ready(), "ai-gateway": _observed_ready(), "ai-gateway-crds": _observed_ready(), + "cluster-agent": _observed_ready(), "dra-driver-critical-pods-quota": _observed_ready(), "envoy-gateway": _observed_ready(), "gaie-crds-inferenceobjectives.inference.networking.x-k8s.io": _observed_ready(), @@ -6256,6 +6403,7 @@ def test_compose(case: ComposeCase) -> None: "usage-cert-manager-by-envoy-gateway": _observed_ready(), "usage-cert-manager-by-gateway-selfsigned-issuer": _observed_ready(), "usage-gateway-namespace-by-gateway-proxy": _observed_ready(), + "usage-gateway-namespace-by-cluster-agent": _observed_ready(), "usage-gateway-namespace-by-gateway-selfsigned-issuer": _observed_ready(), "usage-gateway-selfsigned-issuer-by-trust-manager": _observed_ready(), "cert-manager": _observed_ready(), @@ -6315,6 +6463,7 @@ def test_compose(case: ComposeCase) -> None: # The common components. "ai-gateway", "ai-gateway-crds", + "cluster-agent", "dra-driver-critical-pods-quota", "envoy-gateway", "gaie-crds-inferenceobjectives.inference.networking.x-k8s.io", @@ -6328,6 +6477,7 @@ def test_compose(case: ComposeCase) -> None: "usage-cert-manager-by-envoy-gateway", "usage-cert-manager-by-gateway-selfsigned-issuer", "usage-gateway-namespace-by-gateway-proxy", + "usage-gateway-namespace-by-cluster-agent", "usage-gateway-namespace-by-gateway-selfsigned-issuer", "usage-gateway-selfsigned-issuer-by-trust-manager", # The EKS half, generated from aicr. @@ -6412,6 +6562,7 @@ def test_compose(case: ComposeCase) -> None: "usage-envoy-gateway-by-gateway-class": _observed_ready(), "ai-gateway": _observed_ready(), "ai-gateway-crds": _observed_ready(), + "cluster-agent": _observed_ready(), "dra-driver-critical-pods-quota": _observed_ready(), "envoy-gateway": _observed_ready(), "gaie-crds-inferenceobjectives.inference.networking.x-k8s.io": _observed_ready(), @@ -6425,6 +6576,7 @@ def test_compose(case: ComposeCase) -> None: "usage-cert-manager-by-envoy-gateway": _observed_ready(), "usage-cert-manager-by-gateway-selfsigned-issuer": _observed_ready(), "usage-gateway-namespace-by-gateway-proxy": _observed_ready(), + "usage-gateway-namespace-by-cluster-agent": _observed_ready(), "usage-gateway-namespace-by-gateway-selfsigned-issuer": _observed_ready(), "usage-gateway-selfsigned-issuer-by-trust-manager": _observed_ready(), "cert-manager": _observed_ready(), @@ -6472,6 +6624,7 @@ def test_compose(case: ComposeCase) -> None: # The common components. "ai-gateway", "ai-gateway-crds", + "cluster-agent", "dra-driver-critical-pods-quota", "envoy-gateway", "gaie-crds-inferenceobjectives.inference.networking.x-k8s.io", @@ -6485,6 +6638,7 @@ def test_compose(case: ComposeCase) -> None: "usage-cert-manager-by-envoy-gateway", "usage-cert-manager-by-gateway-selfsigned-issuer", "usage-gateway-namespace-by-gateway-proxy", + "usage-gateway-namespace-by-cluster-agent", "usage-gateway-namespace-by-gateway-selfsigned-issuer", "usage-gateway-selfsigned-issuer-by-trust-manager", # The AKS half, generated from aicr. @@ -6558,6 +6712,7 @@ def test_compose(case: ComposeCase) -> None: "usage-envoy-gateway-by-gateway-class": _observed_ready(), "ai-gateway": _observed_ready(), "ai-gateway-crds": _observed_ready(), + "cluster-agent": _observed_ready(), "dra-driver-critical-pods-quota": _observed_ready(), "envoy-gateway": _observed_ready(), "gaie-crds-inferenceobjectives.inference.networking.x-k8s.io": _observed_ready(), @@ -6571,6 +6726,7 @@ def test_compose(case: ComposeCase) -> None: "usage-cert-manager-by-envoy-gateway": _observed_ready(), "usage-cert-manager-by-gateway-selfsigned-issuer": _observed_ready(), "usage-gateway-namespace-by-gateway-proxy": _observed_ready(), + "usage-gateway-namespace-by-cluster-agent": _observed_ready(), "usage-gateway-namespace-by-gateway-selfsigned-issuer": _observed_ready(), "usage-gateway-selfsigned-issuer-by-trust-manager": _observed_ready(), "cert-manager": _observed_ready(), @@ -6632,6 +6788,7 @@ def test_compose(case: ComposeCase) -> None: # The common components. "ai-gateway", "ai-gateway-crds", + "cluster-agent", "dra-driver-critical-pods-quota", "envoy-gateway", "gaie-crds-inferenceobjectives.inference.networking.x-k8s.io", @@ -6645,6 +6802,7 @@ def test_compose(case: ComposeCase) -> None: "usage-cert-manager-by-envoy-gateway", "usage-cert-manager-by-gateway-selfsigned-issuer", "usage-gateway-namespace-by-gateway-proxy", + "usage-gateway-namespace-by-cluster-agent", "usage-gateway-namespace-by-gateway-selfsigned-issuer", "usage-gateway-selfsigned-issuer-by-trust-manager", # The AKS half, generated from aicr. @@ -6732,6 +6890,7 @@ def test_compose(case: ComposeCase) -> None: "usage-envoy-gateway-by-gateway-class": _observed_ready(), "ai-gateway": _observed_ready(), "ai-gateway-crds": _observed_ready(), + "cluster-agent": _observed_ready(), "dra-driver-critical-pods-quota": _observed_ready(), "envoy-gateway": _observed_ready(), "gaie-crds-inferenceobjectives.inference.networking.x-k8s.io": _observed_ready(), @@ -6745,6 +6904,7 @@ def test_compose(case: ComposeCase) -> None: "usage-cert-manager-by-envoy-gateway": _observed_ready(), "usage-cert-manager-by-gateway-selfsigned-issuer": _observed_ready(), "usage-gateway-namespace-by-gateway-proxy": _observed_ready(), + "usage-gateway-namespace-by-cluster-agent": _observed_ready(), "usage-gateway-namespace-by-gateway-selfsigned-issuer": _observed_ready(), "usage-gateway-selfsigned-issuer-by-trust-manager": _observed_ready(), "cert-manager": _observed_ready(), @@ -6794,6 +6954,7 @@ def test_compose(case: ComposeCase) -> None: # The common components. "ai-gateway", "ai-gateway-crds", + "cluster-agent", "dra-driver-critical-pods-quota", "envoy-gateway", "gaie-crds-inferenceobjectives.inference.networking.x-k8s.io", @@ -6807,6 +6968,7 @@ def test_compose(case: ComposeCase) -> None: "usage-cert-manager-by-envoy-gateway", "usage-cert-manager-by-gateway-selfsigned-issuer", "usage-gateway-namespace-by-gateway-proxy", + "usage-gateway-namespace-by-cluster-agent", "usage-gateway-namespace-by-gateway-selfsigned-issuer", "usage-gateway-selfsigned-issuer-by-trust-manager", # The GKE half, generated from aicr. @@ -6884,6 +7046,7 @@ def test_compose(case: ComposeCase) -> None: "usage-envoy-gateway-by-gateway-class": _observed_ready(), "ai-gateway": _observed_ready(), "ai-gateway-crds": _observed_ready(), + "cluster-agent": _observed_ready(), "dra-driver-critical-pods-quota": _observed_ready(), "envoy-gateway": _observed_ready(), "gaie-crds-inferenceobjectives.inference.networking.x-k8s.io": _observed_ready(), @@ -6897,6 +7060,7 @@ def test_compose(case: ComposeCase) -> None: "usage-cert-manager-by-envoy-gateway": _observed_ready(), "usage-cert-manager-by-gateway-selfsigned-issuer": _observed_ready(), "usage-gateway-namespace-by-gateway-proxy": _observed_ready(), + "usage-gateway-namespace-by-cluster-agent": _observed_ready(), "usage-gateway-namespace-by-gateway-selfsigned-issuer": _observed_ready(), "usage-gateway-selfsigned-issuer-by-trust-manager": _observed_ready(), "cert-manager": _observed_ready(), @@ -6960,6 +7124,7 @@ def test_compose(case: ComposeCase) -> None: # The common components. "ai-gateway", "ai-gateway-crds", + "cluster-agent", "dra-driver-critical-pods-quota", "envoy-gateway", "gaie-crds-inferenceobjectives.inference.networking.x-k8s.io", @@ -6973,6 +7138,7 @@ def test_compose(case: ComposeCase) -> None: "usage-cert-manager-by-envoy-gateway", "usage-cert-manager-by-gateway-selfsigned-issuer", "usage-gateway-namespace-by-gateway-proxy", + "usage-gateway-namespace-by-cluster-agent", "usage-gateway-namespace-by-gateway-selfsigned-issuer", "usage-gateway-selfsigned-issuer-by-trust-manager", # The GKE half, generated from aicr. @@ -7064,6 +7230,7 @@ def test_compose(case: ComposeCase) -> None: "usage-envoy-gateway-by-gateway-class": _observed_ready(), "ai-gateway": _observed_ready(), "ai-gateway-crds": _observed_ready(), + "cluster-agent": _observed_ready(), "dra-driver-critical-pods-quota": _observed_ready(), "envoy-gateway": _observed_ready(), "gaie-crds-inferenceobjectives.inference.networking.x-k8s.io": _observed_ready(), @@ -7077,6 +7244,7 @@ def test_compose(case: ComposeCase) -> None: "usage-cert-manager-by-envoy-gateway": _observed_ready(), "usage-cert-manager-by-gateway-selfsigned-issuer": _observed_ready(), "usage-gateway-namespace-by-gateway-proxy": _observed_ready(), + "usage-gateway-namespace-by-cluster-agent": _observed_ready(), "usage-gateway-namespace-by-gateway-selfsigned-issuer": _observed_ready(), "usage-gateway-selfsigned-issuer-by-trust-manager": _observed_ready(), "cert-manager": _observed_ready(), @@ -7105,6 +7273,7 @@ def test_compose(case: ComposeCase) -> None: # The common components. "ai-gateway", "ai-gateway-crds", + "cluster-agent", "dra-driver-critical-pods-quota", "envoy-gateway", "gaie-crds-inferenceobjectives.inference.networking.x-k8s.io", @@ -7118,6 +7287,7 @@ def test_compose(case: ComposeCase) -> None: "usage-cert-manager-by-envoy-gateway", "usage-cert-manager-by-gateway-selfsigned-issuer", "usage-gateway-namespace-by-gateway-proxy", + "usage-gateway-namespace-by-cluster-agent", "usage-gateway-namespace-by-gateway-selfsigned-issuer", "usage-gateway-selfsigned-issuer-by-trust-manager", # The hand-written Nebius half. @@ -7171,6 +7341,7 @@ def test_compose(case: ComposeCase) -> None: "usage-envoy-gateway-by-gateway-class": _observed_ready(), "ai-gateway": _observed_ready(), "ai-gateway-crds": _observed_ready(), + "cluster-agent": _observed_ready(), "dra-driver-critical-pods-quota": _observed_ready(), "envoy-gateway": _observed_ready(), "gaie-crds-inferenceobjectives.inference.networking.x-k8s.io": _observed_ready(), @@ -7184,6 +7355,7 @@ def test_compose(case: ComposeCase) -> None: "usage-cert-manager-by-envoy-gateway": _observed_ready(), "usage-cert-manager-by-gateway-selfsigned-issuer": _observed_ready(), "usage-gateway-namespace-by-gateway-proxy": _observed_ready(), + "usage-gateway-namespace-by-cluster-agent": _observed_ready(), "usage-gateway-namespace-by-gateway-selfsigned-issuer": _observed_ready(), "usage-gateway-selfsigned-issuer-by-trust-manager": _observed_ready(), "cert-manager": _observed_ready(), @@ -7226,6 +7398,7 @@ def test_compose(case: ComposeCase) -> None: # The common components. "ai-gateway", "ai-gateway-crds", + "cluster-agent", "dra-driver-critical-pods-quota", "envoy-gateway", "gaie-crds-inferenceobjectives.inference.networking.x-k8s.io", @@ -7239,6 +7412,7 @@ def test_compose(case: ComposeCase) -> None: "usage-cert-manager-by-envoy-gateway", "usage-cert-manager-by-gateway-selfsigned-issuer", "usage-gateway-namespace-by-gateway-proxy", + "usage-gateway-namespace-by-cluster-agent", "usage-gateway-namespace-by-gateway-selfsigned-issuer", "usage-gateway-selfsigned-issuer-by-trust-manager", # The hand-written Nebius half. @@ -7306,6 +7480,7 @@ def test_compose(case: ComposeCase) -> None: "usage-envoy-gateway-by-gateway-class": _observed_ready(), "ai-gateway": _observed_ready(), "ai-gateway-crds": _observed_ready(), + "cluster-agent": _observed_ready(), "dra-driver-critical-pods-quota": _observed_ready(), "envoy-gateway": _observed_ready(), "gaie-crds-inferenceobjectives.inference.networking.x-k8s.io": _observed_ready(), @@ -7319,6 +7494,7 @@ def test_compose(case: ComposeCase) -> None: "usage-cert-manager-by-envoy-gateway": _observed_ready(), "usage-cert-manager-by-gateway-selfsigned-issuer": _observed_ready(), "usage-gateway-namespace-by-gateway-proxy": _observed_ready(), + "usage-gateway-namespace-by-cluster-agent": _observed_ready(), "usage-gateway-namespace-by-gateway-selfsigned-issuer": _observed_ready(), "usage-gateway-selfsigned-issuer-by-trust-manager": _observed_ready(), "cert-manager": _observed_ready(), @@ -7346,6 +7522,7 @@ def test_compose(case: ComposeCase) -> None: # The common components. "ai-gateway", "ai-gateway-crds", + "cluster-agent", "dra-driver-critical-pods-quota", "envoy-gateway", "gaie-crds-inferenceobjectives.inference.networking.x-k8s.io", @@ -7359,6 +7536,7 @@ def test_compose(case: ComposeCase) -> None: "usage-cert-manager-by-envoy-gateway", "usage-cert-manager-by-gateway-selfsigned-issuer", "usage-gateway-namespace-by-gateway-proxy", + "usage-gateway-namespace-by-cluster-agent", "usage-gateway-namespace-by-gateway-selfsigned-issuer", "usage-gateway-selfsigned-issuer-by-trust-manager", # The hand-written Vultr half. VKE pre-installs NFD via its managed GPU @@ -7412,6 +7590,7 @@ def test_compose(case: ComposeCase) -> None: "usage-envoy-gateway-by-gateway-class": _observed_ready(), "ai-gateway": _observed_ready(), "ai-gateway-crds": _observed_ready(), + "cluster-agent": _observed_ready(), "dra-driver-critical-pods-quota": _observed_ready(), "envoy-gateway": _observed_ready(), "gaie-crds-inferenceobjectives.inference.networking.x-k8s.io": _observed_ready(), @@ -7425,6 +7604,7 @@ def test_compose(case: ComposeCase) -> None: "usage-cert-manager-by-envoy-gateway": _observed_ready(), "usage-cert-manager-by-gateway-selfsigned-issuer": _observed_ready(), "usage-gateway-namespace-by-gateway-proxy": _observed_ready(), + "usage-gateway-namespace-by-cluster-agent": _observed_ready(), "usage-gateway-namespace-by-gateway-selfsigned-issuer": _observed_ready(), "usage-gateway-selfsigned-issuer-by-trust-manager": _observed_ready(), "cert-manager": _observed_ready(), @@ -7466,6 +7646,7 @@ def test_compose(case: ComposeCase) -> None: # The common components. "ai-gateway", "ai-gateway-crds", + "cluster-agent", "dra-driver-critical-pods-quota", "envoy-gateway", "gaie-crds-inferenceobjectives.inference.networking.x-k8s.io", @@ -7479,6 +7660,7 @@ def test_compose(case: ComposeCase) -> None: "usage-cert-manager-by-envoy-gateway", "usage-cert-manager-by-gateway-selfsigned-issuer", "usage-gateway-namespace-by-gateway-proxy", + "usage-gateway-namespace-by-cluster-agent", "usage-gateway-namespace-by-gateway-selfsigned-issuer", "usage-gateway-selfsigned-issuer-by-trust-manager", # The hand-written Vultr half. VKE pre-installs NFD via its managed GPU @@ -7546,6 +7728,7 @@ def test_compose(case: ComposeCase) -> None: "usage-envoy-gateway-by-gateway-class": _observed_ready(), "ai-gateway": _observed_ready(), "ai-gateway-crds": _observed_ready(), + "cluster-agent": _observed_ready(), "dra-driver-critical-pods-quota": _observed_ready(), "envoy-gateway": _observed_ready(), "gaie-crds-inferenceobjectives.inference.networking.x-k8s.io": _observed_ready(), @@ -7559,6 +7742,7 @@ def test_compose(case: ComposeCase) -> None: "usage-cert-manager-by-envoy-gateway": _observed_ready(), "usage-cert-manager-by-gateway-selfsigned-issuer": _observed_ready(), "usage-gateway-namespace-by-gateway-proxy": _observed_ready(), + "usage-gateway-namespace-by-cluster-agent": _observed_ready(), "usage-gateway-namespace-by-gateway-selfsigned-issuer": _observed_ready(), "usage-gateway-selfsigned-issuer-by-trust-manager": _observed_ready(), "cert-manager": _observed_ready(), @@ -7591,6 +7775,7 @@ def test_compose(case: ComposeCase) -> None: # The common components. "ai-gateway", "ai-gateway-crds", + "cluster-agent", "dra-driver-critical-pods-quota", "envoy-gateway", "gaie-crds-inferenceobjectives.inference.networking.x-k8s.io", @@ -7604,6 +7789,7 @@ def test_compose(case: ComposeCase) -> None: "usage-cert-manager-by-envoy-gateway", "usage-cert-manager-by-gateway-selfsigned-issuer", "usage-gateway-namespace-by-gateway-proxy", + "usage-gateway-namespace-by-cluster-agent", "usage-gateway-namespace-by-gateway-selfsigned-issuer", "usage-gateway-selfsigned-issuer-by-trust-manager", # The hand-written Civo half. Civo pre-installs no GPU stack at all, @@ -7664,6 +7850,7 @@ def test_compose(case: ComposeCase) -> None: "usage-envoy-gateway-by-gateway-class": _observed_ready(), "ai-gateway": _observed_ready(), "ai-gateway-crds": _observed_ready(), + "cluster-agent": _observed_ready(), "dra-driver-critical-pods-quota": _observed_ready(), "envoy-gateway": _observed_ready(), "gaie-crds-inferenceobjectives.inference.networking.x-k8s.io": _observed_ready(), @@ -7677,6 +7864,7 @@ def test_compose(case: ComposeCase) -> None: "usage-cert-manager-by-envoy-gateway": _observed_ready(), "usage-cert-manager-by-gateway-selfsigned-issuer": _observed_ready(), "usage-gateway-namespace-by-gateway-proxy": _observed_ready(), + "usage-gateway-namespace-by-cluster-agent": _observed_ready(), "usage-gateway-namespace-by-gateway-selfsigned-issuer": _observed_ready(), "usage-gateway-selfsigned-issuer-by-trust-manager": _observed_ready(), "cert-manager": _observed_ready(), @@ -7723,6 +7911,7 @@ def test_compose(case: ComposeCase) -> None: # The common components. "ai-gateway", "ai-gateway-crds", + "cluster-agent", "dra-driver-critical-pods-quota", "envoy-gateway", "gaie-crds-inferenceobjectives.inference.networking.x-k8s.io", @@ -7736,6 +7925,7 @@ def test_compose(case: ComposeCase) -> None: "usage-cert-manager-by-envoy-gateway", "usage-cert-manager-by-gateway-selfsigned-issuer", "usage-gateway-namespace-by-gateway-proxy", + "usage-gateway-namespace-by-cluster-agent", "usage-gateway-namespace-by-gateway-selfsigned-issuer", "usage-gateway-selfsigned-issuer-by-trust-manager", # The hand-written Civo half. Civo pre-installs no GPU stack at all, @@ -7810,6 +8000,7 @@ def test_compose(case: ComposeCase) -> None: "usage-envoy-gateway-by-gateway-class": _observed_ready(), "ai-gateway": _observed_ready(), "ai-gateway-crds": _observed_ready(), + "cluster-agent": _observed_ready(), "dra-driver-critical-pods-quota": _observed_ready(), "envoy-gateway": _observed_ready(), "gaie-crds-inferenceobjectives.inference.networking.x-k8s.io": _observed_ready(), @@ -7823,6 +8014,7 @@ def test_compose(case: ComposeCase) -> None: "usage-cert-manager-by-envoy-gateway": _observed_ready(), "usage-cert-manager-by-gateway-selfsigned-issuer": _observed_ready(), "usage-gateway-namespace-by-gateway-proxy": _observed_ready(), + "usage-gateway-namespace-by-cluster-agent": _observed_ready(), "usage-gateway-namespace-by-gateway-selfsigned-issuer": _observed_ready(), "usage-gateway-selfsigned-issuer-by-trust-manager": _observed_ready(), "cert-manager": _observed_ready(), @@ -7851,6 +8043,7 @@ def test_compose(case: ComposeCase) -> None: # The common components. "ai-gateway", "ai-gateway-crds", + "cluster-agent", "dra-driver-critical-pods-quota", "envoy-gateway", "gaie-crds-inferenceobjectives.inference.networking.x-k8s.io", @@ -7864,6 +8057,7 @@ def test_compose(case: ComposeCase) -> None: "usage-cert-manager-by-envoy-gateway", "usage-cert-manager-by-gateway-selfsigned-issuer", "usage-gateway-namespace-by-gateway-proxy", + "usage-gateway-namespace-by-cluster-agent", "usage-gateway-namespace-by-gateway-selfsigned-issuer", "usage-gateway-selfsigned-issuer-by-trust-manager", # The hand-written Existing half. @@ -7917,6 +8111,7 @@ def test_compose(case: ComposeCase) -> None: "usage-envoy-gateway-by-gateway-class": _observed_ready(), "ai-gateway": _observed_ready(), "ai-gateway-crds": _observed_ready(), + "cluster-agent": _observed_ready(), "dra-driver-critical-pods-quota": _observed_ready(), "envoy-gateway": _observed_ready(), "gaie-crds-inferenceobjectives.inference.networking.x-k8s.io": _observed_ready(), @@ -7930,6 +8125,7 @@ def test_compose(case: ComposeCase) -> None: "usage-cert-manager-by-envoy-gateway": _observed_ready(), "usage-cert-manager-by-gateway-selfsigned-issuer": _observed_ready(), "usage-gateway-namespace-by-gateway-proxy": _observed_ready(), + "usage-gateway-namespace-by-cluster-agent": _observed_ready(), "usage-gateway-namespace-by-gateway-selfsigned-issuer": _observed_ready(), "usage-gateway-selfsigned-issuer-by-trust-manager": _observed_ready(), "cert-manager": _observed_ready(), @@ -7972,6 +8168,7 @@ def test_compose(case: ComposeCase) -> None: # The common components. "ai-gateway", "ai-gateway-crds", + "cluster-agent", "dra-driver-critical-pods-quota", "envoy-gateway", "gaie-crds-inferenceobjectives.inference.networking.x-k8s.io", @@ -7985,6 +8182,7 @@ def test_compose(case: ComposeCase) -> None: "usage-cert-manager-by-envoy-gateway", "usage-cert-manager-by-gateway-selfsigned-issuer", "usage-gateway-namespace-by-gateway-proxy", + "usage-gateway-namespace-by-cluster-agent", "usage-gateway-namespace-by-gateway-selfsigned-issuer", "usage-gateway-selfsigned-issuer-by-trust-manager", # The hand-written Existing half. diff --git a/schemas/.lock.json b/schemas/.lock.json index ad2bd8e12..4e083b183 100644 --- a/schemas/.lock.json +++ b/schemas/.lock.json @@ -1,7 +1,8 @@ { "packages": { - "fs://apis": "1e73064210fec7fa5508d4af954e85d2adba93ccb40101324d926f950e8d2e2f", + "fs://apis": "dffeedd4cecb877e11fde90a2f3316ba0690935af6c8d5428bf48fcfe4e92d28", "git://https://github.com/crossplane/crossplane/cluster/crds": "90d8b72ad8b829f0bcd7d7d5a98eaa0d579f244a", + "git://https://github.com/modelplaneai/cluster-agent/cluster/charts/cluster-agent/files/crds": "ca7631d72ef8550a2d2f415f33db1ec52b20c65c", "xpkg://xpkg.upbound.io/upbound/provider-aws-ec2:v2.8.1": "sha256:ca2e9e3b2e3a8b6ca44a9700d5abf7abd733cfa388d1afe9bb2bf7c847cd39ef", "xpkg://xpkg.upbound.io/upbound/provider-aws-efs:v2.8.1": "sha256:ebb1bcd8dc9a7e60e97a324652fbd9d1b0609ddbb1107db518ce999cbc1113bd", "xpkg://xpkg.upbound.io/upbound/provider-aws-eks:v2.8.1": "sha256:45ba27a14d6f8c4b9acdced0c48bec5650aab22bfc275af8bf1f5c770da3fa24", diff --git a/schemas/python/models/ai/modelplane/agent/__init__.py b/schemas/python/models/ai/modelplane/agent/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/schemas/python/models/ai/modelplane/agent/poolobservation/__init__.py b/schemas/python/models/ai/modelplane/agent/poolobservation/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/schemas/python/models/ai/modelplane/agent/poolobservation/v1alpha1.py b/schemas/python/models/ai/modelplane/agent/poolobservation/v1alpha1.py new file mode 100644 index 000000000..e70c00a1f --- /dev/null +++ b/schemas/python/models/ai/modelplane/agent/poolobservation/v1alpha1.py @@ -0,0 +1,282 @@ +# generated by datamodel-codegen: +# filename: workdir/agent_modelplane_ai_v1alpha1_poolobservation.yaml + +from __future__ import annotations + +from typing import Literal + +from pydantic import AwareDatetime, BaseModel, Field, RootModel, conint, constr + +from .....io.k8s.apimachinery.pkg.apis.meta import v1 + + +class Spec(BaseModel): + poolName: constr( + pattern=r'^[A-Za-z0-9]([-A-Za-z0-9_.]*[A-Za-z0-9])?$', + min_length=1, + max_length=63, + ) + """ + PoolName is the pool's name: the value of its nodes' modelplane.ai/pool + label, and of the node selector that pins a pod to the pool. It can't be + changed. + """ + + +class Condition(BaseModel): + lastTransitionTime: AwareDatetime + """ + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + """ + message: constr(max_length=32768) + """ + message is a human readable message indicating details about the transition. + This may be an empty string. + """ + observedGeneration: conint(ge=0) | None = None + """ + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + """ + reason: constr( + pattern=r'^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$', + min_length=1, + max_length=1024, + ) + """ + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + """ + status: Literal['True', 'False', 'Unknown'] + """ + status of the condition, one of True, False, Unknown. + """ + type: constr( + pattern=r'^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$', + max_length=316, + ) + """ + type of condition in CamelCase or in foo.example.com/CamelCase. + """ + + +class Attributes(BaseModel): + bool_: bool | None = Field(None, alias='bool') + """ + BoolValue is a true/false value. + """ + bools: list[bool] | None = None + """ + BoolValues is a non-empty list of true/false values. + """ + int_: int | None = Field(None, alias='int') + """ + IntValue is a number. + """ + ints: list[int] | None = None + """ + IntValues is a non-empty list of numbers. + + This is an alpha field and requires enabling the DRAListTypeAttributes feature gate. + """ + string: str | None = None + """ + StringValue is a string. Must not be longer than 64 characters. + """ + strings: list[str] | None = None + """ + StringValues is a non-empty list of strings. + Each string must not be longer than 64 characters. + + This is an alpha field and requires enabling the DRAListTypeAttributes feature gate. + """ + version: str | None = None + """ + VersionValue is a semantic version according to semver.org spec 2.0.0. + Must not be longer than 64 characters. + """ + versions: list[str] | None = None + """ + VersionValues is a non-empty list of semantic versions according to semver.org spec 2.0.0. + Each version string must not be longer than 64 characters. + + This is an alpha field and requires enabling the DRAListTypeAttributes feature gate. + """ + + +class Device(BaseModel): + allocatable: conint(ge=0) + """ + Allocatable counts the class's untainted devices on schedulable nodes. + """ + allocated: conint(ge=0) + """ + Allocated counts the allocatable devices a ResourceClaim has allocated. + """ + attributes: dict[str, Attributes] | None = Field(None, max_length=32) + """ + Attributes are the device attributes every device of the class in the + pool agrees on, keyed by the attribute names the DRA driver publishes. + """ + deviceClassName: constr(max_length=253) + """ + DeviceClassName is the DeviceClass the devices belong to. + """ + nodes: conint(ge=0) + """ + Nodes counts the pool's nodes that publish at least one device of the + class, whatever their state. + """ + + +class ExtendedResources(RootModel[int]): + root: int + + +class ExtendedResourcesModel( + RootModel[ + constr( + pattern=r'^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$' + ) + ] +): + root: constr( + pattern=r'^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$' + ) + + +class Nodes(BaseModel): + ready: conint(ge=0) + """ + Ready counts the nodes whose Ready condition is True. + """ + schedulable: conint(ge=0) + """ + Schedulable counts the ready nodes that aren't cordoned, and carry no + NoSchedule or NoExecute taint but nvidia.com/gpu with NoSchedule, which + Modelplane's GPU pods tolerate. + """ + total: conint(ge=0) + """ + Total counts every node in the pool. + """ + + +class NodesByFreeDevice(BaseModel): + freeDevices: dict[str, int] | None = Field(None, max_length=32) + """ + FreeDevices counts each of the nodes' free devices, keyed by + DeviceClass. A class the nodes publish no devices of is absent. + """ + nodes: conint(ge=1) + """ + Nodes counts the nodes with these free devices. + """ + + +class Status(BaseModel): + conditions: list[Condition] | None = Field(None, max_length=8) + """ + Conditions of the pool. Healthy is False when any of the pool's nodes + is not ready or cordoned, or any of its devices is tainted. + ScaleUpBlocked is True when the cluster autoscaler reports it can't add + nodes for an unschedulable pod. Both are Unknown, and the status reports + no nodes or devices, when the agent can't observe the pool. Each + condition's lastTransitionTime is when its status last changed. + """ + devices: list[Device] | None = Field(None, max_length=32) + """ + Devices are the DRA devices the pool's nodes publish, by the + DeviceClass they belong to. A device that belongs to more than one + DeviceClass counts toward each. + """ + extendedResources: dict[str, ExtendedResources | ExtendedResourcesModel] | None = ( + Field(None, max_length=64) + ) + """ + ExtendedResources sums the extended resources, such as nvidia.com/gpu, + that the pool's nodes advertise as allocatable. A device plugin + advertises these. One advertising the same hardware a DRA driver + publishes gives the two allocators separate books over one set of + devices. + """ + lastAllocationTime: AwareDatetime | None = None + """ + LastAllocationTime is the latest time the agent first saw a ResourceClaim + allocated devices on the pool's nodes. It never goes back. The agent + observes the pool from a view of the cluster that includes every claim it + saw allocated at or before this time, unless the claim has since been + deallocated. So NodesByFreeDevices accounts for the devices of a + ReplicaObservation whose lastAllocationTime for the pool is no later than + this, unless they've since been deallocated. + """ + nodes: Nodes + """ + Nodes counts the pool's nodes. + """ + nodesByFreeDevices: list[NodesByFreeDevice] | None = Field(None, max_length=256) + """ + NodesByFreeDevices is a histogram of the pool's schedulable nodes, by + how many devices of each DeviceClass are allocatable and not allocated + on them. Nodes with the same number of free devices of every class + share a bucket. + """ + observedGeneration: int | None = None + """ + ObservedGeneration is the generation of the spec the status reports on. + """ + unschedulablePods: conint(ge=0) + """ + UnschedulablePods counts the pods pinned to the pool by a + modelplane.ai/pool node selector that the scheduler couldn't find a + node for. Only pods that carry a modelplane.ai/replica label count. + """ + + +class PoolObservation(BaseModel): + apiVersion: Literal['agent.modelplane.ai/v1alpha1'] | None = ( + 'agent.modelplane.ai/v1alpha1' + ) + """ + APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + """ + kind: Literal['PoolObservation'] | None = 'PoolObservation' + """ + Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + """ + metadata: v1.ObjectMeta | None = None + """ + Standard object's metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + """ + spec: Spec + """ + PoolObservationSpec specifies which pool to observe. + """ + status: Status | None = None + """ + PoolObservationStatus is what the agent observed about the pool. + """ + + +class PoolObservationList(BaseModel): + apiVersion: str | None = None + """ + APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + """ + items: list[PoolObservation] + """ + List of poolobservations. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md + """ + kind: str | None = None + """ + Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + """ + metadata: v1.ListMeta | None = None + """ + Standard list metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + """ \ No newline at end of file diff --git a/schemas/python/models/ai/modelplane/agent/replicaobservation/__init__.py b/schemas/python/models/ai/modelplane/agent/replicaobservation/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/schemas/python/models/ai/modelplane/agent/replicaobservation/v1alpha1.py b/schemas/python/models/ai/modelplane/agent/replicaobservation/v1alpha1.py new file mode 100644 index 000000000..a0dc1b3c7 --- /dev/null +++ b/schemas/python/models/ai/modelplane/agent/replicaobservation/v1alpha1.py @@ -0,0 +1,224 @@ +# generated by datamodel-codegen: +# filename: workdir/agent_modelplane_ai_v1alpha1_replicaobservation.yaml + +from __future__ import annotations + +from typing import Literal + +from pydantic import AwareDatetime, BaseModel, Field, conint, constr + +from .....io.k8s.apimachinery.pkg.apis.meta import v1 + + +class MatchExpression(BaseModel): + key: str + """ + key is the label key that the selector applies to. + """ + operator: str + """ + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + """ + values: list[str] | None = None + """ + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + """ + + +class PodSelector(BaseModel): + matchExpressions: list[MatchExpression] | None = None + """ + matchExpressions is a list of label selector requirements. The requirements are ANDed. + """ + matchLabels: dict[str, str] | None = None + """ + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + """ + + +class Spec(BaseModel): + podSelector: PodSelector + """ + PodSelector selects the replica's pods in the ReplicaObservation's + namespace. The agent only sees pods that carry a modelplane.ai/replica + label, so a selector that matches other pods matches nothing. + """ + + +class Condition(BaseModel): + lastTransitionTime: AwareDatetime + """ + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + """ + message: constr(max_length=32768) + """ + message is a human readable message indicating details about the transition. + This may be an empty string. + """ + observedGeneration: conint(ge=0) | None = None + """ + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + """ + reason: constr( + pattern=r'^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$', + min_length=1, + max_length=1024, + ) + """ + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + """ + status: Literal['True', 'False', 'Unknown'] + """ + status of the condition, one of True, False, Unknown. + """ + type: constr( + pattern=r'^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$', + max_length=316, + ) + """ + type of condition in CamelCase or in foo.example.com/CamelCase. + """ + + +class Device(BaseModel): + device: constr(max_length=63) + """ + Device is the device's name within its DRA pool. + """ + driver: constr(max_length=63) + """ + Driver that publishes the device. + """ + nodeName: constr(max_length=253) | None = None + """ + NodeName is the node the device is attached to. It's omitted for a + device that isn't attached to one node, like a network-attached device, + or that no ResourceSlice currently publishes. + """ + pool: constr(max_length=253) + """ + Pool is the DRA resource pool the device belongs to. This is the + driver's pool, not a Modelplane GPU pool. + """ + uuid: constr(max_length=64) | None = None + """ + UUID is the device's uuid attribute, when the driver publishes one. + """ + + +class Pods(BaseModel): + ready: conint(ge=0) + """ + Ready counts the pods whose Ready condition is True. + """ + scheduled: conint(ge=0) + """ + Scheduled counts the pods bound to a node. + """ + total: conint(ge=0) + """ + Total counts the pods the replica's selector matches that haven't + run to completion. A pod that has succeeded or failed, like an evicted + one, doesn't count. + """ + + +class Pool(BaseModel): + lastAllocationTime: AwareDatetime + """ + LastAllocationTime is the latest time the agent first saw one of the + replica's ResourceClaims that hold devices on the pool allocated. The + pool's PoolObservation accounts for those devices once its own + lastAllocationTime is no earlier than this, unless they've since been + deallocated. + """ + name: constr(max_length=63) + """ + Name of the pool: its nodes' modelplane.ai/pool label. + """ + + +class Status(BaseModel): + conditions: list[Condition] | None = Field(None, max_length=8) + """ + Conditions of the replica. Scheduled is True when every pod is bound to + a node and the agent has seen each pod's ResourceClaims allocated. Ready + is True when every pod is ready, and otherwise its reason says what the + replica is waiting on. Each condition's lastTransitionTime is when its + status last changed. + """ + devices: list[Device] | None = Field(None, max_length=4096) + """ + Devices are the DRA devices allocated to the replica's pods. + """ + observedGeneration: int | None = None + """ + ObservedGeneration is the generation of the spec the status reports on. + """ + pods: Pods + """ + Pods counts the replica's pods. + """ + pools: list[Pool] | None = Field(None, max_length=16) + """ + Pools are the pools whose nodes the replica's devices are attached to, + with when the agent first saw the latest of the replica's ResourceClaims + that hold devices on each allocated. + """ + + +class ReplicaObservation(BaseModel): + apiVersion: Literal['agent.modelplane.ai/v1alpha1'] | None = ( + 'agent.modelplane.ai/v1alpha1' + ) + """ + APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + """ + kind: Literal['ReplicaObservation'] | None = 'ReplicaObservation' + """ + Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + """ + metadata: v1.ObjectMeta | None = None + """ + Standard object's metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + """ + spec: Spec + """ + ReplicaObservationSpec specifies which pods make up the replica. + """ + status: Status | None = None + """ + ReplicaObservationStatus is what the agent observed about the replica. + """ + + +class ReplicaObservationList(BaseModel): + apiVersion: str | None = None + """ + APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + """ + items: list[ReplicaObservation] + """ + List of replicaobservations. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md + """ + kind: str | None = None + """ + Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + """ + metadata: v1.ListMeta | None = None + """ + Standard list metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + """ \ No newline at end of file diff --git a/schemas/python/models/ai/modelplane/inferencecluster/v1alpha1.py b/schemas/python/models/ai/modelplane/inferencecluster/v1alpha1.py index da447c98f..8ea419ad9 100644 --- a/schemas/python/models/ai/modelplane/inferencecluster/v1alpha1.py +++ b/schemas/python/models/ai/modelplane/inferencecluster/v1alpha1.py @@ -252,7 +252,10 @@ class NodePool(BaseModel): Maximum node count for autoscaling. Omit for fixed-size pools. """ minNodeCount: conint(ge=0) | None = None - name: constr(max_length=40) + name: constr(pattern=r'^[a-z0-9]([-a-z0-9]*[a-z0-9])?$', max_length=40) + """ + Name of the pool. Its nodes carry it as the value of the modelplane.ai/pool label, and the cluster agent observes the pool through an object of this name. + """ nodeCount: conint(ge=0) | None = 1 zones: list[str] | None = None """ @@ -357,18 +360,41 @@ class Device(BaseModel): name: constr(max_length=63) +class NodesByFreeDevice(BaseModel): + freeDevices: dict[str, conint(ge=0)] | None = Field(None, max_length=32) + """ + Each node's free devices, keyed by DeviceClass. A class the nodes publish no devices of is absent. + """ + nodes: conint(ge=1) + """ + Nodes with these free devices. + """ + + class GpuPool(BaseModel): devices: list[Device] | None = Field(None, max_length=16) """ Devices copied from the pool's InferenceClass. ModelDeployment.nodeSelector matches against these. """ + lastAllocationTime: AwareDatetime | None = None + """ + The latest time the cluster agent first saw a ResourceClaim allocated devices on this pool's nodes. nodesByFreeDevices accounts for the devices of a ModelReplica whose lastAllocationTime for this pool is no later than this. Published with observedNodes, once a claim has allocated devices on the pool. + """ name: str """ Node pool name, matching spec.nodePools[].name. Used to pin a ModelReplica to a specific pool via spec.nodePoolName. """ nodes: int | None = None """ - Number of nodes in this pool. Derived from maxNodeCount (if autoscaling) or nodeCount. + The most nodes this pool can have. Derived from maxNodeCount (if autoscaling) or nodeCount. + """ + nodesByFreeDevices: list[NodesByFreeDevice] | None = Field(None, max_length=256) + """ + A histogram of this pool's schedulable nodes by how many devices of each DeviceClass are free on them, as the cluster agent observes them. Nodes with the same number of free devices of every class share a bucket. Published with observedNodes. + """ + observedNodes: conint(ge=0) | None = None + """ + The nodes this pool has now, ready or not, as the cluster agent observes them. Absent until the agent reports on the pool, and while it can't observe it. """ @@ -394,7 +420,7 @@ class Status(BaseModel): """ gpuPools: list[GpuPool] | None = Field(None, max_length=8) """ - Schedulable GPU node pools on this cluster, derived from the referenced classes and the per-pool node counts. ModelDeployment scheduling matches against these. + Schedulable GPU node pools on this cluster, derived from the referenced classes, the per-pool node counts, and what the cluster agent observes of each pool. ModelDeployment scheduling matches against these. """ namespace: str | None = None """ diff --git a/schemas/python/models/ai/modelplane/modelreplica/v1alpha1.py b/schemas/python/models/ai/modelplane/modelreplica/v1alpha1.py index 70be7e42e..d1b00b12a 100644 --- a/schemas/python/models/ai/modelplane/modelreplica/v1alpha1.py +++ b/schemas/python/models/ai/modelplane/modelreplica/v1alpha1.py @@ -270,11 +270,26 @@ class Condition(BaseModel): type: str +class Pool(BaseModel): + lastAllocationTime: AwareDatetime + """ + The latest time the cluster agent first saw one of this replica's ResourceClaims allocated devices on the pool. The pool's nodesByFreeDevices accounts for them once its own lastAllocationTime is no earlier than this. + """ + name: str + """ + Node pool name, matching the InferenceCluster's spec.nodePools[].name. + """ + + class Status(BaseModel): conditions: list[Condition] | None = None """ Conditions of the resource. """ + pools: list[Pool] | None = Field(None, max_length=16) + """ + The pools whose nodes this replica's devices are attached to, as the cluster agent observes them. + """ class ModelReplica(BaseModel):