From da0da851a2eb2c034fc158cd000677da6c45ec7c Mon Sep 17 00:00:00 2001 From: Dennis Ramdass Date: Mon, 26 Oct 2026 11:57:38 -0700 Subject: [PATCH 1/4] Let a ModelDeployment set its serving port and engine readiness probe A ModelDeployment could only run an engine that listens on port 8000 and answers GET /health. The XRDs pinned requests to 8000 in their port rules and rejected a readinessProbe on the engine container, so an engine on another port or with another health path never went Ready. This adds serving.port, the port every serving pod receives requests on, and lets the engine container set its own readinessProbe. Both are optional. Unset, the port is 8000 and Modelplane probes /health as before. serving.port has no schema default, so a deployment that doesn't set it stores and propagates exactly what it did. The serving port is one value for the whole replica because a replica's serving pods share one InferencePool, so it lives on serving rather than on each engine. The port rules that checked 8000 move up to the spec, where they can see serving.port. Probe paths must start with "/", and probe ports must be a name or a number from 1 to 65535. A Worker has no http port. Its engine's probe must refer to a port that the engine container declares, or the worker pod never goes Ready. Towards #462. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Dennis Ramdass --- apis/modeldeployments/definition.yaml | 71 ++++++++++++------- apis/modelreplicas/definition.yaml | 62 ++++++++++------ schemas/.lock.json | 2 +- .../ai/modelplane/modeldeployment/v1alpha1.py | 14 ++-- .../ai/modelplane/modelreplica/v1alpha1.py | 10 ++- 5 files changed, 106 insertions(+), 53 deletions(-) diff --git a/apis/modeldeployments/definition.yaml b/apis/modeldeployments/definition.yaml index 0e9bfb60e..d30cfb7f9 100644 --- a/apis/modeldeployments/definition.yaml +++ b/apis/modeldeployments/definition.yaml @@ -111,6 +111,22 @@ spec: message: "PrefillDecode serving requires exactly one engine with phase Decode" - rule: "(has(self.serving) && self.serving.mode == 'PrefillDecode') || !self.engines.exists(e, has(e.phase))" message: "engine phase may only be set when serving.mode is PrefillDecode" + # The ports of a serving pod: a Standalone member or a Leader. A Worker + # doesn't serve, so these skip it. The serving port is serving.port, 8000 + # by default. Keep in sync with apis/modelreplicas/definition.yaml. + # - At most one container declares the serving port, where requests arrive. + # - If a container beside the engine declares the serving port, the engine + # declares its http port elsewhere, and if the engine's http port is + # elsewhere, a container beside it declares the serving port. + # - The engine may name the serving port only http. + - rule: "self.engines.all(e, e.members.all(m, (has(m.role) && m.role == 'Worker') || !has(m.template.spec) || m.template.spec.containers.filter(c, has(c.ports) && c.ports.exists(p, p.containerPort == self.?serving.?port.orValue(8000))).size() <= 1))" + message: "at most one container in a serving pod may declare the serving port (serving.port, 8000 by default)" + - rule: "self.engines.all(e, e.members.all(m, (has(m.role) && m.role == 'Worker') || !has(m.template.spec) || !m.template.spec.containers.exists(c, c.name != 'engine' && has(c.ports) && c.ports.exists(p, p.containerPort == self.?serving.?port.orValue(8000))) || m.template.spec.containers.exists(c, c.name == 'engine' && has(c.ports) && c.ports.exists(p, has(p.name) && p.name == 'http' && p.containerPort != self.?serving.?port.orValue(8000)))))" + message: "when another container declares the serving port, the engine container must declare its 'http' port on a different port" + - rule: "self.engines.all(e, e.members.all(m, (has(m.role) && m.role == 'Worker') || !has(m.template.spec) || !m.template.spec.containers.exists(c, c.name == 'engine' && has(c.ports) && c.ports.exists(p, has(p.name) && p.name == 'http' && p.containerPort != self.?serving.?port.orValue(8000))) || m.template.spec.containers.exists(c, c.name != 'engine' && has(c.ports) && c.ports.exists(p, p.containerPort == self.?serving.?port.orValue(8000)))))" + message: "when the engine container's 'http' port isn't the serving port, another container in its pod must declare the serving port" + - rule: "self.engines.all(e, e.members.all(m, (has(m.role) && m.role == 'Worker') || !has(m.template.spec) || !m.template.spec.containers.exists(c, c.name == 'engine' && has(c.ports) && c.ports.exists(p, p.containerPort == self.?serving.?port.orValue(8000) && has(p.name) && p.name != 'http'))))" + message: "the engine container may name the serving port only 'http'" properties: clusterSelector: type: object @@ -172,6 +188,15 @@ spec: endpoint picker set in endpointPicker, and the user writes any sidecar the picker needs as a container in the member templates. + port: + type: integer + minimum: 1 + maximum: 65535 + description: >- + The port every serving pod receives requests on, where the + InferencePool sends them. The engine serves its API there unless + a container in front of it, such as a sidecar, declares this port. + Defaults to 8000. endpointPicker: type: object description: >- @@ -427,21 +452,8 @@ spec: x-kubernetes-validations: - rule: "!has(self.worker) || self.role == 'Worker'" message: "worker may only be set on a Worker member" - # The ports of a serving pod: a Standalone member or a Leader. A Worker - # doesn't serve, so these skip it. Keep in sync with apis/modelreplicas/definition.yaml. - # - At most one container declares 8000, where requests arrive. - # - If a container beside the engine declares 8000, the engine declares - # its http port elsewhere, and if the engine's http port is elsewhere, - # a container beside it declares 8000. - # - The engine may name port 8000 only http. - - rule: "(has(self.role) && self.role == 'Worker') || !has(self.template.spec) || self.template.spec.containers.filter(c, has(c.ports) && c.ports.exists(p, p.containerPort == 8000)).size() <= 1" - message: "at most one container in a serving pod may declare containerPort 8000" - - rule: "(has(self.role) && self.role == 'Worker') || !has(self.template.spec) || !self.template.spec.containers.exists(c, c.name != 'engine' && has(c.ports) && c.ports.exists(p, p.containerPort == 8000)) || self.template.spec.containers.exists(c, c.name == 'engine' && has(c.ports) && c.ports.exists(p, has(p.name) && p.name == 'http' && p.containerPort != 8000))" - message: "when another container declares containerPort 8000, the engine container must declare its 'http' port on a different port" - - rule: "(has(self.role) && self.role == 'Worker') || !has(self.template.spec) || !self.template.spec.containers.exists(c, c.name == 'engine' && has(c.ports) && c.ports.exists(p, has(p.name) && p.name == 'http' && p.containerPort != 8000)) || self.template.spec.containers.exists(c, c.name != 'engine' && has(c.ports) && c.ports.exists(p, p.containerPort == 8000))" - message: "when the engine container's 'http' port isn't 8000, another container in its pod must declare containerPort 8000" - - rule: "(has(self.role) && self.role == 'Worker') || !has(self.template.spec) || !self.template.spec.containers.exists(c, c.name == 'engine' && has(c.ports) && c.ports.exists(p, p.containerPort == 8000 && has(p.name) && p.name != 'http'))" - message: "the engine container may name containerPort 8000 only 'http'" + - rule: "!has(self.role) || self.role != 'Worker' || !has(self.template.spec) || self.template.spec.containers.all(c, c.name != 'engine' || !has(c.readinessProbe) || ((!has(c.readinessProbe.httpGet) || type(c.readinessProbe.httpGet.port) != string || (has(c.ports) && c.ports.exists(p, has(p.name) && p.name == c.readinessProbe.httpGet.port))) && (!has(c.readinessProbe.tcpSocket) || type(c.readinessProbe.tcpSocket.port) != string || (has(c.ports) && c.ports.exists(p, has(p.name) && p.name == c.readinessProbe.tcpSocket.port)))))" + message: "a Worker's engine readinessProbe may name only a port its engine container declares; Modelplane gives a Worker no http port" properties: role: type: string @@ -629,13 +641,11 @@ spec: Containers for the engine pod. Exactly one is named "engine", the inference engine. The others run beside it, such as a routing sidecar. Modelplane adds device claims, MODELPLANE_ - environment, ModelCache mounts, /dev/shm and the readiness - probe to the engine container only. + environment, ModelCache mounts, /dev/shm and a default + readiness probe to the engine container only. x-kubernetes-validations: - rule: "self.exists_one(c, c.name == 'engine')" message: "exactly one container must be named 'engine'" - - rule: "!self.exists(c, c.name == 'engine' && has(c.readinessProbe))" - message: "the engine container can't set readinessProbe; Modelplane probes it on its http port" items: type: object required: [name, image] @@ -766,9 +776,12 @@ spec: readinessProbe: type: object description: >- - How Kubernetes checks that a container beside the engine is - ready, such as a sidecar on port 8000. The engine container - can't set one: Modelplane probes the engine on its http port. + How Kubernetes checks that the container is ready. Without one, + Modelplane probes the engine container of a Standalone or Leader + member with GET /health on its http port. A probe set here replaces + that default whole, and Kubernetes defaults apply to any field it + leaves out. A Worker has no http port, so its engine's probe may name + only a port the engine container declares. x-kubernetes-validations: - rule: "[has(self.httpGet), has(self.tcpSocket), has(self.grpc)].filter(h, h).size() == 1" message: "readinessProbe sets exactly one of httpGet, tcpSocket and grpc" @@ -776,15 +789,22 @@ spec: httpGet: type: object required: [port] + x-kubernetes-validations: + - rule: "type(self.port) == string || (self.port >= 1 && self.port <= 65535)" + message: "httpGet.port must be a port name or a number from 1 to 65535" properties: path: type: string + pattern: '^/' port: anyOf: [{type: integer}, {type: string}] x-kubernetes-int-or-string: true tcpSocket: type: object required: [port] + x-kubernetes-validations: + - rule: "type(self.port) == string || (self.port >= 1 && self.port <= 65535)" + message: "tcpSocket.port must be a port name or a number from 1 to 65535" properties: port: anyOf: [{type: integer}, {type: string}] @@ -795,6 +815,8 @@ spec: properties: port: type: integer + minimum: 1 + maximum: 65535 service: type: string initialDelaySeconds: @@ -824,8 +846,9 @@ spec: Ports the container listens on. The engine container's port named "http" is where Modelplane probes the engine and scrapes its metrics. Without one, the engine serves - on 8000, and an unnamed port 8000 is its "http" port. A - container in front of the engine declares 8000, and the + on the serving port (serving.port, 8000 by default), and + an unnamed port there is its "http" port. A container in + front of the engine declares the serving port, and the engine then declares its "http" port elsewhere. A Worker's ports pass through as written. items: diff --git a/apis/modelreplicas/definition.yaml b/apis/modelreplicas/definition.yaml index 25342be5b..302ee62cf 100644 --- a/apis/modelreplicas/definition.yaml +++ b/apis/modelreplicas/definition.yaml @@ -48,6 +48,22 @@ spec: message: "PrefillDecode serving requires exactly one engine with phase Decode" - rule: "(has(self.serving) && self.serving.mode == 'PrefillDecode') || !self.engines.exists(e, has(e.phase))" message: "engine phase may only be set when serving.mode is PrefillDecode" + # The ports of a serving pod: a Standalone member or a Leader. A Worker + # doesn't serve, so these skip it. The serving port is serving.port, 8000 + # by default. Keep in sync with apis/modeldeployments/definition.yaml. + # - At most one container declares the serving port, where requests arrive. + # - If a container beside the engine declares the serving port, the engine + # declares its http port elsewhere, and if the engine's http port is + # elsewhere, a container beside it declares the serving port. + # - The engine may name the serving port only http. + - rule: "self.engines.all(e, e.members.all(m, (has(m.role) && m.role == 'Worker') || !has(m.template.spec) || m.template.spec.containers.filter(c, has(c.ports) && c.ports.exists(p, p.containerPort == self.?serving.?port.orValue(8000))).size() <= 1))" + message: "at most one container in a serving pod may declare the serving port (serving.port, 8000 by default)" + - rule: "self.engines.all(e, e.members.all(m, (has(m.role) && m.role == 'Worker') || !has(m.template.spec) || !m.template.spec.containers.exists(c, c.name != 'engine' && has(c.ports) && c.ports.exists(p, p.containerPort == self.?serving.?port.orValue(8000))) || m.template.spec.containers.exists(c, c.name == 'engine' && has(c.ports) && c.ports.exists(p, has(p.name) && p.name == 'http' && p.containerPort != self.?serving.?port.orValue(8000)))))" + message: "when another container declares the serving port, the engine container must declare its 'http' port on a different port" + - rule: "self.engines.all(e, e.members.all(m, (has(m.role) && m.role == 'Worker') || !has(m.template.spec) || !m.template.spec.containers.exists(c, c.name == 'engine' && has(c.ports) && c.ports.exists(p, has(p.name) && p.name == 'http' && p.containerPort != self.?serving.?port.orValue(8000))) || m.template.spec.containers.exists(c, c.name != 'engine' && has(c.ports) && c.ports.exists(p, p.containerPort == self.?serving.?port.orValue(8000)))))" + message: "when the engine container's 'http' port isn't the serving port, another container in its pod must declare the serving port" + - rule: "self.engines.all(e, e.members.all(m, (has(m.role) && m.role == 'Worker') || !has(m.template.spec) || !m.template.spec.containers.exists(c, c.name == 'engine' && has(c.ports) && c.ports.exists(p, p.containerPort == self.?serving.?port.orValue(8000) && has(p.name) && p.name != 'http'))))" + message: "the engine container may name the serving port only 'http'" properties: clusterName: type: string @@ -94,6 +110,15 @@ spec: router: type: string enum: [EndpointPicker] + port: + type: integer + minimum: 1 + maximum: 65535 + description: >- + The port every serving pod receives requests on, where the + InferencePool sends them. The engine serves its API there unless + a container in front of it, such as a sidecar, declares this port. + Defaults to 8000. endpointPicker: type: object required: [template] @@ -280,21 +305,8 @@ spec: x-kubernetes-validations: - rule: "!has(self.worker) || self.role == 'Worker'" message: "worker may only be set on a Worker member" - # The ports of a serving pod: a Standalone member or a Leader. A Worker - # doesn't serve, so these skip it. Keep in sync with apis/modeldeployments/definition.yaml. - # - At most one container declares 8000, where requests arrive. - # - If a container beside the engine declares 8000, the engine declares - # its http port elsewhere, and if the engine's http port is elsewhere, - # a container beside it declares 8000. - # - The engine may name port 8000 only http. - - rule: "(has(self.role) && self.role == 'Worker') || !has(self.template.spec) || self.template.spec.containers.filter(c, has(c.ports) && c.ports.exists(p, p.containerPort == 8000)).size() <= 1" - message: "at most one container in a serving pod may declare containerPort 8000" - - rule: "(has(self.role) && self.role == 'Worker') || !has(self.template.spec) || !self.template.spec.containers.exists(c, c.name != 'engine' && has(c.ports) && c.ports.exists(p, p.containerPort == 8000)) || self.template.spec.containers.exists(c, c.name == 'engine' && has(c.ports) && c.ports.exists(p, has(p.name) && p.name == 'http' && p.containerPort != 8000))" - message: "when another container declares containerPort 8000, the engine container must declare its 'http' port on a different port" - - rule: "(has(self.role) && self.role == 'Worker') || !has(self.template.spec) || !self.template.spec.containers.exists(c, c.name == 'engine' && has(c.ports) && c.ports.exists(p, has(p.name) && p.name == 'http' && p.containerPort != 8000)) || self.template.spec.containers.exists(c, c.name != 'engine' && has(c.ports) && c.ports.exists(p, p.containerPort == 8000))" - message: "when the engine container's 'http' port isn't 8000, another container in its pod must declare containerPort 8000" - - rule: "(has(self.role) && self.role == 'Worker') || !has(self.template.spec) || !self.template.spec.containers.exists(c, c.name == 'engine' && has(c.ports) && c.ports.exists(p, p.containerPort == 8000 && has(p.name) && p.name != 'http'))" - message: "the engine container may name containerPort 8000 only 'http'" + - rule: "!has(self.role) || self.role != 'Worker' || !has(self.template.spec) || self.template.spec.containers.all(c, c.name != 'engine' || !has(c.readinessProbe) || ((!has(c.readinessProbe.httpGet) || type(c.readinessProbe.httpGet.port) != string || (has(c.ports) && c.ports.exists(p, has(p.name) && p.name == c.readinessProbe.httpGet.port))) && (!has(c.readinessProbe.tcpSocket) || type(c.readinessProbe.tcpSocket.port) != string || (has(c.ports) && c.ports.exists(p, has(p.name) && p.name == c.readinessProbe.tcpSocket.port)))))" + message: "a Worker's engine readinessProbe may name only a port its engine container declares; Modelplane gives a Worker no http port" properties: role: type: string @@ -438,8 +450,6 @@ spec: x-kubernetes-validations: - rule: "self.exists_one(c, c.name == 'engine')" message: "exactly one container must be named 'engine'" - - rule: "!self.exists(c, c.name == 'engine' && has(c.readinessProbe))" - message: "the engine container can't set readinessProbe; Modelplane probes it on its http port" items: type: object required: [name, image] @@ -530,9 +540,12 @@ spec: readinessProbe: type: object description: >- - How Kubernetes checks that a container beside the engine is - ready, such as a sidecar on port 8000. The engine container - can't set one: Modelplane probes the engine on its http port. + How Kubernetes checks that the container is ready. Without one, + Modelplane probes the engine container of a Standalone or Leader + member with GET /health on its http port. A probe set here replaces + that default whole, and Kubernetes defaults apply to any field it + leaves out. A Worker has no http port, so its engine's probe may name + only a port the engine container declares. x-kubernetes-validations: - rule: "[has(self.httpGet), has(self.tcpSocket), has(self.grpc)].filter(h, h).size() == 1" message: "readinessProbe sets exactly one of httpGet, tcpSocket and grpc" @@ -540,15 +553,22 @@ spec: httpGet: type: object required: [port] + x-kubernetes-validations: + - rule: "type(self.port) == string || (self.port >= 1 && self.port <= 65535)" + message: "httpGet.port must be a port name or a number from 1 to 65535" properties: path: type: string + pattern: '^/' port: anyOf: [{type: integer}, {type: string}] x-kubernetes-int-or-string: true tcpSocket: type: object required: [port] + x-kubernetes-validations: + - rule: "type(self.port) == string || (self.port >= 1 && self.port <= 65535)" + message: "tcpSocket.port must be a port name or a number from 1 to 65535" properties: port: anyOf: [{type: integer}, {type: string}] @@ -559,6 +579,8 @@ spec: properties: port: type: integer + minimum: 1 + maximum: 65535 service: type: string initialDelaySeconds: diff --git a/schemas/.lock.json b/schemas/.lock.json index ad2bd8e12..1bd4dbb02 100644 --- a/schemas/.lock.json +++ b/schemas/.lock.json @@ -1,6 +1,6 @@ { "packages": { - "fs://apis": "1e73064210fec7fa5508d4af954e85d2adba93ccb40101324d926f950e8d2e2f", + "fs://apis": "37faac3cc263b5386f7a3fbc71cc093f59980339f649b873580ee21fd6fb6cb0", "git://https://github.com/crossplane/crossplane/cluster/crds": "90d8b72ad8b829f0bcd7d7d5a98eaa0d579f244a", "xpkg://xpkg.upbound.io/upbound/provider-aws-ec2:v2.8.1": "sha256:ca2e9e3b2e3a8b6ca44a9700d5abf7abd733cfa388d1afe9bb2bf7c847cd39ef", "xpkg://xpkg.upbound.io/upbound/provider-aws-efs:v2.8.1": "sha256:ebb1bcd8dc9a7e60e97a324652fbd9d1b0609ddbb1107db518ce999cbc1113bd", diff --git a/schemas/python/models/ai/modelplane/modeldeployment/v1alpha1.py b/schemas/python/models/ai/modelplane/modeldeployment/v1alpha1.py index 68d2b0b87..890c5f456 100644 --- a/schemas/python/models/ai/modelplane/modeldeployment/v1alpha1.py +++ b/schemas/python/models/ai/modelplane/modeldeployment/v1alpha1.py @@ -133,12 +133,12 @@ class Port(BaseModel): class Grpc(BaseModel): - port: int + port: conint(ge=1, le=65535) service: str | None = None class HttpGet(BaseModel): - path: str | None = None + path: constr(pattern=r'^/') | None = None port: int | str @@ -188,11 +188,11 @@ class Container(BaseModel): """ ports: list[Port] | None = Field(None, max_length=8) """ - Ports the container listens on. The engine container's port named "http" is where Modelplane probes the engine and scrapes its metrics. Without one, the engine serves on 8000, and an unnamed port 8000 is its "http" port. A container in front of the engine declares 8000, and the engine then declares its "http" port elsewhere. A Worker's ports pass through as written. + Ports the container listens on. The engine container's port named "http" is where Modelplane probes the engine and scrapes its metrics. Without one, the engine serves on the serving port (serving.port, 8000 by default), and an unnamed port there is its "http" port. A container in front of the engine declares the serving port, and the engine then declares its "http" port elsewhere. A Worker's ports pass through as written. """ readinessProbe: ReadinessProbe | None = None """ - How Kubernetes checks that a container beside the engine is ready, such as a sidecar on port 8000. The engine container can't set one: Modelplane probes the engine on its http port. + How Kubernetes checks that the container is ready. Without one, Modelplane probes the engine container of a Standalone or Leader member with GET /health on its http port. A probe set here replaces that default whole, and Kubernetes defaults apply to any field it leaves out. A Worker has no http port, so its engine's probe may name only a port the engine container declares. """ resources: Resources | None = None """ @@ -207,7 +207,7 @@ class ImagePullSecret(BaseModel): class Spec(BaseModel): containers: list[Container] = Field(..., max_length=8, min_length=1) """ - Containers for the engine pod. Exactly one is named "engine", the inference engine. The others run beside it, such as a routing sidecar. Modelplane adds device claims, MODELPLANE_ environment, ModelCache mounts, /dev/shm and the readiness probe to the engine container only. + Containers for the engine pod. Exactly one is named "engine", the inference engine. The others run beside it, such as a routing sidecar. Modelplane adds device claims, MODELPLANE_ environment, ModelCache mounts, /dev/shm and a default readiness probe to the engine container only. """ imagePullSecrets: list[ImagePullSecret] | None = None """ @@ -325,6 +325,10 @@ class Serving(BaseModel): """ Unified serves prefill and decode on one engine. PrefillDecode splits them across two engines, each marking its phase as Prefill or Decode. """ + port: conint(ge=1, le=65535) | None = None + """ + The port every serving pod receives requests on, where the InferencePool sends them. The engine serves its API there unless a container in front of it, such as a sidecar, declares this port. Defaults to 8000. + """ router: Literal['EndpointPicker'] | None = None """ The router the user writes. Required under PrefillDecode. Omitted, under Unified, Modelplane routes the deployment with its default picker configuration. EndpointPicker runs the endpoint picker set in endpointPicker, and the user writes any sidecar the picker needs as a container in the member templates. diff --git a/schemas/python/models/ai/modelplane/modelreplica/v1alpha1.py b/schemas/python/models/ai/modelplane/modelreplica/v1alpha1.py index 70be7e42e..618df5c55 100644 --- a/schemas/python/models/ai/modelplane/modelreplica/v1alpha1.py +++ b/schemas/python/models/ai/modelplane/modelreplica/v1alpha1.py @@ -110,12 +110,12 @@ class Port(BaseModel): class Grpc(BaseModel): - port: int + port: conint(ge=1, le=65535) service: str | None = None class HttpGet(BaseModel): - path: str | None = None + path: constr(pattern=r'^/') | None = None port: int | str @@ -150,7 +150,7 @@ class Container(BaseModel): ports: list[Port] | None = Field(None, max_length=8) readinessProbe: ReadinessProbe | None = None """ - How Kubernetes checks that a container beside the engine is ready, such as a sidecar on port 8000. The engine container can't set one: Modelplane probes the engine on its http port. + How Kubernetes checks that the container is ready. Without one, Modelplane probes the engine container of a Standalone or Leader member with GET /health on its http port. A probe set here replaces that default whole, and Kubernetes defaults apply to any field it leaves out. A Worker has no http port, so its engine's probe may name only a port the engine container declares. """ resources: Resources | None = None """ @@ -235,6 +235,10 @@ class EndpointPicker(BaseModel): class Serving(BaseModel): endpointPicker: EndpointPicker | None = None mode: Literal['Unified', 'PrefillDecode'] | None = 'Unified' + port: conint(ge=1, le=65535) | None = None + """ + The port every serving pod receives requests on, where the InferencePool sends them. The engine serves its API there unless a container in front of it, such as a sidecar, declares this port. Defaults to 8000. + """ router: Literal['EndpointPicker'] | None = None From 87b7bd6f32c9b059aa4ba7755e00ec854703e59f Mon Sep 17 00:00:00 2001 From: Dennis Ramdass Date: Mon, 26 Oct 2026 11:57:38 -0700 Subject: [PATCH 2/4] Serve and probe engines on the replica's serving port compose-model-replica fixed the engine's default http port, its readiness probe and the InferencePool's targetPorts at 8000, and always probed GET /health. It now reads serving.port, defaulting to 8000, for all three, in the native, llm-d and Grove backends alike. An engine that sets its own readinessProbe gets it as written in place of the default probe, the same way a sidecar's probe passes through. Modelplane doesn't probe a Worker, so a probe set on a Worker's engine passes through as written, like its ports. Replicas that set neither compose the same resources as before. Towards #462. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Dennis Ramdass --- .../function/backends/base.py | 53 +- .../function/backends/grove.py | 14 +- .../function/backends/llmd.py | 14 +- .../function/backends/native.py | 5 +- .../compose-model-replica/function/routing.py | 12 +- .../tests/test_backends.py | 1361 ++++++++++++++--- 6 files changed, 1223 insertions(+), 236 deletions(-) diff --git a/functions/compose-model-replica/function/backends/base.py b/functions/compose-model-replica/function/backends/base.py index f962652b5..db882bc84 100644 --- a/functions/compose-model-replica/function/backends/base.py +++ b/functions/compose-model-replica/function/backends/base.py @@ -231,9 +231,10 @@ def remote_namespace(replica: v1alpha1.ModelReplica) -> str: return resource.child_name(_NS_PREFIX, _namespace(replica.metadata)) -# Port the engine serves its OpenAI-compatible API on. A contract shared with -# the ModelEndpoint URLs, so it must not diverge between backends. -ENGINE_PORT = 8000 +# Port a serving pod receives requests on when serving.port is unset. The +# InferencePool sends to it, and an engine with no http port of its own serves +# there. +DEFAULT_SERVING_PORT = 8000 # The name given to the engine's port. Named because the collector's engine # scrape job selects on it: matching by number instead would find a sidecar's @@ -620,24 +621,36 @@ def sidecar_containers(member: v1alpha1.Member) -> list[dict]: return [c.model_dump(exclude_none=True) for c in member.template.spec.containers if c.name != "engine"] -def engine_ports(container: v1alpha1.Container) -> list[dict]: +def serving_port(replica: v1alpha1.ModelReplica) -> int: + """The port the replica's serving pods receive requests on. + + The InferencePool sends every request there, so it's one port for every + engine of the replica. + """ + serving = replica.spec.serving + if serving and serving.port: + return serving.port + return DEFAULT_SERVING_PORT + + +def engine_ports(container: v1alpha1.Container, port: int) -> list[dict]: """A serving engine container's ports, always including the http port. - The engine declares its http port when another container takes - ENGINE_PORT, as the disaggregation sidecar does on a decode pod; the XRD - requires that whenever another container declares 8000. Otherwise the - engine serves on ENGINE_PORT. + port is the serving port. The engine declares its http port when another + container takes the serving port, as the disaggregation sidecar does on a + decode pod; the XRD requires that whenever another container declares it. + Otherwise the engine serves on the serving port. """ ports = declared_ports(container) if any(p.get("name") == ENGINE_PORT_NAME for p in ports): return ports - # An engine that declares ENGINE_PORT without a name serves its API there, - # so that port is the http port. The XRD rejects any other name for it. + # An engine that declares the serving port without a name serves its API + # there, so that port is the http port. The XRD rejects any other name for it. for p in ports: - if p["containerPort"] == ENGINE_PORT: + if p["containerPort"] == port: p["name"] = ENGINE_PORT_NAME return ports - ports.insert(0, {"name": ENGINE_PORT_NAME, "containerPort": ENGINE_PORT}) + ports.insert(0, {"name": ENGINE_PORT_NAME, "containerPort": port}) return ports @@ -646,15 +659,17 @@ def declared_ports(container: v1alpha1.Container) -> list[dict]: return [p.model_dump(exclude_none=True) for p in container.ports or []] -def engine_http_port(container: v1alpha1.Container) -> int: - """The port a serving engine container serves its API, health and metrics on.""" - return next(p["containerPort"] for p in engine_ports(container) if p.get("name") == ENGINE_PORT_NAME) - +def engine_readiness_probe(container: v1alpha1.Container, port: int) -> dict: + """The readiness probe for a serving engine container. -def readiness_probe(port: int) -> dict: - """The readiness probe for a serving engine container on its http port.""" + The engine's own probe if it sets one, as written. Otherwise GET /health on + its http port, which defaults to port, the serving port. + """ + if container.readinessProbe: + return container.readinessProbe.model_dump(exclude_none=True) + http = next(p["containerPort"] for p in engine_ports(container, port) if p.get("name") == ENGINE_PORT_NAME) return { - "httpGet": {"path": "/health", "port": port}, + "httpGet": {"path": "/health", "port": http}, "initialDelaySeconds": 30, "periodSeconds": 10, # Kubernetes defaults the probe timeout to 1s. Some engines' /health diff --git a/functions/compose-model-replica/function/backends/grove.py b/functions/compose-model-replica/function/backends/grove.py index 72d80d75b..184b7e8dc 100644 --- a/functions/compose-model-replica/function/backends/grove.py +++ b/functions/compose-model-replica/function/backends/grove.py @@ -75,6 +75,7 @@ def build( worker_replicas = int(worker.worker.nodes) if worker.worker else 1 cache_volumes, cache_volume_mounts = base.cache_mounts(replica) + port = base.serving_port(replica) def container(member: v1alpha1.Member, *, serving: bool) -> dict: engine_container = base.engine_container(member) @@ -111,10 +112,15 @@ def container(member: v1alpha1.Member, *, serving: bool) -> dict: if security_context: c["securityContext"] = security_context if serving: - c["ports"] = base.engine_ports(engine_container) - c["readinessProbe"] = base.readiness_probe(base.engine_http_port(engine_container)) - elif engine_container.ports: - c["ports"] = base.declared_ports(engine_container) + c["ports"] = base.engine_ports(engine_container, port) + c["readinessProbe"] = base.engine_readiness_probe(engine_container, port) + else: + # A Worker doesn't serve, so it gets no http port or default + # probe. The ports and probe the user wrote pass through. + if engine_container.ports: + c["ports"] = base.declared_ports(engine_container) + if engine_container.readinessProbe: + c["readinessProbe"] = engine_container.readinessProbe.model_dump(exclude_none=True) return c def pod_spec(member: v1alpha1.Member, c: dict) -> dict: diff --git a/functions/compose-model-replica/function/backends/llmd.py b/functions/compose-model-replica/function/backends/llmd.py index d40ef0669..4771c9855 100644 --- a/functions/compose-model-replica/function/backends/llmd.py +++ b/functions/compose-model-replica/function/backends/llmd.py @@ -74,6 +74,7 @@ def build( size = 1 + (int(worker.worker.nodes) if worker.worker else 1) cache_volumes, cache_volume_mounts = base.cache_mounts(replica) + port = base.serving_port(replica) def container(member: v1alpha1.Member, *, serving: bool) -> dict: engine_container = base.engine_container(member) @@ -104,10 +105,15 @@ def container(member: v1alpha1.Member, *, serving: bool) -> dict: env.extend(e.model_dump(exclude_none=True) for e in engine_container.env) c["env"] = env if serving: - c["ports"] = base.engine_ports(engine_container) - c["readinessProbe"] = base.readiness_probe(base.engine_http_port(engine_container)) - elif engine_container.ports: - c["ports"] = base.declared_ports(engine_container) + c["ports"] = base.engine_ports(engine_container, port) + c["readinessProbe"] = base.engine_readiness_probe(engine_container, port) + else: + # A Worker doesn't serve, so it gets no http port or default + # probe. The ports and probe the user wrote pass through. + if engine_container.ports: + c["ports"] = base.declared_ports(engine_container) + if engine_container.readinessProbe: + c["readinessProbe"] = engine_container.readinessProbe.model_dump(exclude_none=True) return c def pod_spec(member: v1alpha1.Member, c: dict) -> dict: diff --git a/functions/compose-model-replica/function/backends/native.py b/functions/compose-model-replica/function/backends/native.py index 5c8a3133a..4e1ef786d 100644 --- a/functions/compose-model-replica/function/backends/native.py +++ b/functions/compose-model-replica/function/backends/native.py @@ -59,15 +59,16 @@ def build( selector = {base.LABEL_WORKLOAD: name} cache_volumes, cache_volume_mounts = base.cache_mounts(replica) + port = base.serving_port(replica) container = { "name": "engine", "image": engine_container.image, "args": list(engine_container.args or []), - "ports": base.engine_ports(engine_container), + "ports": base.engine_ports(engine_container, port), # vLLM tensor parallelism and NIXL KV transfer need a large /dev/shm. "volumeMounts": [{"name": "dshm", "mountPath": "/dev/shm"}, *cache_volume_mounts], - "readinessProbe": base.readiness_probe(base.engine_http_port(engine_container)), + "readinessProbe": base.engine_readiness_probe(engine_container, port), } # GPUs bind via DRA: the container references the pod-level claim # backed by the member's ResourceClaimTemplate. A Standalone member diff --git a/functions/compose-model-replica/function/routing.py b/functions/compose-model-replica/function/routing.py index 69dbc89fb..85214eca7 100644 --- a/functions/compose-model-replica/function/routing.py +++ b/functions/compose-model-replica/function/routing.py @@ -362,7 +362,9 @@ def _unified( container, config_yaml, secrets = _picker(replica, _unified_epp_config_yaml(block_size)) selector = {base.LABEL_SERVING: base.serving_label(replica)} ns = base.remote_namespace(replica) - out["inference-pool"] = base.wrap_object(provider_config, _inference_pool(name, ns, selector)) + out["inference-pool"] = base.wrap_object( + provider_config, _inference_pool(name, ns, selector, base.serving_port(replica)) + ) out[base.ROUTE_KEY] = base.wrap_object(provider_config, _http_route(replica, name)) out.update(_epp_objects(name, ns, provider_config, container, config_yaml, secrets, replica)) return out @@ -402,7 +404,9 @@ def _disaggregated( # EPP's prefill/decode filters, not for pool membership. selector = {base.LABEL_SERVING: base.serving_label(replica)} ns = base.remote_namespace(replica) - out["inference-pool"] = base.wrap_object(provider_config, _inference_pool(name, ns, selector)) + out["inference-pool"] = base.wrap_object( + provider_config, _inference_pool(name, ns, selector, base.serving_port(replica)) + ) out[base.ROUTE_KEY] = base.wrap_object(provider_config, _http_route(replica, name)) out.update(_epp_objects(name, ns, provider_config, container, config_yaml, secrets, replica)) return out @@ -442,14 +446,14 @@ def _label_role(obj: k8sobjv1alpha1.Object, *, role: str) -> None: tmpl["labels"][_LABEL_ROLE] = role -def _inference_pool(name: str, namespace: str, selector: dict[str, str]) -> dict: +def _inference_pool(name: str, namespace: str, selector: dict[str, str], port: int) -> dict: return { "apiVersion": "inference.networking.k8s.io/v1", "kind": "InferencePool", "metadata": {"name": f"{name}-pool", "namespace": namespace}, "spec": { "selector": {"matchLabels": selector}, - "targetPorts": [{"number": base.ENGINE_PORT}], + "targetPorts": [{"number": port}], # Envoy AI Gateway fails closed without a picker whatever the # failure mode, so the pool leaves it at its default, FailClose. "endpointPickerRef": { diff --git a/functions/compose-model-replica/tests/test_backends.py b/functions/compose-model-replica/tests/test_backends.py index 743d86430..056744ac3 100644 --- a/functions/compose-model-replica/tests/test_backends.py +++ b/functions/compose-model-replica/tests/test_backends.py @@ -211,7 +211,7 @@ def _route(*, name: str, path: str, pool: str) -> dict: } -def _inference_pool(*, name: str, serving: str, picker: str) -> dict: +def _inference_pool(*, name: str, serving: str, picker: str, port: int) -> dict: """The Object composing the InferencePool that fronts the replica's serving pods.""" return { "spec": { @@ -224,7 +224,7 @@ def _inference_pool(*, name: str, serving: str, picker: str) -> dict: "metadata": {"name": name, "namespace": "mp-ml-team-51733"}, "spec": { "selector": {"matchLabels": {"modelplane.ai/serving": serving}}, - "targetPorts": [{"number": 8000}], + "targetPorts": [{"number": port}], "endpointPickerRef": {"name": picker, "port": {"number": 9002}}, }, } @@ -796,6 +796,178 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "resource-claim-main-standalone": _claim_template(name="r-main-standalone-devices-f456f", count=1), }, ), + BuildCase( + name="StandaloneEngineProbe", + reason=("A Standalone engine that sets its own readiness probe gets that probe as written, not the default."), + backend=native.NativeBackend(), + replica=_replica( + name="r", + namespace="ml-team", + cluster_name="cluster-a", + labels=None, + model_cache_ref=None, + serving=None, + engines=[ + v1alpha1.Engine( + name="main", + copies=1, + members=[ + v1alpha1.Member( + role="Standalone", + nodePoolName="frontier", + deviceRequests=[ + v1alpha1.DeviceRequest( + name="gpu", + deviceClassName="gpu.nvidia.com", + count=1, + selectors=[ + v1alpha1.Selector( + cel='device.capacity["gpu.nvidia.com"].memory.compareTo(quantity("80Gi")) >= 0' + ) + ], + ) + ], + template=v1alpha1.Template( + spec=v1alpha1.Spec( + containers=[ + v1alpha1.Container( + name="engine", + image="vllm/vllm-openai:latest", + args=["--model=Qwen/Qwen3-0.6B"], + readinessProbe=v1alpha1.ReadinessProbe( + httpGet=v1alpha1.HttpGet(path="/v1/models", port="http"), + timeoutSeconds=5, + ), + ) + ] + ) + ), + ) + ], + ) + ], + ), + provider_config="cluster-a-pc", + serving_label="r", + stack="Standard", + want={ + "model-serving-main": _deployment( + name="r-main-bb4e3", + claim_template_name="r-main-standalone-devices-f456f", + replicas=1, + pod_metadata={ + "labels": { + "modelplane.ai/engine": "main", + "modelplane.ai/role": "Standalone", + "modelplane.ai/serving": "r", + "modelplane.ai/workload": "r-main-bb4e3", + } + }, + containers=[ + { + "name": "engine", + "image": "vllm/vllm-openai:latest", + "args": ["--model=Qwen/Qwen3-0.6B"], + "ports": [{"name": "http", "containerPort": 8000}], + "volumeMounts": [{"name": "dshm", "mountPath": "/dev/shm"}], + "readinessProbe": { + "httpGet": {"path": "/v1/models", "port": "http"}, + "timeoutSeconds": 5, + }, + "resources": {"claims": [{"name": "devices"}]}, + } + ], + volumes=[{"name": "dshm", "emptyDir": {"medium": "Memory"}}], + ), + "resource-claim-main-standalone": _claim_template(name="r-main-standalone-devices-f456f", count=1), + }, + ), + # serving.port moves the engine's default http port, so the probe and the + # collector find it there. + BuildCase( + name="StandaloneServingPort", + reason=("A Standalone engine of a replica with a serving port serves there, and the default probe follows it."), + backend=native.NativeBackend(), + replica=_replica( + name="r", + namespace="ml-team", + cluster_name="cluster-a", + labels=None, + model_cache_ref=None, + serving=v1alpha1.Serving(mode="Unified", port=8080), + engines=[ + v1alpha1.Engine( + name="main", + copies=1, + members=[ + v1alpha1.Member( + role="Standalone", + nodePoolName="frontier", + deviceRequests=[ + v1alpha1.DeviceRequest( + name="gpu", + deviceClassName="gpu.nvidia.com", + count=1, + selectors=[ + v1alpha1.Selector( + cel='device.capacity["gpu.nvidia.com"].memory.compareTo(quantity("80Gi")) >= 0' + ) + ], + ) + ], + template=v1alpha1.Template( + spec=v1alpha1.Spec( + containers=[ + v1alpha1.Container( + name="engine", + image="vllm/vllm-openai:latest", + args=["--model=Qwen/Qwen3-0.6B"], + ) + ] + ) + ), + ) + ], + ) + ], + ), + provider_config="cluster-a-pc", + serving_label="r", + stack="Standard", + want={ + "model-serving-main": _deployment( + name="r-main-bb4e3", + claim_template_name="r-main-standalone-devices-f456f", + replicas=1, + pod_metadata={ + "labels": { + "modelplane.ai/engine": "main", + "modelplane.ai/role": "Standalone", + "modelplane.ai/serving": "r", + "modelplane.ai/workload": "r-main-bb4e3", + } + }, + containers=[ + { + "name": "engine", + "image": "vllm/vllm-openai:latest", + "args": ["--model=Qwen/Qwen3-0.6B"], + "ports": [{"name": "http", "containerPort": 8080}], + "volumeMounts": [{"name": "dshm", "mountPath": "/dev/shm"}], + "readinessProbe": { + "httpGet": {"path": "/health", "port": 8080}, + "initialDelaySeconds": 30, + "periodSeconds": 10, + "timeoutSeconds": 5, + }, + "resources": {"claims": [{"name": "devices"}]}, + } + ], + volumes=[{"name": "dshm", "emptyDir": {"medium": "Memory"}}], + ), + "resource-claim-main-standalone": _claim_template(name="r-main-standalone-devices-f456f", count=1), + }, + ), # A sidecar takes 8000, so the engine declares its http port elsewhere, and # the probe and the collector follow it there. The sidecar is the user's and # passes through as written: no claim, env, mounts or probe. @@ -2190,14 +2362,9 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "resource-claim-main-worker": _claim_template(name="r-main-worker-devices-99b8a", count=8), }, ), - # A Grove leader behind a sidecar, and a worker carrying its own ports and - # an extra container. BuildCase( - name="GroveSidecars", - reason=( - "On Grove, a leader behind a sidecar is probed on the engine's http port, and a worker's engine ports and " - "extra container pass through as written." - ), + name="GroveServingPort", + reason=("A Leader/Worker engine on Grove serves its leader on the replica's serving port, probed there."), backend=grove.GroveBackend(), replica=_replica( name="r", @@ -2205,7 +2372,7 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: cluster_name="cluster-a", labels=None, model_cache_ref=None, - serving=None, + serving=v1alpha1.Serving(mode="Unified", port=8080), engines=[ v1alpha1.Engine( name="main", @@ -2235,15 +2402,9 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: command=[ "/bin/sh", "-c", - "ray start --head --port=6379; exec vllm serve --model=meta-llama/Llama-3.1-405B --tensor-parallel-size=8 --pipeline-parallel-size=2 --port=8001", + "ray start --head --port=6379; exec vllm serve --model=meta-llama/Llama-3.1-405B --tensor-parallel-size=8 --pipeline-parallel-size=2 --port=8080", ], - ports=[v1alpha1.Port(name="http", containerPort=8001)], - ), - v1alpha1.Container( - name="proxy", - image="example.com/proxy:v1", - ports=[v1alpha1.Port(containerPort=8000)], - ), + ) ] ) ), @@ -2275,9 +2436,7 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "-c", "exec ray start --address=$(MODELPLANE_LEADER_ADDRESS):6379 --block", ], - ports=[v1alpha1.Port(name="nixl", containerPort=5557)], - ), - v1alpha1.Container(name="log-tail", image="busybox:1.36"), + ) ] ) ), @@ -2317,7 +2476,7 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "command": [ "/bin/sh", "-c", - "ray start --head --port=6379; exec vllm serve --model=meta-llama/Llama-3.1-405B --tensor-parallel-size=8 --pipeline-parallel-size=2 --port=8001", + "ray start --head --port=6379; exec vllm serve --model=meta-llama/Llama-3.1-405B --tensor-parallel-size=8 --pipeline-parallel-size=2 --port=8080", ], "env": [ { @@ -2325,19 +2484,14 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "value": "$(GROVE_PCSG_NAME)-$(GROVE_PCSG_INDEX)-leader-0.$(GROVE_HEADLESS_SERVICE)", } ], - "ports": [{"name": "http", "containerPort": 8001}], + "ports": [{"name": "http", "containerPort": 8080}], "readinessProbe": { - "httpGet": {"path": "/health", "port": 8001}, + "httpGet": {"path": "/health", "port": 8080}, "initialDelaySeconds": 30, "periodSeconds": 10, "timeoutSeconds": 5, }, - }, - { - "name": "proxy", - "image": "example.com/proxy:v1", - "ports": [{"containerPort": 8000}], - }, + } ], "volumes": [{"name": "dshm", "emptyDir": {"medium": "Memory"}}], "schedulerName": "kai-scheduler", @@ -2387,9 +2541,7 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "value": "$(GROVE_PCSG_NAME)-$(GROVE_PCSG_INDEX)-leader-0.$(GROVE_HEADLESS_SERVICE)", } ], - "ports": [{"name": "nixl", "containerPort": 5557}], - }, - {"name": "log-tail", "image": "busybox:1.36"}, + } ], "volumes": [{"name": "dshm", "emptyDir": {"medium": "Memory"}}], "schedulerName": "kai-scheduler", @@ -2416,9 +2568,14 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "resource-claim-main-worker": _claim_template(name="r-main-worker-devices-99b8a", count=8), }, ), + # A Grove leader behind a sidecar, and a worker carrying its own ports, + # probe and an extra container. BuildCase( - name="GroveMemberEnv", - reason="A Grove leader's own env follows the leader address alias in its container.", + name="GroveSidecars", + reason=( + "On Grove, a leader behind a sidecar is probed on the engine's http port, and a worker's engine ports, " + "engine probe and extra container pass through as written." + ), backend=grove.GroveBackend(), replica=_replica( name="r", @@ -2456,10 +2613,15 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: command=[ "/bin/sh", "-c", - "ray start --head --port=6379; exec vllm serve --model=meta-llama/Llama-3.1-405B --tensor-parallel-size=8 --pipeline-parallel-size=2 --port=8000", + "ray start --head --port=6379; exec vllm serve --model=meta-llama/Llama-3.1-405B --tensor-parallel-size=8 --pipeline-parallel-size=2 --port=8001", ], - env=[v1alpha1.EnvItem(name="HF_TOKEN", value="x")], - ) + ports=[v1alpha1.Port(name="http", containerPort=8001)], + ), + v1alpha1.Container( + name="proxy", + image="example.com/proxy:v1", + ports=[v1alpha1.Port(containerPort=8000)], + ), ] ) ), @@ -2491,7 +2653,12 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "-c", "exec ray start --address=$(MODELPLANE_LEADER_ADDRESS):6379 --block", ], - ) + ports=[v1alpha1.Port(name="nixl", containerPort=5557)], + readinessProbe=v1alpha1.ReadinessProbe( + tcpSocket=v1alpha1.TcpSocket(port="nixl") + ), + ), + v1alpha1.Container(name="log-tail", image="busybox:1.36"), ] ) ), @@ -2531,23 +2698,27 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "command": [ "/bin/sh", "-c", - "ray start --head --port=6379; exec vllm serve --model=meta-llama/Llama-3.1-405B --tensor-parallel-size=8 --pipeline-parallel-size=2 --port=8000", + "ray start --head --port=6379; exec vllm serve --model=meta-llama/Llama-3.1-405B --tensor-parallel-size=8 --pipeline-parallel-size=2 --port=8001", ], "env": [ { "name": "MODELPLANE_LEADER_ADDRESS", "value": "$(GROVE_PCSG_NAME)-$(GROVE_PCSG_INDEX)-leader-0.$(GROVE_HEADLESS_SERVICE)", - }, - {"name": "HF_TOKEN", "value": "x"}, + } ], - "ports": [{"name": "http", "containerPort": 8000}], + "ports": [{"name": "http", "containerPort": 8001}], "readinessProbe": { - "httpGet": {"path": "/health", "port": 8000}, + "httpGet": {"path": "/health", "port": 8001}, "initialDelaySeconds": 30, "periodSeconds": 10, "timeoutSeconds": 5, }, - } + }, + { + "name": "proxy", + "image": "example.com/proxy:v1", + "ports": [{"containerPort": 8000}], + }, ], "volumes": [{"name": "dshm", "emptyDir": {"medium": "Memory"}}], "schedulerName": "kai-scheduler", @@ -2597,7 +2768,10 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "value": "$(GROVE_PCSG_NAME)-$(GROVE_PCSG_INDEX)-leader-0.$(GROVE_HEADLESS_SERVICE)", } ], - } + "ports": [{"name": "nixl", "containerPort": 5557}], + "readinessProbe": {"tcpSocket": {"port": "nixl"}}, + }, + {"name": "log-tail", "image": "busybox:1.36"}, ], "volumes": [{"name": "dshm", "emptyDir": {"medium": "Memory"}}], "schedulerName": "kai-scheduler", @@ -2624,11 +2798,9 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "resource-claim-main-worker": _claim_template(name="r-main-worker-devices-99b8a", count=8), }, ), - # Multi-NIC RDMA nodes need VLLM_HOST_IP from status.podIP so the engine - # binds the right interface (#141). BuildCase( - name="GroveFieldRefEnv", - reason="A Grove leader's env drawn from a pod field passes through unchanged.", + name="GroveMemberEnv", + reason="A Grove leader's own env follows the leader address alias in its container.", backend=grove.GroveBackend(), replica=_replica( name="r", @@ -2668,14 +2840,7 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "-c", "ray start --head --port=6379; exec vllm serve --model=meta-llama/Llama-3.1-405B --tensor-parallel-size=8 --pipeline-parallel-size=2 --port=8000", ], - env=[ - v1alpha1.EnvItem( - name="VLLM_HOST_IP", - valueFrom=v1alpha1.ValueFrom( - fieldRef=v1alpha1.FieldRef(fieldPath="status.podIP") - ), - ) - ], + env=[v1alpha1.EnvItem(name="HF_TOKEN", value="x")], ) ] ) @@ -2755,10 +2920,7 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "name": "MODELPLANE_LEADER_ADDRESS", "value": "$(GROVE_PCSG_NAME)-$(GROVE_PCSG_INDEX)-leader-0.$(GROVE_HEADLESS_SERVICE)", }, - { - "name": "VLLM_HOST_IP", - "valueFrom": {"fieldRef": {"fieldPath": "status.podIP"}}, - }, + {"name": "HF_TOKEN", "value": "x"}, ], "ports": [{"name": "http", "containerPort": 8000}], "readinessProbe": { @@ -2844,7 +3006,227 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "resource-claim-main-worker": _claim_template(name="r-main-worker-devices-99b8a", count=8), }, ), - # Grove propagates a clique's labels and annotations to its pods (#378). + # Multi-NIC RDMA nodes need VLLM_HOST_IP from status.podIP so the engine + # binds the right interface (#141). + BuildCase( + name="GroveFieldRefEnv", + reason="A Grove leader's env drawn from a pod field passes through unchanged.", + backend=grove.GroveBackend(), + replica=_replica( + name="r", + namespace="ml-team", + cluster_name="cluster-a", + labels=None, + model_cache_ref=None, + serving=None, + engines=[ + v1alpha1.Engine( + name="main", + copies=1, + members=[ + v1alpha1.Member( + role="Leader", + nodePoolName="frontier", + deviceRequests=[ + v1alpha1.DeviceRequest( + name="gpu", + deviceClassName="gpu.nvidia.com", + count=8, + selectors=[ + v1alpha1.Selector( + cel='device.capacity["gpu.nvidia.com"].memory.compareTo(quantity("80Gi")) >= 0' + ) + ], + ) + ], + template=v1alpha1.Template( + spec=v1alpha1.Spec( + containers=[ + v1alpha1.Container( + name="engine", + image="vllm/vllm-openai:latest", + command=[ + "/bin/sh", + "-c", + "ray start --head --port=6379; exec vllm serve --model=meta-llama/Llama-3.1-405B --tensor-parallel-size=8 --pipeline-parallel-size=2 --port=8000", + ], + env=[ + v1alpha1.EnvItem( + name="VLLM_HOST_IP", + valueFrom=v1alpha1.ValueFrom( + fieldRef=v1alpha1.FieldRef(fieldPath="status.podIP") + ), + ) + ], + ) + ] + ) + ), + ), + v1alpha1.Member( + role="Worker", + nodePoolName="frontier", + worker=v1alpha1.Worker(nodes=1), + deviceRequests=[ + v1alpha1.DeviceRequest( + name="gpu", + deviceClassName="gpu.nvidia.com", + count=8, + selectors=[ + v1alpha1.Selector( + cel='device.capacity["gpu.nvidia.com"].memory.compareTo(quantity("80Gi")) >= 0' + ) + ], + ) + ], + template=v1alpha1.Template( + spec=v1alpha1.Spec( + containers=[ + v1alpha1.Container( + name="engine", + image="vllm/vllm-openai:latest", + command=[ + "/bin/sh", + "-c", + "exec ray start --address=$(MODELPLANE_LEADER_ADDRESS):6379 --block", + ], + ) + ] + ) + ), + ), + ], + ) + ], + ), + provider_config="cluster-a-pc", + serving_label="r", + stack="Dynamo", + want={ + "scaling-group-main": _scaling_group(pcs_name="r-main-bb4e3"), + "model-serving-main": _pod_clique_set( + name="r-main-bb4e3", + cliques=[ + { + "name": "leader", + "labels": { + "modelplane.ai/engine": "main", + "modelplane.ai/role": "Leader", + "modelplane.ai/serving": "r", + "kai.scheduler/queue": "modelplane", + "modelplane.ai/clique-role": "leader", + }, + "spec": { + "roleName": "leader", + "replicas": 1, + "minAvailable": 1, + "podSpec": { + "containers": [ + { + "name": "engine", + "image": "vllm/vllm-openai:latest", + "volumeMounts": [{"name": "dshm", "mountPath": "/dev/shm"}], + "resources": {"claims": [{"name": "devices"}]}, + "command": [ + "/bin/sh", + "-c", + "ray start --head --port=6379; exec vllm serve --model=meta-llama/Llama-3.1-405B --tensor-parallel-size=8 --pipeline-parallel-size=2 --port=8000", + ], + "env": [ + { + "name": "MODELPLANE_LEADER_ADDRESS", + "value": "$(GROVE_PCSG_NAME)-$(GROVE_PCSG_INDEX)-leader-0.$(GROVE_HEADLESS_SERVICE)", + }, + { + "name": "VLLM_HOST_IP", + "valueFrom": {"fieldRef": {"fieldPath": "status.podIP"}}, + }, + ], + "ports": [{"name": "http", "containerPort": 8000}], + "readinessProbe": { + "httpGet": {"path": "/health", "port": 8000}, + "initialDelaySeconds": 30, + "periodSeconds": 10, + "timeoutSeconds": 5, + }, + } + ], + "volumes": [{"name": "dshm", "emptyDir": {"medium": "Memory"}}], + "schedulerName": "kai-scheduler", + "nodeSelector": {"modelplane.ai/pool": "frontier"}, + "resourceClaims": [ + { + "name": "devices", + "resourceClaimTemplateName": "r-main-leader-devices-f58c6", + } + ], + "tolerations": [ + { + "key": "nvidia.com/gpu", + "operator": "Exists", + "effect": "NoSchedule", + } + ], + }, + }, + }, + { + "name": "worker", + "labels": { + "modelplane.ai/engine": "main", + "modelplane.ai/role": "Worker", + "kai.scheduler/queue": "modelplane", + }, + "spec": { + "roleName": "worker", + "replicas": 1, + "minAvailable": 1, + "podSpec": { + "containers": [ + { + "name": "engine", + "image": "vllm/vllm-openai:latest", + "volumeMounts": [{"name": "dshm", "mountPath": "/dev/shm"}], + "resources": {"claims": [{"name": "devices"}]}, + "command": [ + "/bin/sh", + "-c", + "exec ray start --address=$(MODELPLANE_LEADER_ADDRESS):6379 --block", + ], + "env": [ + { + "name": "MODELPLANE_LEADER_ADDRESS", + "value": "$(GROVE_PCSG_NAME)-$(GROVE_PCSG_INDEX)-leader-0.$(GROVE_HEADLESS_SERVICE)", + } + ], + } + ], + "volumes": [{"name": "dshm", "emptyDir": {"medium": "Memory"}}], + "schedulerName": "kai-scheduler", + "nodeSelector": {"modelplane.ai/pool": "frontier"}, + "resourceClaims": [ + { + "name": "devices", + "resourceClaimTemplateName": "r-main-worker-devices-99b8a", + } + ], + "tolerations": [ + { + "key": "nvidia.com/gpu", + "operator": "Exists", + "effect": "NoSchedule", + } + ], + }, + }, + }, + ], + ), + "resource-claim-main-leader": _claim_template(name="r-main-leader-devices-f58c6", count=8), + "resource-claim-main-worker": _claim_template(name="r-main-worker-devices-99b8a", count=8), + }, + ), + # Grove propagates a clique's labels and annotations to its pods (#378). BuildCase( name="GroveMemberMetadata", reason=( @@ -4411,23 +4793,20 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "resource-claim-main-worker": _claim_template(name="r-main-worker-devices-99b8a", count=8), }, ), - # LWS names a LeaderWorkerSet's headless Service after it, so its webhook - # rejects a name that isn't a DNS-1035 label. A claim template's name may - # be a subdomain, so it keeps the dot. BuildCase( - name="LLMDDottedReplica", + name="LLMDServingPort", reason=( - "A Leader/Worker engine of a dotted replica on llm-d composes a LeaderWorkerSet named by a hashed " - "DNS-1035 label, and claim templates that keep the dot." + "A Leader/Worker engine on llm-d serves on the replica's serving port and gets the leader's own probe, " + "while its worker has neither." ), backend=llmd.LLMDBackend(), replica=_replica( - name="qwen2.5", + name="r", namespace="ml-team", cluster_name="cluster-a", labels=None, model_cache_ref=None, - serving=None, + serving=v1alpha1.Serving(mode="Unified", port=8080), engines=[ v1alpha1.Engine( name="main", @@ -4450,7 +4829,15 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: ], template=v1alpha1.Template( spec=v1alpha1.Spec( - containers=[v1alpha1.Container(name="engine", image="vllm/vllm-openai:latest")] + containers=[ + v1alpha1.Container( + name="engine", + image="vllm/vllm-openai:latest", + readinessProbe=v1alpha1.ReadinessProbe( + httpGet=v1alpha1.HttpGet(path="/v1/models", port="http") + ), + ) + ] ) ), ), @@ -4481,14 +4868,14 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: ], ), provider_config="cluster-a-pc", - serving_label="qwen2.5", + serving_label="r", stack="Standard", want={ "model-serving-main": _leader_worker_set( - name="lws-qwen2-5-main-b2784-6d975", - serving="qwen2.5", - leader_claim_template_name="qwen2.5-main-leader-devices-5743a", - worker_claim_template_name="qwen2.5-main-worker-devices-4d4ac", + name="r-main-bb4e3", + serving="r", + leader_claim_template_name="r-main-leader-devices-f58c6", + worker_claim_template_name="r-main-worker-devices-99b8a", replicas=1, size=2, leader_containers=[ @@ -4504,13 +4891,8 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: }, {"name": "MODELPLANE_RANK", "value": "$(LWS_WORKER_INDEX)"}, ], - "ports": [{"name": "http", "containerPort": 8000}], - "readinessProbe": { - "httpGet": {"path": "/health", "port": 8000}, - "initialDelaySeconds": 30, - "periodSeconds": 10, - "timeoutSeconds": 5, - }, + "ports": [{"name": "http", "containerPort": 8080}], + "readinessProbe": {"httpGet": {"path": "/v1/models", "port": "http"}}, } ], leader_volumes=[{"name": "dshm", "emptyDir": {"medium": "Memory"}}], @@ -4531,8 +4913,132 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: ], worker_volumes=[{"name": "dshm", "emptyDir": {"medium": "Memory"}}], ), - "resource-claim-main-leader": _claim_template(name="qwen2.5-main-leader-devices-5743a", count=8), - "resource-claim-main-worker": _claim_template(name="qwen2.5-main-worker-devices-4d4ac", count=8), + "resource-claim-main-leader": _claim_template(name="r-main-leader-devices-f58c6", count=8), + "resource-claim-main-worker": _claim_template(name="r-main-worker-devices-99b8a", count=8), + }, + ), + # LWS names a LeaderWorkerSet's headless Service after it, so its webhook + # rejects a name that isn't a DNS-1035 label. A claim template's name may + # be a subdomain, so it keeps the dot. + BuildCase( + name="LLMDDottedReplica", + reason=( + "A Leader/Worker engine of a dotted replica on llm-d composes a LeaderWorkerSet named by a hashed " + "DNS-1035 label, and claim templates that keep the dot." + ), + backend=llmd.LLMDBackend(), + replica=_replica( + name="qwen2.5", + namespace="ml-team", + cluster_name="cluster-a", + labels=None, + model_cache_ref=None, + serving=None, + engines=[ + v1alpha1.Engine( + name="main", + copies=1, + members=[ + v1alpha1.Member( + role="Leader", + nodePoolName="frontier", + deviceRequests=[ + v1alpha1.DeviceRequest( + name="gpu", + deviceClassName="gpu.nvidia.com", + count=8, + selectors=[ + v1alpha1.Selector( + cel='device.capacity["gpu.nvidia.com"].memory.compareTo(quantity("80Gi")) >= 0' + ) + ], + ) + ], + template=v1alpha1.Template( + spec=v1alpha1.Spec( + containers=[v1alpha1.Container(name="engine", image="vllm/vllm-openai:latest")] + ) + ), + ), + v1alpha1.Member( + role="Worker", + nodePoolName="frontier", + worker=v1alpha1.Worker(nodes=1), + deviceRequests=[ + v1alpha1.DeviceRequest( + name="gpu", + deviceClassName="gpu.nvidia.com", + count=8, + selectors=[ + v1alpha1.Selector( + cel='device.capacity["gpu.nvidia.com"].memory.compareTo(quantity("80Gi")) >= 0' + ) + ], + ) + ], + template=v1alpha1.Template( + spec=v1alpha1.Spec( + containers=[v1alpha1.Container(name="engine", image="vllm/vllm-openai:latest")] + ) + ), + ), + ], + ) + ], + ), + provider_config="cluster-a-pc", + serving_label="qwen2.5", + stack="Standard", + want={ + "model-serving-main": _leader_worker_set( + name="lws-qwen2-5-main-b2784-6d975", + serving="qwen2.5", + leader_claim_template_name="qwen2.5-main-leader-devices-5743a", + worker_claim_template_name="qwen2.5-main-worker-devices-4d4ac", + replicas=1, + size=2, + leader_containers=[ + { + "name": "engine", + "image": "vllm/vllm-openai:latest", + "volumeMounts": [{"name": "dshm", "mountPath": "/dev/shm"}], + "resources": {"claims": [{"name": "devices"}]}, + "env": [ + { + "name": "MODELPLANE_LEADER_ADDRESS", + "value": "$(LWS_LEADER_ADDRESS)", + }, + {"name": "MODELPLANE_RANK", "value": "$(LWS_WORKER_INDEX)"}, + ], + "ports": [{"name": "http", "containerPort": 8000}], + "readinessProbe": { + "httpGet": {"path": "/health", "port": 8000}, + "initialDelaySeconds": 30, + "periodSeconds": 10, + "timeoutSeconds": 5, + }, + } + ], + leader_volumes=[{"name": "dshm", "emptyDir": {"medium": "Memory"}}], + worker_containers=[ + { + "name": "engine", + "image": "vllm/vllm-openai:latest", + "volumeMounts": [{"name": "dshm", "mountPath": "/dev/shm"}], + "resources": {"claims": [{"name": "devices"}]}, + "env": [ + { + "name": "MODELPLANE_LEADER_ADDRESS", + "value": "$(LWS_LEADER_ADDRESS)", + }, + {"name": "MODELPLANE_RANK", "value": "$(LWS_WORKER_INDEX)"}, + ], + } + ], + worker_volumes=[{"name": "dshm", "emptyDir": {"medium": "Memory"}}], + ), + "resource-claim-main-leader": _claim_template(name="qwen2.5-main-leader-devices-5743a", count=8), + "resource-claim-main-worker": _claim_template(name="qwen2.5-main-worker-devices-4d4ac", count=8), }, ), # Each member has its own template, so a container the user adds to the @@ -4663,10 +5169,11 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "resource-claim-main-worker": _claim_template(name="r-main-worker-devices-99b8a", count=8), }, ), - # A Worker doesn't serve: its ports pass through and nothing probes it. + # A Worker doesn't serve, so Modelplane gives it no http port or default + # probe. What the user writes on it passes through. BuildCase( name="LLMDWorkerSidecar", - reason="A Worker's engine ports and the container beside it pass through as written, with no probe.", + reason="A Worker's engine ports and probe and the container beside it pass through as written.", backend=llmd.LLMDBackend(), replica=_replica( name="r", @@ -4726,6 +5233,9 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: name="engine", image="vllm/vllm-openai:latest", ports=[v1alpha1.Port(name="nixl", containerPort=5557)], + readinessProbe=v1alpha1.ReadinessProbe( + tcpSocket=v1alpha1.TcpSocket(port="nixl") + ), ), v1alpha1.Container(name="log-tail", image="busybox:1.36"), ] @@ -4784,6 +5294,7 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: {"name": "MODELPLANE_RANK", "value": "$(LWS_WORKER_INDEX)"}, ], "ports": [{"name": "nixl", "containerPort": 5557}], + "readinessProbe": {"tcpSocket": {"port": "nixl"}}, }, {"name": "log-tail", "image": "busybox:1.36"}, ], @@ -5886,7 +6397,7 @@ def test_cache_env(case: CacheEnvCase) -> None: ], ), "resource-claim-decode-standalone": _claim_template(name="r-decode-standalone-devices-63392", count=1), - "inference-pool": _inference_pool(name="r-pool", serving="r", picker="r-epp"), + "inference-pool": _inference_pool(name="r-pool", serving="r", picker="r-epp", port=8000), "model-route": _route(name="r", path="/ml-team/r/", pool="r-pool"), "epp-serviceaccount": _epp_service_account(name="r-epp"), "epp-role": _epp_role(name="r-epp"), @@ -5907,10 +6418,10 @@ def test_cache_env(case: CacheEnvCase) -> None: }, ), ApplyCase( - name="EndpointPicker", + name="PrefillDecodeServingPort", reason=( - "serving.endpointPicker sets the picker container, and its config, rendered with sorted keys and whole " - "floats as ints, replaces the default configuration." + "A PrefillDecode replica with a serving port is fronted by an InferencePool that sends to that port, where " + "the prefill engine and the decode sidecar serve." ), composed={ "model-serving-prefill": _deployment( @@ -5929,11 +6440,11 @@ def test_cache_env(case: CacheEnvCase) -> None: { "name": "engine", "image": "vllm/vllm-openai:latest", - "args": ["--model=Qwen/Qwen3-0.6B"], - "ports": [{"name": "http", "containerPort": 8000}], + "args": ["--model=Qwen/Qwen3-0.6B", "--port=8080"], + "ports": [{"name": "http", "containerPort": 8080}], "volumeMounts": [{"name": "dshm", "mountPath": "/dev/shm"}], "readinessProbe": { - "httpGet": {"path": "/health", "port": 8000}, + "httpGet": {"path": "/health", "port": 8080}, "initialDelaySeconds": 30, "periodSeconds": 10, "timeoutSeconds": 5, @@ -5960,17 +6471,23 @@ def test_cache_env(case: CacheEnvCase) -> None: { "name": "engine", "image": "vllm/vllm-openai:latest", - "args": ["--model=m", "--block-size=64"], - "ports": [{"name": "http", "containerPort": 8000}], + "args": ["--model=Qwen/Qwen3-0.6B", "--port=8001"], + "ports": [{"name": "http", "containerPort": 8001}], "volumeMounts": [{"name": "dshm", "mountPath": "/dev/shm"}], "readinessProbe": { - "httpGet": {"path": "/health", "port": 8000}, + "httpGet": {"path": "/health", "port": 8001}, "initialDelaySeconds": 30, "periodSeconds": 10, "timeoutSeconds": 5, }, "resources": {"claims": [{"name": "devices"}]}, - } + }, + { + "name": "sidecar", + "image": "ghcr.io/llm-d/llm-d-routing-sidecar:v0.9.0", + "args": ["--port=8080", "--vllm-port=8001"], + "ports": [{"containerPort": 8080}], + }, ], volumes=[{"name": "dshm", "emptyDir": {"medium": "Memory"}}], ), @@ -5984,6 +6501,7 @@ def test_cache_env(case: CacheEnvCase) -> None: model_cache_ref=None, serving=v1alpha1.Serving( mode="PrefillDecode", + port=8080, router="EndpointPicker", endpointPicker=v1alpha1.EndpointPicker( template=v1alpha1.TemplateModel( @@ -5991,29 +6509,22 @@ def test_cache_env(case: CacheEnvCase) -> None: containers=[ v1alpha1.ContainerModel( name="epp", - image="example.com/picker:v2", + image="ghcr.io/llm-d/llm-d-router-endpoint-picker:v0.9.0", args=[ "--pool-name=$(MODELPLANE_POOL_NAME)", "--pool-namespace=$(MODELPLANE_POOL_NAMESPACE)", + "--pool-group=inference.networking.k8s.io", "--config-file=$(MODELPLANE_PICKER_CONFIG)", + "--grpc-port=$(MODELPLANE_GRPC_PORT)", + "--grpc-health-port=$(MODELPLANE_GRPC_HEALTH_PORT)", + "--metrics-port=$(MODELPLANE_METRICS_PORT)", + "--metrics-endpoint-auth=false", ], ) ] ) ), - # Keys out of order at every level, and whole floats, as a - # config arrives through a protobuf Struct. - config={ - "kind": "EndpointPickerConfig", - "apiVersion": "llm-d.ai/v1alpha1", - "plugins": [ - {"type": "prefix-based-pd-decider", "parameters": {"nonCachedTokens": 16.0}}, - {"parameters": {"blockSizeTokens": 64.0, "autoTune": False}, "type": "prefix-cache-scorer"}, - ], - "schedulingProfiles": [ - {"plugins": [{"weight": 0.25, "pluginRef": "queue-scorer"}], "name": "decode"} - ], - }, + config={"kind": "EndpointPickerConfig"}, ), ), engines=[ @@ -6043,7 +6554,7 @@ def test_cache_env(case: CacheEnvCase) -> None: v1alpha1.Container( name="engine", image="vllm/vllm-openai:latest", - args=["--model=Qwen/Qwen3-0.6B"], + args=["--model=Qwen/Qwen3-0.6B", "--port=8080"], ) ] ) @@ -6077,8 +6588,15 @@ def test_cache_env(case: CacheEnvCase) -> None: v1alpha1.Container( name="engine", image="vllm/vllm-openai:latest", - args=["--model=m", "--block-size=64"], - ) + args=["--model=Qwen/Qwen3-0.6B", "--port=8001"], + ports=[v1alpha1.Port(name="http", containerPort=8001)], + ), + v1alpha1.Container( + name="sidecar", + image="ghcr.io/llm-d/llm-d-routing-sidecar:v0.9.0", + args=["--port=8080", "--vllm-port=8001"], + ports=[v1alpha1.Port(containerPort=8080)], + ), ] ) ), @@ -6106,11 +6624,11 @@ def test_cache_env(case: CacheEnvCase) -> None: { "name": "engine", "image": "vllm/vllm-openai:latest", - "args": ["--model=Qwen/Qwen3-0.6B"], - "ports": [{"name": "http", "containerPort": 8000}], + "args": ["--model=Qwen/Qwen3-0.6B", "--port=8080"], + "ports": [{"name": "http", "containerPort": 8080}], "volumeMounts": [{"name": "dshm", "mountPath": "/dev/shm"}], "readinessProbe": { - "httpGet": {"path": "/health", "port": 8000}, + "httpGet": {"path": "/health", "port": 8080}, "initialDelaySeconds": 30, "periodSeconds": 10, "timeoutSeconds": 5, @@ -6140,100 +6658,43 @@ def test_cache_env(case: CacheEnvCase) -> None: { "name": "engine", "image": "vllm/vllm-openai:latest", - "args": ["--model=m", "--block-size=64"], - "ports": [{"name": "http", "containerPort": 8000}], + "args": ["--model=Qwen/Qwen3-0.6B", "--port=8001"], + "ports": [{"name": "http", "containerPort": 8001}], "volumeMounts": [{"name": "dshm", "mountPath": "/dev/shm"}], "readinessProbe": { - "httpGet": {"path": "/health", "port": 8000}, + "httpGet": {"path": "/health", "port": 8001}, "initialDelaySeconds": 30, "periodSeconds": 10, "timeoutSeconds": 5, }, "resources": {"claims": [{"name": "devices"}]}, }, + { + "name": "sidecar", + "image": "ghcr.io/llm-d/llm-d-routing-sidecar:v0.9.0", + "args": ["--port=8080", "--vllm-port=8001"], + "ports": [{"containerPort": 8080}], + }, ], volumes=[ {"name": "dshm", "emptyDir": {"medium": "Memory"}}, ], ), "resource-claim-decode-standalone": _claim_template(name="r-decode-standalone-devices-63392", count=1), - "inference-pool": _inference_pool(name="r-pool", serving="r", picker="r-epp"), + "inference-pool": _inference_pool(name="r-pool", serving="r", picker="r-epp", port=8080), "model-route": _route(name="r", path="/ml-team/r/", pool="r-pool"), "epp-serviceaccount": _epp_service_account(name="r-epp"), "epp-role": _epp_role(name="r-epp"), "epp-rolebinding": _epp_role_binding(name="r-epp"), - # A user config renders as JSON, which is YAML, with its keys sorted - # and its whole floats as ints, so the rendering and its checksum - # are the same every run and the picker reads 64 where it wants an - # int. - "epp-config": _epp_config( - name="r-epp", - config=( - "{\n" - ' "apiVersion": "llm-d.ai/v1alpha1",\n' - ' "kind": "EndpointPickerConfig",\n' - ' "plugins": [\n' - " {\n" - ' "parameters": {\n' - ' "nonCachedTokens": 16\n' - " },\n" - ' "type": "prefix-based-pd-decider"\n' - " },\n" - " {\n" - ' "parameters": {\n' - ' "autoTune": false,\n' - ' "blockSizeTokens": 64\n' - " },\n" - ' "type": "prefix-cache-scorer"\n' - " }\n" - " ],\n" - ' "schedulingProfiles": [\n' - " {\n" - ' "name": "decode",\n' - ' "plugins": [\n' - " {\n" - ' "pluginRef": "queue-scorer",\n' - ' "weight": 0.25\n' - " }\n" - " ]\n" - " }\n" - " ]\n" - "}" - ), - ), + "epp-config": _epp_config(name="r-epp", config='{\n "kind": "EndpointPickerConfig"\n}'), "epp": _epp( name="r-epp", app="r-epp", labels={"app": "r-epp", "modelplane.ai/role": "picker"}, annotations={ - "modelplane.ai/epp-config-checksum": "270cf06d0aae129a190fd927f416a786033a4343b02cedd4801c27459e0de1c7" + "modelplane.ai/epp-config-checksum": "a9d9c5892fd25b535b127de68f30159b3d132f91f37c4c96b9e784b358d4b842" }, - containers=[ - { - "name": "epp", - "image": "example.com/picker:v2", - "args": [ - "--pool-name=$(MODELPLANE_POOL_NAME)", - "--pool-namespace=$(MODELPLANE_POOL_NAMESPACE)", - "--config-file=$(MODELPLANE_PICKER_CONFIG)", - ], - "env": [ - {"name": "MODELPLANE_POOL_NAME", "value": "r-pool"}, - {"name": "MODELPLANE_POOL_NAMESPACE", "value": "mp-ml-team-51733"}, - {"name": "MODELPLANE_GRPC_PORT", "value": "9002"}, - {"name": "MODELPLANE_GRPC_HEALTH_PORT", "value": "9003"}, - {"name": "MODELPLANE_METRICS_PORT", "value": "9090"}, - {"name": "MODELPLANE_PICKER_CONFIG", "value": "/config/epp-config.yaml"}, - ], - "ports": [ - {"name": "grpc", "containerPort": 9002}, - {"name": "grpc-health", "containerPort": 9003}, - {"name": "http", "containerPort": 9090}, - ], - "readinessProbe": {"grpc": {"port": 9003}, "periodSeconds": 2}, - "volumeMounts": [{"name": "config", "mountPath": "/config"}], - } - ], + containers=[_llm_d_epp_container(pool="r-pool")], volumes=[{"name": "config", "configMap": {"name": "r-epp"}}], image_pull_secrets=None, ), @@ -6241,9 +6702,343 @@ def test_cache_env(case: CacheEnvCase) -> None: }, ), ApplyCase( - name="EngineNamesUnlikePhases", + name="EndpointPicker", reason=( - "Engines named alpha and beta take their PrefillDecode roles from their phases, so alpha, the Decode " + "serving.endpointPicker sets the picker container, and its config, rendered with sorted keys and whole " + "floats as ints, replaces the default configuration." + ), + composed={ + "model-serving-prefill": _deployment( + name="r-prefill-d90b0", + claim_template_name="r-prefill-standalone-devices-f62af", + replicas=1, + pod_metadata={ + "labels": { + "modelplane.ai/engine": "prefill", + "modelplane.ai/role": "Standalone", + "modelplane.ai/serving": "r", + "modelplane.ai/workload": "r-prefill-d90b0", + } + }, + containers=[ + { + "name": "engine", + "image": "vllm/vllm-openai:latest", + "args": ["--model=Qwen/Qwen3-0.6B"], + "ports": [{"name": "http", "containerPort": 8000}], + "volumeMounts": [{"name": "dshm", "mountPath": "/dev/shm"}], + "readinessProbe": { + "httpGet": {"path": "/health", "port": 8000}, + "initialDelaySeconds": 30, + "periodSeconds": 10, + "timeoutSeconds": 5, + }, + "resources": {"claims": [{"name": "devices"}]}, + } + ], + volumes=[{"name": "dshm", "emptyDir": {"medium": "Memory"}}], + ), + "resource-claim-prefill-standalone": _claim_template(name="r-prefill-standalone-devices-f62af", count=1), + "model-serving-decode": _deployment( + name="r-decode-4b27b", + claim_template_name="r-decode-standalone-devices-63392", + replicas=1, + pod_metadata={ + "labels": { + "modelplane.ai/engine": "decode", + "modelplane.ai/role": "Standalone", + "modelplane.ai/serving": "r", + "modelplane.ai/workload": "r-decode-4b27b", + } + }, + containers=[ + { + "name": "engine", + "image": "vllm/vllm-openai:latest", + "args": ["--model=m", "--block-size=64"], + "ports": [{"name": "http", "containerPort": 8000}], + "volumeMounts": [{"name": "dshm", "mountPath": "/dev/shm"}], + "readinessProbe": { + "httpGet": {"path": "/health", "port": 8000}, + "initialDelaySeconds": 30, + "periodSeconds": 10, + "timeoutSeconds": 5, + }, + "resources": {"claims": [{"name": "devices"}]}, + } + ], + volumes=[{"name": "dshm", "emptyDir": {"medium": "Memory"}}], + ), + "resource-claim-decode-standalone": _claim_template(name="r-decode-standalone-devices-63392", count=1), + }, + replica=_replica( + name="r", + namespace="ml-team", + cluster_name="cluster-a", + labels=None, + model_cache_ref=None, + serving=v1alpha1.Serving( + mode="PrefillDecode", + router="EndpointPicker", + endpointPicker=v1alpha1.EndpointPicker( + template=v1alpha1.TemplateModel( + spec=v1alpha1.SpecModel( + containers=[ + v1alpha1.ContainerModel( + name="epp", + image="example.com/picker:v2", + args=[ + "--pool-name=$(MODELPLANE_POOL_NAME)", + "--pool-namespace=$(MODELPLANE_POOL_NAMESPACE)", + "--config-file=$(MODELPLANE_PICKER_CONFIG)", + ], + ) + ] + ) + ), + # Keys out of order at every level, and whole floats, as a + # config arrives through a protobuf Struct. + config={ + "kind": "EndpointPickerConfig", + "apiVersion": "llm-d.ai/v1alpha1", + "plugins": [ + {"type": "prefix-based-pd-decider", "parameters": {"nonCachedTokens": 16.0}}, + {"parameters": {"blockSizeTokens": 64.0, "autoTune": False}, "type": "prefix-cache-scorer"}, + ], + "schedulingProfiles": [ + {"plugins": [{"weight": 0.25, "pluginRef": "queue-scorer"}], "name": "decode"} + ], + }, + ), + ), + engines=[ + v1alpha1.Engine( + name="prefill", + phase="Prefill", + copies=1, + members=[ + v1alpha1.Member( + role="Standalone", + nodePoolName="frontier", + deviceRequests=[ + v1alpha1.DeviceRequest( + name="gpu", + deviceClassName="gpu.nvidia.com", + count=1, + selectors=[ + v1alpha1.Selector( + cel='device.capacity["gpu.nvidia.com"].memory.compareTo(quantity("80Gi")) >= 0' + ) + ], + ) + ], + template=v1alpha1.Template( + spec=v1alpha1.Spec( + containers=[ + v1alpha1.Container( + name="engine", + image="vllm/vllm-openai:latest", + args=["--model=Qwen/Qwen3-0.6B"], + ) + ] + ) + ), + ) + ], + ), + v1alpha1.Engine( + name="decode", + phase="Decode", + copies=1, + members=[ + v1alpha1.Member( + role="Standalone", + nodePoolName="frontier", + deviceRequests=[ + v1alpha1.DeviceRequest( + name="gpu", + deviceClassName="gpu.nvidia.com", + count=1, + selectors=[ + v1alpha1.Selector( + cel='device.capacity["gpu.nvidia.com"].memory.compareTo(quantity("80Gi")) >= 0' + ) + ], + ) + ], + template=v1alpha1.Template( + spec=v1alpha1.Spec( + containers=[ + v1alpha1.Container( + name="engine", + image="vllm/vllm-openai:latest", + args=["--model=m", "--block-size=64"], + ) + ] + ) + ), + ) + ], + ), + ], + ), + provider_config="cluster-a-pc", + want={ + "model-serving-prefill": _deployment( + name="r-prefill-d90b0", + claim_template_name="r-prefill-standalone-devices-f62af", + replicas=1, + pod_metadata={ + "labels": { + "modelplane.ai/engine": "prefill", + "modelplane.ai/role": "Standalone", + "modelplane.ai/serving": "r", + "modelplane.ai/workload": "r-prefill-d90b0", + "llm-d.ai/role": "prefill", + } + }, + containers=[ + { + "name": "engine", + "image": "vllm/vllm-openai:latest", + "args": ["--model=Qwen/Qwen3-0.6B"], + "ports": [{"name": "http", "containerPort": 8000}], + "volumeMounts": [{"name": "dshm", "mountPath": "/dev/shm"}], + "readinessProbe": { + "httpGet": {"path": "/health", "port": 8000}, + "initialDelaySeconds": 30, + "periodSeconds": 10, + "timeoutSeconds": 5, + }, + "resources": {"claims": [{"name": "devices"}]}, + } + ], + volumes=[ + {"name": "dshm", "emptyDir": {"medium": "Memory"}}, + ], + ), + "resource-claim-prefill-standalone": _claim_template(name="r-prefill-standalone-devices-f62af", count=1), + "model-serving-decode": _deployment( + name="r-decode-4b27b", + claim_template_name="r-decode-standalone-devices-63392", + replicas=1, + pod_metadata={ + "labels": { + "modelplane.ai/engine": "decode", + "modelplane.ai/role": "Standalone", + "modelplane.ai/serving": "r", + "modelplane.ai/workload": "r-decode-4b27b", + "llm-d.ai/role": "decode", + } + }, + containers=[ + { + "name": "engine", + "image": "vllm/vllm-openai:latest", + "args": ["--model=m", "--block-size=64"], + "ports": [{"name": "http", "containerPort": 8000}], + "volumeMounts": [{"name": "dshm", "mountPath": "/dev/shm"}], + "readinessProbe": { + "httpGet": {"path": "/health", "port": 8000}, + "initialDelaySeconds": 30, + "periodSeconds": 10, + "timeoutSeconds": 5, + }, + "resources": {"claims": [{"name": "devices"}]}, + }, + ], + volumes=[ + {"name": "dshm", "emptyDir": {"medium": "Memory"}}, + ], + ), + "resource-claim-decode-standalone": _claim_template(name="r-decode-standalone-devices-63392", count=1), + "inference-pool": _inference_pool(name="r-pool", serving="r", picker="r-epp", port=8000), + "model-route": _route(name="r", path="/ml-team/r/", pool="r-pool"), + "epp-serviceaccount": _epp_service_account(name="r-epp"), + "epp-role": _epp_role(name="r-epp"), + "epp-rolebinding": _epp_role_binding(name="r-epp"), + # A user config renders as JSON, which is YAML, with its keys sorted + # and its whole floats as ints, so the rendering and its checksum + # are the same every run and the picker reads 64 where it wants an + # int. + "epp-config": _epp_config( + name="r-epp", + config=( + "{\n" + ' "apiVersion": "llm-d.ai/v1alpha1",\n' + ' "kind": "EndpointPickerConfig",\n' + ' "plugins": [\n' + " {\n" + ' "parameters": {\n' + ' "nonCachedTokens": 16\n' + " },\n" + ' "type": "prefix-based-pd-decider"\n' + " },\n" + " {\n" + ' "parameters": {\n' + ' "autoTune": false,\n' + ' "blockSizeTokens": 64\n' + " },\n" + ' "type": "prefix-cache-scorer"\n' + " }\n" + " ],\n" + ' "schedulingProfiles": [\n' + " {\n" + ' "name": "decode",\n' + ' "plugins": [\n' + " {\n" + ' "pluginRef": "queue-scorer",\n' + ' "weight": 0.25\n' + " }\n" + " ]\n" + " }\n" + " ]\n" + "}" + ), + ), + "epp": _epp( + name="r-epp", + app="r-epp", + labels={"app": "r-epp", "modelplane.ai/role": "picker"}, + annotations={ + "modelplane.ai/epp-config-checksum": "270cf06d0aae129a190fd927f416a786033a4343b02cedd4801c27459e0de1c7" + }, + containers=[ + { + "name": "epp", + "image": "example.com/picker:v2", + "args": [ + "--pool-name=$(MODELPLANE_POOL_NAME)", + "--pool-namespace=$(MODELPLANE_POOL_NAMESPACE)", + "--config-file=$(MODELPLANE_PICKER_CONFIG)", + ], + "env": [ + {"name": "MODELPLANE_POOL_NAME", "value": "r-pool"}, + {"name": "MODELPLANE_POOL_NAMESPACE", "value": "mp-ml-team-51733"}, + {"name": "MODELPLANE_GRPC_PORT", "value": "9002"}, + {"name": "MODELPLANE_GRPC_HEALTH_PORT", "value": "9003"}, + {"name": "MODELPLANE_METRICS_PORT", "value": "9090"}, + {"name": "MODELPLANE_PICKER_CONFIG", "value": "/config/epp-config.yaml"}, + ], + "ports": [ + {"name": "grpc", "containerPort": 9002}, + {"name": "grpc-health", "containerPort": 9003}, + {"name": "http", "containerPort": 9090}, + ], + "readinessProbe": {"grpc": {"port": 9003}, "periodSeconds": 2}, + "volumeMounts": [{"name": "config", "mountPath": "/config"}], + } + ], + volumes=[{"name": "config", "configMap": {"name": "r-epp"}}], + image_pull_secrets=None, + ), + "epp-service": _epp_service(name="r-epp", app="r-epp"), + }, + ), + ApplyCase( + name="EngineNamesUnlikePhases", + reason=( + "Engines named alpha and beta take their PrefillDecode roles from their phases, so alpha, the Decode " "engine, gets the decode role." ), composed={ @@ -6484,7 +7279,7 @@ def test_cache_env(case: CacheEnvCase) -> None: ], ), "resource-claim-beta-standalone": _claim_template(name="r-beta-standalone-devices-9d8b1", count=1), - "inference-pool": _inference_pool(name="r-pool", serving="r", picker="r-epp"), + "inference-pool": _inference_pool(name="r-pool", serving="r", picker="r-epp", port=8000), "model-route": _route(name="r", path="/ml-team/r/", pool="r-pool"), "epp-serviceaccount": _epp_service_account(name="r-epp"), "epp-role": _epp_role(name="r-epp"), @@ -6958,7 +7753,7 @@ def test_cache_env(case: CacheEnvCase) -> None: ), "resource-claim-decode-leader": _claim_template(name="r-decode-leader-devices-d2e70", count=8), "resource-claim-decode-worker": _claim_template(name="r-decode-worker-devices-08d44", count=8), - "inference-pool": _inference_pool(name="r-pool", serving="r", picker="r-epp"), + "inference-pool": _inference_pool(name="r-pool", serving="r", picker="r-epp", port=8000), "model-route": _route(name="r", path="/ml-team/r/", pool="r-pool"), "epp-serviceaccount": _epp_service_account(name="r-epp"), "epp-role": _epp_role(name="r-epp"), @@ -7105,7 +7900,161 @@ def test_cache_env(case: CacheEnvCase) -> None: volumes=[{"name": "dshm", "emptyDir": {"medium": "Memory"}}], ), "resource-claim-main-standalone": _claim_template(name="r-main-standalone-devices-f456f", count=1), - "inference-pool": _inference_pool(name="r-pool", serving="r", picker="r-epp"), + "inference-pool": _inference_pool(name="r-pool", serving="r", picker="r-epp", port=8000), + "model-route": _route(name="r", path="/ml-team/r/", pool="r-pool"), + "epp-serviceaccount": _epp_service_account(name="r-epp"), + "epp-role": _epp_role(name="r-epp"), + "epp-rolebinding": _epp_role_binding(name="r-epp"), + "epp-config": _epp_config( + name="r-epp", + config="apiVersion: llm-d.ai/v1alpha1\n" + "kind: EndpointPickerConfig\n" + "plugins:\n" + "- type: approx-prefix-cache-producer\n" + " parameters:\n" + " autoTune: false\n" + " blockSizeTokens: 64\n" + " maxPrefixTokensToMatch: 131072\n" + " lruCapacityPerServer: 31250\n" + "- type: prefix-cache-scorer\n" + "- type: queue-scorer\n" + "- type: max-score-picker\n" + "schedulingProfiles:\n" + "- name: default\n" + " plugins:\n" + " - pluginRef: max-score-picker\n" + " - pluginRef: prefix-cache-scorer\n" + " weight: 2\n" + " - pluginRef: queue-scorer\n" + " weight: 1\n", + ), + "epp": _epp( + name="r-epp", + app="r-epp", + labels={"app": "r-epp", "modelplane.ai/role": "picker"}, + annotations={ + "modelplane.ai/epp-config-checksum": "c4f1cefa4a1fe67069eaa6e9dde3e5b9bc7e79d3f807ecd076dd021259b2ab39" + }, + containers=[_llm_d_epp_container(pool="r-pool")], + volumes=[{"name": "config", "configMap": {"name": "r-epp"}}], + image_pull_secrets=None, + ), + "epp-service": _epp_service(name="r-epp", app="r-epp"), + }, + ), + ApplyCase( + name="UnifiedServingPort", + reason=("A replica with a serving port is fronted by an InferencePool that sends to that port."), + composed={ + "model-serving-main": _deployment( + name="r-main-bb4e3", + claim_template_name="r-main-standalone-devices-f456f", + replicas=1, + pod_metadata={ + "labels": { + "modelplane.ai/engine": "main", + "modelplane.ai/role": "Standalone", + "modelplane.ai/serving": "r", + "modelplane.ai/workload": "r-main-bb4e3", + } + }, + containers=[ + { + "name": "engine", + "image": "vllm/vllm-openai:latest", + "args": ["--model=Qwen/Qwen3-0.6B"], + "ports": [{"name": "http", "containerPort": 8080}], + "volumeMounts": [{"name": "dshm", "mountPath": "/dev/shm"}], + "readinessProbe": { + "httpGet": {"path": "/health", "port": 8080}, + "initialDelaySeconds": 30, + "periodSeconds": 10, + "timeoutSeconds": 5, + }, + "resources": {"claims": [{"name": "devices"}]}, + } + ], + volumes=[{"name": "dshm", "emptyDir": {"medium": "Memory"}}], + ), + "resource-claim-main-standalone": _claim_template(name="r-main-standalone-devices-f456f", count=1), + }, + replica=_replica( + name="r", + namespace="ml-team", + cluster_name="cluster-a", + labels=None, + model_cache_ref=None, + serving=v1alpha1.Serving(mode="Unified", port=8080), + engines=[ + v1alpha1.Engine( + name="main", + copies=1, + members=[ + v1alpha1.Member( + role="Standalone", + nodePoolName="frontier", + deviceRequests=[ + v1alpha1.DeviceRequest( + name="gpu", + deviceClassName="gpu.nvidia.com", + count=1, + selectors=[ + v1alpha1.Selector( + cel='device.capacity["gpu.nvidia.com"].memory.compareTo(quantity("80Gi")) >= 0' + ) + ], + ) + ], + template=v1alpha1.Template( + spec=v1alpha1.Spec( + containers=[ + v1alpha1.Container( + name="engine", + image="vllm/vllm-openai:latest", + args=["--model=Qwen/Qwen3-0.6B"], + ) + ] + ) + ), + ) + ], + ) + ], + ), + provider_config="cluster-a-pc", + want={ + "model-serving-main": _deployment( + name="r-main-bb4e3", + claim_template_name="r-main-standalone-devices-f456f", + replicas=1, + pod_metadata={ + "labels": { + "modelplane.ai/engine": "main", + "modelplane.ai/role": "Standalone", + "modelplane.ai/serving": "r", + "modelplane.ai/workload": "r-main-bb4e3", + } + }, + containers=[ + { + "name": "engine", + "image": "vllm/vllm-openai:latest", + "args": ["--model=Qwen/Qwen3-0.6B"], + "ports": [{"name": "http", "containerPort": 8080}], + "volumeMounts": [{"name": "dshm", "mountPath": "/dev/shm"}], + "readinessProbe": { + "httpGet": {"path": "/health", "port": 8080}, + "initialDelaySeconds": 30, + "periodSeconds": 10, + "timeoutSeconds": 5, + }, + "resources": {"claims": [{"name": "devices"}]}, + } + ], + volumes=[{"name": "dshm", "emptyDir": {"medium": "Memory"}}], + ), + "resource-claim-main-standalone": _claim_template(name="r-main-standalone-devices-f456f", count=1), + "inference-pool": _inference_pool(name="r-pool", serving="r", picker="r-epp", port=8080), "model-route": _route(name="r", path="/ml-team/r/", pool="r-pool"), "epp-serviceaccount": _epp_service_account(name="r-epp"), "epp-role": _epp_role(name="r-epp"), @@ -7259,7 +8208,7 @@ def test_cache_env(case: CacheEnvCase) -> None: volumes=[{"name": "dshm", "emptyDir": {"medium": "Memory"}}], ), "resource-claim-main-standalone": _claim_template(name="r-main-standalone-devices-f456f", count=1), - "inference-pool": _inference_pool(name="r-pool", serving="r", picker="r-epp"), + "inference-pool": _inference_pool(name="r-pool", serving="r", picker="r-epp", port=8000), "model-route": _route(name="r", path="/ml-team/r/", pool="r-pool"), "epp-serviceaccount": _epp_service_account(name="r-epp"), "epp-role": _epp_role(name="r-epp"), @@ -7413,7 +8362,7 @@ def test_cache_env(case: CacheEnvCase) -> None: volumes=[{"name": "dshm", "emptyDir": {"medium": "Memory"}}], ), "resource-claim-main-standalone": _claim_template(name="r-main-standalone-devices-f456f", count=1), - "inference-pool": _inference_pool(name="r-pool", serving="r", picker="r-epp"), + "inference-pool": _inference_pool(name="r-pool", serving="r", picker="r-epp", port=8000), "model-route": _route(name="r", path="/ml-team/r/", pool="r-pool"), "epp-serviceaccount": _epp_service_account(name="r-epp"), "epp-role": _epp_role(name="r-epp"), @@ -7601,7 +8550,7 @@ def test_cache_env(case: CacheEnvCase) -> None: volumes=[{"name": "dshm", "emptyDir": {"medium": "Memory"}}], ), "resource-claim-main-standalone": _claim_template(name="r-main-standalone-devices-f456f", count=1), - "inference-pool": _inference_pool(name="r-pool", serving="r", picker="r-epp"), + "inference-pool": _inference_pool(name="r-pool", serving="r", picker="r-epp", port=8000), "model-route": _route(name="r", path="/ml-team/r/", pool="r-pool"), "epp-serviceaccount": _epp_service_account(name="r-epp"), "epp-role": _epp_role(name="r-epp"), @@ -7797,7 +8746,7 @@ def test_cache_env(case: CacheEnvCase) -> None: volumes=[{"name": "dshm", "emptyDir": {"medium": "Memory"}}], ), "resource-claim-main-standalone": _claim_template(name="r-main-standalone-devices-f456f", count=1), - "inference-pool": _inference_pool(name="r-pool", serving="r", picker="r-epp"), + "inference-pool": _inference_pool(name="r-pool", serving="r", picker="r-epp", port=8000), "model-route": _route(name="r", path="/ml-team/r/", pool="r-pool"), "epp-serviceaccount": _epp_service_account(name="r-epp"), "epp-role": _epp_role(name="r-epp"), @@ -7955,7 +8904,7 @@ def test_cache_env(case: CacheEnvCase) -> None: volumes=[{"name": "dshm", "emptyDir": {"medium": "Memory"}}], ), "resource-claim-main-standalone": _claim_template(name="r-main-standalone-devices-f456f", count=1), - "inference-pool": _inference_pool(name="r-pool", serving="r", picker="r-epp"), + "inference-pool": _inference_pool(name="r-pool", serving="r", picker="r-epp", port=8000), "model-route": _route(name="r", path="/ml-team/r/", pool="r-pool"), "epp-serviceaccount": _epp_service_account(name="r-epp"), "epp-role": _epp_role(name="r-epp"), @@ -8116,7 +9065,7 @@ def test_cache_env(case: CacheEnvCase) -> None: volumes=[{"name": "dshm", "emptyDir": {"medium": "Memory"}}], ), "resource-claim-main-standalone": _claim_template(name="r-main-standalone-devices-f456f", count=1), - "inference-pool": _inference_pool(name="r-pool", serving="r", picker="r-epp"), + "inference-pool": _inference_pool(name="r-pool", serving="r", picker="r-epp", port=8000), "model-route": _route(name="r", path="/ml-team/r/", pool="r-pool"), "epp-serviceaccount": _epp_service_account(name="r-epp"), "epp-role": _epp_role(name="r-epp"), @@ -8368,7 +9317,7 @@ def test_cache_env(case: CacheEnvCase) -> None: ), "resource-claim-main-leader": _claim_template(name="r-main-leader-devices-f58c6", count=8), "resource-claim-main-worker": _claim_template(name="r-main-worker-devices-99b8a", count=8), - "inference-pool": _inference_pool(name="r-pool", serving="r", picker="r-epp"), + "inference-pool": _inference_pool(name="r-pool", serving="r", picker="r-epp", port=8000), "model-route": _route(name="r", path="/ml-team/r/", pool="r-pool"), "epp-serviceaccount": _epp_service_account(name="r-epp"), "epp-role": _epp_role(name="r-epp"), @@ -8534,7 +9483,10 @@ def test_cache_env(case: CacheEnvCase) -> None: name="qwen2.5-7b-abcde-main-standalone-devices-771a9", count=1 ), "inference-pool": _inference_pool( - name="qwen2.5-7b-abcde-pool", serving="qwen2.5-7b-abcde", picker="epp-qwen2-5-7b-abcde-epp-da596" + name="qwen2.5-7b-abcde-pool", + serving="qwen2.5-7b-abcde", + picker="epp-qwen2-5-7b-abcde-epp-da596", + port=8000, ), "model-route": _route( name="qwen2.5-7b-abcde", path="/ml-team/qwen2.5-7b-abcde/", pool="qwen2.5-7b-abcde-pool" @@ -8699,7 +9651,10 @@ def test_cache_env(case: CacheEnvCase) -> None: ), "resource-claim-main-standalone": _claim_template(name="a" * 57 + "-05ced", count=1), "inference-pool": _inference_pool( - name="a" * 57 + "-abcde-pool", serving="a" * 57 + "-abcde", picker="epp-" + "a" * 53 + "-2dd65" + name="a" * 57 + "-abcde-pool", + serving="a" * 57 + "-abcde", + picker="epp-" + "a" * 53 + "-2dd65", + port=8000, ), "model-route": _route( name="a" * 57 + "-abcde", path="/ml-team/" + "a" * 57 + "-abcde/", pool="a" * 57 + "-abcde-pool" From d4c761c10f79bc120e91a2924ed32d6ada83badf Mon Sep 17 00:00:00 2001 From: Dennis Ramdass Date: Mon, 26 Oct 2026 11:57:38 -0700 Subject: [PATCH 3/4] Run the e2e mock engine on port 8080 with a /v1/models probe The e2e deployment's mock engine listened on 8000 and answered /health, so the end-to-end run only covered the default port and probe. It now listens on 8080, sets serving.port to match, and reports readiness at /v1/models through its own readinessProbe. It no longer answers /health. The pod goes Ready only if the engine's probe replaces the default one, and a request through the gateway succeeds only if the InferencePool targets 8080. Towards #462. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Dennis Ramdass --- e2e/manifests/40-model-deployment.yaml | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/e2e/manifests/40-model-deployment.yaml b/e2e/manifests/40-model-deployment.yaml index 01a724f16..2e75ebcd0 100644 --- a/e2e/manifests/40-model-deployment.yaml +++ b/e2e/manifests/40-model-deployment.yaml @@ -3,7 +3,9 @@ # HTTPRoute), so this exercises real routing. The engine is a mock server: it # needs no GPU and no model, so the pod goes Ready and answers both # /v1/chat/completions (OpenAI) and /v1/messages (Anthropic), the way a vLLM -# server exposes both, without real inference. +# server exposes both, without real inference. It listens on 8080 and reports +# health at /v1/models, so the deployment sets serving.port and the engine's +# readinessProbe, the way an engine other than vLLM would. apiVersion: modelplane.ai/v1alpha1 kind: ModelDeployment metadata: @@ -13,6 +15,8 @@ spec: replicas: 1 template: spec: + serving: + port: 8080 engines: - name: mock members: @@ -51,9 +55,8 @@ spec: self.end_headers() self.wfile.write(b) def do_GET(self): - if self.path == "/health": - self._s({"status": "ok"}) - elif self.path == "/metrics": + # No /health: the readinessProbe below asks /v1/models. + if self.path == "/metrics": # Two real vLLM names, so the collector's built-in # renames have something to act on: one gauge and # one the fleet converts the unit of. @@ -138,7 +141,7 @@ spec: }) def log_message(self, *a): pass - http.server.HTTPServer(("0.0.0.0", 8000), H).serve_forever() + http.server.HTTPServer(("0.0.0.0", 8080), H).serve_forever() # python -c takes the script as its first argument, so this follows # it and arrives as sys.argv[1]. --served-model-name is how a real # engine is told what to answer to, and referencing the env @@ -146,7 +149,7 @@ spec: # The mock rejects a request naming anything else, so a 200 # through the gateway proves the model was rewritten. - --served-model-name=$(MODELPLANE_SERVED_MODEL_NAME) - # The ModelDeployment container template is a curated subset - # (name/image/command/args/env); the composition wires the port, - # readiness, and any GPU resources itself (it assumes port 8000). So - # no ports/readinessProbe/resources here — the XRD rejects them. + readinessProbe: + httpGet: + path: /v1/models + port: http From d483b1012757ccccbc7d2445c5aa369949bb26ca Mon Sep 17 00:00:00 2001 From: Dennis Ramdass Date: Mon, 26 Oct 2026 11:57:38 -0700 Subject: [PATCH 4/4] Document the serving port and engine readiness probe The docs said a serving pod receives requests on port 8000, and some recipe comments said the engine must listen there. Deploy a Model now describes serving.port and the engine's readinessProbe, with an example of an engine on 8080 that reports readiness at /v1/models. The routing guide's port rules refer to the serving port, and the recipe comments call 8000 the default. Fixes #462. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Dennis Ramdass --- docs/content/models/model-deployment.md | 37 ++++++++++++++++++- docs/content/models/routing.md | 22 ++++++----- .../recipes/glm-4.5-air/model-deployment.yaml | 2 +- .../laguna/model-deployment-sglang.yaml | 6 +-- .../recipes/laguna/model-deployment.yaml | 4 +- .../qwen3-coder/model-deployment-fp8.yaml | 2 +- 6 files changed, 55 insertions(+), 18 deletions(-) diff --git a/docs/content/models/model-deployment.md b/docs/content/models/model-deployment.md index e851d998f..a1832a6d2 100644 --- a/docs/content/models/model-deployment.md +++ b/docs/content/models/model-deployment.md @@ -13,7 +13,9 @@ cluster. Modelplane is unopinionated about the engine itself. You bring the container and its flags, and Modelplane shapes a serving topology around it. The engine flags you write carry parallelism, quantization, and KV transfer, never injected by -Modelplane. +Modelplane. By default Modelplane expects the engine to serve on port 8000 and +answer `GET /health` once it's ready. For an engine that doesn't, see +[Engine port and readiness](#engine-port-and-readiness). As with a Kubernetes Deployment, `spec.replicas` sets the replica count and `spec.template` describes each replica. Labels for the composed ModelReplicas @@ -121,6 +123,39 @@ runtime. vLLM's `NixlConnector` (and SGLang's NIXL transfer backend) import the `NIXL is not available`. Recent vanilla `vllm/vllm-openai` images include NIXL, so pin a current tag rather than an old one. +## Engine port and readiness + +A serving pod receives requests on port 8000, and Modelplane checks that the +engine is ready with `GET /health` on that port. For an engine that listens on +another port, set `serving.port`. For one that reports readiness elsewhere, set +a `readinessProbe` on the engine container. It replaces Modelplane's check +whole, and Kubernetes defaults apply to the fields it leaves out, so the timeout +is 1 second unless you set `timeoutSeconds`. This engine listens on 8080 and +answers `/v1/models` once it's ready: + +```yaml {nocopy=true} +serving: + port: 8080 +engines: +- name: main + members: + - role: Standalone + template: + spec: + containers: + - name: engine + image: registry.example.com/my-engine:1.0 + readinessProbe: + httpGet: + path: /v1/models + port: http +``` + +`serving.port` is where the InferencePool sends requests for every engine of +the deployment. When a container in front of the engine, such as the llm-d +sidecar, declares that port, the engine serves elsewhere, as +[Ports]({{< ref "routing.md#ports" >}}) describes. + ## Requesting GPUs You don't specify a cluster or a GPU model. Instead each member's `nodeSelector` diff --git a/docs/content/models/routing.md b/docs/content/models/routing.md index 2dd901eb8..5a860d245 100644 --- a/docs/content/models/routing.md +++ b/docs/content/models/routing.md @@ -200,23 +200,25 @@ don't need the label. ## Ports -A serving pod receives requests on port 8000. The engine's port named `http` is -where Modelplane checks the engine's health and the collector reads its -metrics. It defaults to 8000, so an engine alone in its pod declares nothing. +A serving pod receives requests on the serving port, which is 8000 unless you +set `serving.port`. The engine's port named `http` is where Modelplane checks +the engine's health and the collector reads its metrics. It defaults to the +serving port, so an engine alone in its pod declares nothing. When a container is in front of the engine, such as the sidecar: -- That container declares port 8000, and passes requests to the engine. It must - pass `/metrics` through too, because the picker reads it there. +- That container declares the serving port, and passes requests to the engine. + It must pass `/metrics` through too, because the picker reads it there. - The engine declares its `http` port on another port, such as 8001. The health check must reach the engine itself, because the llm-d sidecar answers `/health` without asking the engine. A container that doesn't serve, such as a log shipper, doesn't declare a port, and the engine doesn't either. Modelplane rejects a `ModelDeployment` with a pod -where more than one container declares 8000, or where the engine moves off 8000 -and no other container declares it. A `Worker` member doesn't need a container -on 8000, because worker pods don't receive requests. +where more than one container declares the serving port, or where the engine +moves off the serving port and no other container declares it. A `Worker` +member doesn't need a container on the serving port, because worker pods don't +receive requests. Behind the sidecar, the decode port appears in three places: the `--port` flag on the engine, its `http` port, and the sidecar flag that names the engine's @@ -472,8 +474,8 @@ and its pods and router keep serving as they were. To resume, add: - `serving.router: EndpointPicker`, with the picker's container and configuration in `serving.endpointPicker`. -- The sidecar in the decode member, on port 8000, with the engine's `http` port - declared. +- The sidecar in the decode member, on the serving port, with the engine's + `http` port declared. - `VLLM_NIXL_SIDE_CHANNEL_HOST` and `VLLM_NIXL_SIDE_CHANNEL_PORT` on both engines, for vLLM with NIXL. diff --git a/docs/manifests/recipes/glm-4.5-air/model-deployment.yaml b/docs/manifests/recipes/glm-4.5-air/model-deployment.yaml index c4e374e0e..0dd9c908f 100644 --- a/docs/manifests/recipes/glm-4.5-air/model-deployment.yaml +++ b/docs/manifests/recipes/glm-4.5-air/model-deployment.yaml @@ -8,7 +8,7 @@ # The model is bigger than one A100's VRAM, so --n-cpu-moe offloads the MoE # expert tensors to host RAM; the GPU runs the active path. That's how a 106B # model fits one A100 instead of a multi-GPU node. --port 8000 because llama.cpp -# defaults to 8080 and Modelplane scrapes 8000. +# defaults to 8080 and Modelplane's default serving port is 8000. apiVersion: modelplane.ai/v1alpha1 kind: ModelDeployment metadata: diff --git a/docs/manifests/recipes/laguna/model-deployment-sglang.yaml b/docs/manifests/recipes/laguna/model-deployment-sglang.yaml index df688a5c1..2774907bc 100644 --- a/docs/manifests/recipes/laguna/model-deployment-sglang.yaml +++ b/docs/manifests/recipes/laguna/model-deployment-sglang.yaml @@ -14,9 +14,9 @@ # Poolside Laguna support (PR #24204, which also added the poolside_v1 parsers), # .post1 is its point bugfix, cu129 for Hopper. The image does not auto-launch a # server, so the launch command is set explicitly; SGLang defaults to port -# 30000, so --port=8000 is required for the Modelplane engine port (Modelplane -# scrapes 8000). Poolside's trtllm_mha and DFLASH cookbook optimizations are -# left off (not validated on H100). +# 30000, so --port=8000 puts it on Modelplane's default serving port. Poolside's +# trtllm_mha and DFLASH cookbook optimizations are left off (not validated on +# H100). apiVersion: modelplane.ai/v1alpha1 kind: ModelDeployment metadata: diff --git a/docs/manifests/recipes/laguna/model-deployment.yaml b/docs/manifests/recipes/laguna/model-deployment.yaml index de4e260f8..fc55b8a5a 100644 --- a/docs/manifests/recipes/laguna/model-deployment.yaml +++ b/docs/manifests/recipes/laguna/model-deployment.yaml @@ -14,8 +14,8 @@ # --enable-auto-tool-choice turns on server-side tool selection. # --max-model-len can be lowered to shrink the KV cache footprint; 262144 # matches Poolside's reference config. -# The engine container must be named `engine` and listen on :8000 (Modelplane -# scrapes 8000). +# The engine container must be named `engine`. It listens on :8000, +# Modelplane's default serving port. # # H100-specific FP8 settings, carried over from a community 4x H100 build: # VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER=0 selects an FP8 GEMM path that is diff --git a/docs/manifests/recipes/qwen3-coder/model-deployment-fp8.yaml b/docs/manifests/recipes/qwen3-coder/model-deployment-fp8.yaml index 288fae50a..4095b96e7 100644 --- a/docs/manifests/recipes/qwen3-coder/model-deployment-fp8.yaml +++ b/docs/manifests/recipes/qwen3-coder/model-deployment-fp8.yaml @@ -17,7 +17,7 @@ # without the first, and v0.5.10 publishes no token histograms without the # second. Modelplane's collector reads both. # --host 0.0.0.0 --port 8000: SGLang defaults to 127.0.0.1:30000, but -# Modelplane's contract is 0.0.0.0:8000 with a /health probe. Args pass +# Modelplane's default is 0.0.0.0:8000 with a /health probe. Args pass # through verbatim - Modelplane injects nothing for a non-vLLM engine. apiVersion: modelplane.ai/v1alpha1 kind: ModelDeployment