From c45635f6c886c241aaad60f5cb963712eea63d4b Mon Sep 17 00:00:00 2001 From: mikemikimike <13286568797@163.com> Date: Sat, 17 Oct 2026 10:34:34 +0800 Subject: [PATCH 1/7] Keep an AI Gateway route attached Refs #531 Signed-off-by: mikemikimike <13286568797@163.com> --- .../compose-inference-gateway/function/fn.py | 74 +++++++++- .../tests/test_fn.py | 134 ++++++++++++++++-- 2 files changed, 192 insertions(+), 16 deletions(-) diff --git a/functions/compose-inference-gateway/function/fn.py b/functions/compose-inference-gateway/function/fn.py index 135d872db..c18a916e6 100644 --- a/functions/compose-inference-gateway/function/fn.py +++ b/functions/compose-inference-gateway/function/fn.py @@ -48,6 +48,7 @@ CONDITION_REASON_GATEWAY_PROGRAMMED = "GatewayProgrammed" CONDITION_REASON_WAITING_FOR_CLUSTER = "WaitingForCluster" CONDITION_REASON_WAITING_FOR_GATEWAY = "WaitingForGateway" +CONDITION_REASON_WAITING_FOR_AI_GATEWAY_ROUTE = "WaitingForAIGatewayRoute" CONDITION_REASON_CLUSTER_TAKEN = "ClusterAlreadyHasGateway" CONDITION_REASON_SECRETS_MISSING = "SecretsMissing" CONDITION_REASON_AUTH_NOT_ACCEPTED = "CallerAuthNotAccepted" @@ -77,6 +78,8 @@ _FAILOVER_NAME = "inference-gateway-failover" _CLIENT_TRAFFIC_NAME = "inference-gateway-client-traffic" _REDIRECT_NAME = "inference-gateway-redirect" +_AI_GATEWAY_ANCHOR_NAME = "inference-gateway-anchor" +_AI_GATEWAY_ANCHOR_MODEL = f"{REMOTE_NAMESPACE}/__gateway_anchor__" # The Gateway's listener names. When the gateway terminates TLS, inference binds # to the HTTPS listener alone (compose-model-route sets the AIGatewayRoute's @@ -151,6 +154,11 @@ "a.conditions.exists(c, c.type == 'Accepted' && c.status == 'True'))" ) +_AI_GATEWAY_ROUTE_ACCEPTED_CEL = ( + "has(object.status) && has(object.status.conditions) && " + "object.status.conditions.exists(c, c.type == 'Accepted' && c.status == 'True')" +) + # Kubernetes injects ndots:5 into every pod, which tells the resolver to try each # search domain before a name with fewer than five dots. Every hostname this # gateway resolves has fewer: a provider like api.together.xyz has two, a cluster @@ -321,6 +329,7 @@ def compose(self) -> None: self.compose_secrets() self.compose_envoy_proxy() self.compose_gateway() + self.compose_ai_gateway_anchor() self.compose_client_pki() self.compose_cluster_names() self.compose_caller_auth() @@ -667,6 +676,48 @@ def compose_gateway(self) -> None: ), ) + def compose_ai_gateway_anchor(self) -> None: + """Keep AI Gateway's ext-proc installed before any model route exists.""" + listener = _LISTENER_HTTPS if self.xr.spec.tls else _LISTENER_HTTP + resource.update( + self.rsp.desired.resources["ai-gateway-anchor"], + _k8s_object( + self.pc, + { + "apiVersion": "aigateway.envoyproxy.io/v1beta1", + "kind": "AIGatewayRoute", + "metadata": {"name": _AI_GATEWAY_ANCHOR_NAME, "namespace": REMOTE_NAMESPACE}, + "spec": { + "parentRefs": [ + { + "group": "gateway.networking.k8s.io", + "kind": "Gateway", + "name": _GATEWAY_NAME, + "namespace": REMOTE_NAMESPACE, + "sectionName": listener, + } + ], + "rules": [ + { + "matches": [ + { + "headers": [ + { + "type": "Exact", + "name": "x-ai-eg-model", + "value": _AI_GATEWAY_ANCHOR_MODEL, + } + ] + } + ] + } + ], + }, + }, + ready_when=_AI_GATEWAY_ROUTE_ACCEPTED_CEL, + ), + ) + def compose_client_pki(self) -> None: """Compose the CA whose certificates a cluster gateway trusts. @@ -1330,16 +1381,16 @@ def shared_caller_key(self) -> tuple[str, str] | None: return None def derive_conditions(self) -> None: - """GatewayReady tracks the Gateway being programmed and, where the XR - asks for auth, its caller policy being accepted. + """GatewayReady tracks the Gateway and AI Gateway route being accepted, + and, where the XR asks for auth, its caller policy being accepted. Both, because a gateway whose policy was rejected answers every request with a 500 while its Gateway still has an address. A missing input Secret is reported here (see resolve_secrets). The - Gateway's own readiness only tracks having an address, so neither a - fail-closed auth policy nor an unprogrammed HTTPS listener would - otherwise show. + Gateway's own readiness only tracks having an address, so an unaccepted + anchor route, fail-closed auth policy, or unprogrammed HTTPS listener + would otherwise show late or not at all. """ if self.xr.spec.auth and not self.caller_secrets: if "caller-secrets" not in self.req.required_resources: @@ -1366,9 +1417,12 @@ def derive_conditions(self) -> None: "authentication policy and every request is refused", ) return + readiness_keys = ["gateway", "ai-gateway-anchor"] + if self.xr.spec.auth: + readiness_keys.append("caller-auth") waiting = [ key - for key in (["gateway", "caller-auth"] if self.xr.spec.auth else ["gateway"]) + for key in readiness_keys if resource.get_condition(self.req.observed.resources.get(key), "Ready").status != "True" ] if not waiting: @@ -1381,12 +1435,18 @@ def derive_conditions(self) -> None: ), ) return - if waiting == ["caller-auth"]: + if "caller-auth" in waiting and "gateway" not in waiting: self.not_ready( CONDITION_REASON_AUTH_NOT_ACCEPTED, "The gateway's caller authentication policy has not been accepted, so every request is refused", ) return + if "ai-gateway-anchor" in waiting and "gateway" not in waiting: + self.not_ready( + CONDITION_REASON_WAITING_FOR_AI_GATEWAY_ROUTE, + f"Waiting for the AI Gateway anchor route on cluster {self.xr.spec.clusterName} to be accepted", + ) + return self.not_ready( CONDITION_REASON_WAITING_FOR_GATEWAY, f"Waiting for the Gateway on cluster {self.xr.spec.clusterName} to be programmed", diff --git a/functions/compose-inference-gateway/tests/test_fn.py b/functions/compose-inference-gateway/tests/test_fn.py index 131ef4c76..63b52fa30 100644 --- a/functions/compose-inference-gateway/tests/test_fn.py +++ b/functions/compose-inference-gateway/tests/test_fn.py @@ -168,6 +168,28 @@ def _observed_caller_auth() -> fnv1.Resource: ) +def _observed_ai_gateway_anchor() -> fnv1.Resource: + """The anchor AIGatewayRoute, observed once Envoy AI Gateway accepts it.""" + return fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "kubernetes.m.crossplane.io/v1alpha1", + "kind": "Object", + "status": { + "conditions": [ + { + "type": "Ready", + "status": "True", + "reason": "Available", + "lastTransitionTime": "2026-06-08T00:00:00Z", + } + ] + }, + } + ) + ) + + # The helpers below build the resources the function composes. Each is a # provider-kubernetes Object that applies one manifest to the gateway's cluster # through its ClusterProviderConfig, and every namespaced manifest lands in @@ -810,6 +832,63 @@ def _healthz_route() -> fnv1.Resource: ) +def _ai_gateway_anchor(*, https: bool, ready: fnv1.Ready) -> fnv1.Resource: + """The AIGatewayRoute keeping AI Gateway's ext-proc installed on this Gateway.""" + return fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "kubernetes.m.crossplane.io/v1alpha1", + "kind": "Object", + "metadata": {"namespace": "modelplane-system"}, + "spec": { + "providerConfigRef": {"kind": "ClusterProviderConfig", "name": "gw-eu-cluster-kubeconfig"}, + "readiness": { + "policy": "DeriveFromCelQuery", + "celQuery": ( + "has(object.status) && has(object.status.conditions) && " + "object.status.conditions.exists(c, c.type == 'Accepted' && c.status == 'True')" + ), + }, + "forProvider": { + "manifest": { + "apiVersion": "aigateway.envoyproxy.io/v1beta1", + "kind": "AIGatewayRoute", + "metadata": {"name": "inference-gateway-anchor", "namespace": "modelplane-system"}, + "spec": { + "parentRefs": [ + { + "group": "gateway.networking.k8s.io", + "kind": "Gateway", + "name": "inference-gateway", + "namespace": "modelplane-system", + "sectionName": "https" if https else "http", + } + ], + "rules": [ + { + "matches": [ + { + "headers": [ + { + "type": "Exact", + "name": "x-ai-eg-model", + "value": "modelplane-system/__gateway_anchor__", + } + ] + } + ] + } + ], + }, + } + }, + }, + } + ), + ready=ready, + ) + + def _healthz_auth() -> fnv1.Resource: """The composed SecurityPolicy letting /healthz past caller auth, so a health check needs no credential.""" return fnv1.Resource( @@ -1122,6 +1201,7 @@ def _to_dict(msg: message.Message) -> dict: "client-traffic-policy": _client_traffic_policy(), "healthz-filter": _healthz_filter(), "healthz-route": _healthz_route(), + "ai-gateway-anchor": _ai_gateway_anchor(https=False, ready=fnv1.READY_UNSPECIFIED), }, ), results=[ @@ -1188,6 +1268,7 @@ def _to_dict(msg: message.Message) -> dict: "client-traffic-policy": _client_traffic_policy(), "healthz-filter": _healthz_filter(), "healthz-route": _healthz_route(), + "ai-gateway-anchor": _ai_gateway_anchor(https=False, ready=fnv1.READY_UNSPECIFIED), }, ), results=[ @@ -1252,6 +1333,7 @@ def _to_dict(msg: message.Message) -> dict: "client-traffic-policy": _client_traffic_policy(), "healthz-filter": _healthz_filter(), "healthz-route": _healthz_route(), + "ai-gateway-anchor": _ai_gateway_anchor(https=False, ready=fnv1.READY_UNSPECIFIED), }, ), results=[ @@ -1319,6 +1401,7 @@ def _to_dict(msg: message.Message) -> dict: "client-traffic-policy": _client_traffic_policy(), "healthz-filter": _healthz_filter(), "healthz-route": _healthz_route(), + "ai-gateway-anchor": _ai_gateway_anchor(https=False, ready=fnv1.READY_UNSPECIFIED), }, ), results=[ @@ -1356,6 +1439,7 @@ def _to_dict(msg: message.Message) -> dict: composite=_xr(name="eu", tls=False, caller_secret_labels=None), resources={ "gateway": _observed_gateway(address="gw.example.org", ready=True), + "ai-gateway-anchor": _observed_ai_gateway_anchor(), "client-ca-configmap": fnv1.Resource( resource=resource.dict_to_struct( { @@ -1406,6 +1490,7 @@ def _to_dict(msg: message.Message) -> dict: "client-traffic-policy": _client_traffic_policy(), "healthz-filter": _healthz_filter(), "healthz-route": _healthz_route(), + "ai-gateway-anchor": _ai_gateway_anchor(https=False, ready=fnv1.READY_TRUE), }, ), context=structpb.Struct(), @@ -1428,7 +1513,10 @@ def _to_dict(msg: message.Message) -> dict: req=fnv1.RunFunctionRequest( observed=fnv1.State( composite=_xr(name="eu", tls=False, caller_secret_labels=None), - resources={"gateway": _observed_gateway(address="gw.example.org", ready=True)}, + resources={ + "gateway": _observed_gateway(address="gw.example.org", ready=True), + "ai-gateway-anchor": _observed_ai_gateway_anchor(), + }, ), required_resources={ "clusters": fnv1.Resources(items=[_cluster(provider_config_ref=True)]), @@ -1460,6 +1548,7 @@ def _to_dict(msg: message.Message) -> dict: "client-traffic-policy": _client_traffic_policy(), "healthz-filter": _healthz_filter(), "healthz-route": _healthz_route(), + "ai-gateway-anchor": _ai_gateway_anchor(https=False, ready=fnv1.READY_TRUE), }, ), context=structpb.Struct(), @@ -1797,6 +1886,7 @@ def _to_dict(msg: message.Message) -> dict: "client-traffic-policy": _client_traffic_policy(), "healthz-filter": _healthz_filter(), "healthz-route": _healthz_route(), + "ai-gateway-anchor": _ai_gateway_anchor(https=False, ready=fnv1.READY_UNSPECIFIED), }, ), results=[ @@ -1854,6 +1944,7 @@ def _to_dict(msg: message.Message) -> dict: "client-traffic-policy": _client_traffic_policy(), "healthz-filter": _healthz_filter(), "healthz-route": _healthz_route(), + "ai-gateway-anchor": _ai_gateway_anchor(https=True, ready=fnv1.READY_UNSPECIFIED), "redirect-route": _redirect_route(), }, ), @@ -1886,11 +1977,8 @@ def _to_dict(msg: message.Message) -> dict: # redirect is exempted because it must happen before auth, or an # unauthenticated caller would get a 401 instead of being sent to HTTPS. Case( - name="TLSAndAuth", - reason=( - "A programmed gateway with TLS and auth serves authenticated HTTPS, copies its Secrets to the " - "cluster, and publishes its address but no URLs." - ), + name="AnchorRouteNotAccepted", + reason=("A programmed TLS gateway with accepted caller auth waits for its permanent AI Gateway route."), req=fnv1.RunFunctionRequest( observed=fnv1.State( composite=_xr(name="eu", tls=True, caller_secret_labels={"modelplane.ai/inference-keys": "true"}), @@ -1924,7 +2012,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(status={"address": "34.56.129.3"}, ready=fnv1.READY_UNSPECIFIED), + composite=_desired_xr(status={"address": "34.56.129.3"}, ready=fnv1.READY_FALSE), resources={ "caller-secret-ml-team-keys": _caller_secret( name="callers-ml-team-keys", data={"ml-team-assistant": "c2stbXAtYTFiMmMz"} @@ -1968,6 +2056,7 @@ def _to_dict(msg: message.Message) -> dict: "client-traffic-policy": _client_traffic_policy(), "healthz-filter": _healthz_filter(), "healthz-route": _healthz_route(), + "ai-gateway-anchor": _ai_gateway_anchor(https=True, ready=fnv1.READY_UNSPECIFIED), "healthz-auth": _healthz_auth(), "redirect-route": _redirect_route(), "redirect-auth": fnv1.Resource( @@ -2015,6 +2104,12 @@ def _to_dict(msg: message.Message) -> dict: ), }, ), + results=[ + fnv1.Result( + severity=fnv1.SEVERITY_NORMAL, + message="Waiting for the AI Gateway anchor route on cluster gw-eu to be accepted", + ) + ], context=structpb.Struct(), requirements=fnv1.Requirements( resources={ @@ -2032,7 +2127,12 @@ def _to_dict(msg: message.Message) -> dict: } ), conditions=[ - fnv1.Condition(type="GatewayReady", status=fnv1.STATUS_CONDITION_TRUE, reason="GatewayProgrammed"), + fnv1.Condition( + type="GatewayReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="WaitingForAIGatewayRoute", + message="Waiting for the AI Gateway anchor route on cluster gw-eu to be accepted", + ), ], ), ), @@ -2050,7 +2150,10 @@ def _to_dict(msg: message.Message) -> dict: req=fnv1.RunFunctionRequest( observed=fnv1.State( composite=_xr(name="eu", tls=False, caller_secret_labels={"modelplane.ai/inference-keys": "true"}), - resources={"gateway": _observed_gateway(address="34.56.129.3", ready=True)}, + resources={ + "gateway": _observed_gateway(address="34.56.129.3", ready=True), + "ai-gateway-anchor": _observed_ai_gateway_anchor(), + }, ), required_resources={ "clusters": fnv1.Resources(items=[_cluster(provider_config_ref=True)]), @@ -2089,6 +2192,7 @@ def _to_dict(msg: message.Message) -> dict: "client-traffic-policy": _client_traffic_policy(), "healthz-filter": _healthz_filter(), "healthz-route": _healthz_route(), + "ai-gateway-anchor": _ai_gateway_anchor(https=False, ready=fnv1.READY_TRUE), "healthz-auth": _healthz_auth(), }, ), @@ -2134,6 +2238,7 @@ def _to_dict(msg: message.Message) -> dict: composite=_xr(name="eu", tls=False, caller_secret_labels={"modelplane.ai/inference-keys": "true"}), resources={ "gateway": _observed_gateway(address="34.56.129.3", ready=True), + "ai-gateway-anchor": _observed_ai_gateway_anchor(), "caller-auth": _observed_caller_auth(), }, ), @@ -2179,6 +2284,7 @@ def _to_dict(msg: message.Message) -> dict: "client-traffic-policy": _client_traffic_policy(), "healthz-filter": _healthz_filter(), "healthz-route": _healthz_route(), + "ai-gateway-anchor": _ai_gateway_anchor(https=False, ready=fnv1.READY_TRUE), "healthz-auth": _healthz_auth(), }, ), @@ -2221,6 +2327,7 @@ def _to_dict(msg: message.Message) -> dict: composite=_xr(name="eu", tls=False, caller_secret_labels={"modelplane.ai/inference-keys": "true"}), resources={ "gateway": _observed_gateway(address="34.56.129.3", ready=True), + "ai-gateway-anchor": _observed_ai_gateway_anchor(), "caller-auth": _observed_caller_auth(), }, ), @@ -2263,6 +2370,7 @@ def _to_dict(msg: message.Message) -> dict: "client-traffic-policy": _client_traffic_policy(), "healthz-filter": _healthz_filter(), "healthz-route": _healthz_route(), + "ai-gateway-anchor": _ai_gateway_anchor(https=False, ready=fnv1.READY_TRUE), "healthz-auth": _healthz_auth(), }, ), @@ -2310,6 +2418,7 @@ def _to_dict(msg: message.Message) -> dict: composite=_xr(name="eu", tls=False, caller_secret_labels={"modelplane.ai/inference-keys": "true"}), resources={ "gateway": _observed_gateway(address="34.56.129.3", ready=True), + "ai-gateway-anchor": _observed_ai_gateway_anchor(), "caller-auth": _observed_caller_auth(), }, ), @@ -2357,6 +2466,7 @@ def _to_dict(msg: message.Message) -> dict: "client-traffic-policy": _client_traffic_policy(), "healthz-filter": _healthz_filter(), "healthz-route": _healthz_route(), + "ai-gateway-anchor": _ai_gateway_anchor(https=False, ready=fnv1.READY_TRUE), "healthz-auth": _healthz_auth(), }, ), @@ -2401,6 +2511,7 @@ def _to_dict(msg: message.Message) -> dict: composite=_xr(name="eu", tls=False, caller_secret_labels={"modelplane.ai/inference-keys": "true"}), resources={ "gateway": _observed_gateway(address="34.56.129.3", ready=True), + "ai-gateway-anchor": _observed_ai_gateway_anchor(), "caller-auth": _observed_caller_auth(), }, ), @@ -2448,6 +2559,7 @@ def _to_dict(msg: message.Message) -> dict: "client-traffic-policy": _client_traffic_policy(), "healthz-filter": _healthz_filter(), "healthz-route": _healthz_route(), + "ai-gateway-anchor": _ai_gateway_anchor(https=False, ready=fnv1.READY_TRUE), "healthz-auth": _healthz_auth(), }, ), @@ -2512,6 +2624,7 @@ def _to_dict(msg: message.Message) -> dict: "client-traffic-policy": _client_traffic_policy(), "healthz-filter": _healthz_filter(), "healthz-route": _healthz_route(), + "ai-gateway-anchor": _ai_gateway_anchor(https=False, ready=fnv1.READY_UNSPECIFIED), "healthz-auth": _healthz_auth(), }, ), @@ -2624,6 +2737,7 @@ def _to_dict(msg: message.Message) -> dict: "client-traffic-policy": _client_traffic_policy(), "healthz-filter": _healthz_filter(), "healthz-route": _healthz_route(), + "ai-gateway-anchor": _ai_gateway_anchor(https=False, ready=fnv1.READY_UNSPECIFIED), "healthz-auth": _healthz_auth(), }, ), @@ -2730,6 +2844,7 @@ def _to_dict(msg: message.Message) -> dict: "client-traffic-policy": _client_traffic_policy(), "healthz-filter": _healthz_filter(), "healthz-route": _healthz_route(), + "ai-gateway-anchor": _ai_gateway_anchor(https=False, ready=fnv1.READY_UNSPECIFIED), "healthz-auth": _healthz_auth(), }, ), @@ -2852,6 +2967,7 @@ def _to_dict(msg: message.Message) -> dict: "client-traffic-policy": _client_traffic_policy(), "healthz-filter": _healthz_filter(), "healthz-route": _healthz_route(), + "ai-gateway-anchor": _ai_gateway_anchor(https=True, ready=fnv1.READY_UNSPECIFIED), "healthz-auth": _healthz_auth(), "redirect-route": _redirect_route(), "redirect-auth": fnv1.Resource( From 1cbb18ab99032a8f27db9676e3672cd0e95487bd Mon Sep 17 00:00:00 2001 From: mikemikimike <13286568797@163.com> Date: Tue, 20 Oct 2026 19:03:00 +0800 Subject: [PATCH 2/7] Wait for Envoy rollout before composing model routes The permanent anchor can be accepted before Envoy rolls out its ext-proc sidecar, so the first ModelRoute can report RoutingReady while the proxy returns 500. Observe the Envoy Deployment rollout in GatewayReady and hold route composition until GatewayReady is true. Match the anchor on a non-model header so it stays out of /v1/models. Fixes #531. Signed-off-by: mikemikimike <13286568797@163.com> --- .../compose-inference-gateway/function/fn.py | 57 +++++- .../tests/test_fn.py | 172 +++++++++++++++++- functions/compose-model-route/function/fn.py | 14 ++ .../compose-model-route/tests/test_fn.py | 162 +++++++++++++++-- 4 files changed, 376 insertions(+), 29 deletions(-) diff --git a/functions/compose-inference-gateway/function/fn.py b/functions/compose-inference-gateway/function/fn.py index c18a916e6..c39fb4e6d 100644 --- a/functions/compose-inference-gateway/function/fn.py +++ b/functions/compose-inference-gateway/function/fn.py @@ -49,6 +49,7 @@ CONDITION_REASON_WAITING_FOR_CLUSTER = "WaitingForCluster" CONDITION_REASON_WAITING_FOR_GATEWAY = "WaitingForGateway" CONDITION_REASON_WAITING_FOR_AI_GATEWAY_ROUTE = "WaitingForAIGatewayRoute" +CONDITION_REASON_WAITING_FOR_ENVOY_DEPLOYMENT = "WaitingForEnvoyDeployment" CONDITION_REASON_CLUSTER_TAKEN = "ClusterAlreadyHasGateway" CONDITION_REASON_SECRETS_MISSING = "SecretsMissing" CONDITION_REASON_AUTH_NOT_ACCEPTED = "CallerAuthNotAccepted" @@ -79,7 +80,10 @@ _CLIENT_TRAFFIC_NAME = "inference-gateway-client-traffic" _REDIRECT_NAME = "inference-gateway-redirect" _AI_GATEWAY_ANCHOR_NAME = "inference-gateway-anchor" -_AI_GATEWAY_ANCHOR_MODEL = f"{REMOTE_NAMESPACE}/__gateway_anchor__" +_AI_GATEWAY_ANCHOR_HEADER = "x-modelplane-gateway-anchor" +_AI_GATEWAY_ANCHOR_HEADER_VALUE = "true" +_ENVOY_GATEWAY_NAMESPACE = "envoy-gateway-system" +_ENVOY_DEPLOYMENT_NAME = "modelplane-inference-gateway-envoy" # The Gateway's listener names. When the gateway terminates TLS, inference binds # to the HTTPS listener alone (compose-model-route sets the AIGatewayRoute's @@ -159,6 +163,17 @@ "object.status.conditions.exists(c, c.type == 'Accepted' && c.status == 'True')" ) +_ENVOY_DEPLOYMENT_ROLLOUT_READY_CEL = ( + "has(object.metadata) && has(object.metadata.generation) && " + "has(object.spec) && has(object.spec.replicas) && has(object.status) && " + "has(object.status.observedGeneration) && object.status.observedGeneration >= object.metadata.generation && " + "has(object.status.updatedReplicas) && object.status.updatedReplicas == object.spec.replicas && " + "has(object.status.readyReplicas) && object.status.readyReplicas == object.spec.replicas && " + "has(object.status.availableReplicas) && object.status.availableReplicas == object.spec.replicas && " + "has(object.status.conditions) && " + "object.status.conditions.exists(c, c.type == 'Available' && c.status == 'True')" +) + # Kubernetes injects ndots:5 into every pod, which tells the resolver to try each # search domain before a name with fewer than five dots. Every hostname this # gateway resolves has fewer: a provider like api.together.xyz has two, a cluster @@ -328,6 +343,7 @@ def compose(self) -> None: self.resolve_secrets() self.compose_secrets() self.compose_envoy_proxy() + self.compose_envoy_deployment() self.compose_gateway() self.compose_ai_gateway_anchor() self.compose_client_pki() @@ -569,6 +585,7 @@ def compose_envoy_proxy(self) -> None: "kubernetes": { "envoyService": {"externalTrafficPolicy": "Cluster"}, "envoyDeployment": { + "name": _ENVOY_DEPLOYMENT_NAME, "replicas": _PROXY_REPLICAS, "patch": {"type": "StrategicMerge", "value": _NDOTS_PATCH}, "pod": { @@ -605,6 +622,20 @@ def compose_envoy_proxy(self) -> None: ), ) + def compose_envoy_deployment(self) -> None: + """Observe Envoy's Deployment so readiness includes its full rollout.""" + obj = _k8s_object( + self.pc, + { + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": {"name": _ENVOY_DEPLOYMENT_NAME, "namespace": _ENVOY_GATEWAY_NAMESPACE}, + }, + ready_when=_ENVOY_DEPLOYMENT_ROLLOUT_READY_CEL, + ) + obj.spec.managementPolicies = ["Observe"] + resource.update(self.rsp.desired.resources["envoy-deployment"], obj) + def compose_gateway(self) -> None: """The Gateway callers connect to. @@ -704,8 +735,8 @@ def compose_ai_gateway_anchor(self) -> None: "headers": [ { "type": "Exact", - "name": "x-ai-eg-model", - "value": _AI_GATEWAY_ANCHOR_MODEL, + "name": _AI_GATEWAY_ANCHOR_HEADER, + "value": _AI_GATEWAY_ANCHOR_HEADER_VALUE, } ] } @@ -1381,7 +1412,7 @@ def shared_caller_key(self) -> tuple[str, str] | None: return None def derive_conditions(self) -> None: - """GatewayReady tracks the Gateway and AI Gateway route being accepted, + """GatewayReady tracks the Gateway, Envoy rollout and AI Gateway route, and, where the XR asks for auth, its caller policy being accepted. Both, because a gateway whose policy was rejected answers every request @@ -1417,7 +1448,7 @@ def derive_conditions(self) -> None: "authentication policy and every request is refused", ) return - readiness_keys = ["gateway", "ai-gateway-anchor"] + readiness_keys = ["gateway", "ai-gateway-anchor", "envoy-deployment"] if self.xr.spec.auth: readiness_keys.append("caller-auth") waiting = [ @@ -1441,11 +1472,17 @@ def derive_conditions(self) -> None: "The gateway's caller authentication policy has not been accepted, so every request is refused", ) return - if "ai-gateway-anchor" in waiting and "gateway" not in waiting: - self.not_ready( - CONDITION_REASON_WAITING_FOR_AI_GATEWAY_ROUTE, - f"Waiting for the AI Gateway anchor route on cluster {self.xr.spec.clusterName} to be accepted", - ) + if ("ai-gateway-anchor" in waiting or "envoy-deployment" in waiting) and "gateway" not in waiting: + if "ai-gateway-anchor" in waiting: + self.not_ready( + CONDITION_REASON_WAITING_FOR_AI_GATEWAY_ROUTE, + f"Waiting for the AI Gateway anchor route on cluster {self.xr.spec.clusterName} to be accepted", + ) + else: + self.not_ready( + CONDITION_REASON_WAITING_FOR_ENVOY_DEPLOYMENT, + f"Waiting for the Envoy Deployment on cluster {self.xr.spec.clusterName} to finish rolling out", + ) return self.not_ready( CONDITION_REASON_WAITING_FOR_GATEWAY, diff --git a/functions/compose-inference-gateway/tests/test_fn.py b/functions/compose-inference-gateway/tests/test_fn.py index 63b52fa30..8cf4072c8 100644 --- a/functions/compose-inference-gateway/tests/test_fn.py +++ b/functions/compose-inference-gateway/tests/test_fn.py @@ -190,6 +190,28 @@ def _observed_ai_gateway_anchor() -> fnv1.Resource: ) +def _observed_envoy_deployment(*, ready: bool) -> fnv1.Resource: + """The Envoy Deployment, observed ready only after its rollout completes.""" + return fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "kubernetes.m.crossplane.io/v1alpha1", + "kind": "Object", + "status": { + "conditions": [ + { + "type": "Ready", + "status": "True" if ready else "False", + "reason": "Available" if ready else "Updating", + "lastTransitionTime": "2026-06-08T00:00:00Z", + } + ] + }, + } + ) + ) + + # The helpers below build the resources the function composes. Each is a # provider-kubernetes Object that applies one manifest to the gateway's cluster # through its ClusterProviderConfig, and every namespaced manifest lands in @@ -233,6 +255,7 @@ def _envoy_proxy() -> fnv1.Resource: "kubernetes": { "envoyService": {"externalTrafficPolicy": "Cluster"}, "envoyDeployment": { + "name": "modelplane-inference-gateway-envoy", "replicas": 2, "patch": { "type": "StrategicMerge", @@ -326,6 +349,57 @@ def _envoy_proxy() -> fnv1.Resource: ) +def _envoy_deployment() -> fnv1.Resource: + """The read-only Object that waits for Envoy's Deployment rollout.""" + return fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "kubernetes.m.crossplane.io/v1alpha1", + "kind": "Object", + "metadata": {"namespace": "modelplane-system"}, + "spec": { + "providerConfigRef": {"kind": "ClusterProviderConfig", "name": "gw-eu-cluster-kubeconfig"}, + "readiness": { + "policy": "DeriveFromCelQuery", + "celQuery": ( + "has(object.metadata) && has(object.metadata.generation) && " + "has(object.spec) && has(object.spec.replicas) && has(object.status) && " + "has(object.status.observedGeneration) && " + "object.status.observedGeneration >= object.metadata.generation && " + "has(object.status.updatedReplicas) && " + "object.status.updatedReplicas == object.spec.replicas && " + "has(object.status.readyReplicas) && " + "object.status.readyReplicas == object.spec.replicas && " + "has(object.status.availableReplicas) && " + "object.status.availableReplicas == object.spec.replicas && " + "has(object.status.conditions) && " + "object.status.conditions.exists(c, c.type == 'Available' && c.status == 'True')" + ), + }, + "managementPolicies": ["Observe"], + "forProvider": { + "manifest": { + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": { + "name": "modelplane-inference-gateway-envoy", + "namespace": "envoy-gateway-system", + }, + } + }, + }, + } + ) + ) + + +def _ready_envoy_deployment() -> fnv1.Resource: + """The desired Envoy Deployment after its observed rollout completes.""" + deployment = _envoy_deployment() + deployment.ready = fnv1.READY_TRUE + return deployment + + def _gateway(*, https: bool, ready: fnv1.Ready) -> fnv1.Resource: """The composed Gateway: an HTTP listener, joined by an HTTPS one terminating eu-tls-0 when https is set.""" http_listener = { @@ -871,8 +945,8 @@ def _ai_gateway_anchor(*, https: bool, ready: fnv1.Ready) -> fnv1.Resource: "headers": [ { "type": "Exact", - "name": "x-ai-eg-model", - "value": "modelplane-system/__gateway_anchor__", + "name": "x-modelplane-gateway-anchor", + "value": "true", } ] } @@ -1191,6 +1265,7 @@ def _to_dict(msg: message.Message) -> dict: composite=_desired_xr(status={}, ready=fnv1.READY_FALSE), resources={ "envoy-proxy": _envoy_proxy(), + "envoy-deployment": _envoy_deployment(), "gateway": _gateway(https=False, ready=fnv1.READY_UNSPECIFIED), "client-selfsigned-issuer": _client_selfsigned_issuer(), "client-ca-certificate": _client_ca_certificate(common_name="Modelplane InferenceGateway CA eu"), @@ -1258,6 +1333,7 @@ def _to_dict(msg: message.Message) -> dict: ), resources={ "envoy-proxy": _envoy_proxy(), + "envoy-deployment": _envoy_deployment(), "gateway": _gateway(https=False, ready=fnv1.READY_UNSPECIFIED), "client-selfsigned-issuer": _client_selfsigned_issuer(), "client-ca-certificate": _client_ca_certificate(common_name="Modelplane InferenceGateway CA eu"), @@ -1323,6 +1399,7 @@ def _to_dict(msg: message.Message) -> dict: ), resources={ "envoy-proxy": _envoy_proxy(), + "envoy-deployment": _envoy_deployment(), "gateway": _gateway(https=False, ready=fnv1.READY_UNSPECIFIED), "client-selfsigned-issuer": _client_selfsigned_issuer(), "client-ca-certificate": _client_ca_certificate(common_name="Modelplane InferenceGateway CA eu"), @@ -1389,6 +1466,7 @@ def _to_dict(msg: message.Message) -> dict: composite=_desired_xr(status={}, ready=fnv1.READY_FALSE), resources={ "envoy-proxy": _envoy_proxy(), + "envoy-deployment": _envoy_deployment(), "gateway": _gateway(https=False, ready=fnv1.READY_UNSPECIFIED), "client-selfsigned-issuer": _client_selfsigned_issuer(), "client-ca-certificate": _client_ca_certificate( @@ -1440,6 +1518,7 @@ def _to_dict(msg: message.Message) -> dict: resources={ "gateway": _observed_gateway(address="gw.example.org", ready=True), "ai-gateway-anchor": _observed_ai_gateway_anchor(), + "envoy-deployment": _observed_envoy_deployment(ready=True), "client-ca-configmap": fnv1.Resource( resource=resource.dict_to_struct( { @@ -1480,6 +1559,7 @@ def _to_dict(msg: message.Message) -> dict: ), resources={ "envoy-proxy": _envoy_proxy(), + "envoy-deployment": _ready_envoy_deployment(), "gateway": _gateway(https=False, ready=fnv1.READY_TRUE), "client-selfsigned-issuer": _client_selfsigned_issuer(), "client-ca-certificate": _client_ca_certificate(common_name="Modelplane InferenceGateway CA eu"), @@ -1516,6 +1596,7 @@ def _to_dict(msg: message.Message) -> dict: resources={ "gateway": _observed_gateway(address="gw.example.org", ready=True), "ai-gateway-anchor": _observed_ai_gateway_anchor(), + "envoy-deployment": _observed_envoy_deployment(ready=True), }, ), required_resources={ @@ -1538,6 +1619,7 @@ def _to_dict(msg: message.Message) -> dict: ), resources={ "envoy-proxy": _envoy_proxy(), + "envoy-deployment": _ready_envoy_deployment(), "gateway": _gateway(https=False, ready=fnv1.READY_TRUE), "client-selfsigned-issuer": _client_selfsigned_issuer(), "client-ca-certificate": _client_ca_certificate(common_name="Modelplane InferenceGateway CA eu"), @@ -1563,6 +1645,78 @@ def _to_dict(msg: message.Message) -> dict: ], ), ), + Case( + name="EnvoyDeploymentRollingOut", + reason=( + "An accepted anchor route and Gateway address do not make the gateway Ready while the Envoy " + "Deployment rollout is incomplete." + ), + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr(name="eu", tls=False, caller_secret_labels=None), + resources={ + "gateway": _observed_gateway(address="gw.example.org", ready=True), + "ai-gateway-anchor": _observed_ai_gateway_anchor(), + "envoy-deployment": _observed_envoy_deployment(ready=False), + }, + ), + required_resources={ + "clusters": fnv1.Resources(items=[_cluster(provider_config_ref=True)]), + "gateways": fnv1.Resources(items=[_inference_gateway(name="eu", address=None)]), + }, + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_xr( + status={ + "address": "gw.example.org", + "endpoints": { + "openAI": "http://gw.example.org/v1", + "anthropic": "http://gw.example.org/anthropic/v1", + }, + }, + ready=fnv1.READY_FALSE, + ), + resources={ + "envoy-proxy": _envoy_proxy(), + "envoy-deployment": _envoy_deployment(), + "gateway": _gateway(https=False, ready=fnv1.READY_TRUE), + "client-selfsigned-issuer": _client_selfsigned_issuer(), + "client-ca-certificate": _client_ca_certificate(common_name="Modelplane InferenceGateway CA eu"), + "client-ca-issuer": _client_ca_issuer(), + "client-ca-bundle": _client_ca_bundle(), + "client-ca-configmap": _client_ca_configmap(), + "failover-policy": _failover_policy(), + "client-traffic-policy": _client_traffic_policy(), + "healthz-filter": _healthz_filter(), + "healthz-route": _healthz_route(), + "ai-gateway-anchor": _ai_gateway_anchor(https=False, ready=fnv1.READY_TRUE), + }, + ), + results=[ + fnv1.Result( + severity=fnv1.SEVERITY_NORMAL, + message="Waiting for the Envoy Deployment on cluster gw-eu to finish rolling out", + ) + ], + context=structpb.Struct(), + requirements=fnv1.Requirements( + resources={ + "gateways": fnv1.ResourceSelector(api_version="modelplane.ai/v1alpha1", kind="InferenceGateway"), + "clusters": fnv1.ResourceSelector(api_version="modelplane.ai/v1alpha1", kind="InferenceCluster"), + } + ), + conditions=[ + fnv1.Condition( + type="GatewayReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="WaitingForEnvoyDeployment", + message="Waiting for the Envoy Deployment on cluster gw-eu to finish rolling out", + ), + ], + ), + ), # A ModelService's backends address a cluster gateway by the name # compose-inference-cluster derived, and this gateway's Envoy resolves it, # so its cluster needs a Service of that name. A hostname is how a cloud @@ -1661,6 +1815,7 @@ def _to_dict(msg: message.Message) -> dict: composite=_desired_xr(status={}, ready=fnv1.READY_FALSE), resources={ "envoy-proxy": _envoy_proxy(), + "envoy-deployment": _envoy_deployment(), "gateway": _gateway(https=False, ready=fnv1.READY_UNSPECIFIED), "client-selfsigned-issuer": _client_selfsigned_issuer(), "client-ca-certificate": _client_ca_certificate(common_name="Modelplane InferenceGateway CA eu"), @@ -1934,6 +2089,7 @@ def _to_dict(msg: message.Message) -> dict: composite=_desired_xr(status={}, ready=fnv1.READY_FALSE), resources={ "envoy-proxy": _envoy_proxy(), + "envoy-deployment": _envoy_deployment(), "gateway": _gateway(https=True, ready=fnv1.READY_UNSPECIFIED), "client-selfsigned-issuer": _client_selfsigned_issuer(), "client-ca-certificate": _client_ca_certificate(common_name="Modelplane InferenceGateway CA eu"), @@ -2043,6 +2199,7 @@ def _to_dict(msg: message.Message) -> dict: ) ), "envoy-proxy": _envoy_proxy(), + "envoy-deployment": _envoy_deployment(), "gateway": _gateway(https=True, ready=fnv1.READY_TRUE), "client-selfsigned-issuer": _client_selfsigned_issuer(), "client-ca-certificate": _client_ca_certificate(common_name="Modelplane InferenceGateway CA eu"), @@ -2179,6 +2336,7 @@ def _to_dict(msg: message.Message) -> dict: resources={ "caller-secret-ml-team-keys": _caller_secret(name="callers-ml-team-keys", data={"a": "c2stMQ=="}), "envoy-proxy": _envoy_proxy(), + "envoy-deployment": _envoy_deployment(), "gateway": _gateway(https=False, ready=fnv1.READY_TRUE), "client-selfsigned-issuer": _client_selfsigned_issuer(), "client-ca-certificate": _client_ca_certificate(common_name="Modelplane InferenceGateway CA eu"), @@ -2239,6 +2397,7 @@ def _to_dict(msg: message.Message) -> dict: resources={ "gateway": _observed_gateway(address="34.56.129.3", ready=True), "ai-gateway-anchor": _observed_ai_gateway_anchor(), + "envoy-deployment": _observed_envoy_deployment(ready=True), "caller-auth": _observed_caller_auth(), }, ), @@ -2270,6 +2429,7 @@ def _to_dict(msg: message.Message) -> dict: "caller-secret-team-a-keys": _caller_secret(name="callers-team-a-keys", data={"a": "c2stMQ=="}), "caller-secret-team-b-keys": _caller_secret(name="callers-team-b-keys", data={"b": "c2stMQ=="}), "envoy-proxy": _envoy_proxy(), + "envoy-deployment": _ready_envoy_deployment(), "gateway": _gateway(https=False, ready=fnv1.READY_TRUE), "client-selfsigned-issuer": _client_selfsigned_issuer(), "client-ca-certificate": _client_ca_certificate(common_name="Modelplane InferenceGateway CA eu"), @@ -2357,6 +2517,7 @@ def _to_dict(msg: message.Message) -> dict: name="callers-team-a-keys", data={"a": "c2stMQ==", "z": "c2stMQ=="} ), "envoy-proxy": _envoy_proxy(), + "envoy-deployment": _envoy_deployment(), "gateway": _gateway(https=False, ready=fnv1.READY_TRUE), "client-selfsigned-issuer": _client_selfsigned_issuer(), "client-ca-certificate": _client_ca_certificate(common_name="Modelplane InferenceGateway CA eu"), @@ -2452,6 +2613,7 @@ def _to_dict(msg: message.Message) -> dict: name="callers-team-b-keys", data={"x": "c2stMg==", "y": "c2stMQ=="} ), "envoy-proxy": _envoy_proxy(), + "envoy-deployment": _envoy_deployment(), "gateway": _gateway(https=False, ready=fnv1.READY_TRUE), "client-selfsigned-issuer": _client_selfsigned_issuer(), "client-ca-certificate": _client_ca_certificate(common_name="Modelplane InferenceGateway CA eu"), @@ -2512,6 +2674,7 @@ def _to_dict(msg: message.Message) -> dict: resources={ "gateway": _observed_gateway(address="34.56.129.3", ready=True), "ai-gateway-anchor": _observed_ai_gateway_anchor(), + "envoy-deployment": _observed_envoy_deployment(ready=True), "caller-auth": _observed_caller_auth(), }, ), @@ -2545,6 +2708,7 @@ def _to_dict(msg: message.Message) -> dict: name="callers-team-b-keys", data={"a": "c2stMQ==", "b": "c2stMg=="} ), "envoy-proxy": _envoy_proxy(), + "envoy-deployment": _ready_envoy_deployment(), "gateway": _gateway(https=False, ready=fnv1.READY_TRUE), "client-selfsigned-issuer": _client_selfsigned_issuer(), "client-ca-certificate": _client_ca_certificate(common_name="Modelplane InferenceGateway CA eu"), @@ -2610,6 +2774,7 @@ def _to_dict(msg: message.Message) -> dict: "caller-secret-team-a-keys": _caller_secret(name="callers-team-a-keys", data={"a": "c2stMQ=="}), "caller-secret-team-b-keys": _caller_secret(name="callers-team-b-keys", data={"b": "c2stMg=="}), "envoy-proxy": _envoy_proxy(), + "envoy-deployment": _envoy_deployment(), "gateway": _gateway(https=False, ready=fnv1.READY_UNSPECIFIED), "client-selfsigned-issuer": _client_selfsigned_issuer(), "client-ca-certificate": _client_ca_certificate(common_name="Modelplane InferenceGateway CA eu"), @@ -2684,6 +2849,7 @@ def _to_dict(msg: message.Message) -> dict: composite=_desired_xr(status={}, ready=fnv1.READY_FALSE), resources={ "envoy-proxy": _envoy_proxy(), + "envoy-deployment": _envoy_deployment(), "gateway": _gateway(https=False, ready=fnv1.READY_UNSPECIFIED), "client-selfsigned-issuer": _client_selfsigned_issuer(), "client-ca-certificate": _client_ca_certificate(common_name="Modelplane InferenceGateway CA eu"), @@ -2791,6 +2957,7 @@ def _to_dict(msg: message.Message) -> dict: composite=_desired_xr(status={}, ready=fnv1.READY_FALSE), resources={ "envoy-proxy": _envoy_proxy(), + "envoy-deployment": _envoy_deployment(), "gateway": _gateway(https=False, ready=fnv1.READY_UNSPECIFIED), "client-selfsigned-issuer": _client_selfsigned_issuer(), "client-ca-certificate": _client_ca_certificate(common_name="Modelplane InferenceGateway CA eu"), @@ -2954,6 +3121,7 @@ def _to_dict(msg: message.Message) -> dict: ) ), "envoy-proxy": _envoy_proxy(), + "envoy-deployment": _envoy_deployment(), "gateway": _gateway(https=True, ready=fnv1.READY_UNSPECIFIED), "client-selfsigned-issuer": _client_selfsigned_issuer(), "client-ca-certificate": _client_ca_certificate(common_name="Modelplane InferenceGateway CA eu"), diff --git a/functions/compose-model-route/function/fn.py b/functions/compose-model-route/function/fn.py index 7d06b3da3..fd3cf6b43 100644 --- a/functions/compose-model-route/function/fn.py +++ b/functions/compose-model-route/function/fn.py @@ -368,6 +368,20 @@ def resolve_gateway(self) -> bool: return False gw = igv1alpha1.InferenceGateway.model_validate(gateways[0]) + gateway_conditions = gw.status.conditions if gw.status else None + gateway_ready = next( + (condition for condition in gateway_conditions or [] if condition.type == "GatewayReady"), + None, + ) + if not gateway_ready or gateway_ready.status != "True": + message = ( + gateway_ready.message + if gateway_ready and gateway_ready.message + else f"InferenceGateway {self.xr.spec.gatewayName} is not ready" + ) + self.not_ready(CONDITION_REASON_WAITING_FOR_GATEWAY, message) + return False + if not (gw.status and gw.status.clientCACertificate): self.not_ready( CONDITION_REASON_WAITING_FOR_GATEWAY, diff --git a/functions/compose-model-route/tests/test_fn.py b/functions/compose-model-route/tests/test_fn.py index c287990f5..83c666a74 100644 --- a/functions/compose-model-route/tests/test_fn.py +++ b/functions/compose-model-route/tests/test_fn.py @@ -67,12 +67,21 @@ def _desired_model_route(*, address: str | None, total_endpoints: int, ready_end return fnv1.Resource(resource=resource.dict_to_struct({"status": status}), ready=fnv1.READY_FALSE) -def _inference_gateway(*, tls: bool, address: str, client_ca_published: bool) -> fnv1.Resource: +def _inference_gateway(*, tls: bool, address: str, client_ca_published: bool, gateway_ready: bool) -> fnv1.Resource: """The InferenceGateway eu on cluster gw-eu, as the gateway requirement returns it.""" spec: dict = {"clusterName": "gw-eu"} if tls: spec["tls"] = {"certificateRefs": [{"name": "eu-tls"}]} status: dict = {"address": address} + status["conditions"] = [ + { + "type": "GatewayReady", + "status": "True" if gateway_ready else "False", + "reason": "GatewayProgrammed" if gateway_ready else "WaitingForEnvoyDeployment", + "message": "" if gateway_ready else "Waiting for Envoy Deployment rollout", + "lastTransitionTime": "2026-06-08T00:00:00Z", + } + ] if client_ca_published: status["clientCACertificate"] = "-----BEGIN CERTIFICATE-----\nclient\n-----END CERTIFICATE-----\n" return fnv1.Resource( @@ -530,6 +539,61 @@ def _to_dict(msg: message.Message) -> dict: # Secret data is base64 encoded, as the API server stores it: c2stMQ== is # "sk-1", c2stdG9n "sk-tog" and c2stcHJvdmlkZXI= "sk-provider". COMPOSE_CASES = [ + Case( + name="GatewayNotReady", + reason="A ModelRoute waits for its InferenceGateway rollout before composing routes or backends.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_model_route( + endpoints=[ + v1alpha1.Endpoint( + name="d", + selector=v1alpha1.Selector(matchLabels={"modelplane.ai/deployment": "d"}), + ) + ] + ) + ), + required_resources={ + "gateway": fnv1.Resources( + items=[ + _inference_gateway( + tls=False, address="203.0.113.1", client_ca_published=True, gateway_ready=False + ) + ] + ), + "clusters": fnv1.Resources(items=[_cluster(gateway_ca_published=True)]), + "endpoints-d": fnv1.Resources(items=[_composed_endpoint(model=None)]), + }, + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State(composite=_desired_model_route(address=None, total_endpoints=0, ready_endpoints=0)), + results=[fnv1.Result(severity=fnv1.SEVERITY_NORMAL, message="Waiting for Envoy Deployment rollout")], + context=structpb.Struct(), + requirements=fnv1.Requirements( + resources={ + "gateway": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", kind="InferenceGateway", match_name="eu" + ), + "clusters": fnv1.ResourceSelector(api_version="modelplane.ai/v1alpha1", kind="InferenceCluster"), + "endpoints-d": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", + kind="ModelEndpoint", + namespace="ml-team", + match_labels=fnv1.MatchLabels(labels={"modelplane.ai/deployment": "d"}), + ), + } + ), + conditions=[ + fnv1.Condition( + type="RoutingReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="WaitingForGateway", + message="Waiting for Envoy Deployment rollout", + ) + ], + ), + ), Case( name="ClientCAUnpublished", reason="A ModelRoute whose gateway hasn't published its client CA composes nothing, because no backend could name its client certificate.", @@ -546,7 +610,11 @@ def _to_dict(msg: message.Message) -> dict: ), required_resources={ "gateway": fnv1.Resources( - items=[_inference_gateway(tls=False, address="203.0.113.1", client_ca_published=False)] + items=[ + _inference_gateway( + tls=False, address="203.0.113.1", client_ca_published=False, gateway_ready=True + ) + ] ), "clusters": fnv1.Resources(items=[_cluster(gateway_ca_published=True)]), "endpoints-d": fnv1.Resources(items=[_composed_endpoint(model=None)]), @@ -604,7 +672,11 @@ def _to_dict(msg: message.Message) -> dict: ), required_resources={ "gateway": fnv1.Resources( - items=[_inference_gateway(tls=False, address="203.0.113.1", client_ca_published=True)] + items=[ + _inference_gateway( + tls=False, address="203.0.113.1", client_ca_published=True, gateway_ready=True + ) + ] ), "clusters": fnv1.Resources(items=[_cluster(gateway_ca_published=True)]), "endpoints-d": fnv1.Resources( @@ -673,7 +745,11 @@ def _to_dict(msg: message.Message) -> dict: ), required_resources={ "gateway": fnv1.Resources( - items=[_inference_gateway(tls=False, address="203.0.113.1", client_ca_published=True)] + items=[ + _inference_gateway( + tls=False, address="203.0.113.1", client_ca_published=True, gateway_ready=True + ) + ] ), "clusters": fnv1.Resources(items=[_cluster(gateway_ca_published=False)]), "endpoints-d": fnv1.Resources(items=[_composed_endpoint(model=None)]), @@ -735,7 +811,11 @@ def _to_dict(msg: message.Message) -> dict: ), required_resources={ "gateway": fnv1.Resources( - items=[_inference_gateway(tls=False, address="203.0.113.1", client_ca_published=True)] + items=[ + _inference_gateway( + tls=False, address="203.0.113.1", client_ca_published=True, gateway_ready=True + ) + ] ), "clusters": fnv1.Resources(items=[_cluster(gateway_ca_published=True)]), "endpoints-a": fnv1.Resources( @@ -818,7 +898,11 @@ def _to_dict(msg: message.Message) -> dict: ), required_resources={ "gateway": fnv1.Resources( - items=[_inference_gateway(tls=False, address="203.0.113.1", client_ca_published=True)] + items=[ + _inference_gateway( + tls=False, address="203.0.113.1", client_ca_published=True, gateway_ready=True + ) + ] ), "clusters": fnv1.Resources(items=[_cluster(gateway_ca_published=True)]), "endpoints-d": fnv1.Resources(items=[_composed_endpoint(model="d")]), @@ -929,7 +1013,11 @@ def _to_dict(msg: message.Message) -> dict: ), required_resources={ "gateway": fnv1.Resources( - items=[_inference_gateway(tls=False, address="203.0.113.1", client_ca_published=True)] + items=[ + _inference_gateway( + tls=False, address="203.0.113.1", client_ca_published=True, gateway_ready=True + ) + ] ), "clusters": fnv1.Resources(items=[_cluster(gateway_ca_published=True)]), "endpoints-d": fnv1.Resources(items=[_composed_endpoint(model=None)]), @@ -996,7 +1084,11 @@ def _to_dict(msg: message.Message) -> dict: ), required_resources={ "gateway": fnv1.Resources( - items=[_inference_gateway(tls=True, address="203.0.113.1", client_ca_published=True)] + items=[ + _inference_gateway( + tls=True, address="203.0.113.1", client_ca_published=True, gateway_ready=True + ) + ] ), "clusters": fnv1.Resources(items=[_cluster(gateway_ca_published=True)]), "endpoints-d": fnv1.Resources(items=[_composed_endpoint(model=None)]), @@ -1064,7 +1156,11 @@ def _to_dict(msg: message.Message) -> dict: ), required_resources={ "gateway": fnv1.Resources( - items=[_inference_gateway(tls=False, address="203.0.113.9", client_ca_published=True)] + items=[ + _inference_gateway( + tls=False, address="203.0.113.9", client_ca_published=True, gateway_ready=True + ) + ] ), "clusters": fnv1.Resources(items=[_cluster(gateway_ca_published=True)]), "endpoints-d": fnv1.Resources(items=[_composed_endpoint(model=None)]), @@ -1137,7 +1233,11 @@ def _to_dict(msg: message.Message) -> dict: ), required_resources={ "gateway": fnv1.Resources( - items=[_inference_gateway(tls=False, address="203.0.113.1", client_ca_published=True)] + items=[ + _inference_gateway( + tls=False, address="203.0.113.1", client_ca_published=True, gateway_ready=True + ) + ] ), "clusters": fnv1.Resources(items=[_cluster(gateway_ca_published=True)]), "endpoints-canary": fnv1.Resources( @@ -1243,7 +1343,11 @@ def _to_dict(msg: message.Message) -> dict: ), required_resources={ "gateway": fnv1.Resources( - items=[_inference_gateway(tls=False, address="203.0.113.1", client_ca_published=True)] + items=[ + _inference_gateway( + tls=False, address="203.0.113.1", client_ca_published=True, gateway_ready=True + ) + ] ), "clusters": fnv1.Resources(items=[_cluster(gateway_ca_published=True)]), "endpoints-a": fnv1.Resources( @@ -1368,7 +1472,11 @@ def _to_dict(msg: message.Message) -> dict: ), required_resources={ "gateway": fnv1.Resources( - items=[_inference_gateway(tls=False, address="203.0.113.1", client_ca_published=True)] + items=[ + _inference_gateway( + tls=False, address="203.0.113.1", client_ca_published=True, gateway_ready=True + ) + ] ), "clusters": fnv1.Resources(items=[_cluster(gateway_ca_published=True)]), "endpoints-big": fnv1.Resources( @@ -1498,7 +1606,11 @@ def _to_dict(msg: message.Message) -> dict: ), required_resources={ "gateway": fnv1.Resources( - items=[_inference_gateway(tls=False, address="203.0.113.1", client_ca_published=True)] + items=[ + _inference_gateway( + tls=False, address="203.0.113.1", client_ca_published=True, gateway_ready=True + ) + ] ), "clusters": fnv1.Resources(items=[_cluster(gateway_ca_published=True)]), "endpoints-many": fnv1.Resources( @@ -1639,7 +1751,11 @@ def _to_dict(msg: message.Message) -> dict: ), required_resources={ "gateway": fnv1.Resources( - items=[_inference_gateway(tls=False, address="203.0.113.1", client_ca_published=True)] + items=[ + _inference_gateway( + tls=False, address="203.0.113.1", client_ca_published=True, gateway_ready=True + ) + ] ), "clusters": fnv1.Resources(items=[_cluster(gateway_ca_published=True)]), "endpoints-big": fnv1.Resources( @@ -1756,7 +1872,11 @@ def _to_dict(msg: message.Message) -> dict: ), required_resources={ "gateway": fnv1.Resources( - items=[_inference_gateway(tls=False, address="203.0.113.1", client_ca_published=True)] + items=[ + _inference_gateway( + tls=False, address="203.0.113.1", client_ca_published=True, gateway_ready=True + ) + ] ), "clusters": fnv1.Resources(items=[_cluster(gateway_ca_published=True)]), "endpoints-d": fnv1.Resources( @@ -1854,7 +1974,11 @@ def _to_dict(msg: message.Message) -> dict: ), required_resources={ "gateway": fnv1.Resources( - items=[_inference_gateway(tls=False, address="203.0.113.1", client_ca_published=True)] + items=[ + _inference_gateway( + tls=False, address="203.0.113.1", client_ca_published=True, gateway_ready=True + ) + ] ), "clusters": fnv1.Resources(items=[_cluster(gateway_ca_published=True)]), "endpoints-d": fnv1.Resources( @@ -1946,7 +2070,11 @@ def _to_dict(msg: message.Message) -> dict: ), required_resources={ "gateway": fnv1.Resources( - items=[_inference_gateway(tls=False, address="203.0.113.1", client_ca_published=True)] + items=[ + _inference_gateway( + tls=False, address="203.0.113.1", client_ca_published=True, gateway_ready=True + ) + ] ), "clusters": fnv1.Resources(items=[_cluster(gateway_ca_published=True)]), "endpoints-d": fnv1.Resources( From e0ccd095c72942af9d9430516e8caffad2b80abb Mon Sep 17 00:00:00 2001 From: mikemikimike <13286568797@163.com> Date: Wed, 21 Oct 2026 18:19:53 +0800 Subject: [PATCH 3/7] Preserve routes during gateway rollout Signed-off-by: mikemikimike <13286568797@163.com> --- .../compose-inference-gateway/function/fn.py | 7 +- .../tests/test_fn.py | 6 +- functions/compose-model-route/function/fn.py | 17 ++- .../compose-model-route/tests/test_fn.py | 104 +++++++++++++++++- 4 files changed, 124 insertions(+), 10 deletions(-) diff --git a/functions/compose-inference-gateway/function/fn.py b/functions/compose-inference-gateway/function/fn.py index c39fb4e6d..2b5200930 100644 --- a/functions/compose-inference-gateway/function/fn.py +++ b/functions/compose-inference-gateway/function/fn.py @@ -83,7 +83,9 @@ _AI_GATEWAY_ANCHOR_HEADER = "x-modelplane-gateway-anchor" _AI_GATEWAY_ANCHOR_HEADER_VALUE = "true" _ENVOY_GATEWAY_NAMESPACE = "envoy-gateway-system" -_ENVOY_DEPLOYMENT_NAME = "modelplane-inference-gateway-envoy" +# Envoy Gateway's default for modelplane-system/inference-gateway. Keep this +# name so observing the Deployment does not replace the one already serving. +_ENVOY_DEPLOYMENT_NAME = "modelplane-system-inference-gateway-5dbfd6c0" # The Gateway's listener names. When the gateway terminates TLS, inference binds # to the HTTPS listener alone (compose-model-route sets the AIGatewayRoute's @@ -168,6 +170,7 @@ "has(object.spec) && has(object.spec.replicas) && has(object.status) && " "has(object.status.observedGeneration) && object.status.observedGeneration >= object.metadata.generation && " "has(object.status.updatedReplicas) && object.status.updatedReplicas == object.spec.replicas && " + "has(object.status.replicas) && object.status.replicas == object.spec.replicas && " "has(object.status.readyReplicas) && object.status.readyReplicas == object.spec.replicas && " "has(object.status.availableReplicas) && object.status.availableReplicas == object.spec.replicas && " "has(object.status.conditions) && " @@ -585,7 +588,6 @@ def compose_envoy_proxy(self) -> None: "kubernetes": { "envoyService": {"externalTrafficPolicy": "Cluster"}, "envoyDeployment": { - "name": _ENVOY_DEPLOYMENT_NAME, "replicas": _PROXY_REPLICAS, "patch": {"type": "StrategicMerge", "value": _NDOTS_PATCH}, "pod": { @@ -634,6 +636,7 @@ def compose_envoy_deployment(self) -> None: ready_when=_ENVOY_DEPLOYMENT_ROLLOUT_READY_CEL, ) obj.spec.managementPolicies = ["Observe"] + obj.spec.watch = True resource.update(self.rsp.desired.resources["envoy-deployment"], obj) def compose_gateway(self) -> None: diff --git a/functions/compose-inference-gateway/tests/test_fn.py b/functions/compose-inference-gateway/tests/test_fn.py index 8cf4072c8..01ea8abdc 100644 --- a/functions/compose-inference-gateway/tests/test_fn.py +++ b/functions/compose-inference-gateway/tests/test_fn.py @@ -255,7 +255,6 @@ def _envoy_proxy() -> fnv1.Resource: "kubernetes": { "envoyService": {"externalTrafficPolicy": "Cluster"}, "envoyDeployment": { - "name": "modelplane-inference-gateway-envoy", "replicas": 2, "patch": { "type": "StrategicMerge", @@ -359,6 +358,7 @@ def _envoy_deployment() -> fnv1.Resource: "metadata": {"namespace": "modelplane-system"}, "spec": { "providerConfigRef": {"kind": "ClusterProviderConfig", "name": "gw-eu-cluster-kubeconfig"}, + "watch": True, "readiness": { "policy": "DeriveFromCelQuery", "celQuery": ( @@ -368,6 +368,8 @@ def _envoy_deployment() -> fnv1.Resource: "object.status.observedGeneration >= object.metadata.generation && " "has(object.status.updatedReplicas) && " "object.status.updatedReplicas == object.spec.replicas && " + "has(object.status.replicas) && " + "object.status.replicas == object.spec.replicas && " "has(object.status.readyReplicas) && " "object.status.readyReplicas == object.spec.replicas && " "has(object.status.availableReplicas) && " @@ -382,7 +384,7 @@ def _envoy_deployment() -> fnv1.Resource: "apiVersion": "apps/v1", "kind": "Deployment", "metadata": { - "name": "modelplane-inference-gateway-envoy", + "name": "modelplane-system-inference-gateway-5dbfd6c0", "namespace": "envoy-gateway-system", }, } diff --git a/functions/compose-model-route/function/fn.py b/functions/compose-model-route/function/fn.py index fd3cf6b43..ad484361e 100644 --- a/functions/compose-model-route/function/fn.py +++ b/functions/compose-model-route/function/fn.py @@ -264,6 +264,8 @@ def __init__(self, req: fnv1.RunFunctionRequest, rsp: fnv1.RunFunctionResponse) self.provider_config = "" self.address: str | None = None self.inference_listener = _LISTENER_HTTP + self.gateway_ready = False + self.gateway_not_ready_message = "" # Endpoints per priority, as (entry weight, endpoints) so weights can be # distributed within a tier. self.tiers: dict[int, list[tuple[int, list[mev1alpha1.ModelEndpoint]]]] = {} @@ -373,14 +375,18 @@ def resolve_gateway(self) -> bool: (condition for condition in gateway_conditions or [] if condition.type == "GatewayReady"), None, ) - if not gateway_ready or gateway_ready.status != "True": - message = ( + self.gateway_ready = gateway_ready is not None and gateway_ready.status == "True" + if not self.gateway_ready: + self.gateway_not_ready_message = ( gateway_ready.message if gateway_ready and gateway_ready.message else f"InferenceGateway {self.xr.spec.gatewayName} is not ready" ) - self.not_ready(CONDITION_REASON_WAITING_FOR_GATEWAY, message) - return False + # Do not create a route before its gateway is serving, but keep an + # existing route in desired state during a gateway rollout. + if "route" not in self.req.observed.resources: + self.not_ready(CONDITION_REASON_WAITING_FOR_GATEWAY, self.gateway_not_ready_message) + return False if not (gw.status and gw.status.clientCACertificate): self.not_ready( @@ -885,6 +891,9 @@ def not_ready(self, reason: str, message: str) -> None: def derive_conditions(self) -> None: """RoutingReady once the composed route has been accepted.""" + if not self.gateway_ready: + self.not_ready(CONDITION_REASON_WAITING_FOR_GATEWAY, self.gateway_not_ready_message) + return if resource.get_condition(self.req.observed.resources.get("route"), "Ready").status != "True": self.not_ready( CONDITION_REASON_WAITING_FOR_ROUTE, diff --git a/functions/compose-model-route/tests/test_fn.py b/functions/compose-model-route/tests/test_fn.py index 83c666a74..02d352e6a 100644 --- a/functions/compose-model-route/tests/test_fn.py +++ b/functions/compose-model-route/tests/test_fn.py @@ -451,9 +451,10 @@ def _client_certificate() -> fnv1.Resource: ) -def _ai_gateway_route(*, section_name: str, backend_refs: list[dict]) -> fnv1.Resource: +def _ai_gateway_route(*, section_name: str, backend_refs: list[dict], ready: fnv1.Ready) -> fnv1.Resource: """The composed AIGatewayRoute matching ml-team/assistant, bound to the gateway's section_name listener.""" return fnv1.Resource( + ready=ready, resource=resource.dict_to_struct( { "apiVersion": "kubernetes.m.crossplane.io/v1alpha1", @@ -518,7 +519,7 @@ def _ai_gateway_route(*, section_name: str, backend_refs: list[dict]) -> fnv1.Re }, }, } - ) + ), ) @@ -947,6 +948,7 @@ def _to_dict(msg: message.Message) -> dict: "cluster-ca-gw-eu": _cluster_ca(), "client-certificate": _client_certificate(), "route": _ai_gateway_route( + ready=fnv1.READY_UNSPECIFIED, section_name="http", backend_refs=[ {"name": "assistant-eu-self-a42e6", "weight": 1, "priority": 0, "modelNameOverride": "d"}, @@ -1033,6 +1035,7 @@ def _to_dict(msg: message.Message) -> dict: "cluster-ca-gw-eu": _cluster_ca(), "client-certificate": _client_certificate(), "route": _ai_gateway_route( + ready=fnv1.READY_UNSPECIFIED, section_name="http", backend_refs=[{"name": "assistant-eu-self-a42e6", "weight": 1, "priority": 0}], ), @@ -1066,6 +1069,93 @@ def _to_dict(msg: message.Message) -> dict: ], ), ), + Case( + name="GatewayRollout", + reason="With an existing route, an unready gateway keeps the route desired and reports it not ready.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_model_route( + endpoints=[ + v1alpha1.Endpoint( + name="d", + selector=v1alpha1.Selector(matchLabels={"modelplane.ai/deployment": "d"}), + ) + ] + ), + resources={ + "route": fnv1.Resource( + resource=resource.dict_to_struct( + { + "status": { + "conditions": [ + { + "type": "Ready", + "status": "True", + "reason": "Accepted", + "lastTransitionTime": "2026-06-08T00:00:00Z", + } + ] + } + } + ), + ready=fnv1.READY_TRUE, + ) + }, + ), + required_resources={ + "gateway": fnv1.Resources( + items=[ + _inference_gateway( + tls=False, address="203.0.113.1", client_ca_published=True, gateway_ready=False + ) + ] + ), + "clusters": fnv1.Resources(items=[_cluster(gateway_ca_published=True)]), + "endpoints-d": fnv1.Resources(items=[_composed_endpoint(model=None)]), + }, + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_model_route(address="203.0.113.1", total_endpoints=1, ready_endpoints=1), + resources={ + "backend-self": _composed_endpoint_backend(), + "aibackend-self": _composed_endpoint_ai_backend(), + "cluster-ca-gw-eu": _cluster_ca(), + "client-certificate": _client_certificate(), + "route": _ai_gateway_route( + ready=fnv1.READY_TRUE, + section_name="http", + backend_refs=[{"name": "assistant-eu-self-a42e6", "weight": 1, "priority": 0}], + ), + }, + ), + results=[fnv1.Result(severity=fnv1.SEVERITY_NORMAL, message="Waiting for Envoy Deployment rollout")], + context=structpb.Struct(), + requirements=fnv1.Requirements( + resources={ + "gateway": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", kind="InferenceGateway", match_name="eu" + ), + "clusters": fnv1.ResourceSelector(api_version="modelplane.ai/v1alpha1", kind="InferenceCluster"), + "endpoints-d": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", + kind="ModelEndpoint", + namespace="ml-team", + match_labels=fnv1.MatchLabels(labels={"modelplane.ai/deployment": "d"}), + ), + } + ), + conditions=[ + fnv1.Condition( + type="RoutingReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="WaitingForGateway", + message="Waiting for Envoy Deployment rollout", + ) + ], + ), + ), # A TLS gateway serves inference on its HTTPS listener alone. Binding to :80 # on a TLS gateway would carry credentials in the clear. Case( @@ -1104,6 +1194,7 @@ def _to_dict(msg: message.Message) -> dict: "cluster-ca-gw-eu": _cluster_ca(), "client-certificate": _client_certificate(), "route": _ai_gateway_route( + ready=fnv1.READY_UNSPECIFIED, section_name="https", backend_refs=[{"name": "assistant-eu-self-a42e6", "weight": 1, "priority": 0}], ), @@ -1176,6 +1267,7 @@ def _to_dict(msg: message.Message) -> dict: "cluster-ca-gw-eu": _cluster_ca(), "client-certificate": _client_certificate(), "route": _ai_gateway_route( + ready=fnv1.READY_UNSPECIFIED, section_name="http", backend_refs=[{"name": "assistant-eu-self-a42e6", "weight": 1, "priority": 0}], ), @@ -1274,6 +1366,7 @@ def _to_dict(msg: message.Message) -> dict: "backend-kimi-a": _third_party_backend(name="assistant-eu-kimi-a-bf6de", hostname="a.example.com"), "aibackend-kimi-a": _third_party_ai_backend(name="assistant-eu-kimi-a-bf6de", schema="OpenAI"), "route": _ai_gateway_route( + ready=fnv1.READY_UNSPECIFIED, section_name="http", backend_refs=[{"name": "assistant-eu-kimi-a-bf6de", "weight": 1, "priority": 0}], ), @@ -1398,6 +1491,7 @@ def _to_dict(msg: message.Message) -> dict: "backend-c-0": _third_party_backend(name="assistant-eu-c-0-782db", hostname="c.example.com"), "aibackend-c-0": _third_party_ai_backend(name="assistant-eu-c-0-782db", schema="OpenAI"), "route": _ai_gateway_route( + ready=fnv1.READY_UNSPECIFIED, section_name="http", backend_refs=[ {"name": "assistant-eu-a-0-b66b8", "weight": 1, "priority": 0}, @@ -1543,6 +1637,7 @@ def _to_dict(msg: message.Message) -> dict: ), "aibackend-small-0": _third_party_ai_backend(name="assistant-eu-small-0-60d20", schema="OpenAI"), "route": _ai_gateway_route( + ready=fnv1.READY_UNSPECIFIED, section_name="http", backend_refs=[ {"name": "assistant-eu-big-0-1e944", "weight": 3, "priority": 0}, @@ -1685,6 +1780,7 @@ def _to_dict(msg: message.Message) -> dict: ), "aibackend-many-4": _third_party_ai_backend(name="assistant-eu-many-4-bd6f9", schema="OpenAI"), "route": _ai_gateway_route( + ready=fnv1.READY_UNSPECIFIED, section_name="http", backend_refs=[ {"name": "assistant-eu-many-0-e4e60", "weight": 1, "priority": 0}, @@ -1810,6 +1906,7 @@ def _to_dict(msg: message.Message) -> dict: ), "aibackend-small-1": _third_party_ai_backend(name="assistant-eu-small-1-cad94", schema="OpenAI"), "route": _ai_gateway_route( + ready=fnv1.READY_UNSPECIFIED, section_name="http", backend_refs=[ {"name": "assistant-eu-big-0-1e944", "weight": 1000000, "priority": 0}, @@ -1921,6 +2018,7 @@ def _to_dict(msg: message.Message) -> dict: "backend-z": _third_party_backend(name="assistant-eu-z-e5d6f", hostname="z.example.com"), "aibackend-z": _third_party_ai_backend(name="assistant-eu-z-e5d6f", schema="OpenAI"), "route": _ai_gateway_route( + ready=fnv1.READY_UNSPECIFIED, section_name="http", backend_refs=[ {"name": "assistant-eu-a-18a56", "weight": 1, "priority": 0}, @@ -2018,6 +2116,7 @@ def _to_dict(msg: message.Message) -> dict: }, ), "route": _ai_gateway_route( + ready=fnv1.READY_UNSPECIFIED, section_name="http", backend_refs=[{"name": "assistant-eu-provider-348cb", "weight": 1, "priority": 0}], ), @@ -2116,6 +2215,7 @@ def _to_dict(msg: message.Message) -> dict: }, ), "route": _ai_gateway_route( + ready=fnv1.READY_UNSPECIFIED, section_name="http", backend_refs=[{"name": "assistant-eu-provider-348cb", "weight": 1, "priority": 0}], ), From 544d32396c51dfeeb883509ef8ab70ff83bee277 Mon Sep 17 00:00:00 2001 From: mikemikimike <13286568797@163.com> Date: Thu, 22 Oct 2026 21:55:47 +0800 Subject: [PATCH 4/7] Observe the Envoy Gateway Deployment name GatewayReady observes the Deployment emitted by Envoy Gateway. Its generated name includes an envoy- prefix, so the previous name never resolved and readiness stayed false. Match the generated name while keeping the existing Deployment. Signed-off-by: mikemikimike <13286568797@163.com> --- functions/compose-inference-gateway/function/fn.py | 7 ++++--- functions/compose-inference-gateway/tests/test_fn.py | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/functions/compose-inference-gateway/function/fn.py b/functions/compose-inference-gateway/function/fn.py index 2b5200930..71c350581 100644 --- a/functions/compose-inference-gateway/function/fn.py +++ b/functions/compose-inference-gateway/function/fn.py @@ -83,9 +83,10 @@ _AI_GATEWAY_ANCHOR_HEADER = "x-modelplane-gateway-anchor" _AI_GATEWAY_ANCHOR_HEADER_VALUE = "true" _ENVOY_GATEWAY_NAMESPACE = "envoy-gateway-system" -# Envoy Gateway's default for modelplane-system/inference-gateway. Keep this -# name so observing the Deployment does not replace the one already serving. -_ENVOY_DEPLOYMENT_NAME = "modelplane-system-inference-gateway-5dbfd6c0" +# Envoy Gateway's default, `envoy---`, for +# modelplane-system/inference-gateway. Keep this name so observing the +# Deployment does not replace the one already serving. +_ENVOY_DEPLOYMENT_NAME = "envoy-modelplane-system-inference-gateway-5dbfd6c0" # The Gateway's listener names. When the gateway terminates TLS, inference binds # to the HTTPS listener alone (compose-model-route sets the AIGatewayRoute's diff --git a/functions/compose-inference-gateway/tests/test_fn.py b/functions/compose-inference-gateway/tests/test_fn.py index 01ea8abdc..b8b8421a7 100644 --- a/functions/compose-inference-gateway/tests/test_fn.py +++ b/functions/compose-inference-gateway/tests/test_fn.py @@ -384,7 +384,7 @@ def _envoy_deployment() -> fnv1.Resource: "apiVersion": "apps/v1", "kind": "Deployment", "metadata": { - "name": "modelplane-system-inference-gateway-5dbfd6c0", + "name": "envoy-modelplane-system-inference-gateway-5dbfd6c0", "namespace": "envoy-gateway-system", }, } From 978b2f2b32cfe6c3a7dfac2c3feaf5f156fc6ffb Mon Sep 17 00:00:00 2001 From: mikemikimike <13286568797@163.com> Date: Tue, 27 Oct 2026 11:20:29 +0800 Subject: [PATCH 5/7] Preserve model routes while gateways roll out Gateway readiness can fall while one Envoy pod is unavailable, and waiting for readiness before composing a model route delays its creation. Keep composing routes while reporting their readiness separately. Pin the Envoy Deployment name for observation. Existing gateways will replace their generated Deployment. Wait for the AI Gateway's ext-proc template annotation and for all replicas to be updated, while allowing one available replica to serve traffic. Towards #531. Signed-off-by: mikemikimike <13286568797@163.com> --- .../compose-inference-gateway/function/fn.py | 21 ++- .../tests/test_fn.py | 169 +++++++++++++++++- functions/compose-model-route/function/fn.py | 5 - .../compose-model-route/tests/test_fn.py | 17 +- 4 files changed, 189 insertions(+), 23 deletions(-) diff --git a/functions/compose-inference-gateway/function/fn.py b/functions/compose-inference-gateway/function/fn.py index 71c350581..d161984d7 100644 --- a/functions/compose-inference-gateway/function/fn.py +++ b/functions/compose-inference-gateway/function/fn.py @@ -83,10 +83,9 @@ _AI_GATEWAY_ANCHOR_HEADER = "x-modelplane-gateway-anchor" _AI_GATEWAY_ANCHOR_HEADER_VALUE = "true" _ENVOY_GATEWAY_NAMESPACE = "envoy-gateway-system" -# Envoy Gateway's default, `envoy---`, for -# modelplane-system/inference-gateway. Keep this name so observing the -# Deployment does not replace the one already serving. -_ENVOY_DEPLOYMENT_NAME = "envoy-modelplane-system-inference-gateway-5dbfd6c0" +# Pin the name so readiness can observe the Deployment without depending on +# Envoy Gateway's generated hash. +_ENVOY_DEPLOYMENT_NAME = "envoy-modelplane-system-inference-gateway" # The Gateway's listener names. When the gateway terminates TLS, inference binds # to the HTTPS listener alone (compose-model-route sets the AIGatewayRoute's @@ -166,16 +165,19 @@ "object.status.conditions.exists(c, c.type == 'Accepted' && c.status == 'True')" ) +# The AI Gateway stamps the pod template when it has processed an active route. +# Once that rollout has no old pods, one available replica can serve traffic. _ENVOY_DEPLOYMENT_ROLLOUT_READY_CEL = ( "has(object.metadata) && has(object.metadata.generation) && " - "has(object.spec) && has(object.spec.replicas) && has(object.status) && " + "has(object.spec) && has(object.spec.replicas) && has(object.spec.template) && " + "has(object.spec.template.metadata) && has(object.spec.template.metadata.annotations) && " + "'aigateway.envoyproxy.io/extproc-config-hash' in object.spec.template.metadata.annotations && " + "object.spec.template.metadata.annotations['aigateway.envoyproxy.io/extproc-config-hash'] != '' && " + "has(object.status) && " "has(object.status.observedGeneration) && object.status.observedGeneration >= object.metadata.generation && " "has(object.status.updatedReplicas) && object.status.updatedReplicas == object.spec.replicas && " "has(object.status.replicas) && object.status.replicas == object.spec.replicas && " - "has(object.status.readyReplicas) && object.status.readyReplicas == object.spec.replicas && " - "has(object.status.availableReplicas) && object.status.availableReplicas == object.spec.replicas && " - "has(object.status.conditions) && " - "object.status.conditions.exists(c, c.type == 'Available' && c.status == 'True')" + "has(object.status.availableReplicas) && object.status.availableReplicas > 0" ) # Kubernetes injects ndots:5 into every pod, which tells the resolver to try each @@ -589,6 +591,7 @@ def compose_envoy_proxy(self) -> None: "kubernetes": { "envoyService": {"externalTrafficPolicy": "Cluster"}, "envoyDeployment": { + "name": _ENVOY_DEPLOYMENT_NAME, "replicas": _PROXY_REPLICAS, "patch": {"type": "StrategicMerge", "value": _NDOTS_PATCH}, "pod": { diff --git a/functions/compose-inference-gateway/tests/test_fn.py b/functions/compose-inference-gateway/tests/test_fn.py index b8b8421a7..8c9577da9 100644 --- a/functions/compose-inference-gateway/tests/test_fn.py +++ b/functions/compose-inference-gateway/tests/test_fn.py @@ -255,6 +255,7 @@ def _envoy_proxy() -> fnv1.Resource: "kubernetes": { "envoyService": {"externalTrafficPolicy": "Cluster"}, "envoyDeployment": { + "name": "envoy-modelplane-system-inference-gateway", "replicas": 2, "patch": { "type": "StrategicMerge", @@ -363,19 +364,19 @@ def _envoy_deployment() -> fnv1.Resource: "policy": "DeriveFromCelQuery", "celQuery": ( "has(object.metadata) && has(object.metadata.generation) && " - "has(object.spec) && has(object.spec.replicas) && has(object.status) && " + "has(object.spec) && has(object.spec.replicas) && has(object.spec.template) && " + "has(object.spec.template.metadata) && has(object.spec.template.metadata.annotations) && " + "'aigateway.envoyproxy.io/extproc-config-hash' in object.spec.template.metadata.annotations && " + "object.spec.template.metadata.annotations['aigateway.envoyproxy.io/extproc-config-hash'] != '' && " + "has(object.status) && " "has(object.status.observedGeneration) && " "object.status.observedGeneration >= object.metadata.generation && " "has(object.status.updatedReplicas) && " "object.status.updatedReplicas == object.spec.replicas && " "has(object.status.replicas) && " "object.status.replicas == object.spec.replicas && " - "has(object.status.readyReplicas) && " - "object.status.readyReplicas == object.spec.replicas && " "has(object.status.availableReplicas) && " - "object.status.availableReplicas == object.spec.replicas && " - "has(object.status.conditions) && " - "object.status.conditions.exists(c, c.type == 'Available' && c.status == 'True')" + "object.status.availableReplicas > 0" ), }, "managementPolicies": ["Observe"], @@ -384,7 +385,7 @@ def _envoy_deployment() -> fnv1.Resource: "apiVersion": "apps/v1", "kind": "Deployment", "metadata": { - "name": "envoy-modelplane-system-inference-gateway-5dbfd6c0", + "name": "envoy-modelplane-system-inference-gateway", "namespace": "envoy-gateway-system", }, } @@ -3216,6 +3217,160 @@ def _to_dict(msg: message.Message) -> dict: ], ), ), + Case( + name="TLSAuthGatewayReady", + reason=( + "A TLS and auth gateway becomes Ready after its policy, route, Gateway, and Envoy Deployment are ready, " + "without observing a Secret." + ), + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr(name="eu", tls=True, caller_secret_labels={"team": "ml"}), + resources={ + "gateway": _observed_gateway(address="34.56.129.3", ready=True), + "caller-auth": _observed_caller_auth(), + "ai-gateway-anchor": _observed_ai_gateway_anchor(), + "envoy-deployment": _observed_envoy_deployment(ready=True), + }, + ), + required_resources={ + "clusters": fnv1.Resources(items=[_cluster(provider_config_ref=True)]), + "gateways": fnv1.Resources(items=[_inference_gateway(name="eu", address=None)]), + "caller-secrets": fnv1.Resources( + items=[_caller_key_secret(name="ml-team-keys", data={"alice": "a2V5"})] + ), + "tls-secret-0": fnv1.Resources( + items=[ + fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "v1", + "kind": "Secret", + "metadata": {"name": "eu-tls-0", "namespace": "modelplane-system"}, + "data": {"tls.crt": "Y2VydA==", "tls.key": "a2V5"}, + } + ) + ) + ] + ), + }, + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_xr(status={"address": "34.56.129.3"}, ready=fnv1.READY_UNSPECIFIED), + resources={ + "caller-secret-ml-team-keys": _caller_secret(name="callers-ml-team-keys", data={"alice": "a2V5"}), + "tls-secret-eu-tls-0": fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "kubernetes.m.crossplane.io/v1alpha1", + "kind": "Object", + "metadata": {"namespace": "modelplane-system"}, + "spec": { + "providerConfigRef": { + "kind": "ClusterProviderConfig", + "name": "gw-eu-cluster-kubeconfig", + }, + "readiness": {"policy": "SuccessfulCreate"}, + "forProvider": { + "manifest": { + "apiVersion": "v1", + "kind": "Secret", + "metadata": {"name": "eu-tls-0", "namespace": "modelplane-system"}, + "type": "kubernetes.io/tls", + "data": {"tls.crt": "Y2VydA==", "tls.key": "a2V5"}, + } + }, + }, + } + ) + ), + "envoy-proxy": _envoy_proxy(), + "envoy-deployment": _ready_envoy_deployment(), + "gateway": _gateway(https=True, ready=fnv1.READY_TRUE), + "client-selfsigned-issuer": _client_selfsigned_issuer(), + "client-ca-certificate": _client_ca_certificate(common_name="Modelplane InferenceGateway CA eu"), + "client-ca-issuer": _client_ca_issuer(), + "client-ca-bundle": _client_ca_bundle(), + "client-ca-configmap": _client_ca_configmap(), + "caller-auth": _caller_auth( + credential_refs=[{"name": "callers-ml-team-keys"}], ready=fnv1.READY_TRUE + ), + "failover-policy": _failover_policy(), + "client-traffic-policy": _client_traffic_policy(), + "healthz-filter": _healthz_filter(), + "healthz-route": _healthz_route(), + "ai-gateway-anchor": _ai_gateway_anchor(https=True, ready=fnv1.READY_TRUE), + "healthz-auth": _healthz_auth(), + "redirect-route": _redirect_route(), + "redirect-auth": fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "kubernetes.m.crossplane.io/v1alpha1", + "kind": "Object", + "metadata": {"namespace": "modelplane-system"}, + "spec": { + "providerConfigRef": { + "kind": "ClusterProviderConfig", + "name": "gw-eu-cluster-kubeconfig", + }, + "readiness": { + "policy": "DeriveFromCelQuery", + "celQuery": ( + "has(object.status) && has(object.status.ancestors) && " + "object.status.ancestors.exists(a, has(a.conditions) && " + "a.conditions.exists(c, c.type == 'Accepted' && c.status == 'True'))" + ), + }, + "forProvider": { + "manifest": { + "apiVersion": "gateway.envoyproxy.io/v1alpha1", + "kind": "SecurityPolicy", + "metadata": { + "name": "inference-gateway-redirect-open", + "namespace": "modelplane-system", + }, + "spec": { + "targetRefs": [ + { + "group": "gateway.networking.k8s.io", + "kind": "HTTPRoute", + "name": "inference-gateway-redirect", + } + ], + "authorization": {"defaultAction": "Allow"}, + }, + } + }, + }, + } + ) + ), + }, + ), + results=[], + context=structpb.Struct(), + requirements=fnv1.Requirements( + resources={ + "gateways": fnv1.ResourceSelector(api_version="modelplane.ai/v1alpha1", kind="InferenceGateway"), + "clusters": fnv1.ResourceSelector(api_version="modelplane.ai/v1alpha1", kind="InferenceCluster"), + "caller-secrets": fnv1.ResourceSelector( + api_version="v1", + kind="Secret", + namespace="modelplane-system", + match_labels=fnv1.MatchLabels(labels={"team": "ml"}), + ), + "tls-secret-0": fnv1.ResourceSelector( + api_version="v1", kind="Secret", namespace="modelplane-system", match_name="eu-tls-0" + ), + } + ), + conditions=[ + fnv1.Condition(type="GatewayReady", status=fnv1.STATUS_CONDITION_TRUE, reason="GatewayProgrammed") + ], + ), + ), ] diff --git a/functions/compose-model-route/function/fn.py b/functions/compose-model-route/function/fn.py index ad484361e..23397d5b6 100644 --- a/functions/compose-model-route/function/fn.py +++ b/functions/compose-model-route/function/fn.py @@ -382,11 +382,6 @@ def resolve_gateway(self) -> bool: if gateway_ready and gateway_ready.message else f"InferenceGateway {self.xr.spec.gatewayName} is not ready" ) - # Do not create a route before its gateway is serving, but keep an - # existing route in desired state during a gateway rollout. - if "route" not in self.req.observed.resources: - self.not_ready(CONDITION_REASON_WAITING_FOR_GATEWAY, self.gateway_not_ready_message) - return False if not (gw.status and gw.status.clientCACertificate): self.not_ready( diff --git a/functions/compose-model-route/tests/test_fn.py b/functions/compose-model-route/tests/test_fn.py index 02d352e6a..004bb51d5 100644 --- a/functions/compose-model-route/tests/test_fn.py +++ b/functions/compose-model-route/tests/test_fn.py @@ -542,7 +542,7 @@ def _to_dict(msg: message.Message) -> dict: COMPOSE_CASES = [ Case( name="GatewayNotReady", - reason="A ModelRoute waits for its InferenceGateway rollout before composing routes or backends.", + reason="A ModelRoute composes a new route during an InferenceGateway rollout and reports it not ready.", req=fnv1.RunFunctionRequest( observed=fnv1.State( composite=_model_route( @@ -568,7 +568,20 @@ def _to_dict(msg: message.Message) -> dict: ), want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), - desired=fnv1.State(composite=_desired_model_route(address=None, total_endpoints=0, ready_endpoints=0)), + desired=fnv1.State( + composite=_desired_model_route(address="203.0.113.1", total_endpoints=1, ready_endpoints=1), + resources={ + "backend-self": _composed_endpoint_backend(), + "aibackend-self": _composed_endpoint_ai_backend(), + "cluster-ca-gw-eu": _cluster_ca(), + "client-certificate": _client_certificate(), + "route": _ai_gateway_route( + ready=fnv1.READY_UNSPECIFIED, + section_name="http", + backend_refs=[{"name": "assistant-eu-self-a42e6", "weight": 1, "priority": 0}], + ), + }, + ), results=[fnv1.Result(severity=fnv1.SEVERITY_NORMAL, message="Waiting for Envoy Deployment rollout")], context=structpb.Struct(), requirements=fnv1.Requirements( From 824e861a9fa0c011702c0c2919aa7fd3248851a9 Mon Sep 17 00:00:00 2001 From: mikemikimike <13286568797@163.com> Date: Tue, 27 Oct 2026 16:39:21 +0800 Subject: [PATCH 6/7] Allow readiness with one available updated replica Signed-off-by: mikemikimike <13286568797@163.com> --- functions/compose-inference-gateway/function/fn.py | 6 +++--- functions/compose-inference-gateway/tests/test_fn.py | 6 ++---- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/functions/compose-inference-gateway/function/fn.py b/functions/compose-inference-gateway/function/fn.py index d161984d7..a9383512b 100644 --- a/functions/compose-inference-gateway/function/fn.py +++ b/functions/compose-inference-gateway/function/fn.py @@ -166,7 +166,7 @@ ) # The AI Gateway stamps the pod template when it has processed an active route. -# Once that rollout has no old pods, one available replica can serve traffic. +# Require every existing pod to be updated, but allow a replica to be unavailable. _ENVOY_DEPLOYMENT_ROLLOUT_READY_CEL = ( "has(object.metadata) && has(object.metadata.generation) && " "has(object.spec) && has(object.spec.replicas) && has(object.spec.template) && " @@ -175,8 +175,8 @@ "object.spec.template.metadata.annotations['aigateway.envoyproxy.io/extproc-config-hash'] != '' && " "has(object.status) && " "has(object.status.observedGeneration) && object.status.observedGeneration >= object.metadata.generation && " - "has(object.status.updatedReplicas) && object.status.updatedReplicas == object.spec.replicas && " - "has(object.status.replicas) && object.status.replicas == object.spec.replicas && " + "has(object.status.updatedReplicas) && has(object.status.replicas) && " + "object.status.updatedReplicas == object.status.replicas && " "has(object.status.availableReplicas) && object.status.availableReplicas > 0" ) diff --git a/functions/compose-inference-gateway/tests/test_fn.py b/functions/compose-inference-gateway/tests/test_fn.py index 8c9577da9..c70e9bc8a 100644 --- a/functions/compose-inference-gateway/tests/test_fn.py +++ b/functions/compose-inference-gateway/tests/test_fn.py @@ -371,10 +371,8 @@ def _envoy_deployment() -> fnv1.Resource: "has(object.status) && " "has(object.status.observedGeneration) && " "object.status.observedGeneration >= object.metadata.generation && " - "has(object.status.updatedReplicas) && " - "object.status.updatedReplicas == object.spec.replicas && " - "has(object.status.replicas) && " - "object.status.replicas == object.spec.replicas && " + "has(object.status.updatedReplicas) && has(object.status.replicas) && " + "object.status.updatedReplicas == object.status.replicas && " "has(object.status.availableReplicas) && " "object.status.availableReplicas > 0" ), From f51ac5da69bfc11917cfd349c4d72787a8861dc1 Mon Sep 17 00:00:00 2001 From: mikemikimike <13286568797@163.com> Date: Wed, 28 Oct 2026 14:17:42 +0800 Subject: [PATCH 7/7] Keep TLS and auth readiness coverage Signed-off-by: mikemikimike <13286568797@163.com> --- functions/compose-inference-gateway/tests/test_fn.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/functions/compose-inference-gateway/tests/test_fn.py b/functions/compose-inference-gateway/tests/test_fn.py index c70e9bc8a..a46982bb9 100644 --- a/functions/compose-inference-gateway/tests/test_fn.py +++ b/functions/compose-inference-gateway/tests/test_fn.py @@ -3216,9 +3216,9 @@ def _to_dict(msg: message.Message) -> dict: ), ), Case( - name="TLSAuthGatewayReady", + name="TLSAndAuthGatewayReady", reason=( - "A TLS and auth gateway becomes Ready after its policy, route, Gateway, and Envoy Deployment are ready, " + "A TLS and auth gateway becomes GatewayReady=True after its policy, route, Gateway, and Envoy Deployment are ready, " "without observing a Secret." ), req=fnv1.RunFunctionRequest(