diff --git a/functions/compose-inference-gateway/function/fn.py b/functions/compose-inference-gateway/function/fn.py index 135d872db..a9383512b 100644 --- a/functions/compose-inference-gateway/function/fn.py +++ b/functions/compose-inference-gateway/function/fn.py @@ -48,6 +48,8 @@ CONDITION_REASON_GATEWAY_PROGRAMMED = "GatewayProgrammed" CONDITION_REASON_WAITING_FOR_CLUSTER = "WaitingForCluster" CONDITION_REASON_WAITING_FOR_GATEWAY = "WaitingForGateway" +CONDITION_REASON_WAITING_FOR_AI_GATEWAY_ROUTE = "WaitingForAIGatewayRoute" +CONDITION_REASON_WAITING_FOR_ENVOY_DEPLOYMENT = "WaitingForEnvoyDeployment" CONDITION_REASON_CLUSTER_TAKEN = "ClusterAlreadyHasGateway" CONDITION_REASON_SECRETS_MISSING = "SecretsMissing" CONDITION_REASON_AUTH_NOT_ACCEPTED = "CallerAuthNotAccepted" @@ -77,6 +79,13 @@ _FAILOVER_NAME = "inference-gateway-failover" _CLIENT_TRAFFIC_NAME = "inference-gateway-client-traffic" _REDIRECT_NAME = "inference-gateway-redirect" +_AI_GATEWAY_ANCHOR_NAME = "inference-gateway-anchor" +_AI_GATEWAY_ANCHOR_HEADER = "x-modelplane-gateway-anchor" +_AI_GATEWAY_ANCHOR_HEADER_VALUE = "true" +_ENVOY_GATEWAY_NAMESPACE = "envoy-gateway-system" +# Pin the name so readiness can observe the Deployment without depending on +# Envoy Gateway's generated hash. +_ENVOY_DEPLOYMENT_NAME = "envoy-modelplane-system-inference-gateway" # The Gateway's listener names. When the gateway terminates TLS, inference binds # to the HTTPS listener alone (compose-model-route sets the AIGatewayRoute's @@ -151,6 +160,26 @@ "a.conditions.exists(c, c.type == 'Accepted' && c.status == 'True'))" ) +_AI_GATEWAY_ROUTE_ACCEPTED_CEL = ( + "has(object.status) && has(object.status.conditions) && " + "object.status.conditions.exists(c, c.type == 'Accepted' && c.status == 'True')" +) + +# The AI Gateway stamps the pod template when it has processed an active route. +# Require every existing pod to be updated, but allow a replica to be unavailable. +_ENVOY_DEPLOYMENT_ROLLOUT_READY_CEL = ( + "has(object.metadata) && has(object.metadata.generation) && " + "has(object.spec) && has(object.spec.replicas) && has(object.spec.template) && " + "has(object.spec.template.metadata) && has(object.spec.template.metadata.annotations) && " + "'aigateway.envoyproxy.io/extproc-config-hash' in object.spec.template.metadata.annotations && " + "object.spec.template.metadata.annotations['aigateway.envoyproxy.io/extproc-config-hash'] != '' && " + "has(object.status) && " + "has(object.status.observedGeneration) && object.status.observedGeneration >= object.metadata.generation && " + "has(object.status.updatedReplicas) && has(object.status.replicas) && " + "object.status.updatedReplicas == object.status.replicas && " + "has(object.status.availableReplicas) && object.status.availableReplicas > 0" +) + # Kubernetes injects ndots:5 into every pod, which tells the resolver to try each # search domain before a name with fewer than five dots. Every hostname this # gateway resolves has fewer: a provider like api.together.xyz has two, a cluster @@ -320,7 +349,9 @@ def compose(self) -> None: self.resolve_secrets() self.compose_secrets() self.compose_envoy_proxy() + self.compose_envoy_deployment() self.compose_gateway() + self.compose_ai_gateway_anchor() self.compose_client_pki() self.compose_cluster_names() self.compose_caller_auth() @@ -560,6 +591,7 @@ def compose_envoy_proxy(self) -> None: "kubernetes": { "envoyService": {"externalTrafficPolicy": "Cluster"}, "envoyDeployment": { + "name": _ENVOY_DEPLOYMENT_NAME, "replicas": _PROXY_REPLICAS, "patch": {"type": "StrategicMerge", "value": _NDOTS_PATCH}, "pod": { @@ -596,6 +628,21 @@ def compose_envoy_proxy(self) -> None: ), ) + def compose_envoy_deployment(self) -> None: + """Observe Envoy's Deployment so readiness includes its full rollout.""" + obj = _k8s_object( + self.pc, + { + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": {"name": _ENVOY_DEPLOYMENT_NAME, "namespace": _ENVOY_GATEWAY_NAMESPACE}, + }, + ready_when=_ENVOY_DEPLOYMENT_ROLLOUT_READY_CEL, + ) + obj.spec.managementPolicies = ["Observe"] + obj.spec.watch = True + resource.update(self.rsp.desired.resources["envoy-deployment"], obj) + def compose_gateway(self) -> None: """The Gateway callers connect to. @@ -667,6 +714,48 @@ def compose_gateway(self) -> None: ), ) + def compose_ai_gateway_anchor(self) -> None: + """Keep AI Gateway's ext-proc installed before any model route exists.""" + listener = _LISTENER_HTTPS if self.xr.spec.tls else _LISTENER_HTTP + resource.update( + self.rsp.desired.resources["ai-gateway-anchor"], + _k8s_object( + self.pc, + { + "apiVersion": "aigateway.envoyproxy.io/v1beta1", + "kind": "AIGatewayRoute", + "metadata": {"name": _AI_GATEWAY_ANCHOR_NAME, "namespace": REMOTE_NAMESPACE}, + "spec": { + "parentRefs": [ + { + "group": "gateway.networking.k8s.io", + "kind": "Gateway", + "name": _GATEWAY_NAME, + "namespace": REMOTE_NAMESPACE, + "sectionName": listener, + } + ], + "rules": [ + { + "matches": [ + { + "headers": [ + { + "type": "Exact", + "name": _AI_GATEWAY_ANCHOR_HEADER, + "value": _AI_GATEWAY_ANCHOR_HEADER_VALUE, + } + ] + } + ] + } + ], + }, + }, + ready_when=_AI_GATEWAY_ROUTE_ACCEPTED_CEL, + ), + ) + def compose_client_pki(self) -> None: """Compose the CA whose certificates a cluster gateway trusts. @@ -1330,16 +1419,16 @@ def shared_caller_key(self) -> tuple[str, str] | None: return None def derive_conditions(self) -> None: - """GatewayReady tracks the Gateway being programmed and, where the XR - asks for auth, its caller policy being accepted. + """GatewayReady tracks the Gateway, Envoy rollout and AI Gateway route, + and, where the XR asks for auth, its caller policy being accepted. Both, because a gateway whose policy was rejected answers every request with a 500 while its Gateway still has an address. A missing input Secret is reported here (see resolve_secrets). The - Gateway's own readiness only tracks having an address, so neither a - fail-closed auth policy nor an unprogrammed HTTPS listener would - otherwise show. + Gateway's own readiness only tracks having an address, so an unaccepted + anchor route, fail-closed auth policy, or unprogrammed HTTPS listener + would otherwise show late or not at all. """ if self.xr.spec.auth and not self.caller_secrets: if "caller-secrets" not in self.req.required_resources: @@ -1366,9 +1455,12 @@ def derive_conditions(self) -> None: "authentication policy and every request is refused", ) return + readiness_keys = ["gateway", "ai-gateway-anchor", "envoy-deployment"] + if self.xr.spec.auth: + readiness_keys.append("caller-auth") waiting = [ key - for key in (["gateway", "caller-auth"] if self.xr.spec.auth else ["gateway"]) + for key in readiness_keys if resource.get_condition(self.req.observed.resources.get(key), "Ready").status != "True" ] if not waiting: @@ -1381,12 +1473,24 @@ def derive_conditions(self) -> None: ), ) return - if waiting == ["caller-auth"]: + if "caller-auth" in waiting and "gateway" not in waiting: self.not_ready( CONDITION_REASON_AUTH_NOT_ACCEPTED, "The gateway's caller authentication policy has not been accepted, so every request is refused", ) return + if ("ai-gateway-anchor" in waiting or "envoy-deployment" in waiting) and "gateway" not in waiting: + if "ai-gateway-anchor" in waiting: + self.not_ready( + CONDITION_REASON_WAITING_FOR_AI_GATEWAY_ROUTE, + f"Waiting for the AI Gateway anchor route on cluster {self.xr.spec.clusterName} to be accepted", + ) + else: + self.not_ready( + CONDITION_REASON_WAITING_FOR_ENVOY_DEPLOYMENT, + f"Waiting for the Envoy Deployment on cluster {self.xr.spec.clusterName} to finish rolling out", + ) + return self.not_ready( CONDITION_REASON_WAITING_FOR_GATEWAY, f"Waiting for the Gateway on cluster {self.xr.spec.clusterName} to be programmed", diff --git a/functions/compose-inference-gateway/tests/test_fn.py b/functions/compose-inference-gateway/tests/test_fn.py index 131ef4c76..a46982bb9 100644 --- a/functions/compose-inference-gateway/tests/test_fn.py +++ b/functions/compose-inference-gateway/tests/test_fn.py @@ -168,6 +168,50 @@ def _observed_caller_auth() -> fnv1.Resource: ) +def _observed_ai_gateway_anchor() -> fnv1.Resource: + """The anchor AIGatewayRoute, observed once Envoy AI Gateway accepts it.""" + return fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "kubernetes.m.crossplane.io/v1alpha1", + "kind": "Object", + "status": { + "conditions": [ + { + "type": "Ready", + "status": "True", + "reason": "Available", + "lastTransitionTime": "2026-06-08T00:00:00Z", + } + ] + }, + } + ) + ) + + +def _observed_envoy_deployment(*, ready: bool) -> fnv1.Resource: + """The Envoy Deployment, observed ready only after its rollout completes.""" + return fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "kubernetes.m.crossplane.io/v1alpha1", + "kind": "Object", + "status": { + "conditions": [ + { + "type": "Ready", + "status": "True" if ready else "False", + "reason": "Available" if ready else "Updating", + "lastTransitionTime": "2026-06-08T00:00:00Z", + } + ] + }, + } + ) + ) + + # The helpers below build the resources the function composes. Each is a # provider-kubernetes Object that applies one manifest to the gateway's cluster # through its ClusterProviderConfig, and every namespaced manifest lands in @@ -211,6 +255,7 @@ def _envoy_proxy() -> fnv1.Resource: "kubernetes": { "envoyService": {"externalTrafficPolicy": "Cluster"}, "envoyDeployment": { + "name": "envoy-modelplane-system-inference-gateway", "replicas": 2, "patch": { "type": "StrategicMerge", @@ -304,6 +349,58 @@ def _envoy_proxy() -> fnv1.Resource: ) +def _envoy_deployment() -> fnv1.Resource: + """The read-only Object that waits for Envoy's Deployment rollout.""" + return fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "kubernetes.m.crossplane.io/v1alpha1", + "kind": "Object", + "metadata": {"namespace": "modelplane-system"}, + "spec": { + "providerConfigRef": {"kind": "ClusterProviderConfig", "name": "gw-eu-cluster-kubeconfig"}, + "watch": True, + "readiness": { + "policy": "DeriveFromCelQuery", + "celQuery": ( + "has(object.metadata) && has(object.metadata.generation) && " + "has(object.spec) && has(object.spec.replicas) && has(object.spec.template) && " + "has(object.spec.template.metadata) && has(object.spec.template.metadata.annotations) && " + "'aigateway.envoyproxy.io/extproc-config-hash' in object.spec.template.metadata.annotations && " + "object.spec.template.metadata.annotations['aigateway.envoyproxy.io/extproc-config-hash'] != '' && " + "has(object.status) && " + "has(object.status.observedGeneration) && " + "object.status.observedGeneration >= object.metadata.generation && " + "has(object.status.updatedReplicas) && has(object.status.replicas) && " + "object.status.updatedReplicas == object.status.replicas && " + "has(object.status.availableReplicas) && " + "object.status.availableReplicas > 0" + ), + }, + "managementPolicies": ["Observe"], + "forProvider": { + "manifest": { + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": { + "name": "envoy-modelplane-system-inference-gateway", + "namespace": "envoy-gateway-system", + }, + } + }, + }, + } + ) + ) + + +def _ready_envoy_deployment() -> fnv1.Resource: + """The desired Envoy Deployment after its observed rollout completes.""" + deployment = _envoy_deployment() + deployment.ready = fnv1.READY_TRUE + return deployment + + def _gateway(*, https: bool, ready: fnv1.Ready) -> fnv1.Resource: """The composed Gateway: an HTTP listener, joined by an HTTPS one terminating eu-tls-0 when https is set.""" http_listener = { @@ -810,6 +907,63 @@ def _healthz_route() -> fnv1.Resource: ) +def _ai_gateway_anchor(*, https: bool, ready: fnv1.Ready) -> fnv1.Resource: + """The AIGatewayRoute keeping AI Gateway's ext-proc installed on this Gateway.""" + return fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "kubernetes.m.crossplane.io/v1alpha1", + "kind": "Object", + "metadata": {"namespace": "modelplane-system"}, + "spec": { + "providerConfigRef": {"kind": "ClusterProviderConfig", "name": "gw-eu-cluster-kubeconfig"}, + "readiness": { + "policy": "DeriveFromCelQuery", + "celQuery": ( + "has(object.status) && has(object.status.conditions) && " + "object.status.conditions.exists(c, c.type == 'Accepted' && c.status == 'True')" + ), + }, + "forProvider": { + "manifest": { + "apiVersion": "aigateway.envoyproxy.io/v1beta1", + "kind": "AIGatewayRoute", + "metadata": {"name": "inference-gateway-anchor", "namespace": "modelplane-system"}, + "spec": { + "parentRefs": [ + { + "group": "gateway.networking.k8s.io", + "kind": "Gateway", + "name": "inference-gateway", + "namespace": "modelplane-system", + "sectionName": "https" if https else "http", + } + ], + "rules": [ + { + "matches": [ + { + "headers": [ + { + "type": "Exact", + "name": "x-modelplane-gateway-anchor", + "value": "true", + } + ] + } + ] + } + ], + }, + } + }, + }, + } + ), + ready=ready, + ) + + def _healthz_auth() -> fnv1.Resource: """The composed SecurityPolicy letting /healthz past caller auth, so a health check needs no credential.""" return fnv1.Resource( @@ -1112,6 +1266,7 @@ def _to_dict(msg: message.Message) -> dict: composite=_desired_xr(status={}, ready=fnv1.READY_FALSE), resources={ "envoy-proxy": _envoy_proxy(), + "envoy-deployment": _envoy_deployment(), "gateway": _gateway(https=False, ready=fnv1.READY_UNSPECIFIED), "client-selfsigned-issuer": _client_selfsigned_issuer(), "client-ca-certificate": _client_ca_certificate(common_name="Modelplane InferenceGateway CA eu"), @@ -1122,6 +1277,7 @@ def _to_dict(msg: message.Message) -> dict: "client-traffic-policy": _client_traffic_policy(), "healthz-filter": _healthz_filter(), "healthz-route": _healthz_route(), + "ai-gateway-anchor": _ai_gateway_anchor(https=False, ready=fnv1.READY_UNSPECIFIED), }, ), results=[ @@ -1178,6 +1334,7 @@ def _to_dict(msg: message.Message) -> dict: ), resources={ "envoy-proxy": _envoy_proxy(), + "envoy-deployment": _envoy_deployment(), "gateway": _gateway(https=False, ready=fnv1.READY_UNSPECIFIED), "client-selfsigned-issuer": _client_selfsigned_issuer(), "client-ca-certificate": _client_ca_certificate(common_name="Modelplane InferenceGateway CA eu"), @@ -1188,6 +1345,7 @@ def _to_dict(msg: message.Message) -> dict: "client-traffic-policy": _client_traffic_policy(), "healthz-filter": _healthz_filter(), "healthz-route": _healthz_route(), + "ai-gateway-anchor": _ai_gateway_anchor(https=False, ready=fnv1.READY_UNSPECIFIED), }, ), results=[ @@ -1242,6 +1400,7 @@ def _to_dict(msg: message.Message) -> dict: ), resources={ "envoy-proxy": _envoy_proxy(), + "envoy-deployment": _envoy_deployment(), "gateway": _gateway(https=False, ready=fnv1.READY_UNSPECIFIED), "client-selfsigned-issuer": _client_selfsigned_issuer(), "client-ca-certificate": _client_ca_certificate(common_name="Modelplane InferenceGateway CA eu"), @@ -1252,6 +1411,7 @@ def _to_dict(msg: message.Message) -> dict: "client-traffic-policy": _client_traffic_policy(), "healthz-filter": _healthz_filter(), "healthz-route": _healthz_route(), + "ai-gateway-anchor": _ai_gateway_anchor(https=False, ready=fnv1.READY_UNSPECIFIED), }, ), results=[ @@ -1307,6 +1467,7 @@ def _to_dict(msg: message.Message) -> dict: composite=_desired_xr(status={}, ready=fnv1.READY_FALSE), resources={ "envoy-proxy": _envoy_proxy(), + "envoy-deployment": _envoy_deployment(), "gateway": _gateway(https=False, ready=fnv1.READY_UNSPECIFIED), "client-selfsigned-issuer": _client_selfsigned_issuer(), "client-ca-certificate": _client_ca_certificate( @@ -1319,6 +1480,7 @@ def _to_dict(msg: message.Message) -> dict: "client-traffic-policy": _client_traffic_policy(), "healthz-filter": _healthz_filter(), "healthz-route": _healthz_route(), + "ai-gateway-anchor": _ai_gateway_anchor(https=False, ready=fnv1.READY_UNSPECIFIED), }, ), results=[ @@ -1356,6 +1518,8 @@ def _to_dict(msg: message.Message) -> dict: composite=_xr(name="eu", tls=False, caller_secret_labels=None), resources={ "gateway": _observed_gateway(address="gw.example.org", ready=True), + "ai-gateway-anchor": _observed_ai_gateway_anchor(), + "envoy-deployment": _observed_envoy_deployment(ready=True), "client-ca-configmap": fnv1.Resource( resource=resource.dict_to_struct( { @@ -1396,6 +1560,7 @@ def _to_dict(msg: message.Message) -> dict: ), resources={ "envoy-proxy": _envoy_proxy(), + "envoy-deployment": _ready_envoy_deployment(), "gateway": _gateway(https=False, ready=fnv1.READY_TRUE), "client-selfsigned-issuer": _client_selfsigned_issuer(), "client-ca-certificate": _client_ca_certificate(common_name="Modelplane InferenceGateway CA eu"), @@ -1406,6 +1571,7 @@ def _to_dict(msg: message.Message) -> dict: "client-traffic-policy": _client_traffic_policy(), "healthz-filter": _healthz_filter(), "healthz-route": _healthz_route(), + "ai-gateway-anchor": _ai_gateway_anchor(https=False, ready=fnv1.READY_TRUE), }, ), context=structpb.Struct(), @@ -1428,7 +1594,11 @@ def _to_dict(msg: message.Message) -> dict: req=fnv1.RunFunctionRequest( observed=fnv1.State( composite=_xr(name="eu", tls=False, caller_secret_labels=None), - resources={"gateway": _observed_gateway(address="gw.example.org", ready=True)}, + resources={ + "gateway": _observed_gateway(address="gw.example.org", ready=True), + "ai-gateway-anchor": _observed_ai_gateway_anchor(), + "envoy-deployment": _observed_envoy_deployment(ready=True), + }, ), required_resources={ "clusters": fnv1.Resources(items=[_cluster(provider_config_ref=True)]), @@ -1450,6 +1620,7 @@ def _to_dict(msg: message.Message) -> dict: ), resources={ "envoy-proxy": _envoy_proxy(), + "envoy-deployment": _ready_envoy_deployment(), "gateway": _gateway(https=False, ready=fnv1.READY_TRUE), "client-selfsigned-issuer": _client_selfsigned_issuer(), "client-ca-certificate": _client_ca_certificate(common_name="Modelplane InferenceGateway CA eu"), @@ -1460,6 +1631,7 @@ def _to_dict(msg: message.Message) -> dict: "client-traffic-policy": _client_traffic_policy(), "healthz-filter": _healthz_filter(), "healthz-route": _healthz_route(), + "ai-gateway-anchor": _ai_gateway_anchor(https=False, ready=fnv1.READY_TRUE), }, ), context=structpb.Struct(), @@ -1474,6 +1646,78 @@ def _to_dict(msg: message.Message) -> dict: ], ), ), + Case( + name="EnvoyDeploymentRollingOut", + reason=( + "An accepted anchor route and Gateway address do not make the gateway Ready while the Envoy " + "Deployment rollout is incomplete." + ), + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr(name="eu", tls=False, caller_secret_labels=None), + resources={ + "gateway": _observed_gateway(address="gw.example.org", ready=True), + "ai-gateway-anchor": _observed_ai_gateway_anchor(), + "envoy-deployment": _observed_envoy_deployment(ready=False), + }, + ), + required_resources={ + "clusters": fnv1.Resources(items=[_cluster(provider_config_ref=True)]), + "gateways": fnv1.Resources(items=[_inference_gateway(name="eu", address=None)]), + }, + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_xr( + status={ + "address": "gw.example.org", + "endpoints": { + "openAI": "http://gw.example.org/v1", + "anthropic": "http://gw.example.org/anthropic/v1", + }, + }, + ready=fnv1.READY_FALSE, + ), + resources={ + "envoy-proxy": _envoy_proxy(), + "envoy-deployment": _envoy_deployment(), + "gateway": _gateway(https=False, ready=fnv1.READY_TRUE), + "client-selfsigned-issuer": _client_selfsigned_issuer(), + "client-ca-certificate": _client_ca_certificate(common_name="Modelplane InferenceGateway CA eu"), + "client-ca-issuer": _client_ca_issuer(), + "client-ca-bundle": _client_ca_bundle(), + "client-ca-configmap": _client_ca_configmap(), + "failover-policy": _failover_policy(), + "client-traffic-policy": _client_traffic_policy(), + "healthz-filter": _healthz_filter(), + "healthz-route": _healthz_route(), + "ai-gateway-anchor": _ai_gateway_anchor(https=False, ready=fnv1.READY_TRUE), + }, + ), + results=[ + fnv1.Result( + severity=fnv1.SEVERITY_NORMAL, + message="Waiting for the Envoy Deployment on cluster gw-eu to finish rolling out", + ) + ], + context=structpb.Struct(), + requirements=fnv1.Requirements( + resources={ + "gateways": fnv1.ResourceSelector(api_version="modelplane.ai/v1alpha1", kind="InferenceGateway"), + "clusters": fnv1.ResourceSelector(api_version="modelplane.ai/v1alpha1", kind="InferenceCluster"), + } + ), + conditions=[ + fnv1.Condition( + type="GatewayReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="WaitingForEnvoyDeployment", + message="Waiting for the Envoy Deployment on cluster gw-eu to finish rolling out", + ), + ], + ), + ), # A ModelService's backends address a cluster gateway by the name # compose-inference-cluster derived, and this gateway's Envoy resolves it, # so its cluster needs a Service of that name. A hostname is how a cloud @@ -1572,6 +1816,7 @@ def _to_dict(msg: message.Message) -> dict: composite=_desired_xr(status={}, ready=fnv1.READY_FALSE), resources={ "envoy-proxy": _envoy_proxy(), + "envoy-deployment": _envoy_deployment(), "gateway": _gateway(https=False, ready=fnv1.READY_UNSPECIFIED), "client-selfsigned-issuer": _client_selfsigned_issuer(), "client-ca-certificate": _client_ca_certificate(common_name="Modelplane InferenceGateway CA eu"), @@ -1797,6 +2042,7 @@ def _to_dict(msg: message.Message) -> dict: "client-traffic-policy": _client_traffic_policy(), "healthz-filter": _healthz_filter(), "healthz-route": _healthz_route(), + "ai-gateway-anchor": _ai_gateway_anchor(https=False, ready=fnv1.READY_UNSPECIFIED), }, ), results=[ @@ -1844,6 +2090,7 @@ def _to_dict(msg: message.Message) -> dict: composite=_desired_xr(status={}, ready=fnv1.READY_FALSE), resources={ "envoy-proxy": _envoy_proxy(), + "envoy-deployment": _envoy_deployment(), "gateway": _gateway(https=True, ready=fnv1.READY_UNSPECIFIED), "client-selfsigned-issuer": _client_selfsigned_issuer(), "client-ca-certificate": _client_ca_certificate(common_name="Modelplane InferenceGateway CA eu"), @@ -1854,6 +2101,7 @@ def _to_dict(msg: message.Message) -> dict: "client-traffic-policy": _client_traffic_policy(), "healthz-filter": _healthz_filter(), "healthz-route": _healthz_route(), + "ai-gateway-anchor": _ai_gateway_anchor(https=True, ready=fnv1.READY_UNSPECIFIED), "redirect-route": _redirect_route(), }, ), @@ -1886,11 +2134,8 @@ def _to_dict(msg: message.Message) -> dict: # redirect is exempted because it must happen before auth, or an # unauthenticated caller would get a 401 instead of being sent to HTTPS. Case( - name="TLSAndAuth", - reason=( - "A programmed gateway with TLS and auth serves authenticated HTTPS, copies its Secrets to the " - "cluster, and publishes its address but no URLs." - ), + name="AnchorRouteNotAccepted", + reason=("A programmed TLS gateway with accepted caller auth waits for its permanent AI Gateway route."), req=fnv1.RunFunctionRequest( observed=fnv1.State( composite=_xr(name="eu", tls=True, caller_secret_labels={"modelplane.ai/inference-keys": "true"}), @@ -1924,7 +2169,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(status={"address": "34.56.129.3"}, ready=fnv1.READY_UNSPECIFIED), + composite=_desired_xr(status={"address": "34.56.129.3"}, ready=fnv1.READY_FALSE), resources={ "caller-secret-ml-team-keys": _caller_secret( name="callers-ml-team-keys", data={"ml-team-assistant": "c2stbXAtYTFiMmMz"} @@ -1955,6 +2200,7 @@ def _to_dict(msg: message.Message) -> dict: ) ), "envoy-proxy": _envoy_proxy(), + "envoy-deployment": _envoy_deployment(), "gateway": _gateway(https=True, ready=fnv1.READY_TRUE), "client-selfsigned-issuer": _client_selfsigned_issuer(), "client-ca-certificate": _client_ca_certificate(common_name="Modelplane InferenceGateway CA eu"), @@ -1968,6 +2214,7 @@ def _to_dict(msg: message.Message) -> dict: "client-traffic-policy": _client_traffic_policy(), "healthz-filter": _healthz_filter(), "healthz-route": _healthz_route(), + "ai-gateway-anchor": _ai_gateway_anchor(https=True, ready=fnv1.READY_UNSPECIFIED), "healthz-auth": _healthz_auth(), "redirect-route": _redirect_route(), "redirect-auth": fnv1.Resource( @@ -2015,6 +2262,12 @@ def _to_dict(msg: message.Message) -> dict: ), }, ), + results=[ + fnv1.Result( + severity=fnv1.SEVERITY_NORMAL, + message="Waiting for the AI Gateway anchor route on cluster gw-eu to be accepted", + ) + ], context=structpb.Struct(), requirements=fnv1.Requirements( resources={ @@ -2032,7 +2285,12 @@ def _to_dict(msg: message.Message) -> dict: } ), conditions=[ - fnv1.Condition(type="GatewayReady", status=fnv1.STATUS_CONDITION_TRUE, reason="GatewayProgrammed"), + fnv1.Condition( + type="GatewayReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="WaitingForAIGatewayRoute", + message="Waiting for the AI Gateway anchor route on cluster gw-eu to be accepted", + ), ], ), ), @@ -2050,7 +2308,10 @@ def _to_dict(msg: message.Message) -> dict: req=fnv1.RunFunctionRequest( observed=fnv1.State( composite=_xr(name="eu", tls=False, caller_secret_labels={"modelplane.ai/inference-keys": "true"}), - resources={"gateway": _observed_gateway(address="34.56.129.3", ready=True)}, + resources={ + "gateway": _observed_gateway(address="34.56.129.3", ready=True), + "ai-gateway-anchor": _observed_ai_gateway_anchor(), + }, ), required_resources={ "clusters": fnv1.Resources(items=[_cluster(provider_config_ref=True)]), @@ -2076,6 +2337,7 @@ def _to_dict(msg: message.Message) -> dict: resources={ "caller-secret-ml-team-keys": _caller_secret(name="callers-ml-team-keys", data={"a": "c2stMQ=="}), "envoy-proxy": _envoy_proxy(), + "envoy-deployment": _envoy_deployment(), "gateway": _gateway(https=False, ready=fnv1.READY_TRUE), "client-selfsigned-issuer": _client_selfsigned_issuer(), "client-ca-certificate": _client_ca_certificate(common_name="Modelplane InferenceGateway CA eu"), @@ -2089,6 +2351,7 @@ def _to_dict(msg: message.Message) -> dict: "client-traffic-policy": _client_traffic_policy(), "healthz-filter": _healthz_filter(), "healthz-route": _healthz_route(), + "ai-gateway-anchor": _ai_gateway_anchor(https=False, ready=fnv1.READY_TRUE), "healthz-auth": _healthz_auth(), }, ), @@ -2134,6 +2397,8 @@ def _to_dict(msg: message.Message) -> dict: composite=_xr(name="eu", tls=False, caller_secret_labels={"modelplane.ai/inference-keys": "true"}), resources={ "gateway": _observed_gateway(address="34.56.129.3", ready=True), + "ai-gateway-anchor": _observed_ai_gateway_anchor(), + "envoy-deployment": _observed_envoy_deployment(ready=True), "caller-auth": _observed_caller_auth(), }, ), @@ -2165,6 +2430,7 @@ def _to_dict(msg: message.Message) -> dict: "caller-secret-team-a-keys": _caller_secret(name="callers-team-a-keys", data={"a": "c2stMQ=="}), "caller-secret-team-b-keys": _caller_secret(name="callers-team-b-keys", data={"b": "c2stMQ=="}), "envoy-proxy": _envoy_proxy(), + "envoy-deployment": _ready_envoy_deployment(), "gateway": _gateway(https=False, ready=fnv1.READY_TRUE), "client-selfsigned-issuer": _client_selfsigned_issuer(), "client-ca-certificate": _client_ca_certificate(common_name="Modelplane InferenceGateway CA eu"), @@ -2179,6 +2445,7 @@ def _to_dict(msg: message.Message) -> dict: "client-traffic-policy": _client_traffic_policy(), "healthz-filter": _healthz_filter(), "healthz-route": _healthz_route(), + "ai-gateway-anchor": _ai_gateway_anchor(https=False, ready=fnv1.READY_TRUE), "healthz-auth": _healthz_auth(), }, ), @@ -2221,6 +2488,7 @@ def _to_dict(msg: message.Message) -> dict: composite=_xr(name="eu", tls=False, caller_secret_labels={"modelplane.ai/inference-keys": "true"}), resources={ "gateway": _observed_gateway(address="34.56.129.3", ready=True), + "ai-gateway-anchor": _observed_ai_gateway_anchor(), "caller-auth": _observed_caller_auth(), }, ), @@ -2250,6 +2518,7 @@ def _to_dict(msg: message.Message) -> dict: name="callers-team-a-keys", data={"a": "c2stMQ==", "z": "c2stMQ=="} ), "envoy-proxy": _envoy_proxy(), + "envoy-deployment": _envoy_deployment(), "gateway": _gateway(https=False, ready=fnv1.READY_TRUE), "client-selfsigned-issuer": _client_selfsigned_issuer(), "client-ca-certificate": _client_ca_certificate(common_name="Modelplane InferenceGateway CA eu"), @@ -2263,6 +2532,7 @@ def _to_dict(msg: message.Message) -> dict: "client-traffic-policy": _client_traffic_policy(), "healthz-filter": _healthz_filter(), "healthz-route": _healthz_route(), + "ai-gateway-anchor": _ai_gateway_anchor(https=False, ready=fnv1.READY_TRUE), "healthz-auth": _healthz_auth(), }, ), @@ -2310,6 +2580,7 @@ def _to_dict(msg: message.Message) -> dict: composite=_xr(name="eu", tls=False, caller_secret_labels={"modelplane.ai/inference-keys": "true"}), resources={ "gateway": _observed_gateway(address="34.56.129.3", ready=True), + "ai-gateway-anchor": _observed_ai_gateway_anchor(), "caller-auth": _observed_caller_auth(), }, ), @@ -2343,6 +2614,7 @@ def _to_dict(msg: message.Message) -> dict: name="callers-team-b-keys", data={"x": "c2stMg==", "y": "c2stMQ=="} ), "envoy-proxy": _envoy_proxy(), + "envoy-deployment": _envoy_deployment(), "gateway": _gateway(https=False, ready=fnv1.READY_TRUE), "client-selfsigned-issuer": _client_selfsigned_issuer(), "client-ca-certificate": _client_ca_certificate(common_name="Modelplane InferenceGateway CA eu"), @@ -2357,6 +2629,7 @@ def _to_dict(msg: message.Message) -> dict: "client-traffic-policy": _client_traffic_policy(), "healthz-filter": _healthz_filter(), "healthz-route": _healthz_route(), + "ai-gateway-anchor": _ai_gateway_anchor(https=False, ready=fnv1.READY_TRUE), "healthz-auth": _healthz_auth(), }, ), @@ -2401,6 +2674,8 @@ def _to_dict(msg: message.Message) -> dict: composite=_xr(name="eu", tls=False, caller_secret_labels={"modelplane.ai/inference-keys": "true"}), resources={ "gateway": _observed_gateway(address="34.56.129.3", ready=True), + "ai-gateway-anchor": _observed_ai_gateway_anchor(), + "envoy-deployment": _observed_envoy_deployment(ready=True), "caller-auth": _observed_caller_auth(), }, ), @@ -2434,6 +2709,7 @@ def _to_dict(msg: message.Message) -> dict: name="callers-team-b-keys", data={"a": "c2stMQ==", "b": "c2stMg=="} ), "envoy-proxy": _envoy_proxy(), + "envoy-deployment": _ready_envoy_deployment(), "gateway": _gateway(https=False, ready=fnv1.READY_TRUE), "client-selfsigned-issuer": _client_selfsigned_issuer(), "client-ca-certificate": _client_ca_certificate(common_name="Modelplane InferenceGateway CA eu"), @@ -2448,6 +2724,7 @@ def _to_dict(msg: message.Message) -> dict: "client-traffic-policy": _client_traffic_policy(), "healthz-filter": _healthz_filter(), "healthz-route": _healthz_route(), + "ai-gateway-anchor": _ai_gateway_anchor(https=False, ready=fnv1.READY_TRUE), "healthz-auth": _healthz_auth(), }, ), @@ -2498,6 +2775,7 @@ def _to_dict(msg: message.Message) -> dict: "caller-secret-team-a-keys": _caller_secret(name="callers-team-a-keys", data={"a": "c2stMQ=="}), "caller-secret-team-b-keys": _caller_secret(name="callers-team-b-keys", data={"b": "c2stMg=="}), "envoy-proxy": _envoy_proxy(), + "envoy-deployment": _envoy_deployment(), "gateway": _gateway(https=False, ready=fnv1.READY_UNSPECIFIED), "client-selfsigned-issuer": _client_selfsigned_issuer(), "client-ca-certificate": _client_ca_certificate(common_name="Modelplane InferenceGateway CA eu"), @@ -2512,6 +2790,7 @@ def _to_dict(msg: message.Message) -> dict: "client-traffic-policy": _client_traffic_policy(), "healthz-filter": _healthz_filter(), "healthz-route": _healthz_route(), + "ai-gateway-anchor": _ai_gateway_anchor(https=False, ready=fnv1.READY_UNSPECIFIED), "healthz-auth": _healthz_auth(), }, ), @@ -2571,6 +2850,7 @@ def _to_dict(msg: message.Message) -> dict: composite=_desired_xr(status={}, ready=fnv1.READY_FALSE), resources={ "envoy-proxy": _envoy_proxy(), + "envoy-deployment": _envoy_deployment(), "gateway": _gateway(https=False, ready=fnv1.READY_UNSPECIFIED), "client-selfsigned-issuer": _client_selfsigned_issuer(), "client-ca-certificate": _client_ca_certificate(common_name="Modelplane InferenceGateway CA eu"), @@ -2624,6 +2904,7 @@ def _to_dict(msg: message.Message) -> dict: "client-traffic-policy": _client_traffic_policy(), "healthz-filter": _healthz_filter(), "healthz-route": _healthz_route(), + "ai-gateway-anchor": _ai_gateway_anchor(https=False, ready=fnv1.READY_UNSPECIFIED), "healthz-auth": _healthz_auth(), }, ), @@ -2677,6 +2958,7 @@ def _to_dict(msg: message.Message) -> dict: composite=_desired_xr(status={}, ready=fnv1.READY_FALSE), resources={ "envoy-proxy": _envoy_proxy(), + "envoy-deployment": _envoy_deployment(), "gateway": _gateway(https=False, ready=fnv1.READY_UNSPECIFIED), "client-selfsigned-issuer": _client_selfsigned_issuer(), "client-ca-certificate": _client_ca_certificate(common_name="Modelplane InferenceGateway CA eu"), @@ -2730,6 +3012,7 @@ def _to_dict(msg: message.Message) -> dict: "client-traffic-policy": _client_traffic_policy(), "healthz-filter": _healthz_filter(), "healthz-route": _healthz_route(), + "ai-gateway-anchor": _ai_gateway_anchor(https=False, ready=fnv1.READY_UNSPECIFIED), "healthz-auth": _healthz_auth(), }, ), @@ -2839,6 +3122,7 @@ def _to_dict(msg: message.Message) -> dict: ) ), "envoy-proxy": _envoy_proxy(), + "envoy-deployment": _envoy_deployment(), "gateway": _gateway(https=True, ready=fnv1.READY_UNSPECIFIED), "client-selfsigned-issuer": _client_selfsigned_issuer(), "client-ca-certificate": _client_ca_certificate(common_name="Modelplane InferenceGateway CA eu"), @@ -2852,6 +3136,7 @@ def _to_dict(msg: message.Message) -> dict: "client-traffic-policy": _client_traffic_policy(), "healthz-filter": _healthz_filter(), "healthz-route": _healthz_route(), + "ai-gateway-anchor": _ai_gateway_anchor(https=True, ready=fnv1.READY_UNSPECIFIED), "healthz-auth": _healthz_auth(), "redirect-route": _redirect_route(), "redirect-auth": fnv1.Resource( @@ -2930,6 +3215,160 @@ def _to_dict(msg: message.Message) -> dict: ], ), ), + Case( + name="TLSAndAuthGatewayReady", + reason=( + "A TLS and auth gateway becomes GatewayReady=True after its policy, route, Gateway, and Envoy Deployment are ready, " + "without observing a Secret." + ), + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr(name="eu", tls=True, caller_secret_labels={"team": "ml"}), + resources={ + "gateway": _observed_gateway(address="34.56.129.3", ready=True), + "caller-auth": _observed_caller_auth(), + "ai-gateway-anchor": _observed_ai_gateway_anchor(), + "envoy-deployment": _observed_envoy_deployment(ready=True), + }, + ), + required_resources={ + "clusters": fnv1.Resources(items=[_cluster(provider_config_ref=True)]), + "gateways": fnv1.Resources(items=[_inference_gateway(name="eu", address=None)]), + "caller-secrets": fnv1.Resources( + items=[_caller_key_secret(name="ml-team-keys", data={"alice": "a2V5"})] + ), + "tls-secret-0": fnv1.Resources( + items=[ + fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "v1", + "kind": "Secret", + "metadata": {"name": "eu-tls-0", "namespace": "modelplane-system"}, + "data": {"tls.crt": "Y2VydA==", "tls.key": "a2V5"}, + } + ) + ) + ] + ), + }, + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_xr(status={"address": "34.56.129.3"}, ready=fnv1.READY_UNSPECIFIED), + resources={ + "caller-secret-ml-team-keys": _caller_secret(name="callers-ml-team-keys", data={"alice": "a2V5"}), + "tls-secret-eu-tls-0": fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "kubernetes.m.crossplane.io/v1alpha1", + "kind": "Object", + "metadata": {"namespace": "modelplane-system"}, + "spec": { + "providerConfigRef": { + "kind": "ClusterProviderConfig", + "name": "gw-eu-cluster-kubeconfig", + }, + "readiness": {"policy": "SuccessfulCreate"}, + "forProvider": { + "manifest": { + "apiVersion": "v1", + "kind": "Secret", + "metadata": {"name": "eu-tls-0", "namespace": "modelplane-system"}, + "type": "kubernetes.io/tls", + "data": {"tls.crt": "Y2VydA==", "tls.key": "a2V5"}, + } + }, + }, + } + ) + ), + "envoy-proxy": _envoy_proxy(), + "envoy-deployment": _ready_envoy_deployment(), + "gateway": _gateway(https=True, ready=fnv1.READY_TRUE), + "client-selfsigned-issuer": _client_selfsigned_issuer(), + "client-ca-certificate": _client_ca_certificate(common_name="Modelplane InferenceGateway CA eu"), + "client-ca-issuer": _client_ca_issuer(), + "client-ca-bundle": _client_ca_bundle(), + "client-ca-configmap": _client_ca_configmap(), + "caller-auth": _caller_auth( + credential_refs=[{"name": "callers-ml-team-keys"}], ready=fnv1.READY_TRUE + ), + "failover-policy": _failover_policy(), + "client-traffic-policy": _client_traffic_policy(), + "healthz-filter": _healthz_filter(), + "healthz-route": _healthz_route(), + "ai-gateway-anchor": _ai_gateway_anchor(https=True, ready=fnv1.READY_TRUE), + "healthz-auth": _healthz_auth(), + "redirect-route": _redirect_route(), + "redirect-auth": fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "kubernetes.m.crossplane.io/v1alpha1", + "kind": "Object", + "metadata": {"namespace": "modelplane-system"}, + "spec": { + "providerConfigRef": { + "kind": "ClusterProviderConfig", + "name": "gw-eu-cluster-kubeconfig", + }, + "readiness": { + "policy": "DeriveFromCelQuery", + "celQuery": ( + "has(object.status) && has(object.status.ancestors) && " + "object.status.ancestors.exists(a, has(a.conditions) && " + "a.conditions.exists(c, c.type == 'Accepted' && c.status == 'True'))" + ), + }, + "forProvider": { + "manifest": { + "apiVersion": "gateway.envoyproxy.io/v1alpha1", + "kind": "SecurityPolicy", + "metadata": { + "name": "inference-gateway-redirect-open", + "namespace": "modelplane-system", + }, + "spec": { + "targetRefs": [ + { + "group": "gateway.networking.k8s.io", + "kind": "HTTPRoute", + "name": "inference-gateway-redirect", + } + ], + "authorization": {"defaultAction": "Allow"}, + }, + } + }, + }, + } + ) + ), + }, + ), + results=[], + context=structpb.Struct(), + requirements=fnv1.Requirements( + resources={ + "gateways": fnv1.ResourceSelector(api_version="modelplane.ai/v1alpha1", kind="InferenceGateway"), + "clusters": fnv1.ResourceSelector(api_version="modelplane.ai/v1alpha1", kind="InferenceCluster"), + "caller-secrets": fnv1.ResourceSelector( + api_version="v1", + kind="Secret", + namespace="modelplane-system", + match_labels=fnv1.MatchLabels(labels={"team": "ml"}), + ), + "tls-secret-0": fnv1.ResourceSelector( + api_version="v1", kind="Secret", namespace="modelplane-system", match_name="eu-tls-0" + ), + } + ), + conditions=[ + fnv1.Condition(type="GatewayReady", status=fnv1.STATUS_CONDITION_TRUE, reason="GatewayProgrammed") + ], + ), + ), ] diff --git a/functions/compose-model-route/function/fn.py b/functions/compose-model-route/function/fn.py index 7d06b3da3..23397d5b6 100644 --- a/functions/compose-model-route/function/fn.py +++ b/functions/compose-model-route/function/fn.py @@ -264,6 +264,8 @@ def __init__(self, req: fnv1.RunFunctionRequest, rsp: fnv1.RunFunctionResponse) self.provider_config = "" self.address: str | None = None self.inference_listener = _LISTENER_HTTP + self.gateway_ready = False + self.gateway_not_ready_message = "" # Endpoints per priority, as (entry weight, endpoints) so weights can be # distributed within a tier. self.tiers: dict[int, list[tuple[int, list[mev1alpha1.ModelEndpoint]]]] = {} @@ -368,6 +370,19 @@ def resolve_gateway(self) -> bool: return False gw = igv1alpha1.InferenceGateway.model_validate(gateways[0]) + gateway_conditions = gw.status.conditions if gw.status else None + gateway_ready = next( + (condition for condition in gateway_conditions or [] if condition.type == "GatewayReady"), + None, + ) + self.gateway_ready = gateway_ready is not None and gateway_ready.status == "True" + if not self.gateway_ready: + self.gateway_not_ready_message = ( + gateway_ready.message + if gateway_ready and gateway_ready.message + else f"InferenceGateway {self.xr.spec.gatewayName} is not ready" + ) + if not (gw.status and gw.status.clientCACertificate): self.not_ready( CONDITION_REASON_WAITING_FOR_GATEWAY, @@ -871,6 +886,9 @@ def not_ready(self, reason: str, message: str) -> None: def derive_conditions(self) -> None: """RoutingReady once the composed route has been accepted.""" + if not self.gateway_ready: + self.not_ready(CONDITION_REASON_WAITING_FOR_GATEWAY, self.gateway_not_ready_message) + return if resource.get_condition(self.req.observed.resources.get("route"), "Ready").status != "True": self.not_ready( CONDITION_REASON_WAITING_FOR_ROUTE, diff --git a/functions/compose-model-route/tests/test_fn.py b/functions/compose-model-route/tests/test_fn.py index c287990f5..004bb51d5 100644 --- a/functions/compose-model-route/tests/test_fn.py +++ b/functions/compose-model-route/tests/test_fn.py @@ -67,12 +67,21 @@ def _desired_model_route(*, address: str | None, total_endpoints: int, ready_end return fnv1.Resource(resource=resource.dict_to_struct({"status": status}), ready=fnv1.READY_FALSE) -def _inference_gateway(*, tls: bool, address: str, client_ca_published: bool) -> fnv1.Resource: +def _inference_gateway(*, tls: bool, address: str, client_ca_published: bool, gateway_ready: bool) -> fnv1.Resource: """The InferenceGateway eu on cluster gw-eu, as the gateway requirement returns it.""" spec: dict = {"clusterName": "gw-eu"} if tls: spec["tls"] = {"certificateRefs": [{"name": "eu-tls"}]} status: dict = {"address": address} + status["conditions"] = [ + { + "type": "GatewayReady", + "status": "True" if gateway_ready else "False", + "reason": "GatewayProgrammed" if gateway_ready else "WaitingForEnvoyDeployment", + "message": "" if gateway_ready else "Waiting for Envoy Deployment rollout", + "lastTransitionTime": "2026-06-08T00:00:00Z", + } + ] if client_ca_published: status["clientCACertificate"] = "-----BEGIN CERTIFICATE-----\nclient\n-----END CERTIFICATE-----\n" return fnv1.Resource( @@ -442,9 +451,10 @@ def _client_certificate() -> fnv1.Resource: ) -def _ai_gateway_route(*, section_name: str, backend_refs: list[dict]) -> fnv1.Resource: +def _ai_gateway_route(*, section_name: str, backend_refs: list[dict], ready: fnv1.Ready) -> fnv1.Resource: """The composed AIGatewayRoute matching ml-team/assistant, bound to the gateway's section_name listener.""" return fnv1.Resource( + ready=ready, resource=resource.dict_to_struct( { "apiVersion": "kubernetes.m.crossplane.io/v1alpha1", @@ -509,7 +519,7 @@ def _ai_gateway_route(*, section_name: str, backend_refs: list[dict]) -> fnv1.Re }, }, } - ) + ), ) @@ -530,6 +540,74 @@ def _to_dict(msg: message.Message) -> dict: # Secret data is base64 encoded, as the API server stores it: c2stMQ== is # "sk-1", c2stdG9n "sk-tog" and c2stcHJvdmlkZXI= "sk-provider". COMPOSE_CASES = [ + Case( + name="GatewayNotReady", + reason="A ModelRoute composes a new route during an InferenceGateway rollout and reports it not ready.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_model_route( + endpoints=[ + v1alpha1.Endpoint( + name="d", + selector=v1alpha1.Selector(matchLabels={"modelplane.ai/deployment": "d"}), + ) + ] + ) + ), + required_resources={ + "gateway": fnv1.Resources( + items=[ + _inference_gateway( + tls=False, address="203.0.113.1", client_ca_published=True, gateway_ready=False + ) + ] + ), + "clusters": fnv1.Resources(items=[_cluster(gateway_ca_published=True)]), + "endpoints-d": fnv1.Resources(items=[_composed_endpoint(model=None)]), + }, + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_model_route(address="203.0.113.1", total_endpoints=1, ready_endpoints=1), + resources={ + "backend-self": _composed_endpoint_backend(), + "aibackend-self": _composed_endpoint_ai_backend(), + "cluster-ca-gw-eu": _cluster_ca(), + "client-certificate": _client_certificate(), + "route": _ai_gateway_route( + ready=fnv1.READY_UNSPECIFIED, + section_name="http", + backend_refs=[{"name": "assistant-eu-self-a42e6", "weight": 1, "priority": 0}], + ), + }, + ), + results=[fnv1.Result(severity=fnv1.SEVERITY_NORMAL, message="Waiting for Envoy Deployment rollout")], + context=structpb.Struct(), + requirements=fnv1.Requirements( + resources={ + "gateway": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", kind="InferenceGateway", match_name="eu" + ), + "clusters": fnv1.ResourceSelector(api_version="modelplane.ai/v1alpha1", kind="InferenceCluster"), + "endpoints-d": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", + kind="ModelEndpoint", + namespace="ml-team", + match_labels=fnv1.MatchLabels(labels={"modelplane.ai/deployment": "d"}), + ), + } + ), + conditions=[ + fnv1.Condition( + type="RoutingReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="WaitingForGateway", + message="Waiting for Envoy Deployment rollout", + ) + ], + ), + ), Case( name="ClientCAUnpublished", reason="A ModelRoute whose gateway hasn't published its client CA composes nothing, because no backend could name its client certificate.", @@ -546,7 +624,11 @@ def _to_dict(msg: message.Message) -> dict: ), required_resources={ "gateway": fnv1.Resources( - items=[_inference_gateway(tls=False, address="203.0.113.1", client_ca_published=False)] + items=[ + _inference_gateway( + tls=False, address="203.0.113.1", client_ca_published=False, gateway_ready=True + ) + ] ), "clusters": fnv1.Resources(items=[_cluster(gateway_ca_published=True)]), "endpoints-d": fnv1.Resources(items=[_composed_endpoint(model=None)]), @@ -604,7 +686,11 @@ def _to_dict(msg: message.Message) -> dict: ), required_resources={ "gateway": fnv1.Resources( - items=[_inference_gateway(tls=False, address="203.0.113.1", client_ca_published=True)] + items=[ + _inference_gateway( + tls=False, address="203.0.113.1", client_ca_published=True, gateway_ready=True + ) + ] ), "clusters": fnv1.Resources(items=[_cluster(gateway_ca_published=True)]), "endpoints-d": fnv1.Resources( @@ -673,7 +759,11 @@ def _to_dict(msg: message.Message) -> dict: ), required_resources={ "gateway": fnv1.Resources( - items=[_inference_gateway(tls=False, address="203.0.113.1", client_ca_published=True)] + items=[ + _inference_gateway( + tls=False, address="203.0.113.1", client_ca_published=True, gateway_ready=True + ) + ] ), "clusters": fnv1.Resources(items=[_cluster(gateway_ca_published=False)]), "endpoints-d": fnv1.Resources(items=[_composed_endpoint(model=None)]), @@ -735,7 +825,11 @@ def _to_dict(msg: message.Message) -> dict: ), required_resources={ "gateway": fnv1.Resources( - items=[_inference_gateway(tls=False, address="203.0.113.1", client_ca_published=True)] + items=[ + _inference_gateway( + tls=False, address="203.0.113.1", client_ca_published=True, gateway_ready=True + ) + ] ), "clusters": fnv1.Resources(items=[_cluster(gateway_ca_published=True)]), "endpoints-a": fnv1.Resources( @@ -818,7 +912,11 @@ def _to_dict(msg: message.Message) -> dict: ), required_resources={ "gateway": fnv1.Resources( - items=[_inference_gateway(tls=False, address="203.0.113.1", client_ca_published=True)] + items=[ + _inference_gateway( + tls=False, address="203.0.113.1", client_ca_published=True, gateway_ready=True + ) + ] ), "clusters": fnv1.Resources(items=[_cluster(gateway_ca_published=True)]), "endpoints-d": fnv1.Resources(items=[_composed_endpoint(model="d")]), @@ -863,6 +961,7 @@ def _to_dict(msg: message.Message) -> dict: "cluster-ca-gw-eu": _cluster_ca(), "client-certificate": _client_certificate(), "route": _ai_gateway_route( + ready=fnv1.READY_UNSPECIFIED, section_name="http", backend_refs=[ {"name": "assistant-eu-self-a42e6", "weight": 1, "priority": 0, "modelNameOverride": "d"}, @@ -929,7 +1028,11 @@ def _to_dict(msg: message.Message) -> dict: ), required_resources={ "gateway": fnv1.Resources( - items=[_inference_gateway(tls=False, address="203.0.113.1", client_ca_published=True)] + items=[ + _inference_gateway( + tls=False, address="203.0.113.1", client_ca_published=True, gateway_ready=True + ) + ] ), "clusters": fnv1.Resources(items=[_cluster(gateway_ca_published=True)]), "endpoints-d": fnv1.Resources(items=[_composed_endpoint(model=None)]), @@ -945,6 +1048,7 @@ def _to_dict(msg: message.Message) -> dict: "cluster-ca-gw-eu": _cluster_ca(), "client-certificate": _client_certificate(), "route": _ai_gateway_route( + ready=fnv1.READY_UNSPECIFIED, section_name="http", backend_refs=[{"name": "assistant-eu-self-a42e6", "weight": 1, "priority": 0}], ), @@ -978,6 +1082,93 @@ def _to_dict(msg: message.Message) -> dict: ], ), ), + Case( + name="GatewayRollout", + reason="With an existing route, an unready gateway keeps the route desired and reports it not ready.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_model_route( + endpoints=[ + v1alpha1.Endpoint( + name="d", + selector=v1alpha1.Selector(matchLabels={"modelplane.ai/deployment": "d"}), + ) + ] + ), + resources={ + "route": fnv1.Resource( + resource=resource.dict_to_struct( + { + "status": { + "conditions": [ + { + "type": "Ready", + "status": "True", + "reason": "Accepted", + "lastTransitionTime": "2026-06-08T00:00:00Z", + } + ] + } + } + ), + ready=fnv1.READY_TRUE, + ) + }, + ), + required_resources={ + "gateway": fnv1.Resources( + items=[ + _inference_gateway( + tls=False, address="203.0.113.1", client_ca_published=True, gateway_ready=False + ) + ] + ), + "clusters": fnv1.Resources(items=[_cluster(gateway_ca_published=True)]), + "endpoints-d": fnv1.Resources(items=[_composed_endpoint(model=None)]), + }, + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_model_route(address="203.0.113.1", total_endpoints=1, ready_endpoints=1), + resources={ + "backend-self": _composed_endpoint_backend(), + "aibackend-self": _composed_endpoint_ai_backend(), + "cluster-ca-gw-eu": _cluster_ca(), + "client-certificate": _client_certificate(), + "route": _ai_gateway_route( + ready=fnv1.READY_TRUE, + section_name="http", + backend_refs=[{"name": "assistant-eu-self-a42e6", "weight": 1, "priority": 0}], + ), + }, + ), + results=[fnv1.Result(severity=fnv1.SEVERITY_NORMAL, message="Waiting for Envoy Deployment rollout")], + context=structpb.Struct(), + requirements=fnv1.Requirements( + resources={ + "gateway": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", kind="InferenceGateway", match_name="eu" + ), + "clusters": fnv1.ResourceSelector(api_version="modelplane.ai/v1alpha1", kind="InferenceCluster"), + "endpoints-d": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", + kind="ModelEndpoint", + namespace="ml-team", + match_labels=fnv1.MatchLabels(labels={"modelplane.ai/deployment": "d"}), + ), + } + ), + conditions=[ + fnv1.Condition( + type="RoutingReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="WaitingForGateway", + message="Waiting for Envoy Deployment rollout", + ) + ], + ), + ), # A TLS gateway serves inference on its HTTPS listener alone. Binding to :80 # on a TLS gateway would carry credentials in the clear. Case( @@ -996,7 +1187,11 @@ def _to_dict(msg: message.Message) -> dict: ), required_resources={ "gateway": fnv1.Resources( - items=[_inference_gateway(tls=True, address="203.0.113.1", client_ca_published=True)] + items=[ + _inference_gateway( + tls=True, address="203.0.113.1", client_ca_published=True, gateway_ready=True + ) + ] ), "clusters": fnv1.Resources(items=[_cluster(gateway_ca_published=True)]), "endpoints-d": fnv1.Resources(items=[_composed_endpoint(model=None)]), @@ -1012,6 +1207,7 @@ def _to_dict(msg: message.Message) -> dict: "cluster-ca-gw-eu": _cluster_ca(), "client-certificate": _client_certificate(), "route": _ai_gateway_route( + ready=fnv1.READY_UNSPECIFIED, section_name="https", backend_refs=[{"name": "assistant-eu-self-a42e6", "weight": 1, "priority": 0}], ), @@ -1064,7 +1260,11 @@ def _to_dict(msg: message.Message) -> dict: ), required_resources={ "gateway": fnv1.Resources( - items=[_inference_gateway(tls=False, address="203.0.113.9", client_ca_published=True)] + items=[ + _inference_gateway( + tls=False, address="203.0.113.9", client_ca_published=True, gateway_ready=True + ) + ] ), "clusters": fnv1.Resources(items=[_cluster(gateway_ca_published=True)]), "endpoints-d": fnv1.Resources(items=[_composed_endpoint(model=None)]), @@ -1080,6 +1280,7 @@ def _to_dict(msg: message.Message) -> dict: "cluster-ca-gw-eu": _cluster_ca(), "client-certificate": _client_certificate(), "route": _ai_gateway_route( + ready=fnv1.READY_UNSPECIFIED, section_name="http", backend_refs=[{"name": "assistant-eu-self-a42e6", "weight": 1, "priority": 0}], ), @@ -1137,7 +1338,11 @@ def _to_dict(msg: message.Message) -> dict: ), required_resources={ "gateway": fnv1.Resources( - items=[_inference_gateway(tls=False, address="203.0.113.1", client_ca_published=True)] + items=[ + _inference_gateway( + tls=False, address="203.0.113.1", client_ca_published=True, gateway_ready=True + ) + ] ), "clusters": fnv1.Resources(items=[_cluster(gateway_ca_published=True)]), "endpoints-canary": fnv1.Resources( @@ -1174,6 +1379,7 @@ def _to_dict(msg: message.Message) -> dict: "backend-kimi-a": _third_party_backend(name="assistant-eu-kimi-a-bf6de", hostname="a.example.com"), "aibackend-kimi-a": _third_party_ai_backend(name="assistant-eu-kimi-a-bf6de", schema="OpenAI"), "route": _ai_gateway_route( + ready=fnv1.READY_UNSPECIFIED, section_name="http", backend_refs=[{"name": "assistant-eu-kimi-a-bf6de", "weight": 1, "priority": 0}], ), @@ -1243,7 +1449,11 @@ def _to_dict(msg: message.Message) -> dict: ), required_resources={ "gateway": fnv1.Resources( - items=[_inference_gateway(tls=False, address="203.0.113.1", client_ca_published=True)] + items=[ + _inference_gateway( + tls=False, address="203.0.113.1", client_ca_published=True, gateway_ready=True + ) + ] ), "clusters": fnv1.Resources(items=[_cluster(gateway_ca_published=True)]), "endpoints-a": fnv1.Resources( @@ -1294,6 +1504,7 @@ def _to_dict(msg: message.Message) -> dict: "backend-c-0": _third_party_backend(name="assistant-eu-c-0-782db", hostname="c.example.com"), "aibackend-c-0": _third_party_ai_backend(name="assistant-eu-c-0-782db", schema="OpenAI"), "route": _ai_gateway_route( + ready=fnv1.READY_UNSPECIFIED, section_name="http", backend_refs=[ {"name": "assistant-eu-a-0-b66b8", "weight": 1, "priority": 0}, @@ -1368,7 +1579,11 @@ def _to_dict(msg: message.Message) -> dict: ), required_resources={ "gateway": fnv1.Resources( - items=[_inference_gateway(tls=False, address="203.0.113.1", client_ca_published=True)] + items=[ + _inference_gateway( + tls=False, address="203.0.113.1", client_ca_published=True, gateway_ready=True + ) + ] ), "clusters": fnv1.Resources(items=[_cluster(gateway_ca_published=True)]), "endpoints-big": fnv1.Resources( @@ -1435,6 +1650,7 @@ def _to_dict(msg: message.Message) -> dict: ), "aibackend-small-0": _third_party_ai_backend(name="assistant-eu-small-0-60d20", schema="OpenAI"), "route": _ai_gateway_route( + ready=fnv1.READY_UNSPECIFIED, section_name="http", backend_refs=[ {"name": "assistant-eu-big-0-1e944", "weight": 3, "priority": 0}, @@ -1498,7 +1714,11 @@ def _to_dict(msg: message.Message) -> dict: ), required_resources={ "gateway": fnv1.Resources( - items=[_inference_gateway(tls=False, address="203.0.113.1", client_ca_published=True)] + items=[ + _inference_gateway( + tls=False, address="203.0.113.1", client_ca_published=True, gateway_ready=True + ) + ] ), "clusters": fnv1.Resources(items=[_cluster(gateway_ca_published=True)]), "endpoints-many": fnv1.Resources( @@ -1573,6 +1793,7 @@ def _to_dict(msg: message.Message) -> dict: ), "aibackend-many-4": _third_party_ai_backend(name="assistant-eu-many-4-bd6f9", schema="OpenAI"), "route": _ai_gateway_route( + ready=fnv1.READY_UNSPECIFIED, section_name="http", backend_refs=[ {"name": "assistant-eu-many-0-e4e60", "weight": 1, "priority": 0}, @@ -1639,7 +1860,11 @@ def _to_dict(msg: message.Message) -> dict: ), required_resources={ "gateway": fnv1.Resources( - items=[_inference_gateway(tls=False, address="203.0.113.1", client_ca_published=True)] + items=[ + _inference_gateway( + tls=False, address="203.0.113.1", client_ca_published=True, gateway_ready=True + ) + ] ), "clusters": fnv1.Resources(items=[_cluster(gateway_ca_published=True)]), "endpoints-big": fnv1.Resources( @@ -1694,6 +1919,7 @@ def _to_dict(msg: message.Message) -> dict: ), "aibackend-small-1": _third_party_ai_backend(name="assistant-eu-small-1-cad94", schema="OpenAI"), "route": _ai_gateway_route( + ready=fnv1.READY_UNSPECIFIED, section_name="http", backend_refs=[ {"name": "assistant-eu-big-0-1e944", "weight": 1000000, "priority": 0}, @@ -1756,7 +1982,11 @@ def _to_dict(msg: message.Message) -> dict: ), required_resources={ "gateway": fnv1.Resources( - items=[_inference_gateway(tls=False, address="203.0.113.1", client_ca_published=True)] + items=[ + _inference_gateway( + tls=False, address="203.0.113.1", client_ca_published=True, gateway_ready=True + ) + ] ), "clusters": fnv1.Resources(items=[_cluster(gateway_ca_published=True)]), "endpoints-d": fnv1.Resources( @@ -1801,6 +2031,7 @@ def _to_dict(msg: message.Message) -> dict: "backend-z": _third_party_backend(name="assistant-eu-z-e5d6f", hostname="z.example.com"), "aibackend-z": _third_party_ai_backend(name="assistant-eu-z-e5d6f", schema="OpenAI"), "route": _ai_gateway_route( + ready=fnv1.READY_UNSPECIFIED, section_name="http", backend_refs=[ {"name": "assistant-eu-a-18a56", "weight": 1, "priority": 0}, @@ -1854,7 +2085,11 @@ def _to_dict(msg: message.Message) -> dict: ), required_resources={ "gateway": fnv1.Resources( - items=[_inference_gateway(tls=False, address="203.0.113.1", client_ca_published=True)] + items=[ + _inference_gateway( + tls=False, address="203.0.113.1", client_ca_published=True, gateway_ready=True + ) + ] ), "clusters": fnv1.Resources(items=[_cluster(gateway_ca_published=True)]), "endpoints-d": fnv1.Resources( @@ -1894,6 +2129,7 @@ def _to_dict(msg: message.Message) -> dict: }, ), "route": _ai_gateway_route( + ready=fnv1.READY_UNSPECIFIED, section_name="http", backend_refs=[{"name": "assistant-eu-provider-348cb", "weight": 1, "priority": 0}], ), @@ -1946,7 +2182,11 @@ def _to_dict(msg: message.Message) -> dict: ), required_resources={ "gateway": fnv1.Resources( - items=[_inference_gateway(tls=False, address="203.0.113.1", client_ca_published=True)] + items=[ + _inference_gateway( + tls=False, address="203.0.113.1", client_ca_published=True, gateway_ready=True + ) + ] ), "clusters": fnv1.Resources(items=[_cluster(gateway_ca_published=True)]), "endpoints-d": fnv1.Resources( @@ -1988,6 +2228,7 @@ def _to_dict(msg: message.Message) -> dict: }, ), "route": _ai_gateway_route( + ready=fnv1.READY_UNSPECIFIED, section_name="http", backend_refs=[{"name": "assistant-eu-provider-348cb", "weight": 1, "priority": 0}], ),