diff --git a/apis/modeldeployments/definition.yaml b/apis/modeldeployments/definition.yaml index 0e9bfb60e..4cfd951af 100644 --- a/apis/modeldeployments/definition.yaml +++ b/apis/modeldeployments/definition.yaml @@ -565,6 +565,7 @@ spec: maximum: 63 template: type: object + required: [spec] description: >- Pod template for this member's engine pods. A curated subset of PodTemplateSpec. diff --git a/apis/modelreplicas/definition.yaml b/apis/modelreplicas/definition.yaml index 25342be5b..ede82a965 100644 --- a/apis/modelreplicas/definition.yaml +++ b/apis/modelreplicas/definition.yaml @@ -390,6 +390,7 @@ spec: maximum: 63 template: type: object + required: [spec] properties: metadata: type: object diff --git a/functions/compose-model-deployment/function/fn.py b/functions/compose-model-deployment/function/fn.py index 5e7cb38b3..de93cdca1 100644 --- a/functions/compose-model-deployment/function/fn.py +++ b/functions/compose-model-deployment/function/fn.py @@ -23,6 +23,7 @@ import enum import grpc +import pydantic from crossplane.function import logging, request, resource, response from crossplane.function.proto.v1 import run_function_pb2 as fnv1 from crossplane.function.proto.v1 import run_function_pb2_grpc as grpcv1 @@ -104,8 +105,6 @@ def _inject_served_model_name(template: mrv1alpha1.Template, served: str) -> Non to. Only the engine answers to the name, so a sidecar beside it doesn't get it. """ - if template.spec is None: - return for container in template.spec.containers: if container.name != "engine": continue @@ -196,11 +195,27 @@ async def RunFunction( log.info("Running function") rsp = response.to(req) - c = Composer(req, rsp) + try: + c = Composer(req, rsp) + except pydantic.ValidationError as err: + # The API server validates a deployment against the XRD, but one + # stored before the XRD gained a rule can still break it. + response.fatal(rsp, f"The ModelDeployment doesn't match its schema: {schema_problems(err)}") + return rsp c.compose() return rsp +def schema_problems(err: pydantic.ValidationError) -> str: + """err's problems as field paths and messages, such as + spec.template.spec.engines[0].members[0].template.spec: Field required.""" + problems = [] + for e in err.errors(include_url=False): + path = "".join(f"[{p}]" if isinstance(p, int) else f".{p}" for p in e["loc"]).lstrip(".") + problems.append(f"{path}: {e['msg']}") + return "; ".join(problems) + + class Composer: def __init__(self, req: fnv1.RunFunctionRequest, rsp: fnv1.RunFunctionResponse) -> None: self.req = req @@ -320,7 +335,32 @@ def resolve_inputs(self) -> bool: return False self.clusters = [_inference_cluster(icv1alpha1.InferenceCluster.model_validate(c)) for c in cluster_dicts] - self.all_replicas = [mrv1alpha1.ModelReplica.model_validate(r) for r in replica_dicts] + # Every deployment reads every replica, so one that doesn't parse is + # skipped with a warning rather than failing them all. The replica's own + # composition reports the problem as a fatal result. The scheduler can't + # count nodes for a replica it can't read, so a skipped replica that is + # still running goes uncharged. A member without template.spec never + # composed a workload, so skipping one of those charges nothing it uses. + # The warning names the replica only when it's in this deployment's + # namespace, so one tenant's events don't name another tenant's objects. + self.all_replicas = [] + namespace = _namespace(self.xr.metadata) + for r in replica_dicts: + try: + self.all_replicas.append(mrv1alpha1.ModelReplica.model_validate(r)) + except pydantic.ValidationError as err: + meta = r.get("metadata") or {} + if meta.get("namespace") != namespace: + response.warning( + self.rsp, + "Ignoring a ModelReplica in another namespace that doesn't match its schema. " + "Its own Synced condition reports the problem.", + ) + continue + response.warning( + self.rsp, + f"Ignoring ModelReplica {meta.get('name')}: it doesn't match its schema: {schema_problems(err)}", + ) # compose-model-cache stages a cache only onto the clusters its selector # matches that report cache storage in status.cache, since the cache's diff --git a/functions/compose-model-deployment/tests/test_fn.py b/functions/compose-model-deployment/tests/test_fn.py index 683b48ad2..1a5290f0f 100644 --- a/functions/compose-model-deployment/tests/test_fn.py +++ b/functions/compose-model-deployment/tests/test_fn.py @@ -2726,6 +2726,335 @@ def _to_dict(msg: message.Message) -> dict: ], ), ), + # The XR is a dict rather than built from the generated model, which + # can't represent the member without template.spec that this case tests. + ComposeCase( + name="TemplateSpecMissing", + reason="A stored deployment whose member has no template.spec stops with a fatal result naming it.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "modelplane.ai/v1alpha1", + "kind": "ModelDeployment", + "metadata": {"name": "my-model", "namespace": "ml-team"}, + "spec": { + "replicas": 1, + "template": { + "spec": { + "engines": [ + { + "name": "main", + "members": [ + { + "role": "Standalone", + "nodeSelector": { + "devices": [ + { + "name": "gpu", + "selectors": [ + {"cel": 'device.driver == "gpu.nvidia.com"'} + ], + }, + ], + }, + "template": {}, + }, + ], + }, + ], + }, + }, + }, + } + ) + ), + ), + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State(), + results=[ + fnv1.Result( + severity=fnv1.SEVERITY_FATAL, + message=( + "The ModelDeployment doesn't match its schema: " + "spec.template.spec.engines[0].members[0].template.spec: Field required" + ), + ), + ], + context=structpb.Struct(), + ), + ), + ComposeCase( + name="ForeignReplicaMalformed", + reason="A replica in another namespace that doesn't parse is skipped with a warning that doesn't name it.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_model_deployment( + replicas=1, + template_labels=None, + cluster_selector=None, + model_cache=None, + serving_mode=None, + endpoint_picker=None, + sidecar=None, + engines=[{"name": "main"}], + args=["--model=Qwen/Qwen3-0.6B"], + ), + ), + required_resources={ + "clusters": fnv1.Resources( + items=[ + _cluster( + name="cluster-a", + ready=True, + gateway_hostname="cluster.clusters.example.com", + nodes=2, + cache_storage=False, + placement_labels=None, + ) + ] + ), + "all-replicas": fnv1.Resources( + items=[ + # A dict rather than the generated model, which can't + # represent the member without template.spec. + fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "modelplane.ai/v1alpha1", + "kind": "ModelReplica", + "metadata": { + "name": "other-model-0a1b2", + "namespace": "other-team", + "labels": {"modelplane.ai/deployment": "other-model"}, + }, + "spec": { + "clusterName": "cluster-a", + "engines": [ + { + "name": "main", + "members": [ + { + "role": "Standalone", + "nodePoolName": "gpu-pool", + "deviceRequests": [ + {"name": "gpu", "deviceClassName": "gpu.nvidia.com"} + ], + "template": {}, + }, + ], + }, + ], + }, + } + ) + ), + ] + ), + }, + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_model_deployment(total_replicas=1, ready_replicas=0, ready=fnv1.READY_FALSE), + resources={ + "replica-cluster-a-0": _composed_replica( + name="my-model-5ab63", + cluster="cluster-a", + labels={ + "modelplane.ai/deployment": "my-model", + "modelplane.ai/cluster": "cluster-a", + "modelplane.ai/replica-index": "0", + }, + model_cache=None, + serving_mode=None, + endpoint_picker=None, + sidecar=None, + engines=[{"name": "main"}], + args=["--model=Qwen/Qwen3-0.6B"], + ready=fnv1.READY_UNSPECIFIED, + ), + }, + ), + results=[ + fnv1.Result( + severity=fnv1.SEVERITY_WARNING, + message=( + "Ignoring a ModelReplica in another namespace that doesn't match its schema. " + "Its own Synced condition reports the problem." + ), + ), + fnv1.Result( + severity=fnv1.SEVERITY_NORMAL, + message="Scheduled 1 replicas across 1 clusters: cluster-a", + ), + ], + context=structpb.Struct(), + requirements=fnv1.Requirements( + resources={ + "clusters": fnv1.ResourceSelector(api_version="modelplane.ai/v1alpha1", kind="InferenceCluster"), + "all-replicas": fnv1.ResourceSelector(api_version="modelplane.ai/v1alpha1", kind="ModelReplica"), + }, + ), + conditions=[ + fnv1.Condition( + type="ReplicasScheduled", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Scheduling", + message="Created 0 of 1 replicas", + ), + fnv1.Condition( + type="ReplicasServing", + status=fnv1.STATUS_CONDITION_FALSE, + reason="ModelStarting", + message="0 of 1 replicas serving", + ), + fnv1.Condition( + type="ReplicasReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="ModelStarting", + message="0 of 1 replicas ready", + ), + ], + ), + ), + ComposeCase( + name="LocalReplicaMalformed", + reason="A replica in the deployment's namespace that doesn't parse is skipped with a warning naming it.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_model_deployment( + replicas=1, + template_labels=None, + cluster_selector=None, + model_cache=None, + serving_mode=None, + endpoint_picker=None, + sidecar=None, + engines=[{"name": "main"}], + args=["--model=Qwen/Qwen3-0.6B"], + ), + ), + required_resources={ + "clusters": fnv1.Resources( + items=[ + _cluster( + name="cluster-a", + ready=True, + gateway_hostname="cluster.clusters.example.com", + nodes=2, + cache_storage=False, + placement_labels=None, + ) + ] + ), + "all-replicas": fnv1.Resources( + items=[ + # A dict rather than the generated model, which can't + # represent the member without template.spec. + fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "modelplane.ai/v1alpha1", + "kind": "ModelReplica", + "metadata": { + "name": "other-model-0a1b2", + "namespace": "ml-team", + "labels": {"modelplane.ai/deployment": "other-model"}, + }, + "spec": { + "clusterName": "cluster-a", + "engines": [ + { + "name": "main", + "members": [ + { + "role": "Standalone", + "nodePoolName": "gpu-pool", + "deviceRequests": [ + {"name": "gpu", "deviceClassName": "gpu.nvidia.com"} + ], + "template": {}, + }, + ], + }, + ], + }, + } + ) + ), + ] + ), + }, + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_model_deployment(total_replicas=1, ready_replicas=0, ready=fnv1.READY_FALSE), + resources={ + "replica-cluster-a-0": _composed_replica( + name="my-model-5ab63", + cluster="cluster-a", + labels={ + "modelplane.ai/deployment": "my-model", + "modelplane.ai/cluster": "cluster-a", + "modelplane.ai/replica-index": "0", + }, + model_cache=None, + serving_mode=None, + endpoint_picker=None, + sidecar=None, + engines=[{"name": "main"}], + args=["--model=Qwen/Qwen3-0.6B"], + ready=fnv1.READY_UNSPECIFIED, + ), + }, + ), + results=[ + fnv1.Result( + severity=fnv1.SEVERITY_WARNING, + message=( + "Ignoring ModelReplica other-model-0a1b2: it doesn't match its schema: " + "spec.engines[0].members[0].template.spec: Field required" + ), + ), + fnv1.Result( + severity=fnv1.SEVERITY_NORMAL, + message="Scheduled 1 replicas across 1 clusters: cluster-a", + ), + ], + context=structpb.Struct(), + requirements=fnv1.Requirements( + resources={ + "clusters": fnv1.ResourceSelector(api_version="modelplane.ai/v1alpha1", kind="InferenceCluster"), + "all-replicas": fnv1.ResourceSelector(api_version="modelplane.ai/v1alpha1", kind="ModelReplica"), + }, + ), + conditions=[ + fnv1.Condition( + type="ReplicasScheduled", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Scheduling", + message="Created 0 of 1 replicas", + ), + fnv1.Condition( + type="ReplicasServing", + status=fnv1.STATUS_CONDITION_FALSE, + reason="ModelStarting", + message="0 of 1 replicas serving", + ), + fnv1.Condition( + type="ReplicasReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="ModelStarting", + message="0 of 1 replicas ready", + ), + ], + ), + ), ] diff --git a/functions/compose-model-replica/function/backends/base.py b/functions/compose-model-replica/function/backends/base.py index f962652b5..65be4d4fc 100644 --- a/functions/compose-model-replica/function/backends/base.py +++ b/functions/compose-model-replica/function/backends/base.py @@ -602,10 +602,6 @@ def engine_container(member: v1alpha1.Member) -> v1alpha1.Container: """Return a member's container named 'engine'. The XRD's CEL validation guarantees exactly one exists per member, so this always succeeds. """ - # An engine member carries its container in template.spec. The XRD types - # spec as optional but a member with no spec defines no pod to serve, so - # reaching here without one is a malformed replica. - assert member.template.spec is not None return next(c for c in member.template.spec.containers if c.name == "engine") @@ -616,7 +612,6 @@ def sidecar_containers(member: v1alpha1.Member) -> list[dict]: so these pass through untouched. Each member has its own template, so the user chooses which pods of a gang run them. """ - assert member.template.spec is not None return [c.model_dump(exclude_none=True) for c in member.template.spec.containers if c.name != "engine"] diff --git a/functions/compose-model-replica/function/backends/grove.py b/functions/compose-model-replica/function/backends/grove.py index 72d80d75b..b31556832 100644 --- a/functions/compose-model-replica/function/backends/grove.py +++ b/functions/compose-model-replica/function/backends/grove.py @@ -130,9 +130,6 @@ def pod_spec(member: v1alpha1.Member, c: dict) -> dict: # claims devices, claims GPUs via the member's DRA template (one # fresh claim per pod). base.place_pod(spec, replica, engine, member) - # The XRD types template.spec as optional, but a member with no spec - # defines no pod to serve, so reaching here without one is malformed. - assert member.template.spec is not None secrets = member.template.spec.imagePullSecrets if secrets: spec["imagePullSecrets"] = [s.model_dump(exclude_none=True) for s in secrets] diff --git a/functions/compose-model-replica/function/backends/llmd.py b/functions/compose-model-replica/function/backends/llmd.py index d40ef0669..fb2bf041c 100644 --- a/functions/compose-model-replica/function/backends/llmd.py +++ b/functions/compose-model-replica/function/backends/llmd.py @@ -119,9 +119,6 @@ def pod_spec(member: v1alpha1.Member, c: dict) -> dict: # claims devices, claims GPUs via the member's DRA template (one # fresh claim per pod). base.place_pod(spec, replica, engine, member) - # The XRD types template.spec as optional, but a member with no spec - # defines no pod to serve, so reaching here without one is malformed. - assert member.template.spec is not None secrets = member.template.spec.imagePullSecrets if secrets: spec["imagePullSecrets"] = [s.model_dump(exclude_none=True) for s in secrets] diff --git a/functions/compose-model-replica/function/backends/native.py b/functions/compose-model-replica/function/backends/native.py index 5c8a3133a..e3dc84c99 100644 --- a/functions/compose-model-replica/function/backends/native.py +++ b/functions/compose-model-replica/function/backends/native.py @@ -97,9 +97,6 @@ def build( # Pin to the member's scheduled pool and claim GPUs via DRA. base.place_pod(pod_spec, replica, engine, member) tmpl = member.template - # The XRD types template.spec as optional, but a member with no spec - # defines no pod to serve, so reaching here without one is malformed. - assert tmpl.spec is not None if tmpl.spec.imagePullSecrets: pod_spec["imagePullSecrets"] = [s.model_dump(exclude_none=True) for s in tmpl.spec.imagePullSecrets] diff --git a/functions/compose-model-replica/function/fn.py b/functions/compose-model-replica/function/fn.py index 6ed0814c1..943f040a1 100644 --- a/functions/compose-model-replica/function/fn.py +++ b/functions/compose-model-replica/function/fn.py @@ -28,6 +28,7 @@ """ import grpc +import pydantic from crossplane.function import logging, request, resource, response from crossplane.function.proto.v1 import run_function_pb2 as fnv1 from crossplane.function.proto.v1 import run_function_pb2_grpc as grpcv1 @@ -78,11 +79,27 @@ async def RunFunction( log.info("Running function") rsp = response.to(req) - c = Composer(req, rsp) + try: + c = Composer(req, rsp) + except pydantic.ValidationError as err: + # The API server validates a replica against the XRD, but one stored + # before the XRD gained a rule can still break it. + response.fatal(rsp, f"The ModelReplica doesn't match its schema: {schema_problems(err)}") + return rsp c.compose() return rsp +def schema_problems(err: pydantic.ValidationError) -> str: + """err's problems as field paths and messages, such as + spec.engines[0].members[0].template.spec: Field required.""" + problems = [] + for e in err.errors(include_url=False): + path = "".join(f"[{p}]" if isinstance(p, int) else f".{p}" for p in e["loc"]).lstrip(".") + problems.append(f"{path}: {e['msg']}") + return "; ".join(problems) + + class Composer: def __init__(self, req: fnv1.RunFunctionRequest, rsp: fnv1.RunFunctionResponse) -> None: self.req = req diff --git a/functions/compose-model-replica/function/routing.py b/functions/compose-model-replica/function/routing.py index 69dbc89fb..87590bae8 100644 --- a/functions/compose-model-replica/function/routing.py +++ b/functions/compose-model-replica/function/routing.py @@ -249,7 +249,7 @@ def decode_without_sidecar(replica: v1alpha1.ModelReplica) -> bool: if decode is None: return False member = base.engine_member(decode, base.ROLE_LEADER) or base.engine_member(decode, base.ROLE_STANDALONE) - if member is None or member.template.spec is None: + if member is None: return False return all(c.name == "engine" for c in member.template.spec.containers) diff --git a/functions/compose-model-replica/tests/test_backends.py b/functions/compose-model-replica/tests/test_backends.py index 743d86430..bddd7bbbb 100644 --- a/functions/compose-model-replica/tests/test_backends.py +++ b/functions/compose-model-replica/tests/test_backends.py @@ -9380,7 +9380,15 @@ def _main_engine() -> v1alpha1.Engine: return v1alpha1.Engine( name="main", copies=2, - members=[v1alpha1.Member(role="Standalone", nodePoolName="frontier", template=v1alpha1.Template())], + members=[ + v1alpha1.Member( + role="Standalone", + nodePoolName="frontier", + template=v1alpha1.Template( + spec=v1alpha1.Spec(containers=[v1alpha1.Container(name="engine", image="vllm/vllm-openai:latest")]) + ), + ) + ], ) diff --git a/functions/compose-model-replica/tests/test_fn.py b/functions/compose-model-replica/tests/test_fn.py index c2fd025c7..f13e28f3c 100644 --- a/functions/compose-model-replica/tests/test_fn.py +++ b/functions/compose-model-replica/tests/test_fn.py @@ -900,6 +900,120 @@ def _to_dict(msg: message.Message) -> dict: ], ), ), + # The XR is a dict rather than built from the generated model, which + # can't represent the member without template.spec that these cases test. + Case( + name="StandaloneTemplateSpecMissing", + reason="A stored replica whose Standalone member has no template.spec stops with a fatal result naming it.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "modelplane.ai/v1alpha1", + "kind": "ModelReplica", + "metadata": {"name": "test-replica", "namespace": "ml-team"}, + "spec": { + "clusterName": "cluster-a", + "engines": [ + { + "name": "main", + "members": [ + { + "role": "Standalone", + "nodePoolName": "frontier", + "deviceRequests": [ + {"name": "gpu", "deviceClassName": "gpu.nvidia.com"} + ], + "template": {}, + }, + ], + }, + ], + }, + } + ) + ), + ), + required_resources={"cluster": fnv1.Resources(items=[_cluster(provider_config_ref="cluster-a-pc")])}, + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State(), + results=[ + fnv1.Result( + severity=fnv1.SEVERITY_FATAL, + message=( + "The ModelReplica doesn't match its schema: " + "spec.engines[0].members[0].template.spec: Field required" + ), + ), + ], + context=structpb.Struct(), + ), + ), + Case( + name="WorkerTemplateSpecMissing", + reason="A stored gang replica whose Worker member has no template.spec stops with a fatal result naming it.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "modelplane.ai/v1alpha1", + "kind": "ModelReplica", + "metadata": {"name": "test-replica", "namespace": "ml-team"}, + "spec": { + "clusterName": "cluster-a", + "engines": [ + { + "name": "main", + "members": [ + { + "role": "Leader", + "nodePoolName": "frontier", + "deviceRequests": [ + {"name": "gpu", "deviceClassName": "gpu.nvidia.com"} + ], + "template": { + "spec": { + "containers": [ + {"name": "engine", "image": "vllm/vllm-openai:latest"}, + ], + }, + }, + }, + { + "role": "Worker", + "nodePoolName": "frontier", + "worker": {"nodes": 1}, + "template": {"metadata": {"labels": {"team": "ml"}}}, + }, + ], + }, + ], + }, + } + ) + ), + ), + required_resources={"cluster": fnv1.Resources(items=[_cluster(provider_config_ref="cluster-a-pc")])}, + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State(), + results=[ + fnv1.Result( + severity=fnv1.SEVERITY_FATAL, + message=( + "The ModelReplica doesn't match its schema: " + "spec.engines[0].members[1].template.spec: Field required" + ), + ), + ], + context=structpb.Struct(), + ), + ), # Modelplane can't tell which container is the disaggregation sidecar, so # it composes the pair as written and warns. The warning comes before the # first reconcile's Composing event. diff --git a/schemas/.lock.json b/schemas/.lock.json index ad2bd8e12..5f4a7f561 100644 --- a/schemas/.lock.json +++ b/schemas/.lock.json @@ -1,6 +1,6 @@ { "packages": { - "fs://apis": "1e73064210fec7fa5508d4af954e85d2adba93ccb40101324d926f950e8d2e2f", + "fs://apis": "4077b5279d8804a7b9eaaec344bee5f9cca2fb6de8cceeb1d9be4a15b11e6326", "git://https://github.com/crossplane/crossplane/cluster/crds": "90d8b72ad8b829f0bcd7d7d5a98eaa0d579f244a", "xpkg://xpkg.upbound.io/upbound/provider-aws-ec2:v2.8.1": "sha256:ca2e9e3b2e3a8b6ca44a9700d5abf7abd733cfa388d1afe9bb2bf7c847cd39ef", "xpkg://xpkg.upbound.io/upbound/provider-aws-efs:v2.8.1": "sha256:ebb1bcd8dc9a7e60e97a324652fbd9d1b0609ddbb1107db518ce999cbc1113bd", diff --git a/schemas/python/models/ai/modelplane/modeldeployment/v1alpha1.py b/schemas/python/models/ai/modelplane/modeldeployment/v1alpha1.py index 68d2b0b87..d7b5965a2 100644 --- a/schemas/python/models/ai/modelplane/modeldeployment/v1alpha1.py +++ b/schemas/python/models/ai/modelplane/modeldeployment/v1alpha1.py @@ -220,7 +220,7 @@ class Template(BaseModel): """ Metadata applied to the member's pods. Useful for labels and annotations that control cluster-level features like service mesh injection. Label keys under the modelplane.ai/ prefix are reserved. """ - spec: Spec | None = None + spec: Spec """ Pod spec for this member's engine pods. """ diff --git a/schemas/python/models/ai/modelplane/modelreplica/v1alpha1.py b/schemas/python/models/ai/modelplane/modelreplica/v1alpha1.py index 70be7e42e..fc428e709 100644 --- a/schemas/python/models/ai/modelplane/modelreplica/v1alpha1.py +++ b/schemas/python/models/ai/modelplane/modelreplica/v1alpha1.py @@ -169,7 +169,7 @@ class Spec(BaseModel): class Template(BaseModel): metadata: Metadata | None = None - spec: Spec | None = None + spec: Spec class Worker(BaseModel):