Sync upstream snapshot and SandboxConfig updates

charts/substrate-crds/templates/ate.dev_workerpools.yaml

@@ -39,6 +39,12 @@ spec:
     - jsonPath: .status.readyReplicas
       name: Ready
       type: integer
+    - jsonPath: .status.sandboxClasses[*].configRef.name
+      name: SandboxConfig
+      type: string
+    - jsonPath: .status.sandboxClasses[*].conditions[?(@.type=="Resolved")].status
+      name: Resolved
+      type: string
     - jsonPath: .metadata.creationTimestamp
       name: Age
       type: date
@@ -75,18 +81,22 @@ spec:
               sandboxClasses:
                 description: |-
                   SandboxClasses lists the sandbox runtime families this pool runs. Exactly
-                  one entry is allowed today.
+                  one entry is allowed today. The names cannot change after creation.
                 items:
                   description: WorkerPoolSandboxClass is one sandbox runtime family
                     a WorkerPool runs.
                   properties:
                     configRef:
                       description: |-
                         ConfigRef names a cluster-scoped SandboxConfig for this sandbox class.
-                        The referenced config's SandboxClass must match Name. Not consumed yet.
+                        The referenced config's SandboxClass must match Name. When omitted, the
+                        pool uses the newest SandboxConfig of this class annotated
+                        sandboxconfig.ate.dev/is-class-default: "true". The controller reports
+                        the resolved config in status.sandboxClasses.
                       properties:
                         name:
                           description: Name is the SandboxConfig's metadata.name.
+                          maxLength: 253
                           minLength: 1
                           type: string
                       required:
@@ -113,6 +123,9 @@ spec:
                 x-kubernetes-list-map-keys:
                 - name
                 x-kubernetes-list-type: map
+                x-kubernetes-validations:
+                - message: sandboxClasses names are immutable
+                  rule: self.map(c, c.name) == oldSelf.map(c, c.name)
               template:
                 description: Template holds optional metadata, scheduling, and resource
                   settings for worker workloads.
@@ -491,6 +504,121 @@ spec:
                 format: int32
                 minimum: 0
                 type: integer
+              sandboxClasses:
+                description: |-
+                  SandboxClasses reports the SandboxConfig each sandbox class of the pool
+                  uses and whether the class's spec resolves.
+                items:
+                  description: |-
+                    WorkerPoolSandboxClassStatus reports the SandboxConfig and resolution state
+                    of one sandbox class of a WorkerPool.
+                  properties:
+                    conditions:
+                      description: Conditions describe the current state of this sandbox
+                        class.
+                      items:
+                        description: Condition contains details for one aspect of
+                          the current state of this API Resource.
+                        properties:
+                          lastTransitionTime:
+                            description: |-
+                              lastTransitionTime is the last time the condition transitioned from one status to another.
+                              This should be when the underlying condition changed.  If that is not known, then using the time when the API field changed is acceptable.
+                            format: date-time
+                            type: string
+                          message:
+                            default: ""
+                            description: |-
+                              message is a human readable message indicating details about the transition.
+                              This may be an empty string.
+                            maxLength: 32768
+                            type: string
+                          observedGeneration:
+                            description: |-
+                              observedGeneration represents the .metadata.generation that the condition was set based upon.
+                              For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date
+                              with respect to the current state of the instance.
+                            format: int64
+                            minimum: 0
+                            type: integer
+                          reason:
+                            description: |-
+                              reason contains a programmatic identifier indicating the reason for the condition's last transition.
+                              Producers of specific condition types may define expected values and meanings for this field,
+                              and whether the values are considered a guaranteed API.
+                              The value should be a CamelCase string.
+                              This field may not be empty.
+                            maxLength: 1024
+                            minLength: 1
+                            pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
+                            type: string
+                          status:
+                            description: status of the condition, one of True, False,
+                              Unknown.
+                            enum:
+                            - "True"
+                            - "False"
+                            - Unknown
+                            type: string
+                          type:
+                            description: type of condition in CamelCase or in foo.example.com/CamelCase.
+                            maxLength: 316
+                            pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
+                            type: string
+                        required:
+                        - lastTransitionTime
+                        - reason
+                        - status
+                        - type
+                        type: object
+                      type: array
+                      x-kubernetes-list-map-keys:
+                      - type
+                      x-kubernetes-list-type: map
+                    configRef:
+                      description: |-
+                        ConfigRef names the cluster-scoped SandboxConfig in use. That is the
+                        config spec.sandboxClasses[].configRef names, or the class default, or
+                        the last resolved config while the spec does not resolve. Omitted when
+                        the class has never resolved.
+                      properties:
+                        name:
+                          description: Name is the SandboxConfig's metadata.name.
+                          maxLength: 253
+                          minLength: 1
+                          type: string
+                      required:
+                      - name
+                      type: object
+                    name:
+                      description: Name is the sandbox runtime family.
+                      enum:
+                      - gvisor
+                      - microvm
+                      type: string
+                    pendingConfigRef:
+                      description: |-
+                        PendingConfigRef names the SandboxConfig the class is switching to
+                        while the controller confirms it against the API server. It protects
+                        the config from deletion like configRef does. The same reconcile
+                        clears it, and configRef either moves to it or stays where it was.
+                      properties:
+                        name:
+                          description: Name is the SandboxConfig's metadata.name.
+                          maxLength: 253
+                          minLength: 1
+                          type: string
+                      required:
+                      - name
+                      type: object
+                  required:
+                  - name
+                  type: object
+                maxItems: 1
+                type: array
+                x-kubernetes-list-map-keys:
+                - name
+                x-kubernetes-list-type: map
               selector:
                 description: Selector is the label selector for the worker pods.
                 type: string

charts/substrate/README.md

@@ -85,6 +85,22 @@ resources now put assets and `pauseImage` under `spec.versions[]`, with a named
 `spec.defaultVersion`. WorkerPools use `spec.sandboxClasses[]`, and actor
 templates continue to select the config with `sandboxConfig.configName`.
 
+WorkerPools now resolve `spec.sandboxClasses[].configRef` or the class default
+before creating their worker Deployment. Ensure each pool names an existing
+config of the same class, or that one is annotated
+`sandboxconfig.ate.dev/is-class-default: "true"`. The chart supplies that
+annotation on `gvisor-default`. Sandbox class names are immutable; create a
+new pool to change runtime families. Configs referenced by a pool are protected
+from deletion until the pool switches configs or is deleted.
+
+Update API clients for the simplified snapshot model: `Snapshot.uuid`,
+`fidelity`, `locality`, and `durableSnapshot` replace the `storage[]` entries,
+and durable snapshot status uses `PENDING` instead of `IN_PROGRESS`. The proto
+field numbers changed, so upgrade clients and components together.
+Persisted Actor and Tag snapshot metadata uses the same protobuf schema and
+is not migrated by this update. For development installs, delete affected
+Actors and Tags with the old version, then recreate them after upgrading.
+
 Snapshot transfers now run in native sidecars in the API server and atelet
 pods. Move external object storage environment settings from
 `atelet.extraEnv` or the API environment ConfigMap to

charts/substrate/templates/role.yaml

@@ -47,6 +47,15 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - ate.dev
+  resources:
+  - sandboxconfigs
+  verbs:
+  - get
+  - list
+  - patch
+  - watch
 - apiGroups:
   - ate.dev
   resources:
@@ -93,6 +102,13 @@ rules:
   - signers
   verbs:
   - attest
+- apiGroups:
+  - events.k8s.io
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
 - apiGroups:
   - networking.k8s.io
   resources:

cmd/ate-setup/internal/steps/benchmarks.go

@@ -110,8 +110,10 @@ func (e *Env) DeleteBenchmarks(ctx context.Context, opts BenchmarkOptions) error
 	if err := e.runScript(ctx, deployLocustScript, "--delete"); err != nil {
 		return err
 	}
-	// Only tear down the microvm SandboxConfig if the caller opted into
-	// microvm: it is cluster-wide and may be in use by something else.
+	// The microvm SandboxConfig is cluster-wide. It is only torn down when
+	// the caller opted into microvm, and only after the benchmark pool is
+	// gone. The script leaves it in place if another pool still uses it.
+	// `delete all` tries again once every demo is gone.
 	if opts.SandboxClass == config.SandboxClassMicrovm {
 		return e.runScript(ctx, installMicrovmDepScript, "--delete")
 	}

cmd/ate-setup/internal/steps/delete.go

@@ -17,11 +17,16 @@ package steps
 import (
 	"context"
 	"fmt"
+	"slices"
 
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/dynamic"
+	"k8s.io/client-go/util/retry"
 
 	"github.com/agent-substrate/substrate/cmd/ate-setup/internal/log"
+	atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1"
 )
 
 // DeleteAteSystem removes the control plane.
@@ -74,9 +79,51 @@ func (e *Env) DeleteAteSystem(ctx context.Context) error {
 	if err := e.Kube.WaitDeleted(ctx, schema.GroupVersionKind{Version: "v1", Kind: "Namespace"}, "", e.Namespace(), e.Cfg.WaitTimeout(BootstrapTimeout)); err != nil {
 		return err
 	}
+	// Removing the protection finalizer by hand is safe here. atecontroller,
+	// which would remove it, went away with the namespace. Deleting the CRDs
+	// above deleted every WorkerPool, so no pool uses a config anymore.
+	if err := releaseSandboxConfigs(ctx, e.Kube.Dynamic.Resource(atev1alpha1.GroupVersion.WithResource("sandboxconfigs"))); err != nil {
+		return err
+	}
 	return e.UnlabelNodesSubstrateVersion(ctx)
 }
 
+// releaseSandboxConfigs removes the WorkerPool protection finalizer from
+// every SandboxConfig. Only atecontroller adds and removes this finalizer,
+// and atecontroller is gone once its namespace is. Without this step no
+// SandboxConfig could finish deleting, and deleting the CRD would hang.
+func releaseSandboxConfigs(ctx context.Context, res dynamic.ResourceInterface) error {
+	list, err := res.List(ctx, metav1.ListOptions{})
+	if apierrors.IsNotFound(err) {
+		return nil
+	}
+	if err != nil {
+		return fmt.Errorf("while listing SandboxConfigs: %w", err)
+	}
+	for _, item := range list.Items {
+		name := item.GetName()
+		err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
+			sc, err := res.Get(ctx, name, metav1.GetOptions{})
+			if err != nil {
+				return err
+			}
+			finalizers := slices.DeleteFunc(sc.GetFinalizers(), func(f string) bool {
+				return f == atev1alpha1.SandboxConfigWorkerPoolProtectionFinalizer
+			})
+			if len(finalizers) == len(sc.GetFinalizers()) {
+				return nil
+			}
+			sc.SetFinalizers(finalizers)
+			_, err = res.Update(ctx, sc, metav1.UpdateOptions{})
+			return err
+		})
+		if err != nil && !apierrors.IsNotFound(err) {
+			return fmt.Errorf("while releasing SandboxConfig %s: %w", name, err)
+		}
+	}
+	return nil
+}
+
 // DeleteAtenet removes the atenet dataplane and the bundled credential
 // provider, deleting the provider first so that a later failure cannot leave
 // its ClusterRole and binding behind. The provider's policy ConfigMap stays:
@@ -108,14 +155,38 @@ type Deleter interface {
 	Delete(ctx context.Context, e *Env) error
 }
 
-// DeleteAll removes every registered demo and then the control plane.
+// DeleteAll removes every demo, then the microvm SandboxConfig, then the
+// control plane.
 func (e *Env) DeleteAll(ctx context.Context, demos []Deleter) error {
 	log.Step("delete_all")
 
+	if err := e.deleteDemos(ctx, demos); err != nil {
+		return err
+	}
+	return e.DeleteAteSystem(ctx)
+}
+
+// deleteDemos removes every demo, then tries to delete the microvm
+// SandboxConfig. An earlier `delete benchmarks` or `delete demo` may have
+// left the config behind because a pool still used it. After the demos, no
+// pool does.
+//
+// The delete is best effort. The script skips a config that is still in use,
+// and a failure is only logged, because DeleteAteSystem deletes the config
+// with the CRD anyway.
+//
+// The gvisor config needs no such step. It is installed and deleted with the
+// control plane, and DeleteAteSystem strips its finalizer itself. The microvm
+// config is managed by hack/install-microvm-deps.sh, whose `kubectl delete`
+// would block while a pool still uses it.
+func (e *Env) deleteDemos(ctx context.Context, demos []Deleter) error {
 	for _, demo := range demos {
 		if err := demo.Delete(ctx, e); err != nil {
 			return err
 		}
 	}
-	return e.DeleteAteSystem(ctx)
+	if err := e.runScript(ctx, installMicrovmDepScript, "--delete"); err != nil {
+		log.Warnf("while deleting the microvm SandboxConfig: %v", err)
+	}
+	return nil
 }