From 49f8d1050ad94e5505ad0bdef9cae62c2468427f Mon Sep 17 00:00:00 2001 From: msau42 Date: Tue, 13 Oct 2026 14:54:34 +0000 Subject: [PATCH] Prefer usable local checkpoint on Restore and fall back to object storage Check the node-local checkpoint directory for the requested SnapshotUuid first during Restore, verifying the manifest and every listed snapshot file via readLocalSnapshot. When the local copy is absent or unusable, fall back to fetching the manifest and snapshot files from Snapshot.Object in object storage, and attribute the restore metric kind to local only when served from the node-local copy. --- cmd/atelet/lifecycle_test.go | 235 ++++++++++++++++++++++++++- cmd/atelet/main.go | 77 ++++++--- cmd/atelet/metrics.go | 14 +- cmd/atelet/metrics_test.go | 37 ++--- internal/proto/ateletpb/atelet.pb.go | 9 +- internal/proto/ateletpb/atelet.proto | 9 +- 6 files changed, 319 insertions(+), 62 deletions(-) diff --git a/cmd/atelet/lifecycle_test.go b/cmd/atelet/lifecycle_test.go index 7d272cfd1e..1d12c0cf57 100644 --- a/cmd/atelet/lifecycle_test.go +++ b/cmd/atelet/lifecycle_test.go @@ -17,6 +17,7 @@ package main import ( "context" "crypto/sha256" + "errors" "fmt" "net" "os" @@ -27,11 +28,13 @@ import ( "github.com/agent-substrate/substrate/cmd/atelet/internal/ateletpath" "github.com/agent-substrate/substrate/cmd/atelet/internal/trustbundle" + "github.com/agent-substrate/substrate/internal/apierror" "github.com/agent-substrate/substrate/internal/nodepath" "github.com/agent-substrate/substrate/internal/proto/ateletpb" "github.com/agent-substrate/substrate/internal/proto/ateompb" "github.com/agent-substrate/substrate/internal/resources" "google.golang.org/grpc" + "google.golang.org/grpc/codes" "google.golang.org/protobuf/proto" ) @@ -244,7 +247,7 @@ func runLocalSnapshotGC(t *testing.T, fidelity ateletpb.SnapshotFidelity) { t.Fatalf("pause did not write the local snapshot: %v", err) } - // Resume: restores from that local snapshot. + // Resume: no Snapshot.Object; atelet must find that local snapshot. if _, err := s.Restore(ctx, &ateletpb.RestoreRequest{ Atespace: atespace, ActorName: actorName, @@ -454,7 +457,7 @@ func TestActivationFailureBeforeRegistration(t *testing.T) { {name: "Run partial reset removes registration", failure: "reset", wantErr: "while removing volume dir"}, {name: "Run record failure removes registration", failure: "record", wantErr: "while recording sandbox assets"}, {name: "Restore partial reset removes registration", restore: true, failure: "reset", wantErr: "while removing volume dir"}, - {name: "Restore manifest failure removes registration", restore: true, failure: "manifest", wantErr: "while reading local snapshot manifest"}, + {name: "Restore manifest failure removes registration", restore: true, failure: "manifest", wantErr: "no snapshot.object to fall back to"}, {name: "Restore asset failure removes registration", restore: true, failure: "asset", wantErr: "asset unavailable"}, } { t.Run(tc.name, func(t *testing.T) { @@ -677,6 +680,234 @@ func TestRestoreFailureAfterRegistrationRemovesOwnRegistration(t *testing.T) { } } +// TestRestorePrefersLocalAndFallsBackToObjectStorage checks that Restore reads +// a usable node-local checkpoint in place even when snapshot.object is also +// populated, and falls back to staging from snapshot.object when the local copy +// is absent or unusable (corrupt manifest, missing listed file, or non-regular +// file). +func TestRestorePrefersLocalAndFallsBackToObjectStorage(t *testing.T) { + const ( + atespace = "ate-demo" + actorName = "counter" + actorUID = "01234567-89ab-cdef-0123-456789abcdef" + workerPodUID = "5f1c0d2e-3a4b-4c5d-8e9f-0a1b2c3d4e5f" + snapshotName = "pause-snap-1" + ) + + ateom := &fakeAteom{snapshotFiles: map[string]string{"checkpoint.img": ""}} + serveFakeAteom(t, ateom) + + host := imageVolumeTestRegistry(t) + pinnedImage := pushTestImage(t, host+"/actor:v1", singleFileLayer(t, "bin/app", "app")) + + runsc := []byte("runsc binary") + assetHash := fmt.Sprintf("%x", sha256.Sum256(runsc)) + sandboxAssets := &ateletpb.SandboxAssets{ + SandboxClass: "gvisor", + PauseImage: pinnedImage, + Assets: map[string]*ateletpb.ArchAssets{ + runtime.GOARCH: {Files: map[string]*ateletpb.AssetFile{ + runscAssetName: { + Url: "gs://test-bucket/runsc", + Sha256: assetHash, + }, + }}, + }, + } + spec := &ateletpb.WorkloadSpec{ + Containers: []*ateletpb.Container{{Name: "app", Image: pinnedImage, Command: []string{"/bin/app"}}}, + } + manifestRec := sandboxAssetsRecord{ + SandboxClass: "gvisor", + PauseImage: pinnedImage, + Assets: map[string]assetEntry{runscAssetName: {URL: "gs://test-bucket/runsc", SHA256: assetHash}}, + SnapshotFiles: []string{"checkpoint.img"}, + } + + // Upload a committed copy to object storage once; every subtest restores + // with snapshot.object pointing at it. + objStore := &recordingObjectStorage{} + s := newPluginHerder(t, objStore) + s.ateomDialer = newAteomDialer(1) + s.imageCache = newImageVolumeStore(t) + s.anonGCSClient = fakeObjectStorage{data: runsc} + s.systemInfoVolumes = newSystemInfoVolumeRefresher(nil, nil) + + uri, err := resources.ParseSnapshotURI(pausedSnapshotURI) + if err != nil { + t.Fatal(err) + } + remoteSrcDir := t.TempDir() + writeLocalSnapshot(t, remoteSrcDir, manifestRec, map[string]string{"checkpoint.img": "object-storage-memory"}) + if err := s.uploadSnapshot(t.Context(), uri, remoteSrcDir, &manifestRec, "default", "counter"); err != nil { + t.Fatalf("uploadSnapshot: %v", err) + } + + for _, tc := range []struct { + name string + prepLocal func(t *testing.T, localDir string) + wantLocal bool + }{{ + name: "usable local copy is preferred over object storage", + prepLocal: func(t *testing.T, localDir string) { + writeLocalSnapshot(t, localDir, manifestRec, map[string]string{"checkpoint.img": "local-memory"}) + }, + wantLocal: true, + }, { + name: "absent local copy falls back to object storage", + prepLocal: func(*testing.T, string) {}, + wantLocal: false, + }, { + name: "local copy with missing listed file falls back to object storage", + prepLocal: func(t *testing.T, localDir string) { + writeLocalSnapshot(t, localDir, manifestRec, nil) + }, + wantLocal: false, + }, { + name: "local copy with corrupt manifest falls back to object storage", + prepLocal: func(t *testing.T, localDir string) { + if err := os.MkdirAll(localDir, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(localDir, sandboxManifestName), []byte("{not-json"), 0o600); err != nil { + t.Fatal(err) + } + }, + wantLocal: false, + }, { + name: "local copy with symlinked snapshot file falls back to object storage", + prepLocal: func(t *testing.T, localDir string) { + writeLocalSnapshot(t, localDir, manifestRec, nil) + target := filepath.Join(t.TempDir(), "outside.img") + if err := os.WriteFile(target, []byte("outside"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Symlink(target, filepath.Join(localDir, "checkpoint.img")); err != nil { + t.Fatal(err) + } + }, + wantLocal: false, + }} { + t.Run(tc.name, func(t *testing.T) { + useTempNodeDirs(t) + localDir := ateletpath.LocalSnapshotDir(actorUID, snapshotName) + tc.prepLocal(t, localDir) + + if _, err := s.Restore(t.Context(), &ateletpb.RestoreRequest{ + Atespace: atespace, + ActorName: actorName, + ActorUid: actorUID, + ActorTemplateAtespace: "default", + ActorTemplateName: "counter", + WorkerPodUid: workerPodUID, + SandboxAssets: sandboxAssets, + Spec: spec, + Snapshot: &ateletpb.Snapshot{ + SnapshotUuid: snapshotName, + Fidelity: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + Object: &ateletpb.ObjectStorage{SnapshotUri: pausedSnapshotURI}, + }, + }); err != nil { + t.Fatalf("Restore: %v", err) + } + + wantContent := "object-storage-memory" + wantRestoreDir := ateletpath.RestoreStateDir(actorUID) + if tc.wantLocal { + wantContent = "local-memory" + wantRestoreDir = localDir + } + if got := ateom.restored["checkpoint.img"]; got != wantContent { + t.Errorf("restored checkpoint.img = %q, want %q", got, wantContent) + } + if ateom.preserveRestoreDir != tc.wantLocal { + t.Errorf("RestoreWorkload preserve_restore_dir = %v, want %v", ateom.preserveRestoreDir, tc.wantLocal) + } + if got := ateom.actorDirs["RestoreWorkload"].GetRestoreDir(); got != wantRestoreDir { + t.Errorf("RestoreWorkload restore_dir = %q, want %q", got, wantRestoreDir) + } + if tc.wantLocal { + if entries, err := os.ReadDir(ateletpath.RestoreStateDir(actorUID)); err != nil || len(entries) != 0 { + t.Errorf("RestoreStateDir entries = %v, err = %v; want empty on local restore", entries, err) + } + } + }) + } +} + +// TestRestoreWithoutUsableLocalCopy checks that a Restore with no usable local +// copy and nothing to fall back to fails as NotFound and wraps the local +// copy's problem, whether the request has no snapshot.object or names one that +// was never uploaded. +func TestRestoreWithoutUsableLocalCopy(t *testing.T) { + const ( + actorUID = "0f6c2b0e-1b7e-4b3f-9a6a-1d2c3e4f5a65" + snapshotName = "pause-snap-1" + ) + assetHash := fmt.Sprintf("%x", sha256.Sum256([]byte("runsc binary"))) + assets := &ateletpb.SandboxAssets{ + SandboxClass: "gvisor", PauseImage: testPauseImage, + Assets: map[string]*ateletpb.ArchAssets{runtime.GOARCH: {Files: map[string]*ateletpb.AssetFile{ + runscAssetName: {Url: "gs://test-bucket/runsc", Sha256: assetHash}, + }}}, + } + for _, tc := range []struct { + name string + // unusableLocal writes a manifest that lists a file that is not there. + unusableLocal bool + object *ateletpb.ObjectStorage + wantErr string + wantCause error + }{{ + name: "no local copy and no object", + wantErr: "no snapshot.object to fall back to", + wantCause: os.ErrNotExist, + }, { + name: "unusable local copy and no object", + unusableLocal: true, + wantErr: "no snapshot.object to fall back to", + wantCause: errUnusableLocalSnapshot, + }, { + name: "unusable local copy and an object never uploaded", + unusableLocal: true, + object: &ateletpb.ObjectStorage{SnapshotUri: pausedSnapshotURI}, + wantErr: "no uploaded copy", + wantCause: errUnusableLocalSnapshot, + }} { + t.Run(tc.name, func(t *testing.T) { + useTempNodeDirs(t) + if tc.unusableLocal { + writeLocalSnapshot(t, ateletpath.LocalSnapshotDir(actorUID, snapshotName), sandboxAssetsRecord{ + SandboxClass: "gvisor", PauseImage: testPauseImage, SnapshotFiles: []string{"checkpoint.img"}, + }, nil) + } + // The object store is empty: nothing was ever uploaded. + s := newPluginHerder(t, &recordingObjectStorage{}) + s.systemInfoVolumes = newSystemInfoVolumeRefresher(nil, nil) + _, err := s.Restore(t.Context(), &ateletpb.RestoreRequest{ + Atespace: "team-a", ActorName: "actor-1", ActorUid: actorUID, + WorkerPodUid: "5f1c0d2e-3a4b-4c5d-8e9f-0a1b2c3d4e5f", + SandboxAssets: assets, + Spec: &ateletpb.WorkloadSpec{}, + Snapshot: &ateletpb.Snapshot{ + SnapshotUuid: snapshotName, + Fidelity: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + Object: tc.object, + }, + }) + if code := apierror.Code(err); code != codes.NotFound { + t.Fatalf("Restore error = %v (code %v), want NotFound", err, code) + } + if !strings.Contains(err.Error(), tc.wantErr) { + t.Errorf("Restore error = %v, want it to say %q", err, tc.wantErr) + } + if !errors.Is(err, tc.wantCause) { + t.Errorf("Restore error = %v, want it to wrap %v", err, tc.wantCause) + } + }) + } +} + // TestTerminateWithoutWorkerPodUID verifies that Terminate with an empty // WorkerPodUid (used when cleaning up the assigned node for a paused or // crashed actor that no longer has a worker pod) skips dialing ateom while diff --git a/cmd/atelet/main.go b/cmd/atelet/main.go index 74cc72b7ab..670355fec0 100644 --- a/cmd/atelet/main.go +++ b/cmd/atelet/main.go @@ -755,8 +755,8 @@ func (s *AteomHerder) Checkpoint(ctx context.Context, req *ateletpb.CheckpointRe // Do not move this above CheckpointWorkload to keep MergeDeltaIntoBase on its // in-place path: that leaves the whole checkpoint window with no local snapshot // while LocalSnapshotInfo still names the pruned one, and a crash there strands - // the actor for good (resume never falls back to object storage, RequiredNodes - // pins it to this node, nothing clears the field). + // the actor for good (a paused actor has no uploaded copy for resume to fall + // back to, RequiredNodes pins it to this node, nothing clears the field). // // Best-effort: if this fail, the actor's terminate prunes again. if err := pruneLocalCheckpoints(ctx, actorUID); err != nil { @@ -1045,6 +1045,33 @@ func (s *AteomHerder) uploadLocalCheckpointDir(ctx context.Context, req *ateletp return rec.SandboxClass, pluginsocket.CallError(s.uploadSnapshot(ctx, uri, localDir, rec, req.GetActorTemplateAtespace(), req.GetActorTemplateName())) } +// errUnusableLocalSnapshot marks a node-local snapshot that is present on disk +// (not simply absent) but cannot be read or validated: its manifest cannot be +// read or parsed, or lists a file that is missing or not a regular file, as an +// interrupted prune can leave. File contents and sizes are not checked. +var errUnusableLocalSnapshot = errors.New("unusable local snapshot") + +// readLocalSnapshot returns the manifest of the node-local snapshot in +// localDir once every file it lists is present. A missing manifest reports +// os.ErrNotExist, and any other failure wraps errUnusableLocalSnapshot. +func readLocalSnapshot(localDir string) (*sandboxAssetsRecord, error) { + manifest, err := readSnapshotManifest(localDir) + if errors.Is(err, os.ErrNotExist) { + return nil, err + } + if err != nil { + return nil, fmt.Errorf("%w: %w", errUnusableLocalSnapshot, err) + } + rec, err := unmarshalSandboxRecord(manifest) + if err != nil { + return nil, fmt.Errorf("%w: %w", errUnusableLocalSnapshot, err) + } + if err := checkLocalSnapshotFiles(localDir, rec.SnapshotFiles); err != nil { + return nil, fmt.Errorf("%w: %w", errUnusableLocalSnapshot, err) + } + return rec, nil +} + func readSnapshotManifest(dir string) ([]byte, error) { root, err := os.OpenRoot(dir) if err != nil { @@ -1122,14 +1149,14 @@ func (s *AteomHerder) Restore(ctx context.Context, req *ateletpb.RestoreRequest) } checkpointDir := ateletpath.RestoreStateDir(actorUID) - directLocal := req.GetSnapshot().GetObject() == nil - if directLocal { - checkpointDir = ateletpath.LocalSnapshotDir(actorUID, req.GetSnapshot().GetSnapshotUuid()) - } + localDir := ateletpath.LocalSnapshotDir(actorUID, req.GetSnapshot().GetSnapshotUuid()) + directLocal := false // Fetch the snapshot manifest stored beside the checkpoint images // first: it lists the checkpoint files to download and records the actor - // identity used to label the restore's metrics. + // identity used to label the restore's metrics. Try the node-local + // checkpoint directory first and fall back to object storage if the local + // copy is absent or unusable. tManifest := time.Now() manifestDone := false defer func() { @@ -1138,16 +1165,27 @@ func (s *AteomHerder) Restore(ctx context.Context, req *ateletpb.RestoreRequest) } }() var sandboxRec *sandboxAssetsRecord - if directLocal { - manifest, err := readSnapshotManifest(checkpointDir) - if err != nil { - return nil, wrapFileSystemErr("while reading local snapshot manifest", err) + localRec, localErr := readLocalSnapshot(localDir) + durable := req.GetSnapshot().GetObject() + if localErr == nil { + directLocal = true + checkpointDir = localDir + sandboxRec = localRec + } else { + if durable == nil { + return nil, apierror.NotFound("snapshot %q has no usable local copy and no snapshot.object to fall back to: %w", + req.GetSnapshot().GetSnapshotUuid(), localErr) } - if sandboxRec, err = unmarshalSandboxRecord(manifest); err != nil { - return nil, fmt.Errorf("while unmarshalling sandbox record: %w", err) + if errors.Is(localErr, errUnusableLocalSnapshot) { + // The next checkpoint or terminate prunes the unusable copy. + slog.WarnContext(ctx, "Local snapshot is unusable; restoring from object storage", + slog.String("local_dir", localDir), slog.Any("err", localErr)) + } + manifest, err := s.fetchManifest(ctx, durable.GetSnapshotUri()) + if status.Code(err) == codes.NotFound { + return nil, apierror.NotFound("snapshot %q has no usable local copy and no uploaded copy: %w", + req.GetSnapshot().GetSnapshotUuid(), localErr) } - } else { - manifest, err := s.fetchManifest(ctx, req.GetSnapshot().GetObject().GetSnapshotUri()) if err != nil { return nil, fmt.Errorf("while fetching snapshot manifest: %w", err) } @@ -1161,7 +1199,7 @@ func (s *AteomHerder) Restore(ctx context.Context, req *ateletpb.RestoreRequest) // The manifest is what tells a golden restore from a latest one, so the // snapshot kind only becomes knowable here. - op.kind = restoreSnapshotKind(req, sandboxRec) + op.kind = restoreSnapshotKind(directLocal, sandboxRec) var registration *registeredActor // Undo the Register if the restore fails. The errgroup join below happens @@ -1192,10 +1230,11 @@ func (s *AteomHerder) Restore(ctx context.Context, req *ateletpb.RestoreRequest) downloadErr = err }() if directLocal { - // Restore in place from LocalSnapshotDir; no staging. - return checkLocalSnapshotFiles(checkpointDir, sandboxRec.SnapshotFiles) + // Restore in place from LocalSnapshotDir, whose files + // readLocalSnapshot checked; no staging. + return nil } - return s.downloadExternalCheckpoint(gctx, req.GetSnapshot().GetObject().GetSnapshotUri(), checkpointDir, sandboxRec.SnapshotFiles) + return s.downloadExternalCheckpoint(gctx, durable.GetSnapshotUri(), checkpointDir, sandboxRec.SnapshotFiles) }) g.Go(func() (err error) { defer func() { prepErr = err }() diff --git a/cmd/atelet/metrics.go b/cmd/atelet/metrics.go index 804020545f..7433497b8b 100644 --- a/cmd/atelet/metrics.go +++ b/cmd/atelet/metrics.go @@ -153,13 +153,13 @@ func assetsAfterCollateral(prepFailedPhase string, assets time.Duration) time.Du return assets } -// restoreSnapshotKind classifies which snapshot a restore reads. A local -// restore is evident from the wire; golden and latest both arrive as an external -// URI prefix, so they are told apart by the identity the manifest records for -// the actor that wrote the snapshot. An empty result means the manifest has not -// been read yet, so the kind is not knowable. -func restoreSnapshotKind(req *ateletpb.RestoreRequest, rec *sandboxAssetsRecord) string { - if req.GetSnapshot().GetObject() == nil { +// restoreSnapshotKind classifies which snapshot a restore reads: local when it +// is served from the node-local copy, and otherwise golden or latest, told +// apart by the identity the manifest records for the actor that wrote the +// snapshot. An empty result means the manifest has not been read yet, so the +// kind is not knowable. +func restoreSnapshotKind(directLocal bool, rec *sandboxAssetsRecord) string { + if directLocal { return ateattr.SnapshotKindLocal } if rec == nil { diff --git a/cmd/atelet/metrics_test.go b/cmd/atelet/metrics_test.go index 8aaf437113..92f5ef34d2 100644 --- a/cmd/atelet/metrics_test.go +++ b/cmd/atelet/metrics_test.go @@ -280,59 +280,48 @@ func TestAssetsAfterCollateral(t *testing.T) { } func TestRestoreSnapshotKind(t *testing.T) { - localSnapshot := &ateletpb.Snapshot{ - SnapshotUuid: "snap-1", - } - externalSnapshot := &ateletpb.Snapshot{ - SnapshotUuid: "snap-1", - Object: &ateletpb.ObjectStorage{SnapshotUri: "gs://bucket/snap"}, - } tests := []struct { - name string - req *ateletpb.RestoreRequest - rec *sandboxAssetsRecord - want string + name string + directLocal bool + rec *sandboxAssetsRecord + want string }{ { - name: "local pause snapshot", - req: &ateletpb.RestoreRequest{Snapshot: localSnapshot}, - rec: &sandboxAssetsRecord{Atespace: "team-a"}, - want: ateattr.SnapshotKindLocal, + name: "restored from the local copy", + directLocal: true, + rec: &sandboxAssetsRecord{Atespace: "team-a"}, + want: ateattr.SnapshotKindLocal, }, { - name: "local restore is classifiable before the manifest is read", - req: &ateletpb.RestoreRequest{Snapshot: localSnapshot}, - rec: nil, - want: ateattr.SnapshotKindLocal, + name: "a golden actor's snapshot served locally is still local", + directLocal: true, + rec: &sandboxAssetsRecord{Atespace: resources.GoldenActorAtespace}, + want: ateattr.SnapshotKindLocal, }, { name: "external snapshot written by a golden actor", - req: &ateletpb.RestoreRequest{Snapshot: externalSnapshot}, rec: &sandboxAssetsRecord{Atespace: resources.GoldenActorAtespace}, want: ateattr.SnapshotKindGolden, }, { name: "external snapshot written by a tenant actor", - req: &ateletpb.RestoreRequest{Snapshot: externalSnapshot}, rec: &sandboxAssetsRecord{Atespace: "team-a"}, want: ateattr.SnapshotKindLatest, }, { name: "manifest predating the identity fields", - req: &ateletpb.RestoreRequest{Snapshot: externalSnapshot}, rec: &sandboxAssetsRecord{}, want: ateattr.SnapshotKindLatest, }, { name: "external kind is unknowable until the manifest is read", - req: &ateletpb.RestoreRequest{Snapshot: externalSnapshot}, rec: nil, want: "", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := restoreSnapshotKind(tt.req, tt.rec); got != tt.want { + if got := restoreSnapshotKind(tt.directLocal, tt.rec); got != tt.want { t.Errorf("restoreSnapshotKind() = %q, want %q", got, tt.want) } }) diff --git a/internal/proto/ateletpb/atelet.pb.go b/internal/proto/ateletpb/atelet.pb.go index 16b2fc7103..d8167a7bae 100644 --- a/internal/proto/ateletpb/atelet.pb.go +++ b/internal/proto/ateletpb/atelet.pb.go @@ -2792,8 +2792,7 @@ type Snapshot struct { // usable local copy. // // TODO: Temporarily, this is optional while we haven't implemented deferred - // uploads. Restore also uses its presence to choose between the local copy - // and object storage; see RestoreRequest.snapshot. + // uploads. // // +k8s:optional Object *ObjectStorage `protobuf:"bytes,4,opt,name=object,proto3" json:"object,omitempty"` @@ -3286,9 +3285,9 @@ type RestoreRequest struct { // detaching the actor, it waits for the detach to finish and then reads // `snapshot.object`. Fails if neither copy exists: that generation is lost. // - // TODO: Temporarily, whether `snapshot.object` is set chooses the copy: - // atelet reads only the local copy when it is unset, and only - // `snapshot.object` when it is set, with no fallback either way. + // TODO: Temporarily, the control plane does not set `snapshot.object` when + // it restores a paused actor, so atelet has nothing to fall back to if that + // actor's local copy is missing or unusable. // // +k8s:required Snapshot *Snapshot `protobuf:"bytes,8,opt,name=snapshot,proto3" json:"snapshot,omitempty"` diff --git a/internal/proto/ateletpb/atelet.proto b/internal/proto/ateletpb/atelet.proto index aff782894d..7e75daa959 100644 --- a/internal/proto/ateletpb/atelet.proto +++ b/internal/proto/ateletpb/atelet.proto @@ -837,8 +837,7 @@ message Snapshot { // usable local copy. // // TODO: Temporarily, this is optional while we haven't implemented deferred - // uploads. Restore also uses its presence to choose between the local copy - // and object storage; see RestoreRequest.snapshot. + // uploads. // // +k8s:optional ObjectStorage object = 4; @@ -1022,9 +1021,9 @@ message RestoreRequest { // detaching the actor, it waits for the detach to finish and then reads // `snapshot.object`. Fails if neither copy exists: that generation is lost. // - // TODO: Temporarily, whether `snapshot.object` is set chooses the copy: - // atelet reads only the local copy when it is unset, and only - // `snapshot.object` when it is set, with no fallback either way. + // TODO: Temporarily, the control plane does not set `snapshot.object` when + // it restores a paused actor, so atelet has nothing to fall back to if that + // actor's local copy is missing or unusable. // // +k8s:required Snapshot snapshot = 8;