From 0fe58271bbd3ebae33754845ccf179699029f27a Mon Sep 17 00:00:00 2001 From: cloudonly Date: Fri, 16 Oct 2026 13:57:59 +0800 Subject: [PATCH 1/3] Run golden snapshots in the template's atespace Place generated actors and published tags beside their templates, using template UIDs as resource names. Remove the system-atespace exceptions for lifecycle operations, snapshot fidelity, authorization, and metrics. Keep the reconciler's existing RPCs and state machine, and cover tenant storage ownership, snapshot inheritance, restore, and cleanup. --- benchmarking/analysis/README.md | 8 +- benchmarking/analysis/phase_report.py | 6 +- benchmarking/analysis/test_phase_report.py | 12 +- .../internal/apivalidation/validation_test.go | 2 +- cmd/ateapi/internal/authz/interceptor_test.go | 4 +- cmd/ateapi/internal/authz/registry.go | 5 +- cmd/ateapi/internal/authz/registry_test.go | 5 +- cmd/ateapi/internal/controlapi/actor.go | 5 - .../controlapi/actor_template_test.go | 6 +- cmd/ateapi/internal/controlapi/actor_test.go | 8 +- .../functionaltest/actor_template_test.go | 6 +- .../controlapi/functionaltest/actor_test.go | 20 +-- .../controlapi/functionaltest/common_test.go | 7 +- .../controlapi/template_reconciler.go | 16 +-- .../controlapi/template_reconciler_test.go | 23 ++-- .../internal/controlapi/workflow_resume.go | 14 -- .../controlapi/workflow_resume_test.go | 2 +- .../internal/controlapi/workflow_suspend.go | 21 +-- .../controlapi/workflow_suspend_async_test.go | 22 --- .../controlapi/workflow_suspend_test.go | 82 ----------- .../controlapi/workflow_template_delete.go | 6 +- cmd/ateapi/internal/workerservice/suspend.go | 10 -- .../internal/workerservice/suspend_test.go | 42 ------ cmd/atelet/internal/apivalidation/validate.go | 10 -- .../internal/apivalidation/validation_test.go | 4 - .../apivalidation/zz_generated.validation.go | 4 - cmd/atelet/main.go | 6 +- cmd/atelet/metrics.go | 17 +-- cmd/atelet/metrics_test.go | 21 ++- .../internal/printer/printer_test.go | 2 +- docs/api-guide.md | 27 +++- docs/metrics/registry/metrics.yaml | 6 +- docs/observability.md | 8 +- internal/ateattr/ateattr.go | 5 +- internal/ateattr/ateattr_test.go | 1 - internal/e2e/suites/demo/golden_test.go | 128 ++++++++++++++++++ internal/proto/ateletpb/atelet.pb.go | 3 - internal/proto/ateletpb/atelet.proto | 3 - internal/resources/actor.go | 3 - pkg/proto/ateapipb/ateapi.proto | 2 +- pkg/proto/ateapipb/ateapi_grpc.pb.go | 4 +- 41 files changed, 242 insertions(+), 344 deletions(-) create mode 100644 internal/e2e/suites/demo/golden_test.go diff --git a/benchmarking/analysis/README.md b/benchmarking/analysis/README.md index 78f6670153..3109de03fb 100644 --- a/benchmarking/analysis/README.md +++ b/benchmarking/analysis/README.md @@ -72,9 +72,11 @@ A failed atelet operation still writes its record, marked with `error.type` ## Reading the report **Phase percentiles.** Per layer, operation, sandbox class, snapshot kind, -scope and phase: count, p50, p90, p95 and max. A `golden` restore downloads -the golden image and a `latest` one the actor's own, so they are separate -rows, as are gVisor and micro-VM operations. The atelet rows split a +scope and phase: count, p50, p90, p95 and max. A `local` restore reads a +node-local snapshot and a `latest` one a durable snapshot from object +storage, so they are separate rows, as are gVisor and micro-VM operations. +A restore from a template's golden snapshot reports `latest`, like the +actor's own durable snapshot. The atelet rows split a checkpoint between `sandbox_assets`, `worker_checkpoint`, `persist` and `cleanup`, and a restore between `volume_mount`, `manifest_fetch`, `sandbox_assets`, diff --git a/benchmarking/analysis/phase_report.py b/benchmarking/analysis/phase_report.py index 61334bf575..2d7c154f40 100755 --- a/benchmarking/analysis/phase_report.py +++ b/benchmarking/analysis/phase_report.py @@ -294,9 +294,9 @@ def fmt_s(seconds: float) -> str: def report_phases(breakdowns: list[Breakdown], writer) -> list[dict]: """Per (source, op, sandbox class, kind, scope, phase) percentiles. A - golden restore (which downloads the golden image) and a latest restore, - or a gVisor and a micro-VM checkpoint, are different distributions and - must not be pooled. Returns the rows for CSV.""" + local restore and a latest one, or a gVisor and a micro-VM checkpoint, + are different distributions and must not be pooled. Returns the rows + for CSV.""" groups: dict[tuple, list[float]] = defaultdict(list) for b in breakdowns: if b.failed: diff --git a/benchmarking/analysis/test_phase_report.py b/benchmarking/analysis/test_phase_report.py index 8712e54d68..40fde9b188 100644 --- a/benchmarking/analysis/test_phase_report.py +++ b/benchmarking/analysis/test_phase_report.py @@ -291,14 +291,14 @@ def test_failed_records_are_flagged_and_excluded_from_percentiles(self): self.assertEqual(downloads[0]["count"], 1) # the failed 30s never entered def test_percentiles_split_by_sandbox_class_and_kind(self): - golden = dict(ATELET_RESTORE, **{"time": "2026-09-23T09:59:00.000000000Z", - "ate.snapshot.kind": "golden", "ate.sandbox.class": "gvisor", - "ate.actor.restore.duration.download": 9.0, - "ate.actor.restore.duration.total": 10.0}) + local = dict(ATELET_RESTORE, **{"time": "2026-09-23T09:59:00.000000000Z", + "ate.snapshot.kind": "local", "ate.sandbox.class": "gvisor", + "ate.actor.restore.duration.download": 9.0, + "ate.actor.restore.duration.total": 10.0}) latest = dict(ATELET_RESTORE, **{"ate.sandbox.class": "gvisor"}) - rows = phase_report.report_phases(parse(golden, latest).breakdowns, lambda _: None) + rows = phase_report.report_phases(parse(local, latest).breakdowns, lambda _: None) downloads = {(r["class"], r["kind"]): r["p50_ms"] for r in rows if r["phase"] == "download"} - self.assertEqual(downloads, {("gvisor", "golden"): 9000.0, ("gvisor", "latest"): 2400.0}) + self.assertEqual(downloads, {("gvisor", "local"): 9000.0, ("gvisor", "latest"): 2400.0}) def test_paired_ateworker_record_inherits_the_atelet_kind(self): out = parse(ATEWORKER_RESTORE, ATELET_RESTORE) diff --git a/cmd/ateapi/internal/apivalidation/validation_test.go b/cmd/ateapi/internal/apivalidation/validation_test.go index 6e1451850e..0b1d5e252c 100644 --- a/cmd/ateapi/internal/apivalidation/validation_test.go +++ b/cmd/ateapi/internal/apivalidation/validation_test.go @@ -1712,7 +1712,7 @@ func TestValidateTagRequestPayloads(t *testing.T) { func validGoldenSnapshotStatus(mutate ...func(*ateapipb.GoldenSnapshotStatus)) *ateapipb.GoldenSnapshotStatus { s := &ateapipb.GoldenSnapshotStatus{ - GoldenTag: &ateapipb.ObjectRef{Atespace: "ate-golden", Name: "01234567-89ab-cdef-0123-456789abcdef"}, + GoldenTag: &ateapipb.ObjectRef{Atespace: "team-a", Name: "01234567-89ab-cdef-0123-456789abcdef"}, SnapshotTime: ×tamppb.Timestamp{Seconds: 867}, } for _, m := range mutate { diff --git a/cmd/ateapi/internal/authz/interceptor_test.go b/cmd/ateapi/internal/authz/interceptor_test.go index 2ac47665b6..52df3035c7 100644 --- a/cmd/ateapi/internal/authz/interceptor_test.go +++ b/cmd/ateapi/internal/authz/interceptor_test.go @@ -559,7 +559,6 @@ func TestUnaryServerInterceptor_LifecycleEgressWorkerAndTagChecks(t *testing.T) } cloneInTeamA := cloneTag("team-a", tag) cloneInTeamB := cloneTag("team-b", tag) - cloneGolden := cloneTag("team-a", &ateapipb.ObjectRef{Atespace: "ate-golden", Name: "golden-v1"}) createWithoutTag := cloneTag("team-a", nil) tests := []struct { @@ -638,14 +637,13 @@ func TestUnaryServerInterceptor_LifecycleEgressWorkerAndTagChecks(t *testing.T) {"bootstrap owner deletes tag", "bootstrap-owner", deleteTag, codes.OK}, // Cloning from a tag needs editor on the tag's atespace, whatever its - // scope; golden tags and actors created without a tag are unaffected. + // scope. An actor created without an explicit tag only needs its template. {"editor clones tag", "editor", cloneInTeamA, codes.OK}, {"viewer cannot clone tag", "viewer", cloneInTeamA, codes.PermissionDenied}, {"editor of another atespace cannot clone tag", "outsider", cloneInTeamB, codes.PermissionDenied}, {"viewer of tag's atespace cannot clone tag into another atespace", "outsider-with-viewer", cloneInTeamB, codes.PermissionDenied}, {"global owner clones tag into another atespace", "global-owner", cloneInTeamB, codes.OK}, {"bootstrap owner clones tag", "bootstrap-owner", cloneInTeamA, codes.OK}, - {"editor clones golden tag", "editor", cloneGolden, codes.OK}, {"editor creates actor without a tag", "editor", createWithoutTag, codes.OK}, // Only the egress gateway mints actor credentials; not even owners. diff --git a/cmd/ateapi/internal/authz/registry.go b/cmd/ateapi/internal/authz/registry.go index eb81b80940..4acacf82cf 100644 --- a/cmd/ateapi/internal/authz/registry.go +++ b/cmd/ateapi/internal/authz/registry.go @@ -124,10 +124,9 @@ func onTag(relation string, ref resourceRef, fldPath *field.Path) ([]check, fiel } // onSourceTag checks can_use_tag on the tag an actor is created from, if the -// request names one. Golden tags are exempt: the handler only accepts one -// built from the actor's own template, which can_use_template already covers. +// request names one. func onSourceTag(ref *ateapipb.ObjectRef, fldPath *field.Path) ([]check, field.ErrorList) { - if ref == nil || ref.GetAtespace() == resources.GoldenActorAtespace { + if ref == nil { return nil, nil } return onTag(RelationCanUseTag, ref, fldPath) diff --git a/cmd/ateapi/internal/authz/registry_test.go b/cmd/ateapi/internal/authz/registry_test.go index be4ea32aa3..05d4e26386 100644 --- a/cmd/ateapi/internal/authz/registry_test.go +++ b/cmd/ateapi/internal/authz/registry_test.go @@ -123,16 +123,17 @@ func TestDefaultRPCPermissions(t *testing.T) { }, }, { - name: "CreateActor from a golden tag does not check the tag", + name: "CreateActor from a template tag checks the tag", fullMethod: ateapipb.Control_CreateActor_FullMethodName, req: &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ Metadata: actorMeta, ActorTemplate: sharedTemplate, - SourceTag: &ateapipb.ObjectRef{Atespace: "ate-golden", Name: "0b5c7d4e-golden"}, + SourceTag: &ateapipb.ObjectRef{Atespace: "shared", Name: "0b5c7d4e-golden"}, }}, want: []check{ {RelationCanCreateActor, "atespace:team-a"}, {RelationCanUseTemplate, "actor_template:shared/tmpl"}, + {RelationCanUseTag, "tag:shared/0b5c7d4e-golden"}, }, }, { diff --git a/cmd/ateapi/internal/controlapi/actor.go b/cmd/ateapi/internal/controlapi/actor.go index 623d7cb15d..42ed43295e 100644 --- a/cmd/ateapi/internal/controlapi/actor.go +++ b/cmd/ateapi/internal/controlapi/actor.go @@ -108,11 +108,6 @@ func (s *ServiceImpl) CreateActor(ctx context.Context, inActor *ateapipb.Actor) if err != nil { return nil, err } - if inActor.GetSourceTag() == nil { - if err := validateGoldenSnapshotFidelity(sourceTag.GetStatus().GetSnapshot()); err != nil { - return nil, err - } - } } atespace := inActor.GetMetadata().GetAtespace() diff --git a/cmd/ateapi/internal/controlapi/actor_template_test.go b/cmd/ateapi/internal/controlapi/actor_template_test.go index 501b1671f4..b848388504 100644 --- a/cmd/ateapi/internal/controlapi/actor_template_test.go +++ b/cmd/ateapi/internal/controlapi/actor_template_test.go @@ -101,7 +101,7 @@ func TestCreateActorTemplateIgnoresServerOwnedFields(t *testing.T) { // Server-owned status a client must not be able to set. tmpl.Status = &ateapipb.ActorTemplateStatus{ GoldenSnapshotStatus: &ateapipb.GoldenSnapshotStatus{ - GoldenTag: &ateapipb.ObjectRef{Atespace: "ate-golden", Name: "golden-tag"}, + GoldenTag: &ateapipb.ObjectRef{Atespace: "ns1", Name: "golden-tag"}, }, } }) @@ -152,7 +152,7 @@ func TestDeleteActorTemplate(t *testing.T) { persistence := newTestPersistence(t) tmpl := seedSubstrateTemplate(t, ctx, persistence, "tmpl") templateRef := resources.ActorTemplateRefFromActorTemplate(tmpl) - goldenRef := resources.ActorRef{Atespace: resources.GoldenActorAtespace, Name: tmpl.GetMetadata().GetUid()} + goldenRef := resources.ActorRef{Atespace: "team-a", Name: tmpl.GetMetadata().GetUid()} actor := storetest.MustCreateActor(t, ctx, persistence, &ateapipb.Actor{ Metadata: &ateapipb.ResourceMetadata{Atespace: goldenRef.Atespace, Name: goldenRef.Name}, ActorTemplate: templateRef.ToObjectRef(), @@ -366,7 +366,7 @@ func TestUpdateActorTemplateMetadata(t *testing.T) { // A server-owned status write passes validation and bumps the version. updated, err := persistence.UpdateActorTemplate(ctx, ref, store.PreconditionFrom(created), func(tmpl *ateapipb.ActorTemplate) error { tmpl.Status = &ateapipb.ActorTemplateStatus{GoldenSnapshotStatus: &ateapipb.GoldenSnapshotStatus{ - GoldenTag: &ateapipb.ObjectRef{Atespace: "ate-golden", Name: "golden-tag"}, + GoldenTag: &ateapipb.ObjectRef{Atespace: "ns1", Name: "golden-tag"}, }} return nil }) diff --git a/cmd/ateapi/internal/controlapi/actor_test.go b/cmd/ateapi/internal/controlapi/actor_test.go index afa3566544..328f6430a4 100644 --- a/cmd/ateapi/internal/controlapi/actor_test.go +++ b/cmd/ateapi/internal/controlapi/actor_test.go @@ -825,16 +825,15 @@ func TestCreateActor_GoldenTagDefault(t *testing.T) { ctx := t.Context() persistence := newTestPersistence(t) storetest.MustCreateAtespace(t, ctx, persistence, "team-a") - storetest.MustCreateAtespace(t, ctx, persistence, resources.GoldenActorAtespace) tmpl := seedSubstrateTemplate(t, ctx, persistence, "tmpl") - ref := &ateapipb.ObjectRef{Atespace: resources.GoldenActorAtespace, Name: "golden"} + ref := &ateapipb.ObjectRef{Atespace: "team-a", Name: tmpl.GetMetadata().GetUid()} tag := &ateapipb.Tag{ Metadata: &ateapipb.ResourceMetadata{Atespace: ref.Atespace, Name: ref.Name}, SourceActor: ref, Scope: ateapipb.TagScope_TAG_SCOPE_PUBLISHED, Status: &ateapipb.TagStatus{ ActorTemplateUid: tmpl.GetMetadata().GetUid(), - Snapshot: newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, tmpl.GetMetadata().GetUid(), someActorUID, "gs://bucket/atespaces/ate-golden/tags/"+someActorUID, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + Snapshot: newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, tmpl.GetMetadata().GetUid(), someActorUID, "gs://bucket/atespaces/team-a/tags/"+someActorUID, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), }, } wantCode := codes.OK @@ -849,7 +848,6 @@ func TestCreateActor_GoldenTagDefault(t *testing.T) { wantCode = codes.FailedPrecondition case "data scope": tag.Status.Snapshot.Fidelity = ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES - wantCode = codes.FailedPrecondition } if scenario != "missing" { if _, err := persistence.CreateTag(ctx, tag); err != nil { @@ -865,7 +863,7 @@ func TestCreateActor_GoldenTagDefault(t *testing.T) { actor := &ateapipb.Actor{Metadata: &ateapipb.ResourceMetadata{Atespace: "team-a", Name: "actor"}, ActorTemplate: resources.ActorTemplateRefFromActorTemplate(tmpl).ToObjectRef()} if scenario == "explicit tag" { tag.Metadata.Name = "explicit" - tag.Status.Snapshot = newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, tmpl.GetMetadata().GetUid(), "explicit", "gs://bucket/atespaces/ate-golden/tags/explicit", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + tag.Status.Snapshot = newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, tmpl.GetMetadata().GetUid(), "explicit", "gs://bucket/atespaces/team-a/tags/explicit", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) if _, err := persistence.CreateTag(ctx, tag); err != nil { t.Fatal(err) } diff --git a/cmd/ateapi/internal/controlapi/functionaltest/actor_template_test.go b/cmd/ateapi/internal/controlapi/functionaltest/actor_template_test.go index cb0736ccf3..68a27b5dba 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/actor_template_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/actor_template_test.go @@ -48,7 +48,7 @@ func TestActorTemplateCRUD(t *testing.T) { // Server-owned status on the request is ignored. Status: &ateapipb.ActorTemplateStatus{ GoldenSnapshotStatus: &ateapipb.GoldenSnapshotStatus{ - GoldenTag: &ateapipb.ObjectRef{Atespace: "ate-golden", Name: "golden-tag"}, + GoldenTag: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "golden-tag"}, }, }, }, @@ -163,7 +163,7 @@ func TestGoldenTagLifecycle(t *testing.T) { if err != nil { t.Fatal(err) } - if !parsed.OwnedBy(resources.TagSnapshotOwner(resources.GoldenActorAtespace, golden.GetMetadata().GetUid())) { + if !parsed.OwnedBy(resources.TagSnapshotOwner(testAtespace, golden.GetMetadata().GetUid())) { t.Fatalf("golden snapshot %q is not under the golden tag's own prefix", uri) } assertSnapshotPresent(t, tc, uri) @@ -247,7 +247,7 @@ func TestActorLifecycleWithoutGoldenSnapshot(t *testing.T) { } workerName := createWorkerPod(t, tc, ns, "worker-1", "node1", "pool1") controlTemplate := createTemplateWithSelector(t, tc, "golden-control", &ateapipb.Selector{MatchLabels: map[string]string{poolLabelKey: ns}}) - goldenRef := &ateapipb.ObjectRef{Atespace: resources.GoldenActorAtespace, Name: tmpl.GetMetadata().GetUid()} + goldenRef := &ateapipb.ObjectRef{Atespace: testAtespace, Name: tmpl.GetMetadata().GetUid()} assertNoGolden := func(ctx context.Context) error { if _, err := tc.client.GetActor(ctx, &ateapipb.GetActorRequest{Actor: goldenRef}); status.Code(err) != codes.NotFound { return fmt.Errorf("disabled template's golden actor: got %v, want NotFound", err) diff --git a/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go b/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go index e7bbf484c3..241ccf83fc 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go @@ -3272,10 +3272,9 @@ func createDataFidelityTemplate(t *testing.T, tc *testContext, ns string) *ateap // Publish the golden snapshot as a tag and point the template at it, as // createTemplateWithContainersAndVolumes does. - createAtespace(t, tc, resources.GoldenActorAtespace) tag, err := tc.persistence.CreateTag(context.Background(), &ateapipb.Tag{ - Metadata: &ateapipb.ResourceMetadata{Atespace: resources.GoldenActorAtespace, Name: created.GetMetadata().GetUid()}, - SourceActor: &ateapipb.ObjectRef{Atespace: resources.GoldenActorAtespace, Name: created.GetMetadata().GetUid()}, + Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: created.GetMetadata().GetUid()}, + SourceActor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: created.GetMetadata().GetUid()}, Scope: ateapipb.TagScope_TAG_SCOPE_PUBLISHED, Status: &ateapipb.TagStatus{ Snapshot: newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, created.GetMetadata().GetUid(), goldenSnapshotName, goldenSnapshotURI(t)), @@ -3943,10 +3942,15 @@ func TestSuspendActor(t *testing.T) { // It stopped borrowing without releasing what it had borrowed. assertSnapshotPresent(t, tc, taggedSnapshotURI) assertSnapshotPresent(t, tc, cloneSnapshotURI) - // The untagged suspend created no tag. + // The untagged suspend created no tag: only the golden and explicit tag remain. listed, err = tc.client.ListTags(context.Background(), &ateapipb.ListTagsRequest{Atespace: testAtespace}) - if err != nil || len(listed.GetTags()) != 1 { - t.Fatalf("ListTags after clone suspension = (%v, %v), want one", listed, err) + if err != nil || len(listed.GetTags()) != 2 { + t.Fatalf("ListTags after clone suspension = (%v, %v), want golden and explicit tags", listed, err) + } + for _, tag := range listed.GetTags() { + if ref := resources.TagRefFromTag(tag).ToObjectRef(); !proto.Equal(ref, tagRef) && !proto.Equal(ref, tmpl.GetStatus().GetGoldenSnapshotStatus().GetGoldenTag()) { + t.Errorf("unexpected tag after clone suspension: %v", ref) + } } getResp, err := tc.client.GetActor(context.Background(), &ateapipb.GetActorRequest{ @@ -6163,8 +6167,8 @@ func TestResumeActor_SandboxRuntimePlacementAndFallback(t *testing.T) { goldenSnap := newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, tmpl.GetMetadata().GetUid(), goldenSnapshotName, goldenSnapshotURI(t)) goldenSnap.SandboxRuntime = proto.Clone(rtA).(*ateapipb.SandboxRuntime) stampedGoldenTag, err := tc.persistence.CreateTag(ctx, &ateapipb.Tag{ - Metadata: &ateapipb.ResourceMetadata{Atespace: resources.GoldenActorAtespace, Name: "golden-rt-a"}, - SourceActor: &ateapipb.ObjectRef{Atespace: resources.GoldenActorAtespace, Name: tmpl.GetMetadata().GetUid()}, + Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "golden-rt-a"}, + SourceActor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: tmpl.GetMetadata().GetUid()}, Scope: ateapipb.TagScope_TAG_SCOPE_PUBLISHED, Status: &ateapipb.TagStatus{ Snapshot: goldenSnap, diff --git a/cmd/ateapi/internal/controlapi/functionaltest/common_test.go b/cmd/ateapi/internal/controlapi/functionaltest/common_test.go index ab7ba9fae1..2040b3719c 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/common_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/common_test.go @@ -383,7 +383,7 @@ const goldenSnapshotName = "9c2f7b41-6d05-4e83-a1f7-3b8c0d5e2a94" // goldenSnapshotURI is the snapshot owned by the test template's golden tag. func goldenSnapshotURI(t *testing.T) string { t.Helper() - uri, err := resources.NewTagSnapshotURI(testStorageLocation, resources.GoldenActorAtespace, goldenSnapshotName) + uri, err := resources.NewTagSnapshotURI(testStorageLocation, testAtespace, goldenSnapshotName) if err != nil { t.Fatalf("NewTagSnapshotURI: %v", err) } @@ -499,10 +499,9 @@ func createTemplateWithContainersAndVolumes(t *testing.T, tc *testContext, ns st t.Fatalf("failed to create actor template: %v", err) } - createAtespace(t, tc, resources.GoldenActorAtespace) tag, err := tc.persistence.CreateTag(context.Background(), &ateapipb.Tag{ - Metadata: &ateapipb.ResourceMetadata{Atespace: resources.GoldenActorAtespace, Name: created.GetMetadata().GetUid()}, - SourceActor: &ateapipb.ObjectRef{Atespace: resources.GoldenActorAtespace, Name: created.GetMetadata().GetUid()}, + Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: created.GetMetadata().GetUid()}, + SourceActor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: created.GetMetadata().GetUid()}, Scope: ateapipb.TagScope_TAG_SCOPE_PUBLISHED, Status: &ateapipb.TagStatus{ Snapshot: newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, created.GetMetadata().GetUid(), goldenSnapshotName, goldenSnapshotURI(t)), diff --git a/cmd/ateapi/internal/controlapi/template_reconciler.go b/cmd/ateapi/internal/controlapi/template_reconciler.go index 2a3a3fd95c..350f6954ef 100644 --- a/cmd/ateapi/internal/controlapi/template_reconciler.go +++ b/cmd/ateapi/internal/controlapi/template_reconciler.go @@ -72,7 +72,6 @@ type goldenActorControl interface { CreateTag(ctx context.Context, req *ateapipb.CreateTagRequest) (*ateapipb.Tag, error) DeleteTag(ctx context.Context, req *ateapipb.DeleteTagRequest) (*ateapipb.Tag, error) DeleteActor(ctx context.Context, req *ateapipb.DeleteActorRequest) (*ateapipb.Actor, error) - CreateAtespace(ctx context.Context, req *ateapipb.CreateAtespaceRequest) (*ateapipb.Atespace, error) CreateActor(ctx context.Context, req *ateapipb.CreateActorRequest) (*ateapipb.Actor, error) GetActor(ctx context.Context, req *ateapipb.GetActorRequest) (*ateapipb.Actor, error) ResumeActor(ctx context.Context, req *ateapipb.ResumeActorRequest) (*ateapipb.ResumeActorResponse, error) @@ -198,11 +197,9 @@ func (r *ActorTemplateReconciler) reconcileOne(ctx context.Context, ref resource } goldenActorRef := &ateapipb.ObjectRef{ - // Golden actors live in the reserved ate-golden atespace, because - // the suspend workflow relies on the ate-golden system atespace to - // always take a full snapshot of the golden actor. - // https://github.com/agent-substrate/substrate/blob/cb7c8385ef2bb489c3d5f7bfa71820fd33935d91/cmd/ateapi/internal/controlapi/workflow_suspend.go#L170-L173 - Atespace: resources.GoldenActorAtespace, + // Golden actors live in the template's atespace, under the same + // policies and storage grants as ordinary actors. + Atespace: tmpl.GetMetadata().GetAtespace(), // Use the template's UID as golden actor's name to prevent collision // when templates are recreated with the same name. Name: tmpl.GetMetadata().GetUid(), @@ -435,13 +432,6 @@ func (r *ActorTemplateReconciler) ensureActorExists(ctx context.Context, tmpl *a if apierror.Code(err) != codes.NotFound { return nil, fmt.Errorf("while getting golden actor: %w", err) } - // Golden actor has not yet been created. Its reserved atespace is - // system-owned, so ensure it exists rather than assuming bootstrap did. - if _, err := r.control.CreateAtespace(ctx, &ateapipb.CreateAtespaceRequest{ - Atespace: &ateapipb.Atespace{Metadata: &ateapipb.ResourceMetadata{Name: goldenActorRef.GetAtespace()}}, - }); err != nil && apierror.Code(err) != codes.AlreadyExists { - return nil, fmt.Errorf("while ensuring atespace %q: %w", goldenActorRef.GetAtespace(), err) - } actor, err = r.control.CreateActor(ctx, &ateapipb.CreateActorRequest{ Actor: &ateapipb.Actor{ Metadata: &ateapipb.ResourceMetadata{ diff --git a/cmd/ateapi/internal/controlapi/template_reconciler_test.go b/cmd/ateapi/internal/controlapi/template_reconciler_test.go index 18b66adcfe..8a53f51c89 100644 --- a/cmd/ateapi/internal/controlapi/template_reconciler_test.go +++ b/cmd/ateapi/internal/controlapi/template_reconciler_test.go @@ -363,7 +363,7 @@ func withSnapshotDeadline(at time.Time) func(*ateapipb.ActorTemplate) { func withGoldenTag() func(*ateapipb.ActorTemplate) { return func(tmpl *ateapipb.ActorTemplate) { - seededGoldenStatus(tmpl).GoldenTag = &ateapipb.ObjectRef{Atespace: resources.GoldenActorAtespace, Name: testTemplateUID} + seededGoldenStatus(tmpl).GoldenTag = &ateapipb.ObjectRef{Atespace: testTemplateRef.Atespace, Name: testTemplateUID} } } @@ -398,7 +398,7 @@ func TestGoldenSnapshotWarmupFor(t *testing.T) { } func TestReconcileOne(t *testing.T) { - goldenSnapshot := "gs://bucket/root/atespaces/ate-golden/actors/" + someActorUID + "/snapshots/snap-1" + goldenSnapshot := "gs://bucket/root/atespaces/ns/actors/" + someActorUID + "/snapshots/snap-1" tests := []struct { name string template *ateapipb.ActorTemplate // nil: template absent from the store @@ -641,13 +641,12 @@ func TestReconcileOne(t *testing.T) { // TestReconcileOne_GoldenActorRequests pins the shape of the control-plane // requests the happy path issues: the golden actor is named after the // template UID so recreated templates with the same name never collide, and -// lives in the reserved ate-golden atespace so the suspend workflow commits -// it Full regardless of the template's preferredFidelity scope. +// lives in the template's atespace under ordinary tenant policies. func TestReconcileOne_GoldenActorRequests(t *testing.T) { ctx := context.Background() st := newFakeTemplateStore(testTemplate()) - goldenSnapshot := "gs://bucket/root/atespaces/ate-golden/actors/" + someActorUID + "/snapshots/snap-1" + goldenSnapshot := "gs://bucket/root/atespaces/ns/actors/" + someActorUID + "/snapshots/snap-1" control := &fakeGoldenControl{snapshot: goldenSnapshot} r := newTestTemplateReconciler(st, control) @@ -659,14 +658,14 @@ func TestReconcileOne_GoldenActorRequests(t *testing.T) { if got := created.GetMetadata().GetName(); got != testTemplateUID { t.Errorf("golden actor name = %q, want template UID %q", got, testTemplateUID) } - if got := created.GetMetadata().GetAtespace(); got != resources.GoldenActorAtespace { - t.Errorf("golden actor atespace = %q, want %q", got, resources.GoldenActorAtespace) + if got := created.GetMetadata().GetAtespace(); got != testTemplateRef.Atespace { + t.Errorf("golden actor atespace = %q, want %q", got, testTemplateRef.Atespace) } if got := created.GetActorTemplate().GetName(); got != testTemplateName { t.Errorf("golden actor template ref = %q, want %q", got, testTemplateName) } - if len(control.atespaceReqs) != 1 || control.atespaceReqs[0].GetAtespace().GetMetadata().GetName() != resources.GoldenActorAtespace { - t.Errorf("atespace ensure requests = %v, want one for %q", control.atespaceReqs, resources.GoldenActorAtespace) + if len(control.atespaceReqs) != 0 { + t.Errorf("unexpected atespace creation requests: %v", control.atespaceReqs) } if got := control.resumeReqs[0].GetActor().GetName(); got != testTemplateUID { t.Errorf("resumed actor = %q, want %q", got, testTemplateUID) @@ -891,7 +890,7 @@ func TestResync_FollowsPagination(t *testing.T) { } func TestReconcileOne_GoldenTagRecovery(t *testing.T) { - ref := &ateapipb.ObjectRef{Atespace: resources.GoldenActorAtespace, Name: testTemplateUID} + ref := &ateapipb.ObjectRef{Atespace: testTemplateRef.Atespace, Name: testTemplateUID} completed := &ateapipb.Tag{ Metadata: &ateapipb.ResourceMetadata{Atespace: ref.Atespace, Name: ref.Name}, SourceActor: ref, @@ -980,8 +979,8 @@ func TestReconcileOne_GoldenTagConflict(t *testing.T) { templateUID string sourceActor *ateapipb.ObjectRef }{ - {name: "template", templateUID: "another-template", sourceActor: &ateapipb.ObjectRef{Atespace: resources.GoldenActorAtespace, Name: testTemplateUID}}, - {name: "actor name", templateUID: testTemplateUID, sourceActor: &ateapipb.ObjectRef{Atespace: resources.GoldenActorAtespace, Name: "another-actor"}}, + {name: "template", templateUID: "another-template", sourceActor: &ateapipb.ObjectRef{Atespace: testTemplateRef.Atespace, Name: testTemplateUID}}, + {name: "actor name", templateUID: testTemplateUID, sourceActor: &ateapipb.ObjectRef{Atespace: testTemplateRef.Atespace, Name: "another-actor"}}, {name: "actor atespace", templateUID: testTemplateUID, sourceActor: &ateapipb.ObjectRef{Atespace: "another-atespace", Name: testTemplateUID}}, } for _, tt := range tests { diff --git a/cmd/ateapi/internal/controlapi/workflow_resume.go b/cmd/ateapi/internal/controlapi/workflow_resume.go index fd156896ca..ad8e18048b 100644 --- a/cmd/ateapi/internal/controlapi/workflow_resume.go +++ b/cmd/ateapi/internal/controlapi/workflow_resume.go @@ -141,20 +141,6 @@ func (w *ActorWorkflow) ResumeActor(ctx context.Context, actorRef resources.Acto return actor, true, nil } -// validateGoldenSnapshotFidelity rejects a golden snapshot that does not carry -// the guest state (memory + fs delta) a restore needs. Golden actors always -// commit MEMORY (preferredFidelity), so this only trips on a golden snapshot -// whose record drifted — surface a clear error instead of shipping a restore -// request atelet would reject (or that would boot an empty guest). -func validateGoldenSnapshotFidelity(snapshot *ateapipb.Snapshot) error { - if fidelity := snapshot.GetFidelity(); fidelity != ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY { - return apierror.FailedPrecondition( - "ActorTemplate golden snapshot %q was taken with fidelity %s, not MEMORY; regenerate the golden snapshot", - snapshot.GetDurableSnapshot().GetObject().GetSnapshotUri(), fidelity) - } - return nil -} - // loadActorForResume fetches the current actor record and its template, and // resolves the boot source for the pending restore. func (w *ActorWorkflow) loadActorForResume(ctx context.Context, actorRef resources.ActorRef) (_ *ateapipb.Actor, _ *ateapipb.ActorTemplate, _ resumeSnapshotSource, err error) { diff --git a/cmd/ateapi/internal/controlapi/workflow_resume_test.go b/cmd/ateapi/internal/controlapi/workflow_resume_test.go index b1b5d8e30d..a735dc49ad 100644 --- a/cmd/ateapi/internal/controlapi/workflow_resume_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_resume_test.go @@ -1234,7 +1234,7 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { const malformedURI = "not-a-valid-snapshot-uri" actorURI := someActorSnapshotURI(t, testStorageLocation, "team-a", durableSnapshotName) - goldenURI := someActorSnapshotURI(t, "gs://bucket/golden-root", "ate-golden", "golden-1") + goldenURI := someActorSnapshotURI(t, "gs://bucket/golden-root", "team-a", "golden-1") fullScope := ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY rootfsScope := ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS diff --git a/cmd/ateapi/internal/controlapi/workflow_suspend.go b/cmd/ateapi/internal/controlapi/workflow_suspend.go index d84ce5b1b7..87ed43dcef 100644 --- a/cmd/ateapi/internal/controlapi/workflow_suspend.go +++ b/cmd/ateapi/internal/controlapi/workflow_suspend.go @@ -164,12 +164,6 @@ func (w *ActorWorkflow) ensureMarkedSuspending(ctx context.Context, actorRef res if state != ateapipb.ActorState_ACTOR_STATE_RUNNING && state != ateapipb.ActorState_ACTOR_STATE_SUSPENDING { return nil, apierror.FailedPrecondition("MarkSuspending prerequisite not met for Actor: %s (got: %v, want %s)", actorRef, state, ateapipb.ActorState_ACTOR_STATE_RUNNING) } - // Golden actors are only suspended, never paused. Checked before the skip, - // so an async retry can't finish a golden actor's suspend as a pause. - // TODO: Remove this check: the suspend workflow should not special-case golden actors. - if mode == suspendAsync && actorRef.Atespace == resources.GoldenActorAtespace { - return nil, apierror.FailedPrecondition("actors in atespace %q are golden actors, which cannot be paused", actorRef.Atespace) - } if state == ateapipb.ActorState_ACTOR_STATE_SUSPENDING { markSkipped(ctx, "actor already SUSPENDING") return actor, nil @@ -193,7 +187,7 @@ func (w *ActorWorkflow) ensureMarkedSuspending(ctx context.Context, actorRef res snap := newDurableSnapshot( gen, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, - preferredFidelity(actorRef.Atespace, actorTemplate), + actorTemplate.GetSnapshotConfig().GetPreferredFidelity(), actorTemplate.GetMetadata().GetUid(), uri.Name(), uri.String(), @@ -213,17 +207,6 @@ func (w *ActorWorkflow) ensureMarkedSuspending(ctx context.Context, actorRef res return storedActor, nil } -// preferredFidelity returns the fidelity a suspend snapshot is taken with. -// Golden actors always commit MEMORY regardless of the template's -// preferredFidelity: new actors borrow the golden snapshot and resume it -// with memory, so it must carry the guest memory and filesystem. -func preferredFidelity(atespace string, tmpl *ateapipb.ActorTemplate) ateapipb.SnapshotFidelity { - if atespace == resources.GoldenActorAtespace { - return ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY - } - return tmpl.GetSnapshotConfig().GetPreferredFidelity() -} - // ensureAteletSuspended checkpoints the workload to the actor's persisted // in-progress snapshot location. A sync suspend has atelet upload it before // returning; an async suspend has atelet keep it on the node. It returns the @@ -259,7 +242,7 @@ func (w *ActorWorkflow) ensureAteletSuspended(ctx context.Context, actorRef reso // actor is currently running (recorded on-node at Run/Restore) and pins it // into the snapshot manifest. inProgressSnap := snapshotAtLatestGeneration(actor.GetStatus()) - fidelity := fidelityToAtelet(preferredFidelity(actor.GetMetadata().GetAtespace(), actorTemplate)) + fidelity := fidelityToAtelet(actorTemplate.GetSnapshotConfig().GetPreferredFidelity()) req := &ateletpb.CheckpointRequest{ WorkerPodUid: assignment.GetWorkerPodUid(), Atespace: actor.GetMetadata().GetAtespace(), diff --git a/cmd/ateapi/internal/controlapi/workflow_suspend_async_test.go b/cmd/ateapi/internal/controlapi/workflow_suspend_async_test.go index 1aa1d88401..c1bc646e6d 100644 --- a/cmd/ateapi/internal/controlapi/workflow_suspend_async_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_suspend_async_test.go @@ -427,28 +427,6 @@ func TestSuspendActorAsync_CrashesWhenPausingActorMissingWorkerPod(t *testing.T) } } -// TestSuspendActorAsync_MarkedPausing_GoldenAtespaceRejected verifies golden actors cannot be -// paused: by design they can only be suspended (committed). -func TestSuspendActorAsync_MarkedPausing_GoldenAtespaceRejected(t *testing.T) { - ctx := context.Background() - st := newTestPersistence(t) - seedSuspendTemplate(t, ctx, st, nil) - w := newMarkOnlyWorkflow(st) - - actorRef := resources.ActorRef{Atespace: resources.GoldenActorAtespace, Name: "golden-1"} - seedWorkflowActor(t, ctx, st, actorRef, "ns", "tmpl1", ateapipb.ActorState_ACTOR_STATE_RUNNING, func(a *ateapipb.Actor) { - a.Status.AssignedNode = "node-1" - a.Status.WorkerAssignment = wireTestAssignment() - }) - _, err := w.SuspendActor(ctx, actorRef, suspendAsync) - if got := apierror.Code(err); got != codes.FailedPrecondition { - t.Fatalf("apierror.Code = %v (err %v), want FailedPrecondition", got, err) - } - if got := mustReadActor(t, ctx, st, actorRef).GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_RUNNING { - t.Errorf("stored state = %v, want RUNNING untouched", got) - } -} - // TestSuspendActorAsync_AteletPaused_ForwardsPreferredFidelity verifies that local pause // checkpoints forward the template's PreferredFidelity to atelet, and label the // pause's lifecycle-op metric with it. diff --git a/cmd/ateapi/internal/controlapi/workflow_suspend_test.go b/cmd/ateapi/internal/controlapi/workflow_suspend_test.go index 165c22567d..3e5675ecaf 100644 --- a/cmd/ateapi/internal/controlapi/workflow_suspend_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_suspend_test.go @@ -203,49 +203,6 @@ func TestSuspendActorWorkflow_SuspendedReturnsAsIs(t *testing.T) { } } -// TestEnsureMarkedSuspending_GoldenActor verifies the golden check runs after -// the state check and before the SUSPENDING skip: an async suspend of a golden -// actor in a state no suspend takes reports that state, and an async retry of a -// golden actor's suspend is refused rather than finished as a pause. None of -// these cases writes the store. -func TestEnsureMarkedSuspending_GoldenActor(t *testing.T) { - goldenRef := resources.ActorRef{Atespace: resources.GoldenActorAtespace, Name: "golden"} - tests := []struct { - name string - mode suspendMode - state ateapipb.ActorState - // wantErr is a substring of the error, "" for none. - wantErr string - }{ - {name: "async of a crashed actor reports its state", mode: suspendAsync, state: ateapipb.ActorState_ACTOR_STATE_CRASHED, wantErr: "got: ACTOR_STATE_CRASHED"}, - {name: "async of a deleting actor reports its state", mode: suspendAsync, state: ateapipb.ActorState_ACTOR_STATE_DELETING, wantErr: "got: ACTOR_STATE_DELETING"}, - {name: "async of a running actor is refused", mode: suspendAsync, state: ateapipb.ActorState_ACTOR_STATE_RUNNING, wantErr: "cannot be paused"}, - {name: "async retry of a suspending actor is refused", mode: suspendAsync, state: ateapipb.ActorState_ACTOR_STATE_SUSPENDING, wantErr: "cannot be paused"}, - {name: "sync retry of a suspending actor skips", mode: suspendSync, state: ateapipb.ActorState_ACTOR_STATE_SUSPENDING}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - w := &ActorWorkflow{} - actor := &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{Atespace: goldenRef.Atespace, Name: goldenRef.Name}, - Status: &ateapipb.ActorStatus{State: tt.state}, - } - got, err := w.ensureMarkedSuspending(context.Background(), goldenRef, actor, &ateapipb.ActorTemplate{}, tt.mode) - if tt.wantErr == "" { - if err != nil { - t.Fatalf("ensureMarkedSuspending() error = %v, want nil", err) - } - if got != actor { - t.Errorf("ensureMarkedSuspending() = %v, want the actor as it is", got) - } - return - } - if apierror.Code(err) != codes.FailedPrecondition || !strings.Contains(err.Error(), tt.wantErr) { - t.Fatalf("ensureMarkedSuspending() error = %v, want FailedPrecondition containing %q", err, tt.wantErr) - } - }) - } -} // TestSuspendActor_MarkedSuspending_StateMatrix verifies the suspend edge's // state gating against every actor state, in either mode: RUNNING takes the @@ -860,36 +817,6 @@ func TestSuspendActor_Finalized_ReleasesOnlyOwnWorker(t *testing.T) { } } -// TestPreferredFidelity verifies golden actors always commit Full — new -// actors resume the golden snapshot Full, so the template's preferredFidelity must not -// thin it down to a data-only capture. -func TestPreferredFidelity(t *testing.T) { - tmpl := func(fidelity ateapipb.SnapshotFidelity) *ateapipb.ActorTemplate { - return &ateapipb.ActorTemplate{ - SnapshotConfig: &ateapipb.SnapshotConfig{PreferredFidelity: fidelity}, - } - } - fullScope := ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY - dataScope := ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES - tests := []struct { - name string - atespace string - fidelity ateapipb.SnapshotFidelity - want ateapipb.SnapshotFidelity - }{ - {"golden actor ignores Data fidelity", resources.GoldenActorAtespace, dataScope, fullScope}, - {"golden actor keeps Full fidelity", resources.GoldenActorAtespace, fullScope, fullScope}, - {"regular actor uses Data fidelity", "team-a", dataScope, dataScope}, - {"regular actor uses Full fidelity", "team-a", fullScope, fullScope}, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - if got := preferredFidelity(tc.atespace, tmpl(tc.fidelity)); got != tc.want { - t.Errorf("preferredFidelity(%q, fidelity=%s) = %s, want %s", tc.atespace, tc.fidelity, got, tc.want) - } - }) - } -} // TestSuspendActor_Finalized_MissingDurableSnapshotClearsAssignment verifies that if the // latest generation snapshot entry is missing when the suspend finalizes, the @@ -1107,15 +1034,6 @@ func TestSuspendAndPause_CharacteristicMatrix(t *testing.T) { sync: rejected(suspendOutcome{state: ateapipb.ActorState_ACTOR_STATE_DELETING}), async: rejected(suspendOutcome{state: ateapipb.ActorState_ACTOR_STATE_DELETING}), }, - { - // A golden actor commits MEMORY whatever the template prefers, and - // cannot be paused at all. - name: "golden RUNNING", - atespace: resources.GoldenActorAtespace, - status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_RUNNING, AssignedNode: node, WorkerAssignment: wireTestAssignment()}, - sync: suspendOutcome{code: codes.OK, atelet: ateletCall{checkpoints: 1, fidelity: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, uploadMode: immediate, object: true}, state: suspended, generation: 1, snapshots: []snapshotOutcome{{1, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", completed}}}, - async: rejected(suspendOutcome{state: ateapipb.ActorState_ACTOR_STATE_RUNNING, assignedNode: node, workerHeld: true}), - }, } for i, tc := range tests { diff --git a/cmd/ateapi/internal/controlapi/workflow_template_delete.go b/cmd/ateapi/internal/controlapi/workflow_template_delete.go index 553ee0585a..acdb37b0aa 100644 --- a/cmd/ateapi/internal/controlapi/workflow_template_delete.go +++ b/cmd/ateapi/internal/controlapi/workflow_template_delete.go @@ -53,12 +53,12 @@ func (w *ActorWorkflow) DeleteActorTemplate(ctx context.Context, templateRef res return nil, apierror.Aborted("concurrent update conflict, please retry") } - // Both are named after the template's uid, in the reserved atespace. + // Both are named after the template's UID, in the template's atespace. goldenName := tmpl.GetMetadata().GetUid() - if err := w.ensureGoldenActorDeleted(ctx, resources.ActorRef{Atespace: resources.GoldenActorAtespace, Name: goldenName}); err != nil { + if err := w.ensureGoldenActorDeleted(ctx, resources.ActorRef{Atespace: templateRef.Atespace, Name: goldenName}); err != nil { return nil, err } - if err := w.ensureGoldenTagDeleted(ctx, resources.TagRef{Atespace: resources.GoldenActorAtespace, Name: goldenName}); err != nil { + if err := w.ensureGoldenTagDeleted(ctx, resources.TagRef{Atespace: templateRef.Atespace, Name: goldenName}); err != nil { return nil, err } return w.finalizeTemplateDeleted(ctx, templateRef, precondition) diff --git a/cmd/ateapi/internal/workerservice/suspend.go b/cmd/ateapi/internal/workerservice/suspend.go index 35d1a4c3f7..c59d0d9622 100644 --- a/cmd/ateapi/internal/workerservice/suspend.go +++ b/cmd/ateapi/internal/workerservice/suspend.go @@ -39,16 +39,6 @@ func (s *Server) RequestActorSuspend(ctx context.Context, req *ateapipb.RequestA return nil, resources.ToAPIError(errs) } - // A golden actor may be suspended, but only by the template controller: - // the suspend is what commits its snapshot, and the controller runs it once - // the warmup window it set has elapsed. A golden actor boots and waits, so - // it looks idle from the moment it starts, and honoring its own request - // would commit the snapshot before that window ends -- leaving every actor - // restored from the template to start from a workload that never warmed up. - if req.GetActor().GetAtespace() == resources.GoldenActorAtespace { - return nil, apierror.FailedPrecondition("actors in atespace %q are golden actors, which cannot request their own suspend", resources.GoldenActorAtespace) - } - workerName := req.GetWorker().GetName() actorRef := resources.ActorRefFromObjectRef(req.GetActor()) diff --git a/cmd/ateapi/internal/workerservice/suspend_test.go b/cmd/ateapi/internal/workerservice/suspend_test.go index 83866ec88b..57bb082095 100644 --- a/cmd/ateapi/internal/workerservice/suspend_test.go +++ b/cmd/ateapi/internal/workerservice/suspend_test.go @@ -209,48 +209,6 @@ func TestRequestActorSuspend_Errors(t *testing.T) { } } -// A golden actor is committed by the template controller once the warmup it -// timed has elapsed, and the suspend is that commit. The actor idles from the -// moment it boots, so it would ask immediately and snapshot a workload that -// never warmed up. Control.SuspendActor still serves the controller; only the -// Worker's request for one is refused. -func TestRequestActorSuspend_GoldenActorCannotSelfSuspend(t *testing.T) { - st, cleanup := storetest.SetupTestStore(t) - defer cleanup() - seedReportedWorker(t, st, testNode, &ateapipb.WorkerResources{Actors: 1}) - golden := storetest.MustCreateActor(t, context.Background(), st, &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{Atespace: resources.GoldenActorAtespace, Name: suspendActor}, - Status: &ateapipb.ActorStatus{ - State: ateapipb.ActorState_ACTOR_STATE_RUNNING, - WorkerAssignment: &ateapipb.WorkerAssignment{ - Worker: &ateapipb.ObjectRef{Name: testWorkerName}, - WorkerNamespace: "ate-system", - WorkerPool: "pool-1", - WorkerPod: "worker-pod-1", - WorkerPodUid: testWorkerName, - }, - }, - }) - suspender := &fakeSuspender{} - s := New(st, suspender, testAteletSPIFFEID, nil, "", nil) - - // Everything else about the request is in order: the Worker really does - // host this actor, so being golden is the only reason it is refused. - _, err := s.RequestActorSuspend( - ateletauthtest.ContextWith(ateletauthtest.CertOn(t, testNode)), - &ateapipb.RequestActorSuspendRequest{ - Worker: &ateapipb.ObjectRef{Name: testWorkerName}, - Actor: &ateapipb.ObjectRef{Atespace: resources.GoldenActorAtespace, Name: suspendActor}, - ActorUid: golden.GetMetadata().GetUid(), - }) - if got := apierror.Code(err); got != codes.FailedPrecondition { - t.Fatalf("code = %v (err %v), want FailedPrecondition", got, err) - } - if len(suspender.calls) != 0 { - t.Errorf("golden actor request still reached the suspend: %v", suspender.calls) - } -} - // The suspend is a proposal, and the control plane's refusal is the answer: a // request that loses a race to a resume, pause, or delete comes back as what // the workflow said, not as something the worker has to interpret. diff --git a/cmd/atelet/internal/apivalidation/validate.go b/cmd/atelet/internal/apivalidation/validate.go index bc67bff8dd..f6c530ee16 100644 --- a/cmd/atelet/internal/apivalidation/validate.go +++ b/cmd/atelet/internal/apivalidation/validate.go @@ -16,7 +16,6 @@ package apivalidation import ( "context" - "fmt" "go.ate.dev/substrate/internal/proto/ateletpb" "go.ate.dev/substrate/internal/resources" @@ -105,15 +104,6 @@ func ValidateCustom_AssetFile_Sha256(_ context.Context, _ operation.Operation, f return nil } -// ValidateCustom_PromoteSnapshotRequest_Atespace rejects the golden atespace: -// golden actors are never paused, so a paused checkpoint is never promoted to -// a golden snapshot. -func ValidateCustom_PromoteSnapshotRequest_Atespace(_ context.Context, _ operation.Operation, fldPath *field.Path, value, _ *string) field.ErrorList { - if *value == resources.GoldenActorAtespace { - return field.ErrorList{field.Forbidden(fldPath, fmt.Sprintf("atespace %q holds golden actors, which are never paused", *value))} - } - return nil -} func ValidateCustom_ObjectStorage_SnapshotUri(_ context.Context, _ operation.Operation, fldPath *field.Path, value, _ *string) field.ErrorList { return validateSnapshotURI(fldPath, *value) diff --git a/cmd/atelet/internal/apivalidation/validation_test.go b/cmd/atelet/internal/apivalidation/validation_test.go index d890d426d4..3f2d63d14a 100644 --- a/cmd/atelet/internal/apivalidation/validation_test.go +++ b/cmd/atelet/internal/apivalidation/validation_test.go @@ -910,10 +910,6 @@ func TestValidatePromoteSnapshotRequest(t *testing.T) { name: "invalid atespace: path escape", obj: valid(func(r *ateletpb.PromoteSnapshotRequest) { r.Atespace = "../escape" }), want: field.ErrorList{field.Invalid(field.NewPath("atespace"), nil, "").WithOrigin("format=k8s-short-name")}, - }, { - name: "golden atespace", - obj: valid(func(r *ateletpb.PromoteSnapshotRequest) { r.Atespace = resources.GoldenActorAtespace }), - want: field.ErrorList{field.Forbidden(field.NewPath("atespace"), "")}, }, { name: "invalid actor_name: uppercase", obj: valid(func(r *ateletpb.PromoteSnapshotRequest) { r.ActorName = "UPPER" }), diff --git a/cmd/atelet/internal/apivalidation/zz_generated.validation.go b/cmd/atelet/internal/apivalidation/zz_generated.validation.go index a4c32678a0..d69ca8cd5c 100644 --- a/cmd/atelet/internal/apivalidation/zz_generated.validation.go +++ b/cmd/atelet/internal/apivalidation/zz_generated.validation.go @@ -1966,10 +1966,6 @@ func Validate_PromoteSnapshotRequest( if earlyReturn { return // do not proceed } - // custom validation - if e := ValidateCustom_PromoteSnapshotRequest_Atespace(ctx, op, fldPath, obj, oldObj); len(e) != 0 { - errs = append(errs, e...) - } if e := validate.ShortName(ctx, op, fldPath, obj, oldObj); len(e) != 0 { errs = append(errs, e...) } diff --git a/cmd/atelet/main.go b/cmd/atelet/main.go index dc8b5c6e49..8ccd671cce 100644 --- a/cmd/atelet/main.go +++ b/cmd/atelet/main.go @@ -1185,8 +1185,7 @@ func (s *Atelet) PromoteSnapshot(ctx context.Context, req *ateletpb.PromoteSnaps op := snapshotOp{ templateNamespace: req.GetActorTemplateAtespace(), templateName: req.GetActorTemplateName(), - // Always the actor's durable latest: golden actors are never paused - // (validation above rejects the golden atespace). + // Uploading a paused checkpoint creates a durable latest snapshot. kind: ateattr.SnapshotKindLatest, fidelity: ateattr.SnapshotFidelityValue(req.GetSnapshot().GetFidelity()), } @@ -1472,8 +1471,7 @@ func (s *Atelet) Restore(ctx context.Context, req *ateletpb.RestoreRequest) (res dManifest = time.Since(tManifest) manifestDone = true - // The manifest is what tells a golden restore from a latest one, so the - // snapshot kind only becomes knowable here. + // The snapshot kind is recorded once the manifest has been read. op.kind = restoreSnapshotKind(directLocal, sandboxRec) var registration *registeredActor diff --git a/cmd/atelet/metrics.go b/cmd/atelet/metrics.go index d7d65c2359..ed31bad2f5 100644 --- a/cmd/atelet/metrics.go +++ b/cmd/atelet/metrics.go @@ -24,7 +24,6 @@ import ( "go.ate.dev/substrate/internal/ateattr" "go.ate.dev/substrate/internal/proto/ateletpb" - "go.ate.dev/substrate/internal/resources" ) const ( @@ -154,9 +153,8 @@ func assetsAfterCollateral(prepFailedPhase string, assets time.Duration) time.Du } // restoreSnapshotKind classifies which snapshot a restore reads: local when it -// is served from the node-local copy, and otherwise golden or latest, told -// apart by the identity the manifest records for the actor that wrote the -// snapshot. An empty result means the manifest has not been read yet, so the +// is served from the node-local copy, and otherwise latest. An empty result +// means the manifest has not been read yet, so the // kind is not knowable. func restoreSnapshotKind(directLocal bool, rec *sandboxAssetsRecord) string { if directLocal { @@ -165,23 +163,14 @@ func restoreSnapshotKind(directLocal bool, rec *sandboxAssetsRecord) string { if rec == nil { return "" } - // Manifests written before the identity fields existed carry no atespace and - // fall through to latest, which is the common case for them anyway. - if rec.Atespace == resources.GoldenActorAtespace { - return ateattr.SnapshotKindGolden - } return ateattr.SnapshotKindLatest } // checkpointSnapshotKind classifies which snapshot a checkpoint writes: a pause -// writes the node-local one, a suspend the actor's durable latest, and a commit -// by an actor in the golden atespace the template's golden image. +// writes the node-local one, and a suspend writes a durable latest snapshot. func checkpointSnapshotKind(req *ateletpb.CheckpointRequest) string { if req.GetUploadConfig().GetMode() == ateletpb.UploadMode_UPLOAD_MODE_DEFERRED { return ateattr.SnapshotKindLocal } - if req.GetAtespace() == resources.GoldenActorAtespace { - return ateattr.SnapshotKindGolden - } return ateattr.SnapshotKindLatest } diff --git a/cmd/atelet/metrics_test.go b/cmd/atelet/metrics_test.go index b95115f84c..4145e6e91e 100644 --- a/cmd/atelet/metrics_test.go +++ b/cmd/atelet/metrics_test.go @@ -28,7 +28,6 @@ import ( "go.ate.dev/substrate/internal/ateattr" "go.ate.dev/substrate/internal/proto/ateletpb" - "go.ate.dev/substrate/internal/resources" ) const ( @@ -357,15 +356,15 @@ func TestRestoreSnapshotKind(t *testing.T) { want: ateattr.SnapshotKindLocal, }, { - name: "a golden actor's snapshot served locally is still local", + name: "a snapshot from another atespace served locally is still local", directLocal: true, - rec: &sandboxAssetsRecord{Atespace: resources.GoldenActorAtespace}, + rec: &sandboxAssetsRecord{Atespace: "team-b"}, want: ateattr.SnapshotKindLocal, }, { - name: "external snapshot written by a golden actor", - rec: &sandboxAssetsRecord{Atespace: resources.GoldenActorAtespace}, - want: ateattr.SnapshotKindGolden, + name: "external snapshot written in another atespace", + rec: &sandboxAssetsRecord{Atespace: "team-b"}, + want: ateattr.SnapshotKindLatest, }, { name: "external snapshot written by a tenant actor", @@ -411,13 +410,13 @@ func TestCheckpointSnapshotKind(t *testing.T) { want: ateattr.SnapshotKindLatest, }, { - name: "a golden actor's commit writes the template's golden", - req: &ateletpb.CheckpointRequest{UploadConfig: immediate, Atespace: resources.GoldenActorAtespace}, - want: ateattr.SnapshotKindGolden, + name: "a durable checkpoint in another atespace is latest", + req: &ateletpb.CheckpointRequest{UploadConfig: immediate, Atespace: "team-b"}, + want: ateattr.SnapshotKindLatest, }, { - name: "a local checkpoint in the golden atespace is still local", - req: &ateletpb.CheckpointRequest{UploadConfig: deferred, Atespace: resources.GoldenActorAtespace}, + name: "a local checkpoint in another atespace is still local", + req: &ateletpb.CheckpointRequest{UploadConfig: deferred, Atespace: "team-b"}, want: ateattr.SnapshotKindLocal, }, } diff --git a/cmd/kubectl-ate/internal/printer/printer_test.go b/cmd/kubectl-ate/internal/printer/printer_test.go index 8e97b9b2a2..3137bc69b8 100644 --- a/cmd/kubectl-ate/internal/printer/printer_test.go +++ b/cmd/kubectl-ate/internal/printer/printer_test.go @@ -436,7 +436,7 @@ func TestPrintActorTemplatesTo_Table(t *testing.T) { }, Status: &ateapipb.ActorTemplateStatus{ GoldenSnapshotStatus: &ateapipb.GoldenSnapshotStatus{ - GoldenTag: &ateapipb.ObjectRef{Atespace: "ate-golden", Name: "golden-tag"}, + GoldenTag: &ateapipb.ObjectRef{Atespace: "ate-demo-counter-substrate", Name: "golden-tag"}, }, }, }, diff --git a/docs/api-guide.md b/docs/api-guide.md index a427d5118a..d8c3070740 100644 --- a/docs/api-guide.md +++ b/docs/api-guide.md @@ -342,7 +342,7 @@ snapshotConfig: /atespaces//tags/ ``` -The objects of a snapshot (its manifest, memory image, durable-data tar) are named below it. So for the template above, a snapshot of an actor in atespace `team-a` is stored at `gs://my-bucket/secret-agent/atespaces/team-a/actors/3f8b…/snapshots/f47ac10b-…`, and the template's golden snapshot — the golden tag lives in the reserved `ate-golden` atespace — under `gs://my-bucket/secret-agent/atespaces/ate-golden/tags/`. +The objects of a snapshot (its manifest, memory image, durable-data tar) are named below it. So for the template above, a snapshot of an actor in atespace `team-a` is stored at `gs://my-bucket/secret-agent/atespaces/team-a/actors/3f8b…/snapshots/f47ac10b-…`, and the template's golden snapshot under `gs://my-bucket/secret-agent/atespaces/team-a/tags/`. An actor takes a series of snapshots over its life, so it gets a prefix of its own and each snapshot sits below it. A tag holds exactly one, so the tag's @@ -414,7 +414,7 @@ An `ActorTemplate` references its golden tag with the `ObjectRef` in `status.goldenSnapshotStatus.goldenTag`. These status fields are server-owned and ignored on input. Parse a URI only against the scheme above. -An `ActorTemplate` belongs to one atespace, but one `storageLocation` still holds snapshots for many atespaces: the golden actor lives in the reserved `ate-golden` atespace, and a `PUBLISHED` snapshot may be cloned from other atespaces. The `` level exists so that access can be granted per tenant: an object-storage policy can only condition on an **object-name prefix**, and cannot read the identity recorded inside a snapshot's manifest. Binding a per-atespace grant on GCS looks like: +An `ActorTemplate` belongs to one atespace, but one `storageLocation` still holds snapshots for many atespaces: the golden actor and tag live in the template's atespace, and a `PUBLISHED` snapshot may be cloned from other atespaces. The `` level exists so that access can be granted per tenant: an object-storage policy can only condition on an **object-name prefix**, and cannot read the identity recorded inside a snapshot's manifest. Binding a per-atespace grant on GCS looks like: ```yaml # Read-only on team-a's snapshots for this template, and nothing else. @@ -528,15 +528,28 @@ pause, suspend, and resume using their own snapshots. for these templates. When an `ActorTemplate` is created with golden snapshots enabled: -1. Substrate creates and resumes a temporary golden actor in `ate-golden`. -2. It waits for readiness (or the warm-up interval), then suspends the actor. -3. It creates a published tag named after the template UID, copying the snapshot into tag-owned storage. +1. Substrate creates and resumes a temporary golden actor in the template's atespace, using the same atespace-scoped policies and storage grants as ordinary actors. +2. It waits for readiness plus a 20-second warmup, then suspends the actor at the template's `preferredFidelity`. The extra interval is zero when every container has a wakeup probe, since Resume already waits for readiness. +3. It creates a published tag named after the template UID, copying the snapshot into tag-owned storage in the same atespace. The golden actor uses that name too. 4. It deletes the golden actor and records the tag reference in the template status. -`CreateActor` uses an explicit `sourceTag` when supplied; otherwise it resolves the template's golden tag and records that snapshot on the new actor. If the golden tag is not ready yet, the actor starts without a snapshot and cold-boots even if the tag becomes ready before its first resume. The default does not populate the caller-owned `sourceTag` field. Deleting the template collects its golden tag and any unfinished golden actor. +Golden actors follow ordinary lifecycle and ingress policies and appear in +Actor lists. Leave these resources to the controller: modifying or deleting +them can interrupt Golden creation, and deleting a published Golden tag +invalidates the template's recorded reference. `DeleteActorTemplate` collects +its Golden tag and any unfinished Golden actor. + +Golden capture follows `preferredFidelity`. MEMORY preserves processes and memory; +VOLUMES preserves startup data in mounted volumes and starts new processes on +restore. ROOTFS preserves a micro-VM's filesystem and also starts new processes. + +`CreateActor` uses an explicit `sourceTag` when supplied; otherwise it resolves the template's golden tag and records that snapshot on the new actor. If the golden tag is not ready yet, the actor starts without a snapshot and cold-boots even if the tag becomes ready before its first resume. The default does not populate the caller-owned `sourceTag` field. ### Resumption Lifecycle -Once a template is `Ready`, creating an actor logically (via `kubectl ate create actor`) allows it to be resumed instantly on any free worker in the referenced `WorkerPool`. Substrate bypasses the standard container boot and restores the process directly from its last saved state. +CreateActor fixes the Actor's initial snapshot source at creation. Resume places +it on an eligible worker. A MEMORY snapshot restores saved processes; VOLUMES and +ROOTFS snapshots restore their persisted data and start new processes. An Actor +created without a snapshot boots from its template's images. ### Eviction A worker pod can go away while it hosts running actors: when its `WorkerPool` rolls to a new worker image or pod template, as in an [upgrade](upgrade.md), or scales down, when its node is drained, or when its Spot VM is reclaimed. diff --git a/docs/metrics/registry/metrics.yaml b/docs/metrics/registry/metrics.yaml index e2c16b0e9b..286e27b952 100644 --- a/docs/metrics/registry/metrics.yaml +++ b/docs/metrics/registry/metrics.yaml @@ -302,11 +302,7 @@ groups: - id: latest stability: development value: latest - brief: The durable snapshot of the actor in object storage. - - id: golden - stability: development - value: golden - brief: The golden image of the ActorTemplate. + brief: A durable snapshot in object storage, including a template's golden snapshot. - id: boot stability: development value: boot diff --git a/docs/observability.md b/docs/observability.md index 15d52cb1cc..5a82a6994f 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -312,10 +312,16 @@ For `ate.imagecache.requests`: **No keys** means the operation has no pool. The actor-centric instruments omit the pair, so a crash before the actor reached a worker, or the `no_capacity` outcome, names no pool. No instrument sends the pair with both keys empty. The three snapshot labels are orthogonal and mean the same thing on every histogram that carries them: -* `ate.snapshot.kind`: which snapshot the operation reads or writes. `local` (node-local, written by a pause), `latest` (the actor's own durable snapshot), `golden` (the template's image), or `boot` (from scratch, so it never appears on the atelet histograms). +* `ate.snapshot.kind`: which snapshot the operation reads or writes. `local` (node-local, written by a pause), `latest` (a durable snapshot, including a template's golden snapshot), or `boot` (from scratch, so it never appears on the atelet histograms). * `ate.snapshot.fidelity`: what state it holds. `volumes`, `rootfs`, or `memory`. * `ate.snapshot.phase`: which step was timed. `volume_mount`, `manifest_fetch`, `sandbox_assets`, `download`, `oci_unpack`, `worker_restore` on restore; `sandbox_assets`, `worker_checkpoint`, `persist`, `cleanup` on checkpoint; `total` on both. +Golden snapshots use `latest` on both control-plane and atelet histograms, so +the resulting series also includes other durable restores and does not isolate +golden first restores. To investigate an individual activation, correlate the +Actor's recorded snapshots with its logs and traces instead of adding Actor +identity to metric labels. + **Phases overlap and do not sum to `total`.** The download runs concurrently with the asset fetch and OCI unpack, so each is an independent observation; use `total` as the denominator. A phase that never started is absent rather than zero. The `ate.*` control-plane metric labels are either fixed value sets (operation, outcome, state, class, kind, scope, phase) or scoped to the deployment catalog (template and pool names are operator-created, never derived from request payloads), and the label set varies per operation: resume carries the most dimensions, delete only the operation and error type. `ate.sandbox.class` is derived from the runtime on the actor's worker assignment or snapshot (each worker pool has exactly one class), so it adds no extra series next to the pool label; it exists so dashboards can aggregate by class without enumerating pool names. High-cardinality actor identity (name/uid/atespace) stays off metrics entirely and lives on logs and traces instead — for resource usage, on the [per-actor usage events](#per-actor-usage-events). diff --git a/internal/ateattr/ateattr.go b/internal/ateattr/ateattr.go index 3c1968045f..207e2a5c76 100644 --- a/internal/ateattr/ateattr.go +++ b/internal/ateattr/ateattr.go @@ -310,12 +310,11 @@ const ( // Values for SnapshotKindKey, set by ateapi from its own resume branching, so // the label is bounded at the producer: Local restores an in-node snapshot, -// Latest pulls the actor's durable snapshot from object storage, Golden pulls the -// template's golden image, Boot is a from-scratch start (not a restore). +// Latest pulls a durable snapshot from object storage (including a template's +// golden snapshot), Boot is a from-scratch start (not a restore). // atelet derives the same values for its own histograms, where the kind is the // snapshot a restore reads or a checkpoint writes; Boot never appears there. const ( - SnapshotKindGolden = "golden" SnapshotKindLatest = "latest" SnapshotKindLocal = "local" SnapshotKindBoot = "boot" diff --git a/internal/ateattr/ateattr_test.go b/internal/ateattr/ateattr_test.go index eb5dcf04e3..6810a486ed 100644 --- a/internal/ateattr/ateattr_test.go +++ b/internal/ateattr/ateattr_test.go @@ -391,7 +391,6 @@ func TestMetricLabelValues(t *testing.T) { {SchedulerOutcomeNoCapacity, "no_capacity"}, {SchedulerOutcomeError, "error"}, - {SnapshotKindGolden, "golden"}, {SnapshotKindLatest, "latest"}, {SnapshotKindLocal, "local"}, {SnapshotKindBoot, "boot"}, diff --git a/internal/e2e/suites/demo/golden_test.go b/internal/e2e/suites/demo/golden_test.go new file mode 100644 index 0000000000..23f1de0796 --- /dev/null +++ b/internal/e2e/suites/demo/golden_test.go @@ -0,0 +1,128 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package demo + +import ( + "context" + "testing" + "time" + + "go.ate.dev/substrate/internal/e2e" + "go.ate.dev/substrate/internal/resources" + "go.ate.dev/substrate/pkg/proto/ateapipb" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func TestGoldenSnapshotLifecycle(t *testing.T) { + for _, fidelity := range []ateapipb.SnapshotFidelity{ + ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, + } { + t.Run(fidelity.String(), func(t *testing.T) { + ctx := t.Context() + clients := e2e.GetClients() + ns := e2e.CreateNamespace(t) + env, err := e2e.CheckEnv("BUCKET_NAME") + if err != nil { + t.Fatal(err) + } + // A distinct atespace catches placement in a system atespace and + // keeps cleanup independent of the suite's shared demo atespace. + atespace := "golden-" + ns.Name + t.Cleanup(func() { + cleanupCtx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + _, _ = clients.SubstrateAPI.DeleteAtespace(cleanupCtx, &ateapipb.DeleteAtespaceRequest{Atespace: &ateapipb.ObjectRef{Name: atespace}}) + }) + created := e2e.CreateSubstrateCounterTemplate(ctx, t, clients, ns.Name, e2e.SubstrateTemplateOptions{ + Atespace: atespace, Name: "counter", PoolName: "counter", PoolReplicas: 1, + Labels: map[string]string{"golden-e2e": ns.Name}, + SnapshotConfig: &ateapipb.SnapshotConfig{ + StorageLocation: "gs://" + env["BUCKET_NAME"] + "/" + atespace, + PreferredFidelity: fidelity, + }, + }) + + templateRef := e2e.TemplateRef(created) + + tmpl, err := clients.SubstrateAPI.GetActorTemplate(ctx, &ateapipb.GetActorTemplateRequest{ActorTemplate: templateRef}) + if err != nil { + t.Fatal(err) + } + tagRef := tmpl.GetStatus().GetGoldenSnapshotStatus().GetGoldenTag() + if tagRef.GetAtespace() != atespace { + t.Fatalf("golden tag atespace = %q, want %q", tagRef.GetAtespace(), atespace) + } + if got := tagRef.GetName(); got != tmpl.GetMetadata().GetUid() { + t.Fatalf("golden tag name = %q, want template UID %q", got, tmpl.GetMetadata().GetUid()) + } + tag, err := clients.SubstrateAPI.GetTag(ctx, &ateapipb.GetTagRequest{Tag: tagRef}) + if err != nil { + t.Fatal(err) + } + if source := tag.GetSourceActor(); source.GetAtespace() != atespace || source.GetName() != tmpl.GetMetadata().GetUid() { + t.Fatalf("golden source actor = %v, want %s/%s", source, atespace, tmpl.GetMetadata().GetUid()) + } + if got := tag.GetStatus().GetSnapshot().GetFidelity(); got != fidelity { + t.Fatalf("golden fidelity = %v, want %v", got, fidelity) + } + uri, err := resources.ParseSnapshotURI(tagSnapshotURI(tag)) + if err != nil { + t.Fatal(err) + } + if !uri.OwnedBy(resources.TagSnapshotOwner(atespace, tag.GetMetadata().GetUid())) { + t.Fatalf("golden snapshot %s is not owned by its tenant tag", uri) + } + actorRef := &ateapipb.ObjectRef{Atespace: atespace, Name: "clone"} + t.Cleanup(func() { + cleanupCtx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + _, _ = clients.SubstrateAPI.DeleteActor(cleanupCtx, &ateapipb.DeleteActorRequest{Actor: actorRef, AnyState: true}) + }) + actor, err := clients.SubstrateAPI.CreateActor(ctx, &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: atespace, Name: actorRef.Name}, ActorTemplate: templateRef, + }}) + if err != nil { + t.Fatal(err) + } + if got := durableSnapshotURI(actor.GetStatus()); got != uri.String() { + t.Fatalf("clone snapshot = %q, want golden %q", got, uri) + } + if _, err := e2e.ResumeActorAwaitCapacity(t, ctx, clients, &ateapipb.ResumeActorRequest{Actor: actorRef}); err != nil { + t.Fatal(err) + } + response, err := callActor(t, resources.ActorRefFromObjectRef(actorRef)) + if err != nil { + t.Fatal(err) + } + validateCounterResponse(t, response, "golden restore", 1, 1) + // Delete the borrower before collecting the template's snapshot. + if _, err := clients.SubstrateAPI.DeleteActor(ctx, &ateapipb.DeleteActorRequest{Actor: actorRef, AnyState: true}); err != nil { + t.Fatal(err) + } + if _, err := clients.SubstrateAPI.DeleteActorTemplate(ctx, &ateapipb.DeleteActorTemplateRequest{ActorTemplate: templateRef}); err != nil { + t.Fatal(err) + } + if _, err := clients.SubstrateAPI.GetTag(ctx, &ateapipb.GetTagRequest{Tag: tagRef}); status.Code(err) != codes.NotFound { + t.Fatalf("golden tag after template deletion = %v, want NotFound", err) + } + if _, err := clients.SubstrateAPI.GetActor(ctx, &ateapipb.GetActorRequest{Actor: tagRef}); status.Code(err) != codes.NotFound { + t.Fatalf("golden actor after template deletion = %v, want NotFound", err) + } + t.Logf("golden lifecycle passed: fidelity=%v, snapshot=%s", fidelity, uri) + }) + } +} diff --git a/internal/proto/ateletpb/atelet.pb.go b/internal/proto/ateletpb/atelet.pb.go index b272c10d93..5f085380aa 100644 --- a/internal/proto/ateletpb/atelet.pb.go +++ b/internal/proto/ateletpb/atelet.pb.go @@ -3874,11 +3874,8 @@ func (*RestoreResponse) Descriptor() ([]byte, []int) { type PromoteSnapshotRequest struct { state protoimpl.MessageState `protogen:"open.v1"` - // Golden actors are never paused, so the golden atespace is rejected. - // // +k8s:required // +k8s:format=k8s-short-name - // +k8s:customValidation # not the golden atespace Atespace string `protobuf:"bytes,1,opt,name=atespace,proto3" json:"atespace,omitempty"` // +k8s:required // +k8s:format=k8s-short-name diff --git a/internal/proto/ateletpb/atelet.proto b/internal/proto/ateletpb/atelet.proto index fb41fb9b33..c00ee06159 100644 --- a/internal/proto/ateletpb/atelet.proto +++ b/internal/proto/ateletpb/atelet.proto @@ -1166,11 +1166,8 @@ message RestoreResponse { } message PromoteSnapshotRequest { - // Golden actors are never paused, so the golden atespace is rejected. - // // +k8s:required // +k8s:format=k8s-short-name - // +k8s:customValidation # not the golden atespace string atespace = 1; // +k8s:required diff --git a/internal/resources/actor.go b/internal/resources/actor.go index 13682a2d40..cf2413565a 100644 --- a/internal/resources/actor.go +++ b/internal/resources/actor.go @@ -18,7 +18,4 @@ const ( // ResourceNameRegexPattern is the regular expression pattern for a valid // Substrate resource name. ResourceNameRegexPattern = `[a-z0-9]([-a-z0-9]*[a-z0-9])?` - // GoldenActorAtespace is the reserved system atespace that per-template golden - // actors live in. - GoldenActorAtespace = "ate-golden" ) diff --git a/pkg/proto/ateapipb/ateapi.proto b/pkg/proto/ateapipb/ateapi.proto index 7ef8c2f921..8eefdd647e 100644 --- a/pkg/proto/ateapipb/ateapi.proto +++ b/pkg/proto/ateapipb/ateapi.proto @@ -225,7 +225,7 @@ service Control { } // Delete an ActorTemplate together with its golden actor and golden - // tag in the reserved ate-golden atespace. + // tag in the template's atespace. rpc DeleteActorTemplate(DeleteActorTemplateRequest) returns (ActorTemplate) { option (method).resource = "ActorTemplate"; } diff --git a/pkg/proto/ateapipb/ateapi_grpc.pb.go b/pkg/proto/ateapipb/ateapi_grpc.pb.go index e95a2aa683..a40dd7f2f2 100644 --- a/pkg/proto/ateapipb/ateapi_grpc.pb.go +++ b/pkg/proto/ateapipb/ateapi_grpc.pb.go @@ -181,7 +181,7 @@ type ControlClient interface { GetActorTemplate(ctx context.Context, in *GetActorTemplateRequest, opts ...grpc.CallOption) (*ActorTemplate, error) ListActorTemplates(ctx context.Context, in *ListActorTemplatesRequest, opts ...grpc.CallOption) (*ListActorTemplatesResponse, error) // Delete an ActorTemplate together with its golden actor and golden - // tag in the reserved ate-golden atespace. + // tag in the template's atespace. DeleteActorTemplate(ctx context.Context, in *DeleteActorTemplateRequest, opts ...grpc.CallOption) (*ActorTemplate, error) // Get the deployment-wide global access policy singleton. Returns NotFound // until the policy has been created. @@ -745,7 +745,7 @@ type ControlServer interface { GetActorTemplate(context.Context, *GetActorTemplateRequest) (*ActorTemplate, error) ListActorTemplates(context.Context, *ListActorTemplatesRequest) (*ListActorTemplatesResponse, error) // Delete an ActorTemplate together with its golden actor and golden - // tag in the reserved ate-golden atespace. + // tag in the template's atespace. DeleteActorTemplate(context.Context, *DeleteActorTemplateRequest) (*ActorTemplate, error) // Get the deployment-wide global access policy singleton. Returns NotFound // until the policy has been created. From f9925310a573a74ee1a739433e235b259728f88a Mon Sep 17 00:00:00 2001 From: cloudonly Date: Fri, 16 Oct 2026 14:11:25 +0800 Subject: [PATCH 2/3] Remove unused test imports after dropping Golden checks --- cmd/ateapi/internal/workerservice/suspend_test.go | 1 - cmd/atelet/internal/apivalidation/validation_test.go | 1 - 2 files changed, 2 deletions(-) diff --git a/cmd/ateapi/internal/workerservice/suspend_test.go b/cmd/ateapi/internal/workerservice/suspend_test.go index 57bb082095..426a406274 100644 --- a/cmd/ateapi/internal/workerservice/suspend_test.go +++ b/cmd/ateapi/internal/workerservice/suspend_test.go @@ -26,7 +26,6 @@ import ( "go.ate.dev/substrate/cmd/ateapi/internal/store" "go.ate.dev/substrate/cmd/ateapi/internal/store/storetest" "go.ate.dev/substrate/internal/apierror" - "go.ate.dev/substrate/internal/resources" "go.ate.dev/substrate/pkg/proto/ateapipb" ) diff --git a/cmd/atelet/internal/apivalidation/validation_test.go b/cmd/atelet/internal/apivalidation/validation_test.go index 3f2d63d14a..1978ec860c 100644 --- a/cmd/atelet/internal/apivalidation/validation_test.go +++ b/cmd/atelet/internal/apivalidation/validation_test.go @@ -21,7 +21,6 @@ import ( "testing" "go.ate.dev/substrate/internal/proto/ateletpb" - "go.ate.dev/substrate/internal/resources" "k8s.io/apimachinery/pkg/api/operation" "k8s.io/apimachinery/pkg/util/validation/field" ) From 703cf68519545dd1ae5ebf4cfacf1056bd2c15de Mon Sep 17 00:00:00 2001 From: cloudonly Date: Sat, 17 Oct 2026 08:47:40 +0800 Subject: [PATCH 3/3] Match the Golden fixture to the template's VOLUMES fidelity --- cmd/ateapi/internal/controlapi/functionaltest/actor_test.go | 2 +- cmd/ateapi/internal/controlapi/workflow_suspend_test.go | 2 -- cmd/atelet/internal/apivalidation/validate.go | 1 - 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go b/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go index 241ccf83fc..4a6cefcf38 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go @@ -3277,7 +3277,7 @@ func createDataFidelityTemplate(t *testing.T, tc *testContext, ns string) *ateap SourceActor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: created.GetMetadata().GetUid()}, Scope: ateapipb.TagScope_TAG_SCOPE_PUBLISHED, Status: &ateapipb.TagStatus{ - Snapshot: newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, created.GetMetadata().GetUid(), goldenSnapshotName, goldenSnapshotURI(t)), + Snapshot: newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, created.GetMetadata().GetUid(), goldenSnapshotName, goldenSnapshotURI(t)), ActorTemplateUid: created.GetMetadata().GetUid(), }, }) diff --git a/cmd/ateapi/internal/controlapi/workflow_suspend_test.go b/cmd/ateapi/internal/controlapi/workflow_suspend_test.go index 3e5675ecaf..58b355c81b 100644 --- a/cmd/ateapi/internal/controlapi/workflow_suspend_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_suspend_test.go @@ -203,7 +203,6 @@ func TestSuspendActorWorkflow_SuspendedReturnsAsIs(t *testing.T) { } } - // TestSuspendActor_MarkedSuspending_StateMatrix verifies the suspend edge's // state gating against every actor state, in either mode: RUNNING takes the // edge and marks a snapshot with a PENDING durable copy, SUSPENDING skips the @@ -817,7 +816,6 @@ func TestSuspendActor_Finalized_ReleasesOnlyOwnWorker(t *testing.T) { } } - // TestSuspendActor_Finalized_MissingDurableSnapshotClearsAssignment verifies that if the // latest generation snapshot entry is missing when the suspend finalizes, the // actor transitions to CRASHED and clears WorkerAssignment after releasing the diff --git a/cmd/atelet/internal/apivalidation/validate.go b/cmd/atelet/internal/apivalidation/validate.go index f6c530ee16..4e73e734b2 100644 --- a/cmd/atelet/internal/apivalidation/validate.go +++ b/cmd/atelet/internal/apivalidation/validate.go @@ -104,7 +104,6 @@ func ValidateCustom_AssetFile_Sha256(_ context.Context, _ operation.Operation, f return nil } - func ValidateCustom_ObjectStorage_SnapshotUri(_ context.Context, _ operation.Operation, fldPath *field.Path, value, _ *string) field.ErrorList { return validateSnapshotURI(fldPath, *value) }