From b7ba6f6991769f7fbe1340912677eede0b83c7fd Mon Sep 17 00:00:00 2001 From: Jeff Luo Date: Thu, 8 Oct 2026 14:21:50 +0000 Subject: [PATCH 01/44] ateapi: record one scheduler assignment for each resume (#2244) Part of https://github.com/agent-substrate/substrate/issues/2243 A worker can hold more than one actor, so two resumes can race for its last free slot. The loser is retried transparently, but each losing attempt was recorded on `ate.scheduler.assignment.duration` as `ate.scheduler.outcome="error"`. One resume gave two records instead of one. - [x] Tests pass - [x] Appropriate changes to documentation are included in the PR --- cmd/ateapi/internal/controlapi/metrics.go | 5 +- .../internal/controlapi/metrics_test.go | 2 +- .../internal/controlapi/workflow_resume.go | 88 +++- .../controlapi/workflow_resume_test.go | 425 ++++++++++++++++++ docs/metrics/registry/metrics.yaml | 12 +- docs/observability.md | 4 +- 6 files changed, 507 insertions(+), 29 deletions(-) diff --git a/cmd/ateapi/internal/controlapi/metrics.go b/cmd/ateapi/internal/controlapi/metrics.go index dfe1b523cd..291767fd74 100644 --- a/cmd/ateapi/internal/controlapi/metrics.go +++ b/cmd/ateapi/internal/controlapi/metrics.go @@ -233,7 +233,8 @@ func lifecycleOpAttrs(actor *ateapipb.Actor, template *ateapipb.ActorTemplate, s // set on every outcome it is known for, so no_capacity names the capacity // that ran out and stays comparable with assigned. // The pool keys are set together or not at all; see ateattr.WorkerPoolAttributes. -func (i *Instruments) recordSchedulerAssignment(ctx context.Context, start time.Time, outcome, poolNamespace, pool, class string, err error) { +// elapsed is the time that the attempt took. +func (i *Instruments) recordSchedulerAssignment(ctx context.Context, elapsed time.Duration, outcome, poolNamespace, pool, class string, err error) { if i == nil || i.schedulerAssignmentDuration == nil { return } @@ -246,5 +247,5 @@ func (i *Instruments) recordSchedulerAssignment(ctx context.Context, start time. if outcome == ateattr.SchedulerOutcomeError && err != nil { attrs = append(attrs, ateattr.ErrorTypeKey.String(apierror.Code(err).String())) } - i.schedulerAssignmentDuration.Record(ctx, time.Since(start).Seconds(), metric.WithAttributes(attrs...)) + i.schedulerAssignmentDuration.Record(ctx, elapsed.Seconds(), metric.WithAttributes(attrs...)) } diff --git a/cmd/ateapi/internal/controlapi/metrics_test.go b/cmd/ateapi/internal/controlapi/metrics_test.go index d0fd5453e8..8784b9c2ad 100644 --- a/cmd/ateapi/internal/controlapi/metrics_test.go +++ b/cmd/ateapi/internal/controlapi/metrics_test.go @@ -381,7 +381,7 @@ func TestSchedulerAssignmentShapeAndOutcomes(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { inst, reader := newTestInstruments(t) - inst.recordSchedulerAssignment(context.Background(), time.Now(), tt.outcome, tt.poolNamespace, tt.pool, tt.class, tt.err) + inst.recordSchedulerAssignment(context.Background(), 0, tt.outcome, tt.poolNamespace, tt.pool, tt.class, tt.err) dp := singleHistogramDP(t, reader, schedulerAssignmentMetric) assertAttrKeys(t, dp, tt.wantKeys...) diff --git a/cmd/ateapi/internal/controlapi/workflow_resume.go b/cmd/ateapi/internal/controlapi/workflow_resume.go index 2f4b509e2d..c9e5238cad 100644 --- a/cmd/ateapi/internal/controlapi/workflow_resume.go +++ b/cmd/ateapi/internal/controlapi/workflow_resume.go @@ -260,21 +260,26 @@ func (w *ActorWorkflow) ensureWorkerAssigned(ctx context.Context, actorRef resou return nil, nil, apierror.FailedPrecondition("AssignWorker prerequisite not met for Actor: %s (got: %v, want %s or %s)", actorRef, actor.GetStatus().GetState(), ateapipb.ActorState_ACTOR_STATE_SUSPENDED, ateapipb.ActorState_ACTOR_STATE_PAUSED) } - // Bound contention retries to about three seconds. - backoff := wait.Backoff{ - Steps: 12, - Duration: 15 * time.Millisecond, - Factor: 2.0, - Jitter: 1.0, - Cap: 250 * time.Millisecond, - } var assignedActor *ateapipb.Actor var assignedWorker *ateapipb.Worker - err = wait.ExponentialBackoff(backoff, func() (bool, error) { + // ctxErr marks the loop ending between attempts, where no attempt ran and + // so none recorded. + var ctxErr error + // lastAttempt times only the last attempt that ran, as every other record + // on the histogram does. The backoff can sleep once more after the last + // attempt before it gives up, so the time since that attempt began is too + // long. attempted is false if the context ended before the first attempt. + var lastAttempt time.Duration + attempted := false + err = wait.ExponentialBackoff(assignmentBackoff, func() (bool, error) { if err := ctx.Err(); err != nil { + ctxErr = err return false, err } + attemptStart := time.Now() attemptActor, attemptWorker, attemptErr := w.assignWorkerAttempt(ctx, actorRef, actor, actorTemplate) + lastAttempt = time.Since(attemptStart) + attempted = true if attemptErr == nil { assignedActor, assignedWorker = attemptActor, attemptWorker return true, nil @@ -287,13 +292,55 @@ func (w *ActorWorkflow) ensureWorkerAssigned(ctx context.Context, actorRef resou } return false, attemptErr }) - if err != nil { - if wait.Interrupted(err) && !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) { - return nil, nil, store.ErrVersionConflict + // The retried attempts hid their own records, so the resume would leave no + // trace on the histogram without one here. + // + // wait.Interrupted also matches a context error, which an attempt returns + // when the context ends inside it; that attempt recorded itself. Only the + // timeout of the backoff means that the retries ran out. + exhausted := wait.Interrupted(err) && !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) + if exhausted && ctx.Err() != nil { + // The context ended in the sleep after the last attempt. + ctxErr = ctx.Err() + err = ctxErr + } + switch { + case err == nil: + return assignedActor, assignedWorker, nil + case ctxErr != nil: + if attempted { + w.recordExhaustedAssignment(ctx, lastAttempt, actorTemplate, ctxErr) } return nil, nil, err + case exhausted: + err = apierror.Aborted("concurrent update conflict, please retry: %w", store.ErrVersionConflict) + w.recordExhaustedAssignment(ctx, lastAttempt, actorTemplate, err) + return nil, nil, err + default: + // assignWorkerAttempt recorded this one itself. + return nil, nil, err + } +} + +// assignmentBackoff bounds the retries of an assignment attempt that loses a +// race for a worker. The delay reaches Cap after 5 attempts, which ends the +// retries before Steps runs out, so the loop takes about 0.5 to 1 s. +var assignmentBackoff = wait.Backoff{ + Steps: 12, + Duration: 15 * time.Millisecond, + Factor: 2.0, + Jitter: 1.0, + Cap: 250 * time.Millisecond, +} + +// recordExhaustedAssignment writes the one record of an assignment loop that +// ended without assigning. No worker was assigned, so no pool is named. +func (w *ActorWorkflow) recordExhaustedAssignment(ctx context.Context, elapsed time.Duration, actorTemplate *ateapipb.ActorTemplate, err error) { + class := "" + if actorTemplate != nil { + class = sandboxClassString(actorTemplate.GetSandboxConfig().GetSandboxClass()) } - return assignedActor, assignedWorker, nil + w.instruments.recordSchedulerAssignment(ctx, elapsed, ateattr.SchedulerOutcomeError, "", "", class, err) } // validateAssignedWorker checks a RESUMING actor's persisted assignment @@ -403,12 +450,12 @@ func (w *ActorWorkflow) workerHoldingStaleClaim(ctx context.Context, actor *atea return nil, nil } -// schedulerRecordable excludes retried version conflicts: the assignment loop -// re-runs attempts transparently on store.ErrVersionConflict, so counting -// those attempts would inflate the error rate and double-count the eventual -// success. +// schedulerRecordable excludes the two errors the assignment loop retries +// transparently. Counting a retried attempt would inflate the error rate and +// double-count the eventual success; ensureWorkerAssigned records a loop that +// never succeeds. func schedulerRecordable(err error) bool { - return !errors.Is(err, store.ErrVersionConflict) + return !errors.Is(err, store.ErrVersionConflict) && !errors.Is(err, errWorkerFilledUp) } // assignWorkerAttempt makes one attempt at claiming a worker for the actor @@ -427,7 +474,7 @@ func (w *ActorWorkflow) assignWorkerAttempt(ctx context.Context, actorRef resour } defer func() { if schedulerRecordable(err) { - w.instruments.recordSchedulerAssignment(ctx, start, outcome, poolNamespace, pool, class, err) + w.instruments.recordSchedulerAssignment(ctx, time.Since(start), outcome, poolNamespace, pool, class, err) } }() @@ -480,6 +527,9 @@ func (w *ActorWorkflow) assignWorkerAttempt(ctx context.Context, actorRef resour w.workerCache.Forget(assignedWorker.GetMetadata().GetName()) return nil, nil, fmt.Errorf("selected worker disappeared before claim: %w", store.ErrVersionConflict) } + // errWorkerFilledUp leaves the cache alone: it reports no room for + // this actor's size, so forgetting the worker would hide it from + // every smaller actor until the next watch event or relist. return nil, nil, err } diff --git a/cmd/ateapi/internal/controlapi/workflow_resume_test.go b/cmd/ateapi/internal/controlapi/workflow_resume_test.go index cf155ce4c3..12fcf684d0 100644 --- a/cmd/ateapi/internal/controlapi/workflow_resume_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_resume_test.go @@ -28,6 +28,7 @@ import ( "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest" "github.com/agent-substrate/substrate/cmd/ateapi/internal/workercache" "github.com/agent-substrate/substrate/internal/apierror" + "github.com/agent-substrate/substrate/internal/ateattr" "github.com/agent-substrate/substrate/internal/installdefaults" "github.com/agent-substrate/substrate/internal/proto/ateletpb" "github.com/agent-substrate/substrate/internal/resources" @@ -35,6 +36,8 @@ import ( atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/google/go-cmp/cmp" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials/insecure" @@ -44,6 +47,7 @@ import ( "google.golang.org/protobuf/testing/protocmp" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" ) // TestSchedulerRecordable guards the retry-dedup rule: the assignment loop @@ -59,6 +63,8 @@ func TestSchedulerRecordable(t *testing.T) { {name: "success is recorded", err: nil, want: true}, {name: "version conflict is skipped", err: store.ErrVersionConflict, want: false}, {name: "wrapped version conflict is skipped", err: fmt.Errorf("update worker: %w", store.ErrVersionConflict), want: false}, + {name: "filled up worker is skipped", err: errWorkerFilledUp, want: false}, + {name: "wrapped filled up worker is skipped", err: fmt.Errorf("bind actor: %w", errWorkerFilledUp), want: false}, {name: "real error is recorded", err: status.Error(codes.Internal, "boom"), want: true}, } for _, tt := range tests { @@ -1620,3 +1626,422 @@ func TestEnsureVolumesAttached_ReturnsPublishContext(t *testing.T) { t.Errorf("attach wrote to the stored actor (-before +after):\n%s", diff) } } + +// filledUpScheduler hands out workers in order and refuses room on the ones +// named in full. It stands in for a stale worker cache: Schedule offers a +// worker that looked free, and the admit callback that BindActorToWorker runs +// under the Worker's row lock then refuses it with errWorkerFilledUp. +type filledUpScheduler struct { + workers []*ateapipb.Worker + full map[string]bool + calls int + // onSchedule runs after each Schedule call, with the number of calls made + // so far, so a test can act between attempts. + onSchedule func(calls int) + // err, if set, is what every Schedule call returns. + err error +} + +// Schedule repeats the last worker, so a test that offers no free one keeps +// returning a full worker until the backoff ends. +func (s *filledUpScheduler) Schedule(context.Context, scheduling.Constraints) (*ateapipb.Worker, error) { + i := min(s.calls, len(s.workers)-1) + s.calls++ + if s.onSchedule != nil { + s.onSchedule(s.calls) + } + if s.err != nil { + return nil, s.err + } + return s.workers[i], nil +} + +func (s *filledUpScheduler) Applies(*ateapipb.Worker, scheduling.Constraints) bool { return true } + +func (s *filledUpScheduler) HasRoom(worker *ateapipb.Worker, _ scheduling.Constraints) bool { + return !s.full[worker.GetWorkerPod()] +} + +// contentionStore is the smallest store that an assignment attempt needs. The +// bug under test is in the metric path, so nothing here reaches a database. +type contentionStore struct { + actorWorkflowStore + workers map[string]*ateapipb.Worker + actor *ateapipb.Actor +} + +func (s *contentionStore) FindWorkerHostingActor(context.Context, string) (string, error) { + return "", store.ErrNotFound +} + +// BindActorToWorker runs admit as the real store does under the Worker's row +// lock, and returns its refusal unchanged. +func (s *contentionStore) BindActorToWorker(_ context.Context, workerName string, _ *ateapipb.ActorAssignment, admit func(*ateapipb.Worker) error) error { + worker, ok := s.workers[workerName] + if !ok { + return store.ErrNotFound + } + if admit != nil { + return admit(worker) + } + return nil +} + +func (s *contentionStore) UpdateActor(_ context.Context, _ resources.ActorRef, _ store.Precondition, mutate func(*ateapipb.Actor) error) (*ateapipb.Actor, error) { + updated := proto.Clone(s.actor).(*ateapipb.Actor) + if err := mutate(updated); err != nil { + return nil, err + } + s.actor = updated + return updated, nil +} + +// newAssignmentReader gives a workflow its own ManualReader-backed +// Instruments, so a test reads ate.scheduler.assignment.duration without the +// global meter provider. +func newAssignmentReader(t *testing.T) (*Instruments, *sdkmetric.ManualReader) { + t.Helper() + reader := sdkmetric.NewManualReader() + mp := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)) + instruments, err := NewInstruments(mp.Meter("ateapi")) + if err != nil { + t.Fatalf("NewInstruments: %v", err) + } + return instruments, reader +} + +// assignmentObservations returns how many observations reached +// ate.scheduler.assignment.duration, and the outcome of each one. An outcome +// that carries error.type is reported as "outcome/error.type". +func assignmentObservations(t *testing.T, reader *sdkmetric.ManualReader) (int, []string) { + t.Helper() + m, ok := collectMetric(t, reader, "ate.scheduler.assignment.duration") + if !ok { + return 0, nil + } + hist, ok := m.Data.(metricdata.Histogram[float64]) + if !ok { + t.Fatalf("ate.scheduler.assignment.duration data = %T, want Histogram[float64]", m.Data) + } + total := 0 + var outcomes []string + for _, dp := range hist.DataPoints { + total += int(dp.Count) + outcome, _ := dp.Attributes.Value(ateattr.SchedulerOutcomeKey) + label := outcome.AsString() + if errType, ok := dp.Attributes.Value(ateattr.ErrorTypeKey); ok { + label += "/" + errType.AsString() + } + for range dp.Count { + outcomes = append(outcomes, label) + } + } + return total, outcomes +} + +// assignmentSum returns the total of every observation on +// ate.scheduler.assignment.duration, in seconds. +func assignmentSum(t *testing.T, reader *sdkmetric.ManualReader) float64 { + t.Helper() + m, ok := collectMetric(t, reader, "ate.scheduler.assignment.duration") + if !ok { + return 0 + } + hist, ok := m.Data.(metricdata.Histogram[float64]) + if !ok { + t.Fatalf("ate.scheduler.assignment.duration data = %T, want Histogram[float64]", m.Data) + } + var sum float64 + for _, dp := range hist.DataPoints { + sum += dp.Sum + } + return sum +} + +func contendedWorker(podName string) *ateapipb.Worker { + return &ateapipb.Worker{ + Metadata: &ateapipb.ResourceMetadata{Name: testWorkerUID(podName)}, + WorkerNamespace: "worker-ns", + WorkerPool: "pool", + WorkerPod: podName, + WorkerPodUid: testWorkerUID(podName), + SandboxClass: "gvisor", + Status: &ateapipb.WorkerStatus{ + State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, + Capacity: &ateapipb.WorkerResources{Actors: 1}, + }, + } +} + +// staticWorkerSource feeds a worker cache a fixed fleet and a watch that +// never fires, which is the stale cache this contention arises from. +type staticWorkerSource struct { + workers []*ateapipb.Worker +} + +func (s *staticWorkerSource) WatchWorkers(context.Context) (*store.WorkerWatch, error) { + return store.NewWorkerWatch(make(chan store.WorkerEvent), func() {}), nil +} + +func (s *staticWorkerSource) ListWorkers(context.Context, store.ListOptions) (store.ListResponse[*ateapipb.Worker], error) { + return store.ListResponse[*ateapipb.Worker]{Items: s.workers}, nil +} + +// newContendedWorkflow builds a workflow around sched, with a SUSPENDED actor +// ready to resume and a cache that holds every worker sched offers. +func newContendedWorkflow(t *testing.T, sched *filledUpScheduler) (*ActorWorkflow, *ateapipb.Actor, *sdkmetric.ManualReader) { + t.Helper() + actor := &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: "team-a", Name: "id1", Uid: "actor-uid", Version: 1}, + Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED}, + } + byName := make(map[string]*ateapipb.Worker, len(sched.workers)) + for _, worker := range sched.workers { + byName[worker.GetMetadata().GetName()] = worker + } + instruments, reader := newAssignmentReader(t) + + cacheCtx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + wc := workercache.New(&staticWorkerSource{workers: sched.workers}, time.Minute) + if err := wc.Start(cacheCtx); err != nil { + t.Fatalf("workercache.Start: %v", err) + } + w := &ActorWorkflow{ + store: &contentionStore{workers: byName, actor: actor}, + workerCache: wc, + scheduler: sched, + instruments: instruments, + } + return w, actor, reader +} + +func gvisorTemplate() *ateapipb.ActorTemplate { + return &ateapipb.ActorTemplate{ + SandboxConfig: &ateapipb.SandboxConfig{SandboxClass: ateapipb.SandboxClass_SANDBOX_CLASS_GVISOR}, + } +} + +// TestEnsureWorkerAssigned_ContentionRecordsOneObservation holds the note on +// ate.scheduler.assignment.duration: ateapi records only the last attempt. A +// resume that loses a race for the last slot on a worker retries, and the +// retried attempt must not reach the histogram. Losing that race is a +// capacity condition, and the registry says a lack of room is not a failure. +func TestEnsureWorkerAssigned_ContentionRecordsOneObservation(t *testing.T) { + sched := &filledUpScheduler{ + workers: []*ateapipb.Worker{contendedWorker("full-pod"), contendedWorker("free-pod")}, + full: map[string]bool{"full-pod": true}, + } + w, actor, reader := newContendedWorkflow(t, sched) + + _, worker, err := w.ensureWorkerAssigned(context.Background(), resources.ActorRef{Atespace: "team-a", Name: "id1"}, actor, gvisorTemplate()) + if err != nil { + t.Fatalf("ensureWorkerAssigned: %v", err) + } + if got := worker.GetWorkerPod(); got != "free-pod" { + t.Errorf("assigned worker = %q, want %q", got, "free-pod") + } + + got, outcomes := assignmentObservations(t, reader) + if got != 1 { + t.Errorf("observations on ate.scheduler.assignment.duration = %d %v, want 1: the retried attempt must not be recorded", got, outcomes) + } + for _, outcome := range outcomes { + if outcome != ateattr.SchedulerOutcomeAssigned { + t.Errorf("recorded outcome %q, want only %q", outcome, ateattr.SchedulerOutcomeAssigned) + } + } +} + +// TestEnsureWorkerAssigned_ExhaustedBackoffRecordsOneError covers the other +// end: every attempt loses the race, the retries run out, and the resume +// fails. The histogram carries one observation for the resume, and not one +// for each attempt. error.type is the code RPCService.ResumeActor returns, so +// the scheduler histogram agrees with the router. The returned error carries +// that code too, so ate.actor.lifecycle.operation.duration, which classifies +// the same error, also records Aborted. The record times the last attempt and +// not the backoff, as every other record on the histogram does. +func TestEnsureWorkerAssigned_ExhaustedBackoffRecordsOneError(t *testing.T) { + sched := &filledUpScheduler{ + workers: []*ateapipb.Worker{contendedWorker("full-pod")}, + full: map[string]bool{"full-pod": true}, + } + w, actor, reader := newContendedWorkflow(t, sched) + + _, _, err := w.ensureWorkerAssigned(context.Background(), resources.ActorRef{Atespace: "team-a", Name: "id1"}, actor, gvisorTemplate()) + if err == nil { + t.Fatal("ensureWorkerAssigned succeeded, want a failure after the backoff ended") + } + if got := apierror.Code(err); got != codes.Aborted { + t.Errorf("apierror.Code(err) = %v, want %v (err: %v)", got, codes.Aborted, err) + } + if !errors.Is(err, store.ErrVersionConflict) { + t.Errorf("ensureWorkerAssigned error = %v, want one that wraps store.ErrVersionConflict for RPCService.ResumeActor", err) + } + + got, outcomes := assignmentObservations(t, reader) + if got != 1 { + t.Fatalf("observations on ate.scheduler.assignment.duration = %d %v, want 1 for the whole resume", got, outcomes) + } + want := ateattr.SchedulerOutcomeError + "/" + codes.Aborted.String() + if outcomes[0] != want { + t.Errorf("recorded outcome %q, want %q: error.type must be the code ResumeActor returns", outcomes[0], want) + } + + // One attempt against the stubs takes microseconds, so a record that + // includes the sleep after the last attempt goes above half of it. + _, sleep := replayBackoff(t, assignmentBackoff) + if recorded := assignmentSum(t, reader); recorded >= sleep.Seconds()/2 { + t.Errorf("recorded duration = %.3fs, want less than %.3fs: the record must time the last attempt, not the backoff", recorded, sleep.Seconds()/2) + } +} + +// replayBackoff returns how many attempts wait.ExponentialBackoff makes with +// b, and the shortest sleep it makes after the last attempt before it gives +// up. It replays the steps of ExponentialBackoff without jitter. +func replayBackoff(t *testing.T, b wait.Backoff) (attempts int, lastSleep time.Duration) { + t.Helper() + b.Jitter = 0 + for b.Steps > 0 { + attempts++ + if b.Steps == 1 { + t.Fatal("the backoff ends on Steps, with no sleep after the last attempt; these tests need a backoff that ends on Cap") + } + lastSleep = b.Step() + } + return attempts, lastSleep +} + +// TestEnsureWorkerAssigned_CancelledDuringBackoffRecordsOne covers the loop +// ending between attempts. The caller goes away while an attempt that lost a +// race is waiting to retry, so no attempt records, and the resume would leave +// no trace on the histogram without the one record the loop writes. That +// record times the attempt that lost the race, not the context check after it. +func TestEnsureWorkerAssigned_CancelledDuringBackoffRecordsOne(t *testing.T) { + sched := &filledUpScheduler{ + workers: []*ateapipb.Worker{contendedWorker("full-pod")}, + full: map[string]bool{"full-pod": true}, + } + w, actor, reader := newContendedWorkflow(t, sched) + + // Cancel once the first attempt has lost its race, so the loop ends in the + // pre-check of the second attempt and not before the first. The sleep + // makes that attempt long enough to tell apart from the context check. + const attemptTime = 50 * time.Millisecond + ctx, cancel := context.WithCancel(context.Background()) + sched.onSchedule = func(calls int) { + if calls == 1 { + time.Sleep(attemptTime) + cancel() + } + } + defer cancel() + + _, _, err := w.ensureWorkerAssigned(ctx, resources.ActorRef{Atespace: "team-a", Name: "id1"}, actor, gvisorTemplate()) + if !errors.Is(err, context.Canceled) { + t.Fatalf("ensureWorkerAssigned error = %v, want context.Canceled", err) + } + + got, outcomes := assignmentObservations(t, reader) + if got != 1 { + t.Fatalf("observations on ate.scheduler.assignment.duration = %d %v, want 1 for the whole resume", got, outcomes) + } + want := ateattr.SchedulerOutcomeError + "/" + codes.Canceled.String() + if outcomes[0] != want { + t.Errorf("recorded outcome %q, want %q", outcomes[0], want) + } + if recorded := assignmentSum(t, reader); recorded < attemptTime.Seconds() { + t.Errorf("recorded duration = %.3fs, want at least %.3fs: the record must time the attempt that lost the race", recorded, attemptTime.Seconds()) + } +} + +// TestEnsureWorkerAssigned_CancelledBeforeFirstAttemptRecordsNothing covers a +// context that ended before the loop began. No attempt ran, so there is no +// attempt to time and the histogram gets no record. +func TestEnsureWorkerAssigned_CancelledBeforeFirstAttemptRecordsNothing(t *testing.T) { + sched := &filledUpScheduler{ + workers: []*ateapipb.Worker{contendedWorker("free-pod")}, + } + w, actor, reader := newContendedWorkflow(t, sched) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + _, _, err := w.ensureWorkerAssigned(ctx, resources.ActorRef{Atespace: "team-a", Name: "id1"}, actor, gvisorTemplate()) + if !errors.Is(err, context.Canceled) { + t.Fatalf("ensureWorkerAssigned error = %v, want context.Canceled", err) + } + if sched.calls != 0 { + t.Fatalf("Schedule calls = %d, want 0: no attempt may run after the context ended", sched.calls) + } + if got, outcomes := assignmentObservations(t, reader); got != 0 { + t.Errorf("observations on ate.scheduler.assignment.duration = %d %v, want 0", got, outcomes) + } +} + +// TestEnsureWorkerAssigned_CancelledInLastSleepRecordsOne covers the context +// ending in the sleep after the last attempt. The backoff then gives up with +// its own timeout, but the caller went away, so the resume reports the +// context error and not that the retries ran out. +func TestEnsureWorkerAssigned_CancelledInLastSleepRecordsOne(t *testing.T) { + sched := &filledUpScheduler{ + workers: []*ateapipb.Worker{contendedWorker("full-pod")}, + full: map[string]bool{"full-pod": true}, + } + w, actor, reader := newContendedWorkflow(t, sched) + + attempts, _ := replayBackoff(t, assignmentBackoff) + ctx, cancel := context.WithCancel(context.Background()) + sched.onSchedule = func(calls int) { + if calls == attempts { + cancel() + } + } + defer cancel() + + _, _, err := w.ensureWorkerAssigned(ctx, resources.ActorRef{Atespace: "team-a", Name: "id1"}, actor, gvisorTemplate()) + if !errors.Is(err, context.Canceled) { + t.Fatalf("ensureWorkerAssigned error = %v, want context.Canceled", err) + } + if sched.calls != attempts { + t.Fatalf("Schedule calls = %d, want %d: the context must end after the last attempt", sched.calls, attempts) + } + + got, outcomes := assignmentObservations(t, reader) + if got != 1 { + t.Fatalf("observations on ate.scheduler.assignment.duration = %d %v, want 1 for the whole resume", got, outcomes) + } + want := ateattr.SchedulerOutcomeError + "/" + codes.Canceled.String() + if outcomes[0] != want { + t.Errorf("recorded outcome %q, want %q", outcomes[0], want) + } +} + +// TestEnsureWorkerAssigned_ContextEndsInsideAttemptRecordsOne covers an +// attempt that fails because the context ended inside it. The attempt records +// itself, so the loop must not add a second record, and the caller gets the +// context error and not Aborted. +func TestEnsureWorkerAssigned_ContextEndsInsideAttemptRecordsOne(t *testing.T) { + sched := &filledUpScheduler{ + workers: []*ateapipb.Worker{contendedWorker("free-pod")}, + err: fmt.Errorf("list workers: %w", context.DeadlineExceeded), + } + w, actor, reader := newContendedWorkflow(t, sched) + + _, _, err := w.ensureWorkerAssigned(context.Background(), resources.ActorRef{Atespace: "team-a", Name: "id1"}, actor, gvisorTemplate()) + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("ensureWorkerAssigned error = %v, want context.DeadlineExceeded", err) + } + if got := apierror.Code(err); got != codes.DeadlineExceeded { + t.Errorf("apierror.Code(err) = %v, want %v (err: %v)", got, codes.DeadlineExceeded, err) + } + + got, outcomes := assignmentObservations(t, reader) + if got != 1 { + t.Fatalf("observations on ate.scheduler.assignment.duration = %d %v, want 1 for the whole resume", got, outcomes) + } + want := ateattr.SchedulerOutcomeError + "/" + codes.DeadlineExceeded.String() + if outcomes[0] != want { + t.Errorf("recorded outcome %q, want %q", outcomes[0], want) + } +} diff --git a/docs/metrics/registry/metrics.yaml b/docs/metrics/registry/metrics.yaml index 78e450d32a..b34b2525f5 100644 --- a/docs/metrics/registry/metrics.yaml +++ b/docs/metrics/registry/metrics.yaml @@ -808,11 +808,13 @@ groups: stability: development brief: The time of one attempt of the scheduler to assign a worker during a resume. note: > - If a version conflict causes a second attempt, ateapi records only the - last attempt. The buckets are smaller than the buckets of the lifecycle - histogram. This step is a read of the cache and some writes to the store. - It is not a restore of some seconds. But the buckets go to 5 s. Thus you - can see a large delay in the store. + One resume gives one record. ateapi retries an attempt that loses a race + for a worker, and records only the last attempt. If the retries run out, + the outcome is error, and error.type is the code that the caller gets. + The buckets are smaller than the buckets of the lifecycle histogram. + This step is a read of the cache and some writes to the store. It is not + a restore of some seconds. But the buckets go to 5 s. Thus you can see a + large delay in the store. annotations: substrate: emitted_by: [ateapi] diff --git a/docs/observability.md b/docs/observability.md index a75d54e166..5c3049e30e 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -251,7 +251,7 @@ Agent Substrate emits foundational OpenTelemetry system and server metrics to mo `ate.workerpool.namespace`, `ate.workerpool.name`) | | `ate.workerpool.workers` | ateapi | up/down counter | live worker count per pool, split by state (`idle`/`partial`/`at_capacity`/`unschedulable`) and sandbox class to provide fleet capacity and saturation at a glance | | `ate.actor.lifecycle.operation.duration` | ateapi | histogram | how long each actor operation (create/resume/suspend/pause/delete/revert) takes and whether it failed (`error.type` present = failure, absent = success); labeled by operation, template, pool (`ate.workerpool.namespace` + `ate.workerpool.name`), sandbox class, and snapshot kind and scope on resume; already-running resume no-ops are not recorded so the histogram tracks actual activations, not router traffic | -| `ate.scheduler.assignment.duration` | ateapi | histogram | time it takes for an actor to be assigned to a worker, per attempt (version-conflict retries record only the final attempt), with the outcome (`assigned` / `no_capacity` / `error`), the assigned pool (`ate.workerpool.namespace` + `ate.workerpool.name`) and sandbox class to catch scheduling latency and capacity starvation problems | +| `ate.scheduler.assignment.duration` | ateapi | histogram | time of one attempt to assign a worker to an actor, with one record for each resume (ateapi retries an attempt that loses a race for a worker and records only the last attempt), with the outcome (`assigned` / `no_capacity` / `error`), the assigned pool (`ate.workerpool.namespace` + `ate.workerpool.name`) and sandbox class to catch scheduling latency and capacity starvation problems | | `ate.actor.restore.duration` | atelet | histogram | how long each phase of a restore takes on the worker node, which is where cold-start latency actually goes once ateapi hands off (labels `ate.snapshot.phase`, `ate.snapshot.kind`, `ate.snapshot.scope`, `ate.template.atespace`, `ate.template.name`, `ate.sandbox.class`) | | `ate.actor.checkpoint.duration` | atelet | histogram | the same phase breakdown for writing a snapshot, so a slow suspend can be attributed to ateom or to the upload (same labels as the restore histogram) | | `ate.imagecache.requests` | atelet | counter | image lookups in the node-local image cache, by outcome (`ate.imagecache.outcome`), with `error.type` on the `error` outcome. A miss pays for the pull and the unpack, so the hit ratio per node is a leading indicator of resume latency | @@ -264,7 +264,7 @@ For `ate.workerpool.desired_workers` and `ate.workerpool.ready_workers`: For `atenet.router.route.duration`: * `ate.router.outcome` categorizes the route attempt result: `ok`, `cancelled`, `timeout`, `no_capacity`, `failed_precondition`, `lock_conflict`, `not_found`, `unavailable`, `rate_limited`, or `resume_error`. -* The router and the scheduler both report `no_capacity` for one condition, but they count different events. The scheduler records one sample on `ate.scheduler.assignment.duration` for each assignment attempt. The router records one sample for each request. Many requests can join one resume, and one parked request can cause more than one attempt. Thus do not compare the two counts directly. +* The router and the scheduler both report `no_capacity` for one condition, but they count different events. The scheduler records one sample on `ate.scheduler.assignment.duration` for each resume. The router records one sample for each request. Many requests can join one resume, and one parked request can cause more than one resume. Thus do not compare the two counts directly. * `ate.router.resume` indicates the singleflight execution state of actor resumption: `none` (the resume found the actor already running), `triggered` (this request completed a cold activation), `joined` (this request waited on another request's resume, which activated the actor), or `unknown` (the resume did not complete, so whether an activation ran is unknown). `ate.template.atespace` and `ate.template.name` hold `unknown` when the router has no template to name. For `ate.imagecache.requests`: From 2f9b30dfafeeb451e3b35f6f537e17541d92e5fe Mon Sep 17 00:00:00 2001 From: Quentin Bisson Date: Thu, 8 Oct 2026 14:34:15 +0000 Subject: [PATCH 02/44] Components honor OTEL_TRACES_EXPORTER=none and treat OTEL_METRICS_EXPORTER=prometheus as none where they serve /metrics (#2190) Follow-up to #1878 (#1877). Uses the shared exporter parser from #2193, as agreed in #2067. - `OTEL_METRICS_EXPORTER=prometheus` stops the OTLP push on the components that serve `/metrics`, since the specification defines it as the pull exporter they already provide. Before, it warned and kept the push, so each series arrived twice. `otlp,prometheus` keeps the push. ateom serves no `/metrics`, so there `prometheus` is skipped as unknown with a warning and the push stays on. - `OTEL_TRACES_EXPORTER` is honored with the same rules, knowing `otlp` (the default) and `none`. With `none` the provider keeps sampling and propagating trace context but exports nothing. - [x] Tests pass - [x] Appropriate changes to documentation are included in the PR --- docs/observability.md | 4 +- internal/serverboot/exporters.go | 4 + internal/serverboot/serverboot.go | 89 ++++++++------- internal/serverboot/serverboot_test.go | 149 ++++++++++++++++++++++--- 4 files changed, 194 insertions(+), 52 deletions(-) diff --git a/docs/observability.md b/docs/observability.md index 5c3049e30e..cd1c086760 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -333,7 +333,7 @@ A queue that has never processed an item bridges as an exponential histogram wit ### Scraping instead of pushing -ateapi, atelet, atenet-router and the credential provider also serve every instrument on their Prometheus `/metrics` endpoint. A cluster that scrapes those endpoints sets `OTEL_METRICS_EXPORTER=none` on the components, so each series reaches the backend once. With `none` the components install no OTLP metric reader and keep the Prometheus one; traces and logs are unaffected. atecontroller then registers its instruments (`ate.workerpool.*`) on controller-runtime's registry, so the manager's `:8080` serves them next to the controller-runtime families. ateom serves no endpoint of its own, so leave the variable unset on the worker pods, or it exports no metrics at all. The variable accepts `otlp` (the default) and `none`; unlike `OTEL_LOGS_EXPORTER`, it takes no list and no `console`. Any other value logs a warning and keeps the OTLP export. +ateapi, atelet, atenet-router and the credential provider also serve every instrument on their Prometheus `/metrics` endpoint. A cluster that scrapes those endpoints sets `OTEL_METRICS_EXPORTER=none` on the components, so each series reaches the backend once. With `none` the components install no OTLP metric reader and keep the Prometheus one; traces and logs are unaffected. atecontroller then registers its instruments (`ate.workerpool.*`) on controller-runtime's registry, so the manager's `:8080` serves them next to the controller-runtime families. ateom serves no endpoint of its own, so leave the variable unset on the worker pods, or it exports no metrics at all. The variable takes a comma-separated list, with the same rules as `OTEL_LOGS_EXPORTER`. `otlp` (the default) keeps the push, and `prometheus` or `none` stops it. `/metrics` stays on for each value. ateom serves no `/metrics`, so there `prometheus` is skipped as unknown with a warning, and the push stays on unless the variable is `none`. ### Local Metrics with Prometheus (Kind Cluster) @@ -363,6 +363,8 @@ Distributed tracing tracks the end-to-end flow of requests as they pass through Agent Substrate samples traces by default. Each component roots parentless requests at a per-component ratio (10% on the control plane components, 1% at the atenet router), overridable per component through the standard `OTEL_TRACES_SAMPLER` / `OTEL_TRACES_SAMPLER_ARG` environment variables. Every component uses a parent based sampler, so a client can also force a request to be traced end to end (e.g. via the `--trace` flag). Agent Substrate leverages OpenTelemetry (OTel) for context propagation across the call stack. Each traced request generates a unique trace hash/ID, which you can use to inspect the detailed request lifecycle and span hierarchy inside Google Cloud Trace or Jaeger. See the per-component defaults table in [Tracing Best Practices](dev/best-practices/tracing.md). +A component with `OTEL_TRACES_EXPORTER=none` exports no spans. It still samples and propagates trace context, so the traces of the components around it stay whole. The variable takes a comma-separated list, with the same rules as `OTEL_LOGS_EXPORTER`, and knows `otlp` (the default) and `none`. + ### Local Tracing with Jaeger (Kind Cluster) For local development inside a `kind` cluster, Agent Substrate automatically provisions a local OpenTelemetry Collector and Jaeger instance. diff --git a/internal/serverboot/exporters.go b/internal/serverboot/exporters.go index 5e69e39e9f..d124ecf178 100644 --- a/internal/serverboot/exporters.go +++ b/internal/serverboot/exporters.go @@ -27,6 +27,10 @@ import ( // exporterNone selects the empty set; it is not an exporter. const exporterNone = "none" +// exporterPrometheus is the specification's pull exporter, a known name only +// for a component that serves its own /metrics. +const exporterPrometheus = "prometheus" + // The exporter names the OTEL_*_EXPORTER variables use, for checking a // resolved set with Has. const ( diff --git a/internal/serverboot/serverboot.go b/internal/serverboot/serverboot.go index c051ce9eac..f673b6ca7c 100644 --- a/internal/serverboot/serverboot.go +++ b/internal/serverboot/serverboot.go @@ -26,7 +26,6 @@ import ( "log/slog" "net/http" "os" - "strings" "sync/atomic" "github.com/agent-substrate/substrate/internal/ateattr" @@ -174,50 +173,64 @@ func InitTracing(ctx context.Context, opts TracingOptions) (*sdktrace.TracerProv sdktrace.WithResource(res), sdktrace.WithSampler(opts.Sampling.Sampler()), } - expOpts := []otlptracegrpc.Option{ - // GKE managed traces doesn't support validating the TLS certs of the collector. - otlptracegrpc.WithInsecure(), - } - if opts.ExporterConn != nil { - // WithGRPCConn takes precedence over endpoint/credential options, so - // WithInsecure above is inert on this path. - expOpts = append(expOpts, otlptracegrpc.WithGRPCConn(opts.ExporterConn)) - } - exporter, err := otlptracegrpc.New(ctx, expOpts...) - if err != nil { - return nil, fmt.Errorf("create OTLP exporter: %w", err) + // Without an exporter the provider still samples and propagates trace + // context, so a component that exports nothing keeps the traces of the + // components around it whole. + export := tracesPushEnabled(ctx) + if export { + expOpts := []otlptracegrpc.Option{ + // GKE managed traces doesn't support validating the TLS certs of the collector. + otlptracegrpc.WithInsecure(), + } + if opts.ExporterConn != nil { + // WithGRPCConn takes precedence over endpoint/credential options, so + // WithInsecure above is inert on this path. + expOpts = append(expOpts, otlptracegrpc.WithGRPCConn(opts.ExporterConn)) + } + exporter, err := otlptracegrpc.New(ctx, expOpts...) + if err != nil { + return nil, fmt.Errorf("create OTLP exporter: %w", err) + } + tpOpts = append(tpOpts, sdktrace.WithBatcher(exporter)) } - tpOpts = append(tpOpts, sdktrace.WithBatcher(exporter)) tp := sdktrace.NewTracerProvider(tpOpts...) otel.SetTracerProvider(tp) otel.SetTextMapPropagator(propagation.TraceContext{}) - slog.InfoContext(ctx, "Tracing initialized", slog.String("sampler", opts.Sampling.Sampler().Description())) + slog.InfoContext(ctx, "Tracing initialized", + slog.String("sampler", opts.Sampling.Sampler().Description()), + slog.Bool("export", export)) return tp, nil } -const metricsExporterEnv = "OTEL_METRICS_EXPORTER" - -// metricsPushEnabled applies OTEL_METRICS_EXPORTER: otlp, the default, or none, -// which drops the OTLP reader for a component whose metrics are scraped -// instead. An unrecognized value keeps the OTLP export and logs. -func metricsPushEnabled(ctx context.Context) bool { - switch value := strings.ToLower(strings.TrimSpace(os.Getenv(metricsExporterEnv))); value { - case "", "otlp": - return true - case "none": - return false - default: - slog.WarnContext(ctx, "Unsupported metrics exporter, keeping the OTLP export", - slog.String("env", metricsExporterEnv), - slog.String("exporter", value)) - return true +const ( + metricsExporterEnv = "OTEL_METRICS_EXPORTER" + tracesExporterEnv = "OTEL_TRACES_EXPORTER" +) + +// metricsPushEnabled applies OTEL_METRICS_EXPORTER with the shared exporter +// rules: the OTLP reader is installed while otlp is selected, as it is when the +// variable is unset. prometheus, the specification's pull exporter, is known +// only when pullServed, i.e. the component serves its own /metrics; elsewhere +// it is skipped as unknown, so a push-only component keeps exporting. +func metricsPushEnabled(ctx context.Context, pullServed bool) bool { + known := []string{ExporterOTLP} + if pullServed { + known = append(known, exporterPrometheus) } + return resolveExportersEnv(ctx, metricsExporterEnv, known, Exporters{ExporterOTLP: true}).Has(ExporterOTLP) +} + +// tracesPushEnabled applies OTEL_TRACES_EXPORTER with the shared exporter +// rules: the OTLP span exporter is installed while otlp, the default, is +// selected. +func tracesPushEnabled(ctx context.Context) bool { + return resolveExportersEnv(ctx, tracesExporterEnv, []string{ExporterOTLP}, Exporters{ExporterOTLP: true}).Has(ExporterOTLP) } // InitMetrics registers a global MeterProvider with both a Prometheus // reader (exposed via StartMetricsServer's /metrics handler) and an -// OTLP periodic reader, the latter unless OTEL_METRICS_EXPORTER is "none". +// OTLP periodic reader, the latter while OTEL_METRICS_EXPORTER selects otlp. // The Prometheus reader registers on the default registry. func InitMetrics(ctx context.Context, serviceName string) (*sdkmetric.MeterProvider, error) { if serviceName == "" { @@ -227,16 +240,16 @@ func InitMetrics(ctx context.Context, serviceName string) (*sdkmetric.MeterProvi if err != nil { return nil, fmt.Errorf("create Prometheus metric exporter: %w", err) } - return newMeterProvider(ctx, serviceName, metricsPushEnabled(ctx), false, nil, nil, promReader) + return newMeterProvider(ctx, serviceName, metricsPushEnabled(ctx, true), false, nil, nil, promReader) } // InitMetricsBridged is for a binary whose scraped endpoint serves a registry // it does not own (atecontroller: controller-runtime's), so every metric // reaches the backend exactly once on whichever path is active. With the OTLP // export, reg is bridged onto the push and the OTel instruments stay off it, -// since the bridge would otherwise push them a second time. With -// OTEL_METRICS_EXPORTER=none nothing is pushed and the OTel instruments -// register on reg, so its endpoint serves both. +// since the bridge would otherwise push them a second time. When +// OTEL_METRICS_EXPORTER does not select otlp, nothing is pushed and the OTel +// instruments register on reg, so its endpoint serves both. // // wrapProducer, when non-nil, wraps the bridge producer on the OTLP path, for // a rewrite the push backend needs but the scraped endpoint does not. @@ -247,7 +260,7 @@ func InitMetricsBridged(ctx context.Context, serviceName string, reg interface { if serviceName == "" { return nil, fmt.Errorf("serviceName is required") } - if metricsPushEnabled(ctx) { + if metricsPushEnabled(ctx, true) { producer := prombridge.NewMetricProducer(prombridge.WithGatherer(reg)) if wrapProducer != nil { producer = wrapProducer(producer) @@ -277,7 +290,7 @@ func InitMetricsBridged(ctx context.Context, serviceName string, reg interface { // TracingOptions.RelayCapable, implied rather than a parameter because only a // caller that has a relay to pass reaches for this function in the first place. func InitMetricsPushOnlyVia(ctx context.Context, serviceName string, conn *grpc.ClientConn) (*sdkmetric.MeterProvider, error) { - return newMeterProvider(ctx, serviceName, metricsPushEnabled(ctx), true, conn, nil) + return newMeterProvider(ctx, serviceName, metricsPushEnabled(ctx, false), true, conn, nil) } func newMeterProvider(ctx context.Context, serviceName string, push, relayCapable bool, conn *grpc.ClientConn, producers []sdkmetric.Producer, extraReaders ...sdkmetric.Reader) (*sdkmetric.MeterProvider, error) { diff --git a/internal/serverboot/serverboot_test.go b/internal/serverboot/serverboot_test.go index 720f481691..b685a434b0 100644 --- a/internal/serverboot/serverboot_test.go +++ b/internal/serverboot/serverboot_test.go @@ -34,6 +34,7 @@ import ( "go.opentelemetry.io/otel/sdk/resource" semconv "go.opentelemetry.io/otel/semconv/v1.21.0" colmetricspb "go.opentelemetry.io/proto/otlp/collector/metrics/v1" + coltracepb "go.opentelemetry.io/proto/otlp/collector/trace/v1" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" @@ -346,19 +347,41 @@ func gatheredNames(t *testing.T, reg prometheus.Gatherer) map[string]bool { } func TestMetricsPushEnabled(t *testing.T) { - for value, wantPush := range map[string]bool{"": true, "otlp": true, " OTLP ": true, "none": false, " None ": false, "prometheus": true} { - var buf bytes.Buffer - prev := slog.Default() - slog.SetDefault(slog.New(slog.NewTextHandler(&buf, nil))) - t.Setenv(metricsExporterEnv, value) - push := metricsPushEnabled(t.Context()) - slog.SetDefault(prev) - - if push != wantPush { - t.Errorf("%s=%q: push = %t, want %t", metricsExporterEnv, value, push, wantPush) - } - if warned, wantWarn := strings.Contains(buf.String(), "level=WARN"), value == "prometheus"; warned != wantWarn { - t.Errorf("%s=%q: warned = %t, want %t:\n%s", metricsExporterEnv, value, warned, wantWarn, buf.String()) + type want struct{ push, warn bool } + tests := []struct { + value string + pullServed want + pushOnly want + }{ + {"", want{true, false}, want{true, false}}, + {"otlp", want{true, false}, want{true, false}}, + {" OTLP ", want{true, false}, want{true, false}}, + {"none", want{false, false}, want{false, false}}, + {" None ", want{false, false}, want{false, false}}, + {"prometheus", want{false, false}, want{true, true}}, + {" Prometheus ", want{false, false}, want{true, true}}, + {"console", want{true, true}, want{true, true}}, + {"otlp,prometheus", want{true, false}, want{true, true}}, + {"prometheus, otlp", want{true, false}, want{true, true}}, + {"prometheus,console", want{false, true}, want{true, true}}, + {"none,otlp", want{true, true}, want{true, true}}, + {"none,prometheus", want{true, true}, want{false, true}}, + } + for _, tt := range tests { + for pullServed, w := range map[bool]want{true: tt.pullServed, false: tt.pushOnly} { + var buf bytes.Buffer + prev := slog.Default() + slog.SetDefault(slog.New(slog.NewTextHandler(&buf, nil))) + t.Setenv(metricsExporterEnv, tt.value) + push := metricsPushEnabled(t.Context(), pullServed) + slog.SetDefault(prev) + + if push != w.push { + t.Errorf("%s=%q, pullServed %t: push = %t, want %t", metricsExporterEnv, tt.value, pullServed, push, w.push) + } + if warned := strings.Contains(buf.String(), "level=WARN"); warned != w.warn { + t.Errorf("%s=%q, pullServed %t: warned = %t, want %t:\n%s", metricsExporterEnv, tt.value, pullServed, warned, w.warn, buf.String()) + } } } } @@ -560,3 +583,103 @@ func TestSetLogLevel(t *testing.T) { t.Errorf("SetLogLevel(\"\") changed the level to %v", got) } } + +func TestTracesPushEnabled(t *testing.T) { + tests := []struct { + value string + push, warn bool + }{ + {"", true, false}, + {"otlp", true, false}, + {" OTLP ", true, false}, + {"none", false, false}, + {" None ", false, false}, + {"console", true, true}, + {"otlp,console", true, true}, + {"none,otlp", true, true}, + } + for _, tt := range tests { + var buf bytes.Buffer + prev := slog.Default() + slog.SetDefault(slog.New(slog.NewTextHandler(&buf, nil))) + t.Setenv(tracesExporterEnv, tt.value) + push := tracesPushEnabled(t.Context()) + slog.SetDefault(prev) + + if push != tt.push { + t.Errorf("%s=%q: push = %t, want %t", tracesExporterEnv, tt.value, push, tt.push) + } + if warned := strings.Contains(buf.String(), "level=WARN"); warned != tt.warn { + t.Errorf("%s=%q: warned = %t, want %t:\n%s", tracesExporterEnv, tt.value, warned, tt.warn, buf.String()) + } + } +} + +type traceCollector struct { + coltracepb.UnimplementedTraceServiceServer + spans atomic.Int64 +} + +func (c *traceCollector) Export(_ context.Context, req *coltracepb.ExportTraceServiceRequest) (*coltracepb.ExportTraceServiceResponse, error) { + for _, rs := range req.GetResourceSpans() { + for _, ss := range rs.GetScopeSpans() { + c.spans.Add(int64(len(ss.GetSpans()))) + } + } + return &coltracepb.ExportTraceServiceResponse{}, nil +} + +// startTraceCollector serves a traceCollector on a loopback port, points +// OTEL_EXPORTER_OTLP_ENDPOINT at it, and sets OTEL_TRACES_EXPORTER. +func startTraceCollector(t *testing.T, exporter string) *traceCollector { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + collector := &traceCollector{} + srv := grpc.NewServer() + coltracepb.RegisterTraceServiceServer(srv, collector) + go func() { _ = srv.Serve(ln) }() + t.Cleanup(srv.Stop) + t.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://"+ln.Addr().String()) + t.Setenv(tracesExporterEnv, exporter) + return collector +} + +// endOneSpan ends a sampled span, then flushes and shuts the provider down, so +// a span exporter, if there is one, has exported it. +func endOneSpan(t *testing.T, serviceName string) { + t.Helper() + tp, err := InitTracing(t.Context(), TracingOptions{ServiceName: serviceName, Sampling: ParentRatioSampling(1)}) + if err != nil { + t.Fatalf("InitTracing: %v", err) + } + _, span := tp.Tracer("test").Start(t.Context(), "test") + if !span.SpanContext().IsSampled() { + t.Fatal("the span was not sampled") + } + span.End() + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + _ = tp.ForceFlush(ctx) + _ = tp.Shutdown(ctx) +} + +func TestInitTracingExportsOverOTLPByDefault(t *testing.T) { + collector := startTraceCollector(t, "") + endOneSpan(t, "test-traces-default") + if got := collector.spans.Load(); got != 1 { + t.Errorf("the collector received %d span(s), want 1", got) + } +} + +// With OTEL_TRACES_EXPORTER=none the span is still sampled, so the context a +// component propagates keeps its sampled flag, and nothing is exported. +func TestInitTracingExporterNoneExportsNothing(t *testing.T) { + collector := startTraceCollector(t, "none") + endOneSpan(t, "test-traces-none") + if got := collector.spans.Load(); got != 0 { + t.Errorf("OTEL_TRACES_EXPORTER=none still exported %d span(s)", got) + } +} From 7de36091e5ed4b85c0bd40c5e296f4e7f636b64d Mon Sep 17 00:00:00 2001 From: haiyanmeng Date: Thu, 8 Oct 2026 15:21:53 +0000 Subject: [PATCH 03/44] fake-dataplane: let the fake data plane host actors (#2300) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR builds on the fake data plane skeleton (#2272) so actors can actually be placed and resumed on fake Workers. With the skeleton, every resume failed with `no_capacity`. All changes are benchmark-only, under `cmd/benchmarking/isolate/`. **No production code changes.** atelet, ateom, ate-api-server and atecontroller are untouched, and nothing here runs in a normal install. The fake data plane should cost ate-api-server what a real one does on every call a benchmark exercises. Otherwise control-plane throughput measured on it would look better than it really is. Each commit closes one gap where a fake Worker behaved differently from a real one. ## What's in this PR Six commits, best reviewed one at a time: 1. **fake-atelet: write a placeholder object for each external snapshot.** ate-api-server copies an actor's snapshot when it creates a tag, golden tags included, and refuses to copy an empty one. On an external `Checkpoint` and on `UploadPausedCheckpoint`, fake-atelet now writes one empty object under the snapshot URI. It writes to the backend atelet would use, chosen by `ATE_STORAGE_BACKEND`. The write runs inside the configured delay. If the write fails, the call fails, because a later tag of that snapshot would fail too. 2. **Report fake Worker capacity through fake-atelet.** This fixes the known limitation from #2272. ate-api-server accepts a capacity report only from the atelet on the Worker's node. So fake-workersync plays each fake Worker's ateom and sends the report to the fake-atelet on that Worker's node, which relays `RegisterWorker` under its atelet identity. - **Same values a real Worker reports:** cpu and memory come from the pool's `template.resources.limits`, or from the node's allocatable when no limit is set, which is what the downward API gives ateom. Max actors is ateom's default of 1000 unless the `ate.dev/fake-max-actors` annotation overrides it. The hardware identity, which ate-api-server requires, comes from `hardware.ProbeHost()`, as ateom's does. Each Worker reports once, when it is made, and keeps that capacity, as ateom reads its limits once at startup. - **`readyReplicas` means usable:** a pool counts a Worker ready only once ate-api-server has accepted its capacity. - **A bad annotation doesn't remove the pool:** if a pool's annotation doesn't parse, or is out of range (1 to 2³¹−1), the controller reports an error and leaves the pool's Workers as they are. - **Relay security:** the relay listens on its own port (`--relay-port`, default 8086) and implements only `RegisterWorker`. Both ends use mutual TLS with the pod-identity certificates and check the peer's SPIFFE ID. fake-atelet accepts only fake-workersync, and fake-workersync accepts only the atelet identity. Without that check, any caller could set any Worker's capacity. Both ends reread the trust bundle at each handshake, as the herder does, so a CA rotation during a long run doesn't need a restart. - **Relay connections:** fake-workersync finds each node's relay by listing the fake-atelet pods and keeps one connection per relay. A connection is closed once its relay leaves the pod list. - **Tests:** `fakeworkertest` issues certificates shaped like pod-identity ones, so both ends of the relay are tested over real TLS handshakes, including a trust bundle rotation. 3. **fake-atelet: mint each actor's certificate on activation.** In production, every `Run` and `Restore` with tunneled egress costs ate-api-server a certificate mint. The default install turns on tunneled egress. fake-atelet now mints under its atelet identity, and the call fails if the mint fails, just as tunneled egress fails closed in ateom. The mint time is added on top of `--delay` rather than counted inside it, so if mints slow down under load, resumes slow down by the same amount. `--mint-actor-certificate=false` leaves the mint out, to measure what it costs. 4. **fake-workersync: drain a Worker before deleting it.** Before this change, a Worker that no pool wanted was deleted at once, even with Actors still on it. Now it is retired the way a real worker pod is: drained, then deleted once `ListWorkerActorAssignments` finds no Actors on it. In production, ateom stops a terminating pod's Actors, but nothing does that here. So after `--drain-grace` (default 1m) the Worker is deleted anyway, and `DeleteWorker` releases the Actors still on it, as the real syncer's delete does. A draining Worker doesn't count as a replica. After a restart, `adopt` reads the draining state back from ate-api-server. 5. **fake-workersync: follow WorkerPool label and template edits.** These match what the real worker syncer and a Deployment rollout do: - A label edit on a pool is written onto its Workers in place with `UpdateWorker`. The Worker's version is the precondition, and a conflicting write is retried on the next pass. - An edit to the sandbox class, `template.resources.limits` or the `ate.dev/fake-max-actors` annotation replaces the pool's Workers. `sandbox_class` can't be changed on a Worker, and a real Worker's capacity never changes, since ateom reads its limits and actor count once at startup. A change in a node's allocatable leaves its Workers as they are, as in production. - Edits made while fake-workersync was down are still applied after it restarts. An adopted Worker keeps the capacity ate-api-server recorded and isn't reported again; one whose recorded capacity no longer matches its pool is replaced. 6. **fake-workersync: register each replacement as a new Worker.** A real Worker is named after its pod's UID, so a replacement pod is always a new Worker. Before this change, a replaced fake Worker came back under the same name, pod UID and IP. Then anything ate-api-server keyed by the old Worker could carry over to its successor, which production never allows. - **New identity for each Worker:** each fake Worker is now named with a fresh UUID, which is also its pod UID. `fakeworker.Name` becomes the stand-in pod name, shared by every Worker that fills the same index of a pool, and `adopt` finds this run's Workers by that pod name. - **Replacement while the old Worker drains:** a replaced pod gets its new Worker in the same pass, as a Deployment creates a replacement pod without waiting for the old one to terminate. This covers sandbox class, limits and max-actors edits, scaling down and back up, and Workers whose node has left the benchmark set; those are replaced on a benchmark node, as a pod is when its node goes away. If no benchmark node is listed at all, Workers aren't moved off their nodes, since nothing could replace them. - **Status counts only usable Workers:** a pool's status counts only the Worker currently serving each wanted pod. A Worker made from an older template (sandbox class, limits or max actors), past `spec.replicas`, or off the benchmark nodes doesn't count as a replica even if its drain failed. It gets no label write or capacity report while the drain is retried. ## Known limitations - **Other template edits:** only sandbox class, limits and max-actors edits replace fake Workers. Edits to the image, env or template labels roll a real pool's pods but leave fake Workers in place. - **One mint per activation:** fake-atelet mints an actor certificate once per `Run`/`Restore`. Real ateom also renews it while the actor runs, so multi-hour runs with long-lived actors under-count mint load. - **List calls while draining:** each draining Worker costs one `ListWorkerActorAssignments` per pass until its Actors leave or `--drain-grace` runs out. Large scale-downs therefore add some load during a measurement. - **Storage client at startup:** fake-atelet creates its object-storage client when it starts and exits if that fails. It runs as the atelet service account, which already needs storage access. - **Relay node check:** the relay client checks the relay's SPIFFE ID and chain, but not that the relay is on the Worker's node, which `internal/ateletdial` does check. A misrouted report is rejected by ate-api-server and retried on the next pass. ## Not in this PR The deploy manifest and the `benchmarking/workloads/deploy.sh --fake-data-plane` mode come in a follow-up PR. Deploying these binaries needs the following, which that PR adds: - the fake-atelet relay port - the servicedns trust bundle mount, for fake-atelet to verify ate-api-server - a Role that lets fake-workersync list the fake-atelet pods in `ate-system` Addresses #2261 - [x] Tests pass - [x] Appropriate changes to documentation are included in the PR --- .../isolate/fake-atelet/herder.go | 128 ++- .../isolate/fake-atelet/herder_test.go | 280 +++++- cmd/benchmarking/isolate/fake-atelet/main.go | 93 +- cmd/benchmarking/isolate/fake-atelet/relay.go | 81 ++ .../isolate/fake-atelet/relay_test.go | 169 ++++ .../isolate/fake-workersync/controller.go | 397 +++++++-- .../fake-workersync/controller_test.go | 833 +++++++++++++++++- .../isolate/fake-workersync/main.go | 73 +- .../isolate/fake-workersync/relayclient.go | 141 +++ .../fake-workersync/relayclient_test.go | 199 +++++ .../isolate/internal/fakeworker/fakeworker.go | 57 +- .../internal/fakeworker/fakeworker_test.go | 51 +- .../fakeworkertest/fakeworkertest.go | 121 +++ 13 files changed, 2450 insertions(+), 173 deletions(-) create mode 100644 cmd/benchmarking/isolate/fake-atelet/relay.go create mode 100644 cmd/benchmarking/isolate/fake-atelet/relay_test.go create mode 100644 cmd/benchmarking/isolate/fake-workersync/relayclient.go create mode 100644 cmd/benchmarking/isolate/fake-workersync/relayclient_test.go create mode 100644 cmd/benchmarking/isolate/internal/fakeworker/fakeworkertest/fakeworkertest.go diff --git a/cmd/benchmarking/isolate/fake-atelet/herder.go b/cmd/benchmarking/isolate/fake-atelet/herder.go index f088f6702b..42c2d8ad66 100644 --- a/cmd/benchmarking/isolate/fake-atelet/herder.go +++ b/cmd/benchmarking/isolate/fake-atelet/herder.go @@ -16,23 +16,96 @@ package main import ( "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "fmt" + "io" + "log/slog" + "strings" "time" + "github.com/agent-substrate/substrate/internal/objectstore" "github.com/agent-substrate/substrate/internal/proto/ateletpb" + "github.com/agent-substrate/substrate/internal/resources" + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "google.golang.org/grpc" ) +// placeholderObject is the one object written under each snapshot URI. +const placeholderObject = "fake-dataplane-placeholder" + +// objectWriter is the part of pkg/objectstorage.ObjectStorage the herder +// uses. +type objectWriter interface { + PutObject(ctx context.Context, bucket, object string, reader io.Reader) error +} + +// actorCertMinter is the part of ateapipb.WorkerServiceClient the herder uses. +type actorCertMinter interface { + MintAteomActorCertificate(ctx context.Context, in *ateapipb.MintAteomActorCertificateRequest, opts ...grpc.CallOption) (*ateapipb.MintAteomActorCertificateResponse, error) +} + // delays is how long each AteomHerder call takes before it succeeds: the -// data plane's share of it. +// data plane's share of it, not counting an actor certificate mint. type delays struct { run, restore, checkpoint, uploadPausedCheckpoint, terminate time.Duration } // herder answers every AteomHerder call with success after its delay, without // running or saving any workload. Actors it "runs" do not exist. +// +// The one side effect it keeps is a placeholder object under each snapshot +// URI it is asked to write: ate-api-server copies an actor's snapshot when it +// creates a tag, golden tags included, and refuses to copy an empty one. +// +// The one call back it keeps is the actor certificate mint: before ateom +// starts a workload with tunneled egress it has atelet mint the actor's +// certificate from ate-api-server, so in production every Run and Restore +// that names an egress gateway costs ate-api-server a mint. type herder struct { ateletpb.UnimplementedAteomHerderServer - delays delays + delays delays + storage objectWriter + // minter mints actor certificates under this pod's atelet identity; nil + // leaves the mint out. csr is sent with every mint. + minter actorCertMinter + csr []byte +} + +// newActorCSR returns a CSR for a fresh key. ate-api-server takes only the +// public key from it, so one serves every mint; ateom makes a key per +// activation, but that cost is the data plane's, not ate-api-server's. +func newActorCSR() ([]byte, error) { + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return nil, fmt.Errorf("while generating the actor key: %w", err) + } + csr, err := x509.CreateCertificateRequest(rand.Reader, &x509.CertificateRequest{}, key) + if err != nil { + return nil, fmt.Errorf("while creating the actor CSR: %w", err) + } + return csr, nil +} + +// mintActorCertificate mints the actor's certificate when the call names an +// egress gateway, as ateom does before it starts the workload. Its time is +// ate-api-server's, so it adds to the call's delay rather than running inside +// it; a failure fails the call, as tunneled egress fails closed in ateom. +func (h *herder) mintActorCertificate(ctx context.Context, atespace, name, uid string, gateway *ateletpb.EgressGateway) error { + if h.minter == nil || gateway == nil { + return nil + } + if _, err := h.minter.MintAteomActorCertificate(ctx, &ateapipb.MintAteomActorCertificateRequest{ + Actor: &ateapipb.ObjectRef{Atespace: atespace, Name: name}, + ActorUid: uid, + CertificateSigningRequest: h.csr, + }); err != nil { + return fmt.Errorf("while minting the certificate of actor %s/%s: %w", atespace, name, err) + } + return nil } // wait sleeps until start+d, or returns the context's error if the caller @@ -52,29 +125,68 @@ func wait(ctx context.Context, start time.Time, d time.Duration) error { } } -func (h *herder) Run(ctx context.Context, _ *ateletpb.RunRequest) (*ateletpb.RunResponse, error) { +// writePlaceholder writes the placeholder object under snapshotURI. It runs +// inside the call's delay; a write slower than a nonzero delay is logged, since +// the call then takes longer than configured. With no delay, every write +// outlasts it, so there is nothing to report. +func (h *herder) writePlaceholder(ctx context.Context, start time.Time, d time.Duration, snapshotURI string) error { + uri, err := resources.ParseSnapshotURI(snapshotURI) + if err != nil { + return fmt.Errorf("while parsing snapshot URI %q: %w", snapshotURI, err) + } + bucket, prefix, err := objectstore.BucketPrefix(uri.Prefix()) + if err != nil { + return err + } + if err := h.storage.PutObject(ctx, bucket, prefix+placeholderObject, strings.NewReader("")); err != nil { + return fmt.Errorf("while writing the placeholder under %s: %w", snapshotURI, err) + } + if elapsed := time.Since(start); d > 0 && elapsed > d { + slog.WarnContext(ctx, "Placeholder write outlasted the configured delay", + slog.String("snapshot_uri", snapshotURI), slog.Duration("elapsed", elapsed), slog.Duration("delay", d)) + } + return nil +} + +func (h *herder) Run(ctx context.Context, req *ateletpb.RunRequest) (*ateletpb.RunResponse, error) { + if err := h.mintActorCertificate(ctx, req.GetAtespace(), req.GetActorName(), req.GetActorUid(), req.GetEgressGateway()); err != nil { + return nil, err + } if err := wait(ctx, time.Now(), h.delays.run); err != nil { return nil, err } return &ateletpb.RunResponse{}, nil } -func (h *herder) Restore(ctx context.Context, _ *ateletpb.RestoreRequest) (*ateletpb.RestoreResponse, error) { +func (h *herder) Restore(ctx context.Context, req *ateletpb.RestoreRequest) (*ateletpb.RestoreResponse, error) { + if err := h.mintActorCertificate(ctx, req.GetAtespace(), req.GetActorName(), req.GetActorUid(), req.GetEgressGateway()); err != nil { + return nil, err + } if err := wait(ctx, time.Now(), h.delays.restore); err != nil { return nil, err } return &ateletpb.RestoreResponse{}, nil } -func (h *herder) Checkpoint(ctx context.Context, _ *ateletpb.CheckpointRequest) (*ateletpb.CheckpointResponse, error) { - if err := wait(ctx, time.Now(), h.delays.checkpoint); err != nil { +func (h *herder) Checkpoint(ctx context.Context, req *ateletpb.CheckpointRequest) (*ateletpb.CheckpointResponse, error) { + start := time.Now() + if req.GetType() == ateletpb.CheckpointType_CHECKPOINT_TYPE_EXTERNAL { + if err := h.writePlaceholder(ctx, start, h.delays.checkpoint, req.GetExternalConfig().GetSnapshotUri()); err != nil { + return nil, err + } + } + if err := wait(ctx, start, h.delays.checkpoint); err != nil { return nil, err } return &ateletpb.CheckpointResponse{}, nil } -func (h *herder) UploadPausedCheckpoint(ctx context.Context, _ *ateletpb.UploadPausedCheckpointRequest) (*ateletpb.UploadPausedCheckpointResponse, error) { - if err := wait(ctx, time.Now(), h.delays.uploadPausedCheckpoint); err != nil { +func (h *herder) UploadPausedCheckpoint(ctx context.Context, req *ateletpb.UploadPausedCheckpointRequest) (*ateletpb.UploadPausedCheckpointResponse, error) { + start := time.Now() + if err := h.writePlaceholder(ctx, start, h.delays.uploadPausedCheckpoint, req.GetDestinationSnapshotUri()); err != nil { + return nil, err + } + if err := wait(ctx, start, h.delays.uploadPausedCheckpoint); err != nil { return nil, err } return &ateletpb.UploadPausedCheckpointResponse{}, nil diff --git a/cmd/benchmarking/isolate/fake-atelet/herder_test.go b/cmd/benchmarking/isolate/fake-atelet/herder_test.go index ebbe3a7881..4c90f158f3 100644 --- a/cmd/benchmarking/isolate/fake-atelet/herder_test.go +++ b/cmd/benchmarking/isolate/fake-atelet/herder_test.go @@ -15,19 +15,62 @@ package main import ( + "bytes" "context" + "crypto/x509" "errors" + "io" + "log/slog" + "slices" + "strings" + "sync" "testing" "time" "github.com/agent-substrate/substrate/internal/proto/ateletpb" + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "google.golang.org/grpc" + "google.golang.org/protobuf/proto" ) +const testSnapshotURI = "gs://bench-bucket/benchmark-workloads/sleep/atespaces/team-a/actors/0f9c1d2e-3b4a-4c5d-8e6f-7a8b9c0d1e2f/snapshots/snap-1" + +// recordingStorage records every object written. +type recordingStorage struct { + mu sync.Mutex + written []string // bucket + "/" + object + err error + delay time.Duration +} + +func (s *recordingStorage) PutObject(_ context.Context, bucket, object string, r io.Reader) error { + if s.delay > 0 { + time.Sleep(s.delay) + } + if _, err := io.ReadAll(r); err != nil { + return err + } + s.mu.Lock() + defer s.mu.Unlock() + if s.err != nil { + return s.err + } + s.written = append(s.written, bucket+"/"+object) + return nil +} + +func (s *recordingStorage) objects() []string { + s.mu.Lock() + defer s.mu.Unlock() + return slices.Clone(s.written) +} + func uniformDelays(d time.Duration) delays { return delays{run: d, restore: d, checkpoint: d, uploadPausedCheckpoint: d, terminate: d} } -// herderCalls invokes every AteomHerder method the herder implements. +// herderCalls invokes every AteomHerder method the herder implements. The +// checkpoint calls ask for an external snapshot, so each writes a placeholder. func herderCalls(h *herder) map[string]func(context.Context) error { return map[string]func(context.Context) error{ "Run": func(ctx context.Context) error { @@ -39,11 +82,11 @@ func herderCalls(h *herder) map[string]func(context.Context) error { return err }, "Checkpoint": func(ctx context.Context) error { - _, err := h.Checkpoint(ctx, &ateletpb.CheckpointRequest{}) + _, err := h.Checkpoint(ctx, externalCheckpoint(testSnapshotURI)) return err }, "UploadPausedCheckpoint": func(ctx context.Context) error { - _, err := h.UploadPausedCheckpoint(ctx, &ateletpb.UploadPausedCheckpointRequest{}) + _, err := h.UploadPausedCheckpoint(ctx, &ateletpb.UploadPausedCheckpointRequest{DestinationSnapshotUri: testSnapshotURI}) return err }, "Terminate": func(ctx context.Context) error { @@ -53,9 +96,16 @@ func herderCalls(h *herder) map[string]func(context.Context) error { } } +func externalCheckpoint(uri string) *ateletpb.CheckpointRequest { + return &ateletpb.CheckpointRequest{ + Type: ateletpb.CheckpointType_CHECKPOINT_TYPE_EXTERNAL, + Config: &ateletpb.CheckpointRequest_ExternalConfig{ExternalConfig: &ateletpb.ExternalCheckpointConfiguration{SnapshotUri: uri}}, + } +} + func TestHerderSucceedsAfterDelay(t *testing.T) { const d = 20 * time.Millisecond - for name, call := range herderCalls(&herder{delays: uniformDelays(d)}) { + for name, call := range herderCalls(&herder{delays: uniformDelays(d), storage: &recordingStorage{}}) { t.Run(name, func(t *testing.T) { start := time.Now() if err := call(context.Background()); err != nil { @@ -69,7 +119,7 @@ func TestHerderSucceedsAfterDelay(t *testing.T) { } func TestHerderZeroDelay(t *testing.T) { - for name, call := range herderCalls(&herder{}) { + for name, call := range herderCalls(&herder{storage: &recordingStorage{}}) { if err := call(context.Background()); err != nil { t.Errorf("%s: %v", name, err) } @@ -77,7 +127,7 @@ func TestHerderZeroDelay(t *testing.T) { } func TestHerderStopsWhenCallerGivesUp(t *testing.T) { - for name, call := range herderCalls(&herder{delays: uniformDelays(time.Hour)}) { + for name, call := range herderCalls(&herder{delays: uniformDelays(time.Hour), storage: &recordingStorage{}}) { t.Run(name, func(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) defer cancel() @@ -89,7 +139,7 @@ func TestHerderStopsWhenCallerGivesUp(t *testing.T) { } func TestHerderUsesEachCallsOwnDelay(t *testing.T) { - h := &herder{delays: delays{run: time.Hour}} + h := &herder{delays: delays{run: time.Hour}, storage: &recordingStorage{}} if _, err := h.Restore(context.Background(), &ateletpb.RestoreRequest{}); err != nil { t.Fatalf("Restore with no delay of its own: %v", err) } @@ -99,3 +149,219 @@ func TestHerderUsesEachCallsOwnDelay(t *testing.T) { t.Errorf("Run with an hour's delay: got %v, want %v", err, context.DeadlineExceeded) } } + +// ate-api-server copies an actor's snapshot when it creates a tag and refuses +// to copy an empty one, so both calls that would upload a snapshot must leave +// an object under its URI. +func TestHerderWritesPlaceholderUnderSnapshotURI(t *testing.T) { + const want = "bench-bucket/benchmark-workloads/sleep/atespaces/team-a/actors/0f9c1d2e-3b4a-4c5d-8e6f-7a8b9c0d1e2f/snapshots/snap-1/" + placeholderObject + for _, name := range []string{"Checkpoint", "UploadPausedCheckpoint"} { + t.Run(name, func(t *testing.T) { + storage := &recordingStorage{} + if err := herderCalls(&herder{storage: storage})[name](context.Background()); err != nil { + t.Fatalf("%s: %v", name, err) + } + if got := storage.objects(); len(got) != 1 || got[0] != want { + t.Errorf("%s wrote %v, want [%s]", name, got, want) + } + }) + } +} + +func TestHerderWritesNothingForLocalCheckpoint(t *testing.T) { + storage := &recordingStorage{} + h := &herder{storage: storage} + if _, err := h.Checkpoint(context.Background(), &ateletpb.CheckpointRequest{Type: ateletpb.CheckpointType_CHECKPOINT_TYPE_LOCAL}); err != nil { + t.Fatalf("Checkpoint: %v", err) + } + for name, call := range herderCalls(h) { + if name == "Checkpoint" || name == "UploadPausedCheckpoint" { + continue + } + if err := call(context.Background()); err != nil { + t.Fatalf("%s: %v", name, err) + } + } + if got := storage.objects(); len(got) != 0 { + t.Errorf("wrote %v, want nothing outside external checkpoints", got) + } +} + +func TestHerderFailsWhenPlaceholderCannotBeWritten(t *testing.T) { + h := &herder{storage: &recordingStorage{err: errors.New("bucket unavailable")}} + if _, err := h.Checkpoint(context.Background(), externalCheckpoint(testSnapshotURI)); err == nil { + t.Error("Checkpoint succeeded with no placeholder written; a later tag of this snapshot would fail") + } + if _, err := h.Checkpoint(context.Background(), externalCheckpoint("not a snapshot uri")); err == nil { + t.Error("Checkpoint succeeded with an unparseable snapshot URI") + } +} + +// The write runs inside the delay, so it does not add to it. +func TestHerderPlaceholderWriteRunsInsideDelay(t *testing.T) { + const d = 100 * time.Millisecond + h := &herder{delays: uniformDelays(d), storage: &recordingStorage{delay: 60 * time.Millisecond}} + start := time.Now() + if _, err := h.Checkpoint(context.Background(), externalCheckpoint(testSnapshotURI)); err != nil { + t.Fatalf("Checkpoint: %v", err) + } + if got := time.Since(start); got < d || got > d+50*time.Millisecond { + t.Errorf("Checkpoint took %v, want about %v", got, d) + } +} + +// A write slower than a nonzero delay is worth a warning; with no delay every +// write is slower, and warning on each would flood the log. +func TestHerderWarnsOnlyWhenWriteOutlastsANonzeroDelay(t *testing.T) { + for name, tc := range map[string]struct { + delay time.Duration + wantWarn bool + }{ + "no delay": {delay: 0, wantWarn: false}, + "delay outlasted": {delay: time.Millisecond, wantWarn: true}, + "delay not reached": {delay: time.Hour, wantWarn: false}, + } { + t.Run(name, func(t *testing.T) { + var logs bytes.Buffer + prev := slog.Default() + slog.SetDefault(slog.New(slog.NewTextHandler(&logs, nil))) + t.Cleanup(func() { slog.SetDefault(prev) }) + + h := &herder{storage: &recordingStorage{delay: 20 * time.Millisecond}} + if err := h.writePlaceholder(context.Background(), time.Now(), tc.delay, testSnapshotURI); err != nil { + t.Fatalf("writePlaceholder: %v", err) + } + if got := strings.Contains(logs.String(), "outlasted the configured delay"); got != tc.wantWarn { + t.Errorf("warned = %v, want %v; log:\n%s", got, tc.wantWarn, logs.String()) + } + }) + } +} + +// recordingMinter records every mint, as ate-api-server's WorkerService. +type recordingMinter struct { + mu sync.Mutex + reqs []*ateapipb.MintAteomActorCertificateRequest + err error + delay time.Duration +} + +func (m *recordingMinter) MintAteomActorCertificate(_ context.Context, in *ateapipb.MintAteomActorCertificateRequest, _ ...grpc.CallOption) (*ateapipb.MintAteomActorCertificateResponse, error) { + if m.delay > 0 { + time.Sleep(m.delay) + } + m.mu.Lock() + defer m.mu.Unlock() + m.reqs = append(m.reqs, proto.CloneOf(in)) + if m.err != nil { + return nil, m.err + } + return &ateapipb.MintAteomActorCertificateResponse{ActorCertificates: [][]byte{[]byte("leaf")}}, nil +} + +func (m *recordingMinter) mints() []*ateapipb.MintAteomActorCertificateRequest { + m.mu.Lock() + defer m.mu.Unlock() + return slices.Clone(m.reqs) +} + +const testActorUID = "0f9c1d2e-3b4a-4c5d-8e6f-7a8b9c0d1e2f" + +var testGateway = &ateletpb.EgressGateway{Address: "atenet-egress.ate-system.svc:443"} + +// activations invokes Run and Restore for one actor, naming gateway. +func activations(h *herder, gateway *ateletpb.EgressGateway) map[string]func(context.Context) error { + return map[string]func(context.Context) error{ + "Run": func(ctx context.Context) error { + _, err := h.Run(ctx, &ateletpb.RunRequest{Atespace: "team-a", ActorName: "sleep-1", ActorUid: testActorUID, EgressGateway: gateway}) + return err + }, + "Restore": func(ctx context.Context) error { + _, err := h.Restore(ctx, &ateletpb.RestoreRequest{Atespace: "team-a", ActorName: "sleep-1", ActorUid: testActorUID, EgressGateway: gateway}) + return err + }, + } +} + +func newMintingHerder(t *testing.T, m *recordingMinter, d delays) *herder { + t.Helper() + csr, err := newActorCSR() + if err != nil { + t.Fatal(err) + } + return &herder{delays: d, storage: &recordingStorage{}, minter: m, csr: csr} +} + +// ateom mints the actor's certificate before starting any workload with +// tunneled egress, so each Run and Restore that names a gateway costs +// ate-api-server one mint, for that actor, with a CSR it accepts. +func TestHerderMintsActorCertificateOnActivation(t *testing.T) { + for _, name := range []string{"Run", "Restore"} { + t.Run(name, func(t *testing.T) { + m := &recordingMinter{} + if err := activations(newMintingHerder(t, m, delays{}), testGateway)[name](context.Background()); err != nil { + t.Fatalf("%s: %v", name, err) + } + got := m.mints() + if len(got) != 1 { + t.Fatalf("%s minted %d times, want once", name, len(got)) + } + want := &ateapipb.ObjectRef{Atespace: "team-a", Name: "sleep-1"} + if !proto.Equal(got[0].GetActor(), want) || got[0].GetActorUid() != testActorUID { + t.Errorf("%s minted for %v/%q, want %v/%q", name, got[0].GetActor(), got[0].GetActorUid(), want, testActorUID) + } + // ate-api-server parses the CSR and checks its signature. + csr, err := x509.ParseCertificateRequest(got[0].GetCertificateSigningRequest()) + if err != nil { + t.Fatalf("CSR does not parse: %v", err) + } + if err := csr.CheckSignature(); err != nil { + t.Errorf("CSR signature: %v", err) + } + }) + } +} + +// ateom mints only for tunneled egress, which ate-api-server asks for by +// naming a gateway; and --mint-actor-certificate=false leaves the mint out. +func TestHerderSkipsMint(t *testing.T) { + m := &recordingMinter{} + for name, call := range activations(newMintingHerder(t, m, delays{}), nil) { + if err := call(context.Background()); err != nil { + t.Fatalf("%s without a gateway: %v", name, err) + } + } + if got := len(m.mints()); got != 0 { + t.Errorf("minted %d times with no egress gateway, want none", got) + } + for name, call := range activations(&herder{storage: &recordingStorage{}}, testGateway) { + if err := call(context.Background()); err != nil { + t.Errorf("%s with minting off: %v", name, err) + } + } +} + +// A failed mint fails the activation, as tunneled egress fails closed in ateom. +func TestHerderFailsWhenMintFails(t *testing.T) { + m := &recordingMinter{err: errors.New("actor not found")} + for name, call := range activations(newMintingHerder(t, m, delays{}), testGateway) { + if err := call(context.Background()); err == nil { + t.Errorf("%s succeeded with the mint refused", name) + } + } +} + +// The mint is ate-api-server's time, so it adds to the data plane's delay: a +// slower mint makes a slower resume, as in production. +func TestHerderMintAddsToDelay(t *testing.T) { + const d, mint = 100 * time.Millisecond, 60 * time.Millisecond + for name, call := range activations(newMintingHerder(t, &recordingMinter{delay: mint}, uniformDelays(d)), testGateway) { + start := time.Now() + if err := call(context.Background()); err != nil { + t.Fatalf("%s: %v", name, err) + } + if got := time.Since(start); got < d+mint { + t.Errorf("%s took %v, want at least the mint plus the delay, %v", name, got, d+mint) + } + } +} diff --git a/cmd/benchmarking/isolate/fake-atelet/main.go b/cmd/benchmarking/isolate/fake-atelet/main.go index f8f07db296..609629ff02 100644 --- a/cmd/benchmarking/isolate/fake-atelet/main.go +++ b/cmd/benchmarking/isolate/fake-atelet/main.go @@ -15,7 +15,9 @@ // Command fake-atelet stands in for atelet in control-plane benchmarks. It // runs as atelet's DaemonSet would, on the benchmark nodes only, so // ate-api-server dials it as the node's atelet. It answers every AteomHerder -// call with success after a delay, without running or saving any workload. +// call with success after a delay, without running or saving any workload, +// and relays fake-workersync's capacity reports for the fake Workers on its +// node, as atelet relays ateom's. // // Actors it "runs" do not exist. Never run it on a cluster that serves real // actors. @@ -29,16 +31,24 @@ import ( "log/slog" "net" "net/http" + "os" "os/signal" "strconv" "syscall" "time" + "github.com/agent-substrate/substrate/cmd/benchmarking/isolate/internal/fakeworker" + "github.com/agent-substrate/substrate/internal/ateapiauth" "github.com/agent-substrate/substrate/internal/atelet" "github.com/agent-substrate/substrate/internal/credbundle" + "github.com/agent-substrate/substrate/internal/installdefaults" "github.com/agent-substrate/substrate/internal/proto/ateletpb" "github.com/agent-substrate/substrate/internal/serverboot" "github.com/agent-substrate/substrate/internal/version" + "github.com/agent-substrate/substrate/pkg/objectstorage" + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/spf13/pflag" "google.golang.org/grpc" "google.golang.org/grpc/credentials" @@ -46,17 +56,23 @@ import ( var ( port = pflag.Int("port", atelet.DefaultPort, "Port to serve AteomHerder on. ate-api-server dials atelet.DefaultPort.") + relayPort = pflag.Int("relay-port", fakeworker.DefaultRelayPort, "Port to serve fake-workersync's capacity reports on.") + relayAllowedID = pflag.String("relay-allowed-spiffe-id", installdefaults.SPIFFEID("benchmark-workloads", "fake-workersync"), "The only SPIFFE ID allowed to send capacity reports.") healthListenAddr = pflag.String("health-listen-addr", ":9090", "Address to serve /healthz and /readyz on.") - grpcServerCredBundle = pflag.String("grpc-server-cred-bundle", "/run/podidentity.podcert.ate.dev/credential-bundle.pem", "Credential bundle presented as the serving certificate.") + grpcServerCredBundle = pflag.String("grpc-server-cred-bundle", "/run/podidentity.podcert.ate.dev/credential-bundle.pem", "Credential bundle presented as the serving certificate and the ate-api-server client certificate.") clientCACerts = pflag.String("client-ca-certs", "/run/podidentity.podcert.ate.dev/trust-bundle.pem", "CA bundle used to verify client certificates.") + ateapiAddress = pflag.String("ateapi-address", "dns:///api.ate-system.svc:443", "ate-api-server gRPC target for capacity reports.") + ateapiCAFile = pflag.String("ateapi-ca-file", "/run/servicedns.podcert.ate.dev/trust-bundle.pem", "CA bundle used to verify ate-api-server.") + ateapiServerName = pflag.String("ateapi-server-name", "api.ate-system.svc", "DNS name expected on the ate-api-server certificate.") - delay = pflag.Duration("delay", 0, "How long each AteomHerder call takes before it succeeds.") + delay = pflag.Duration("delay", 0, "How long each AteomHerder call takes before it succeeds, not counting the actor certificate mint on Run and Restore.") delayRun = pflag.Duration("delay-run", -1, "Override --delay for Run. Negative uses --delay.") delayRestore = pflag.Duration("delay-restore", -1, "Override --delay for Restore. Negative uses --delay.") delayCheckpoint = pflag.Duration("delay-checkpoint", -1, "Override --delay for Checkpoint. Negative uses --delay.") delayUploadPausedCheckpoint = pflag.Duration("delay-upload-paused-checkpoint", -1, "Override --delay for UploadPausedCheckpoint. Negative uses --delay.") delayTerminate = pflag.Duration("delay-terminate", -1, "Override --delay for Terminate. Negative uses --delay.") + mintActorCertificate = pflag.Bool("mint-actor-certificate", true, "On each Run and Restore that names an egress gateway, mint the actor's certificate from ate-api-server before the delay, as ateom does before starting the workload.") showVersion = pflag.Bool("version", false, "Print version and exit.") logLevelFlag = pflag.String("log-level", "info", "Minimum log level: debug, info, warn, or error.") @@ -79,6 +95,27 @@ func main() { serverboot.Fatal(ctx, "Invalid --delay", err) } + storage, err := newObjectStorage(ctx) + if err != nil { + serverboot.Fatal(ctx, "Failed to create the object storage client", err) + } + + // No Kubernetes client: fake-atelet never calls the Kubernetes API, so the + // ate-api-server target is resolved through DNS rather than EndpointSlices. + dialOpts, err := ateapiauth.DialOptions(ateapiauth.ClientConfig{ + CAFile: *ateapiCAFile, + ServerName: *ateapiServerName, + ClientCredBundle: *grpcServerCredBundle, + }) + if err != nil { + serverboot.Fatal(ctx, "Failed to build ate-api-server client credentials", err) + } + ateapiConn, err := grpc.NewClient(*ateapiAddress, dialOpts...) + if err != nil { + serverboot.Fatal(ctx, "Failed to create ate-api-server client", err) + } + defer ateapiConn.Close() + herderTLS, err := serverTLSConfig(*grpcServerCredBundle, *clientCACerts) if err != nil { serverboot.Fatal(ctx, "Failed to build server TLS config", err) @@ -87,19 +124,45 @@ func main() { if err != nil { serverboot.Fatal(ctx, "Failed to listen", err) } - h := &herder{delays: callDelays} + workers := ateapipb.NewWorkerServiceClient(ateapiConn) + h := &herder{delays: callDelays, storage: storage} + if *mintActorCertificate { + if h.csr, err = newActorCSR(); err != nil { + serverboot.Fatal(ctx, "Failed to create the actor CSR", err) + } + h.minter = workers + } srv := grpc.NewServer(grpc.Creds(credentials.NewTLS(herderTLS))) ateletpb.RegisterAteomHerderServer(srv, h) + + relayTLS, err := relayTLSConfig(*grpcServerCredBundle, *clientCACerts, *relayAllowedID) + if err != nil { + serverboot.Fatal(ctx, "Failed to build relay TLS config", err) + } + relayLis, err := net.Listen("tcp", ":"+strconv.Itoa(*relayPort)) + if err != nil { + serverboot.Fatal(ctx, "Failed to listen for the capacity relay", err) + } + // Its own server and listener, so each port admits only its own caller: + // ate-api-server on the herder's, fake-workersync on the relay's. + relaySrv := grpc.NewServer(grpc.Creds(credentials.NewTLS(relayTLS))) + ateapipb.RegisterWorkerServiceServer(relaySrv, &relay{client: workers}) health := &http.Server{Addr: *healthListenAddr, Handler: healthHandler(), ReadHeaderTimeout: 10 * time.Second} slog.WarnContext(ctx, "Serving the fake atelet: actor lifecycle calls succeed without running any workload", - slog.String("delays", fmt.Sprintf("%+v", callDelays))) + slog.String("delays", fmt.Sprintf("%+v", callDelays)), slog.Bool("mint_actor_certificate", *mintActorCertificate), + slog.String("relay_allowed_id", *relayAllowedID)) go func() { if err := srv.Serve(lis); err != nil { serverboot.Fatal(ctx, "Failed to serve AteomHerder", err) } }() + go func() { + if err := relaySrv.Serve(relayLis); err != nil { + serverboot.Fatal(ctx, "Failed to serve the capacity relay", err) + } + }() go func() { if err := health.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { serverboot.Fatal(ctx, "Failed to serve health endpoints", err) @@ -109,6 +172,7 @@ func main() { <-ctx.Done() slog.InfoContext(context.Background(), "Shutting down") srv.GracefulStop() + relaySrv.GracefulStop() _ = health.Shutdown(context.Background()) } @@ -148,3 +212,22 @@ func serverTLSConfig(servingBundlePath, clientCAPath string) (*tls.Config, error }, }, nil } + +// newObjectStorage picks the backend the way atelet does, from +// ATE_STORAGE_BACKEND, so placeholders land where atelet would write. +func newObjectStorage(ctx context.Context) (objectstorage.ObjectStorage, error) { + switch os.Getenv("ATE_STORAGE_BACKEND") { + case "s3": + cfg, err := config.LoadDefaultConfig(ctx) + if err != nil { + return nil, fmt.Errorf("loading S3 config: %w", err) + } + return objectstorage.NewS3Client(s3.NewFromConfig(cfg, func(o *s3.Options) { + if os.Getenv("AWS_S3_USE_PATH_STYLE") == "true" { + o.UsePathStyle = true + } + })), nil + default: + return objectstorage.NewGCSClient(ctx) + } +} diff --git a/cmd/benchmarking/isolate/fake-atelet/relay.go b/cmd/benchmarking/isolate/fake-atelet/relay.go new file mode 100644 index 0000000000..96ca89f0a7 --- /dev/null +++ b/cmd/benchmarking/isolate/fake-atelet/relay.go @@ -0,0 +1,81 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "crypto/tls" + "log/slog" + + "github.com/agent-substrate/substrate/cmd/benchmarking/isolate/internal/fakeworker" + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "google.golang.org/grpc" +) + +// capacityClient is the part of ateapipb.WorkerServiceClient the relay uses. +type capacityClient interface { + RegisterWorker(ctx context.Context, in *ateapipb.RegisterWorkerRequest, opts ...grpc.CallOption) (*ateapipb.RegisterWorkerResponse, error) +} + +// relay forwards fake-workersync's capacity reports to ate-api-server under +// this pod's atelet identity. It stands where atelet's AteomSupport stands in +// production: ateom knows its capacity and atelet relays it, because +// ate-api-server accepts a report only from the atelet on the Worker's node. +// The relay decides nothing; fake-workersync names the Worker and its +// capacity, and ate-api-server checks the Worker is on this node. +// +// It serves WorkerService so the request and ate-api-server's status pass +// through unchanged; every other WorkerService call is Unimplemented. +type relay struct { + ateapipb.UnimplementedWorkerServiceServer + + client capacityClient +} + +func (r *relay) RegisterWorker(ctx context.Context, in *ateapipb.RegisterWorkerRequest) (*ateapipb.RegisterWorkerResponse, error) { + resp, err := r.client.RegisterWorker(ctx, in) + if err != nil { + slog.WarnContext(ctx, "Capacity report rejected", + slog.String("worker", in.GetWorker().GetName()), slog.Any("err", err)) + return nil, err + } + slog.DebugContext(ctx, "Relayed capacity report", + slog.String("worker", in.GetWorker().GetName()), slog.Any("capacity", in.GetCapacity())) + return resp, nil +} + +// relayTLSConfig serves the pod-identity certificate and admits only a client +// whose certificate chains to the pod-identity trust bundle and carries the +// SPIFFE ID allowed: anything else could set any Worker's capacity. Like the +// herder's, it reads the trust bundle at each handshake, so a CA rotation is +// picked up without a restart. +func relayTLSConfig(servingBundlePath, clientCAPath, allowed string) (*tls.Config, error) { + cfg, err := serverTLSConfig(servingBundlePath, clientCAPath) + if err != nil { + return nil, err + } + perClient := cfg.GetConfigForClient + cfg.GetConfigForClient = func(hello *tls.ClientHelloInfo) (*tls.Config, error) { + c, err := perClient(hello) + if err != nil { + return nil, err + } + c.VerifyConnection = func(cs tls.ConnectionState) error { + return fakeworker.VerifyPeerID(cs, allowed) + } + return c, nil + } + return cfg, nil +} diff --git a/cmd/benchmarking/isolate/fake-atelet/relay_test.go b/cmd/benchmarking/isolate/fake-atelet/relay_test.go new file mode 100644 index 0000000000..e6b3658dbf --- /dev/null +++ b/cmd/benchmarking/isolate/fake-atelet/relay_test.go @@ -0,0 +1,169 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "crypto/tls" + "net" + "strings" + "testing" + + "github.com/agent-substrate/substrate/cmd/benchmarking/isolate/internal/fakeworker/fakeworkertest" + "github.com/agent-substrate/substrate/internal/credbundle" + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" +) + +// recordingWorkerService records the last report and answers with err. +type recordingWorkerService struct { + got *ateapipb.RegisterWorkerRequest + err error +} + +func (f *recordingWorkerService) RegisterWorker(_ context.Context, in *ateapipb.RegisterWorkerRequest, _ ...grpc.CallOption) (*ateapipb.RegisterWorkerResponse, error) { + f.got = in + if f.err != nil { + return nil, f.err + } + return &ateapipb.RegisterWorkerResponse{}, nil +} + +func report() *ateapipb.RegisterWorkerRequest { + return &ateapipb.RegisterWorkerRequest{ + Worker: &ateapipb.ObjectRef{Name: "fake-r1-c52b9bd3-0"}, + Capacity: &ateapipb.WorkerResources{ + Actors: 1000, + Resources: &ateapipb.Resources{Limits: []*ateapipb.Limits{ + {Name: "cpu", Quantity: "64"}, + {Name: "memory", Quantity: "256Gi"}, + }}, + }, + } +} + +func TestRelayForwardsTheReportUnchanged(t *testing.T) { + svc := &recordingWorkerService{} + if _, err := (&relay{client: svc}).RegisterWorker(context.Background(), report()); err != nil { + t.Fatalf("RegisterWorker: %v", err) + } + if !proto.Equal(svc.got, report()) { + t.Errorf("forwarded %v, want %v", svc.got, report()) + } +} + +// fake-workersync retries a Worker ate-api-server has not seen yet and gives up +// on a rejected report, so the relay must keep ate-api-server's status. +func TestRelayPassesThroughAteAPIServersStatus(t *testing.T) { + for _, code := range []codes.Code{codes.NotFound, codes.InvalidArgument, codes.PermissionDenied, codes.Unavailable, codes.Internal} { + svc := &recordingWorkerService{err: status.Error(code, "from ate-api-server")} + _, err := (&relay{client: svc}).RegisterWorker(context.Background(), report()) + if got := status.Code(err); got != code { + t.Errorf("%v: code = %v", code, got) + } + if !strings.Contains(status.Convert(err).Message(), "from ate-api-server") { + t.Errorf("%v: %v does not carry ate-api-server's message", code, err) + } + } +} + +// The relay calls ate-api-server under atelet's identity, so it must forward +// nothing but capacity reports: a minted actor certificate, say, would hand +// fake-workersync an identity it does not hold. +func TestRelayServesOnlyCapacityReports(t *testing.T) { + _, err := (&relay{client: &recordingWorkerService{}}).MintAteomActorCertificate(context.Background(), &ateapipb.MintAteomActorCertificateRequest{}) + if got := status.Code(err); got != codes.Unimplemented { + t.Errorf("MintAteomActorCertificate code = %v, want Unimplemented", got) + } +} + +// The relay admits fake-workersync and no one else: any other workload with a +// pod certificate could otherwise set any Worker's capacity. +func TestRelayTLSAdmitsOnlyTheAllowedID(t *testing.T) { + const allowed = "spiffe://cluster.local/ns/benchmark-workloads/sa/fake-workersync" + ca := fakeworkertest.NewCA(t) + serverCfg, err := relayTLSConfig(ca.Issue(t, "spiffe://cluster.local/ns/ate-system/sa/atelet"), ca.TrustBundle, allowed) + if err != nil { + t.Fatalf("relayTLSConfig: %v", err) + } + lis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + srv := grpc.NewServer(grpc.Creds(credentials.NewTLS(serverCfg))) + ateapipb.RegisterWorkerServiceServer(srv, &relay{client: &recordingWorkerService{}}) + go func() { _ = srv.Serve(lis) }() + defer srv.Stop() + + call := func(clientBundle string) error { + conn, err := grpc.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{ + GetClientCertificate: credbundle.ClientLoader(clientBundle), + InsecureSkipVerify: true, // the server's identity is not under test here + }))) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + _, err = ateapipb.NewWorkerServiceClient(conn).RegisterWorker(context.Background(), report()) + return err + } + if err := call(ca.Issue(t, allowed)); err != nil { + t.Errorf("fake-workersync refused: %v", err) + } + if err := call(ca.Issue(t, "spiffe://cluster.local/ns/benchmarking/sa/default")); err == nil { + t.Error("another workload's certificate was admitted") + } + other := fakeworkertest.NewCA(t) + if err := call(other.Issue(t, allowed)); err == nil { + t.Error("a certificate from an untrusted signer was admitted") + } +} + +// A pod-identity CA rotation during a long run must not cut fake-workersync +// off: the relay reads the trust bundle at each handshake. +func TestRelayTLSFollowsATrustBundleRotation(t *testing.T) { + const allowed = "spiffe://cluster.local/ns/benchmark-workloads/sa/fake-workersync" + ca := fakeworkertest.NewCA(t) + serverCfg, err := relayTLSConfig(ca.Issue(t, "spiffe://cluster.local/ns/ate-system/sa/atelet"), ca.TrustBundle, allowed) + if err != nil { + t.Fatalf("relayTLSConfig: %v", err) + } + lis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + srv := grpc.NewServer(grpc.Creds(credentials.NewTLS(serverCfg))) + ateapipb.RegisterWorkerServiceServer(srv, &relay{client: &recordingWorkerService{}}) + go func() { _ = srv.Serve(lis) }() + defer srv.Stop() + + next := fakeworkertest.NewCA(t) + fakeworkertest.Rotate(t, ca.TrustBundle, next) + conn, err := grpc.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{ + GetClientCertificate: credbundle.ClientLoader(next.Issue(t, allowed)), + InsecureSkipVerify: true, // the server's identity is not under test here + }))) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + if _, err := ateapipb.NewWorkerServiceClient(conn).RegisterWorker(context.Background(), report()); err != nil { + t.Errorf("a certificate from the rotated-in CA was refused: %v", err) + } +} diff --git a/cmd/benchmarking/isolate/fake-workersync/controller.go b/cmd/benchmarking/isolate/fake-workersync/controller.go index aa4b334dad..64c6a695d1 100644 --- a/cmd/benchmarking/isolate/fake-workersync/controller.go +++ b/cmd/benchmarking/isolate/fake-workersync/controller.go @@ -20,17 +20,25 @@ import ( "fmt" "log/slog" "maps" + "math" "slices" + "strconv" "strings" "sync" + "time" "github.com/agent-substrate/substrate/cmd/benchmarking/isolate/internal/fakeworker" + "github.com/agent-substrate/substrate/internal/hardware" + "github.com/agent-substrate/substrate/internal/resources" atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "github.com/google/uuid" "golang.org/x/sync/errgroup" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/labels" ) @@ -38,12 +46,24 @@ import ( type workerClient interface { CreateWorker(ctx context.Context, in *ateapipb.CreateWorkerRequest, opts ...grpc.CallOption) (*ateapipb.Worker, error) DeleteWorker(ctx context.Context, in *ateapipb.DeleteWorkerRequest, opts ...grpc.CallOption) (*ateapipb.Worker, error) + DrainWorker(ctx context.Context, in *ateapipb.DrainWorkerRequest, opts ...grpc.CallOption) (*ateapipb.Worker, error) + GetWorker(ctx context.Context, in *ateapipb.GetWorkerRequest, opts ...grpc.CallOption) (*ateapipb.Worker, error) + UpdateWorker(ctx context.Context, in *ateapipb.UpdateWorkerRequest, opts ...grpc.CallOption) (*ateapipb.Worker, error) ListWorkers(ctx context.Context, in *ateapipb.ListWorkersRequest, opts ...grpc.CallOption) (*ateapipb.ListWorkersResponse, error) + ListWorkerActorAssignments(ctx context.Context, in *ateapipb.ListWorkerActorAssignmentsRequest, opts ...grpc.CallOption) (*ateapipb.ListWorkerActorAssignmentsResponse, error) } -// node is a benchmark node. +// capacityRelay sends a capacity report to the fake-atelet relay at addr. +type capacityRelay interface { + Report(ctx context.Context, addr string, req *ateapipb.RegisterWorkerRequest) error + // Prune lets go of every relay not in live, the relay addresses by node. + Prune(live map[string]string) +} + +// node is a benchmark node and what it can allocate. type node struct { - name string + name string + allocatable corev1.ResourceList } // cluster is what the controller reads from and writes to Kubernetes. @@ -52,12 +72,22 @@ type cluster interface { Pools() ([]*atev1alpha1.WorkerPool, error) // Nodes returns the benchmark nodes, sorted by name. Nodes(ctx context.Context) ([]node, error) + // Relays returns the fake-atelet relay address on each node that has one. + Relays(ctx context.Context) (map[string]string, error) // UpdateStatus writes a WorkerPool's status. UpdateStatus(ctx context.Context, wp *atev1alpha1.WorkerPool) error } // fakeWorker is a fake Worker the controller has registered. type fakeWorker struct { + // name is the Worker's name and pod UID, fresh for each Worker, as a real + // Worker is named after its pod's UID. A replaced Worker's successor is a + // new Worker, so nothing ate-api-server keyed by the old one carries over. + name string + // pod is the stand-in pod name, fakeworker.Name, shared by every Worker + // that fills the same index of a pool. It is how adopt finds this run's + // Workers. + pod string namespace, pool string index int node string @@ -65,28 +95,49 @@ type fakeWorker struct { // from before its create, so a create the server commits but the caller // sees fail is still retired, or deleted at shutdown. created bool + // labels and sandboxClass are what the Worker is registered with. + labels map[string]string + sandboxClass string + // tmpl is the pool template the Worker was made from; nil for a Worker + // adopted after a restart until reconcile matches it to its pool. + tmpl *template + // capacity is fixed when the Worker is made, as ateom reads its limits + // once at startup; reported is whether ate-api-server has accepted it. + capacity *ateapipb.WorkerResources + reported bool + draining bool + // drainedAt is when the controller drained the Worker, or adopted it + // draining. + drainedAt time.Time } // controller stands in for atecontroller's WorkerPool controller and worker // syncer together. For every WorkerPool it keeps spec.replicas fake Workers, -// none backed by a pod, and writes the pool's status from them. +// none backed by a pod, has each one's capacity reported through the +// fake-atelet on its node, and writes the pool's status from them. // // It keeps the registered Workers in memory and lists them from ate-api-server -// only at startup (adopt). A pass with nothing changed makes no -// ate-api-server calls, so a steady fleet adds no load to the system under -// test. +// only at startup (adopt). A pass with nothing changed and nothing draining +// makes no ate-api-server calls, so a steady fleet adds no load to the system +// under test. type controller struct { client workerClient + relay capacityRelay cluster cluster run string concurrency int + // drainGrace is how long a draining Worker waits for its Actors to leave + // before it is deleted with them still assigned. + drainGrace time.Duration + now func() time.Time + newUID func() string mu sync.Mutex workers map[string]*fakeWorker } -func newController(client workerClient, cl cluster, run string, concurrency int) *controller { - return &controller{client: client, cluster: cl, run: run, concurrency: concurrency, workers: map[string]*fakeWorker{}} +func newController(client workerClient, relay capacityRelay, cl cluster, run string, concurrency int, drainGrace time.Duration) *controller { + return &controller{client: client, relay: relay, cluster: cl, run: run, concurrency: concurrency, drainGrace: drainGrace, now: time.Now, newUID: uuid.NewString, workers: map[string]*fakeWorker{}} } // adopt loads the fake Workers of this run that are already registered, as @@ -100,20 +151,32 @@ func (c *controller) adopt(ctx context.Context) error { return fmt.Errorf("while listing Workers: %w", err) } for _, w := range page.GetWorkers() { - name := w.GetMetadata().GetName() - if !strings.HasPrefix(name, prefix) { + pod := w.GetWorkerPod() + if !strings.HasPrefix(pod, prefix) { continue } - index, ok := fakeworker.Index(c.run, w.GetWorkerNamespace(), w.GetWorkerPool(), name) + index, ok := fakeworker.Index(c.run, w.GetWorkerNamespace(), w.GetWorkerPool(), pod) if !ok { continue } + name := w.GetMetadata().GetName() c.workers[name] = &fakeWorker{ - namespace: w.GetWorkerNamespace(), - pool: w.GetWorkerPool(), - index: index, - node: w.GetNodeName(), - created: true, + name: name, + pod: pod, + namespace: w.GetWorkerNamespace(), + pool: w.GetWorkerPool(), + index: index, + node: w.GetNodeName(), + labels: w.GetLabels(), + sandboxClass: w.GetSandboxClass(), + capacity: w.GetStatus().GetCapacity(), + reported: w.GetStatus().GetCapacity() != nil, + created: true, + draining: w.GetStatus().GetState() == ateapipb.WorkerState_WORKER_STATE_DRAINING, + } + if c.workers[name].draining { + // When it was drained is not recorded, so its grace restarts. + c.workers[name].drainedAt = c.now() } } if token = page.GetNextPageToken(); token == "" { @@ -142,14 +205,30 @@ func (c *controller) reconcile(ctx context.Context) error { if err != nil { return fmt.Errorf("while listing nodes: %w", err) } + relays, err := c.cluster.Relays(ctx) + if err != nil { + return fmt.Errorf("while listing fake-atelet pods: %w", err) + } + c.relay.Prune(relays) nodeNames := make([]string, len(nodes)) + allocatable := map[string]corev1.ResourceList{} for i, n := range nodes { nodeNames[i] = n.name + allocatable[n.name] = n.allocatable } desired := map[string]desiredWorker{} + // held names the pools whose spec cannot be read. Their Workers are left + // as they are rather than retired, so an annotation typo reports an error + // instead of tearing the pool down. + held := map[string]bool{} var errs []error for _, wp := range pools { + if _, err := maxActors(wp); err != nil { + errs = append(errs, err) + held[wp.Namespace+"/"+wp.Name] = true + continue + } for i := range int(wp.Spec.Replicas) { desired[fakeworker.Name(c.run, wp.Namespace, wp.Name, i)] = desiredWorker{pool: wp, index: i} } @@ -162,12 +241,35 @@ func (c *controller) reconcile(ctx context.Context) error { errs = append(errs, err) } + // Each desired pod is served by one current Worker; every other Worker is + // retired. A draining Worker is retired until it is gone, even if its pool + // wants its pod again: it takes no new actors and cannot be undrained. + // A real pool replaces every worker pod when its sandbox class, limits or + // actor count change, and each new pod reports its capacity once, so its + // fake Workers are replaced (see fits). So is a Worker whose node has left + // the benchmark set, as a pod is when its node goes away; with no + // benchmark node listed, Workers are left where they are, since nothing + // could replace them. A replaced pod gets a new Worker + // in the same pass, while the old one drains, as a Deployment creates a + // replacement pod without waiting for the old one to terminate. + onBenchmarkNode := func(w *fakeWorker) bool { + _, ok := allocatable[w.node] + return ok || len(nodes) == 0 + } c.mu.Lock() + current := map[string]*fakeWorker{} var unwanted []string - for name := range c.workers { - if _, ok := desired[name]; !ok { - unwanted = append(unwanted, name) + for _, name := range slices.Sorted(maps.Keys(c.workers)) { + w := c.workers[name] + if held[w.namespace+"/"+w.pool] && !w.draining { + continue + } + d, ok := desired[w.pod] + if ok && !w.draining && onBenchmarkNode(w) && fits(w, d.pool, allocatable) && current[w.pod] == nil { + current[w.pod] = w + continue } + unwanted = append(unwanted, name) } c.mu.Unlock() g, gctx := errgroup.WithContext(ctx) @@ -184,10 +286,10 @@ func (c *controller) reconcile(ctx context.Context) error { g, gctx = errgroup.WithContext(ctx) g.SetLimit(c.concurrency) - for _, name := range slices.Sorted(maps.Keys(desired)) { - d := desired[name] + for _, pod := range slices.Sorted(maps.Keys(desired)) { + d, w := desired[pod], current[pod] g.Go(func() error { - if err := c.ensure(gctx, name, d, nodeNames); err != nil { + if err := c.ensure(gctx, pod, d, w, nodeNames, allocatable, relays); err != nil { fail(err) } return nil @@ -196,44 +298,92 @@ func (c *controller) reconcile(ctx context.Context) error { _ = g.Wait() for _, wp := range pools { - if err := c.syncStatus(ctx, wp); err != nil { + if err := c.syncStatus(ctx, wp, onBenchmarkNode, allocatable); err != nil { errs = append(errs, err) } } return errors.Join(errs...) } -// ensure registers the desired Worker if it is not yet. The Worker is -// tracked before CreateWorker is called and marked created after, so a failed -// create is retried by the next pass, and one the server committed anyway is -// never lost: retire and deleteAll take a NotFound as done. -func (c *controller) ensure(ctx context.Context, name string, d desiredWorker, nodes []string) error { - c.mu.Lock() - w := c.workers[name] - created := w != nil && w.created - c.mu.Unlock() - if created { - return nil - } +// ensure registers a Worker for the desired pod if w, its current Worker, is +// nil, brings the Worker's labels in line with the pool's, then has its +// capacity reported until ate-api-server accepts it. The capacity is fixed +// when the Worker is made, as ateom reads its limits once at startup, and is +// reported once. The Worker is tracked before CreateWorker is called and +// marked created after, so a failed create is retried by the next pass under +// the same name, and one the server committed anyway is never lost: retire +// and deleteAll take a NotFound as done. +func (c *controller) ensure(ctx context.Context, pod string, d desiredWorker, w *fakeWorker, nodes []string, allocatable map[string]corev1.ResourceList, relays map[string]string) error { if w == nil { if len(nodes) == 0 { return errors.New("no benchmark node to place fake Workers on") } w = &fakeWorker{ - namespace: d.pool.Namespace, - pool: d.pool.Name, - index: d.index, - node: fakeworker.Node(d.index, nodes), + name: c.newUID(), + pod: pod, + namespace: d.pool.Namespace, + pool: d.pool.Name, + index: d.index, + node: fakeworker.Node(d.index, nodes), + labels: maps.Clone(d.pool.GetLabels()), + sandboxClass: string(d.pool.Spec.SandboxClass), } c.mu.Lock() - c.workers[name] = w + c.workers[w.name] = w c.mu.Unlock() } - if err := c.create(ctx, name, w, d.pool); err != nil { - return err + name := w.name + c.mu.Lock() + created, tmpl, fixed, reported := w.created, w.tmpl, w.capacity, w.reported + c.mu.Unlock() + // A new Worker takes its template and capacity from the pool now. An + // adopted Worker fits its pool, or it would not be current, so it takes + // the pool's template and keeps the capacity it reported, if it did. + if tmpl == nil || fixed == nil { + t, err := templateOf(d.pool) + if err != nil { + return err + } + if fixed == nil { + alloc, ok := allocatable[w.node] + if !ok { + return fmt.Errorf("node %s of Worker %s is not a benchmark node", w.node, name) + } + fixed = t.capacity(alloc) + } + c.mu.Lock() + w.tmpl, w.capacity = &t, fixed + c.mu.Unlock() + } + if !created { + if err := c.create(ctx, w, d.pool); err != nil { + return err + } + c.mu.Lock() + w.created = true + c.mu.Unlock() + } + if !maps.Equal(w.labels, d.pool.GetLabels()) { + if err := c.updateLabels(ctx, name, w, d.pool.GetLabels()); err != nil { + return err + } + } + if reported { + return nil + } + addr := relays[w.node] + if addr == "" { + return fmt.Errorf("no fake-atelet on node %s yet for Worker %s", w.node, name) + } + if err := c.relay.Report(ctx, addr, &ateapipb.RegisterWorkerRequest{ + Worker: &ateapipb.ObjectRef{Name: name}, + Capacity: fixed, + Hardware: hardware.ProbeHost(), + }); err != nil { + return fmt.Errorf("while reporting capacity for Worker %s: %w", name, err) } c.mu.Lock() - w.created = true + w.reported = true c.mu.Unlock() return nil } @@ -241,13 +391,14 @@ func (c *controller) ensure(ctx context.Context, name string, d desiredWorker, n // create registers the Worker with the fields the worker syncer would copy // from the pool and its pod. Epoch stays 0: a rising epoch makes // ate-api-server crash every Actor on the Worker. -func (c *controller) create(ctx context.Context, name string, w *fakeWorker, wp *atev1alpha1.WorkerPool) error { +func (c *controller) create(ctx context.Context, w *fakeWorker, wp *atev1alpha1.WorkerPool) error { + name := w.name _, err := c.client.CreateWorker(ctx, &ateapipb.CreateWorkerRequest{Worker: &ateapipb.Worker{ Metadata: &ateapipb.ResourceMetadata{Name: name}, WorkerNamespace: wp.Namespace, WorkerPool: wp.Name, - WorkerPod: name, - WorkerPodUid: fakeworker.PodUID(name), + WorkerPod: w.pod, + WorkerPodUid: name, NodeName: w.node, Ips: []string{fakeworker.IP(w.index)}, SandboxClass: string(wp.Spec.SandboxClass), @@ -259,9 +410,65 @@ func (c *controller) create(ctx context.Context, name string, w *fakeWorker, wp return nil } -// retire removes a Worker no pool wants. +// updateLabels writes the pool's labels onto a registered Worker, as the +// worker syncer does when a pool's labels change. UpdateWorker replaces the +// whole resource and takes the version it was read at as its precondition, so +// the Worker is read first; a conflicting write fails the call and the next +// pass retries it. +func (c *controller) updateLabels(ctx context.Context, name string, w *fakeWorker, want map[string]string) error { + got, err := c.client.GetWorker(ctx, &ateapipb.GetWorkerRequest{Worker: &ateapipb.ObjectRef{Name: name}}) + if err != nil { + return fmt.Errorf("while reading Worker %s: %w", name, err) + } + if !maps.Equal(got.GetLabels(), want) { + got.Labels = maps.Clone(want) + if _, err := c.client.UpdateWorker(ctx, &ateapipb.UpdateWorkerRequest{Worker: got}); err != nil { + return fmt.Errorf("while updating Worker %s labels: %w", name, err) + } + } + c.mu.Lock() + w.labels = maps.Clone(want) + c.mu.Unlock() + return nil +} + +// retire removes a Worker no pool wants, the way a real one goes when its pod +// does: drained first so nothing new lands on it, deleted once no Actor is +// assigned to it. In production ateom stops the pod's Actors within its +// termination grace period, and the Worker is deleted when the pod is gone, +// which releases any Actor still bound. No ateom stops them here, so after +// drainGrace the Worker is deleted with its Actors still assigned, and +// DeleteWorker releases them the same way. func (c *controller) retire(ctx context.Context, name string) error { - if _, err := c.client.DeleteWorker(ctx, &ateapipb.DeleteWorkerRequest{Worker: &ateapipb.ObjectRef{Name: name}}); err != nil && status.Code(err) != codes.NotFound { + c.mu.Lock() + w := c.workers[name] + c.mu.Unlock() + ref := &ateapipb.ObjectRef{Name: name} + if !w.draining { + if _, err := c.client.DrainWorker(ctx, &ateapipb.DrainWorkerRequest{Worker: ref}); err != nil { + if status.Code(err) == codes.NotFound { + c.forget(name) + return nil + } + return fmt.Errorf("while draining Worker %s: %w", name, err) + } + c.mu.Lock() + w.draining = true + w.drainedAt = c.now() + c.mu.Unlock() + } + assigned, err := c.client.ListWorkerActorAssignments(ctx, &ateapipb.ListWorkerActorAssignmentsRequest{Worker: ref, PageSize: 1}) + if err != nil { + if status.Code(err) == codes.NotFound { + c.forget(name) + return nil + } + return fmt.Errorf("while listing Actors on Worker %s: %w", name, err) + } + if len(assigned.GetActorAssignments()) > 0 && c.now().Sub(w.drainedAt) < c.drainGrace { + return nil + } + if _, err := c.client.DeleteWorker(ctx, &ateapipb.DeleteWorkerRequest{Worker: ref}); err != nil && status.Code(err) != codes.NotFound { return fmt.Errorf("while deleting Worker %s: %w", name, err) } c.forget(name) @@ -275,19 +482,24 @@ func (c *controller) forget(name string) { } // syncStatus writes the pool's status as the WorkerPool controller does from -// its Deployment: a replica is a registered Worker, and the selector is the -// one worker pods would carry. -func (c *controller) syncStatus(ctx context.Context, wp *atev1alpha1.WorkerPool) error { +// its Deployment: a replica is a registered Worker, ready once its capacity +// is reported, and the selector is the one worker pods would carry. +func (c *controller) syncStatus(ctx context.Context, wp *atev1alpha1.WorkerPool, onBenchmarkNode func(*fakeWorker) bool, allocatable map[string]corev1.ResourceList) error { want := atev1alpha1.WorkerPoolStatus{ Selector: labels.SelectorFromSet(labels.Set{fakeworker.WorkerPoolLabel: wp.Name}).String(), } c.mu.Lock() for _, w := range c.workers { - if w.namespace != wp.Namespace || w.pool != wp.Name || !w.created { + // A Worker its pool no longer wants, made from an older template, or + // off the benchmark nodes is no replica even before its drain lands. + if w.namespace != wp.Namespace || w.pool != wp.Name || !w.created || w.draining || + w.index >= int(wp.Spec.Replicas) || !onBenchmarkNode(w) || !fits(w, wp, allocatable) { continue } want.Replicas++ - want.ReadyReplicas++ + if w.reported { + want.ReadyReplicas++ + } } c.mu.Unlock() if wp.Status == want { @@ -329,3 +541,82 @@ func (c *controller) deleteAll(ctx context.Context) error { slog.InfoContext(ctx, "Deleted fake Workers", slog.Int("workers", len(names))) return nil } + +// template is what a pool's spec fixes about each Worker it makes: the +// sandbox class, the cpu and memory limits, zero where unset, and the actor +// count. Changing any of them replaces a real pool's worker pods. +type template struct { + sandboxClass string + cpuMilli, memoryBytes int64 + actors int +} + +// templateOf reads wp's template; it fails only on a bad max-actors +// annotation. +func templateOf(wp *atev1alpha1.WorkerPool) (template, error) { + actors, err := maxActors(wp) + if err != nil { + return template{}, err + } + t := template{sandboxClass: string(wp.Spec.SandboxClass), actors: actors} + if wp.Spec.Template != nil && wp.Spec.Template.Resources != nil { + limits := wp.Spec.Template.Resources.Limits + t.cpuMilli = limits.Cpu().MilliValue() + t.memoryBytes = limits.Memory().Value() + } + return t, nil +} + +// capacity is what a worker made from t reports on a node with allocatable: +// cpu and memory from its limits, or, with no limit set, the node's +// allocatable, which is what the downward API projects into ateom for an +// unset limit. +func (t template) capacity(allocatable corev1.ResourceList) *ateapipb.WorkerResources { + cpu, memory := t.cpuMilli, t.memoryBytes + if cpu == 0 { + cpu = allocatable.Cpu().MilliValue() + } + if memory == 0 { + memory = allocatable.Memory().Value() + } + return &ateapipb.WorkerResources{Actors: int32(t.actors), Resources: resources.CPUMemory(cpu, memory)} +} + +// fits reports whether w was made from wp's current template, so a real pool +// would keep its pod. A Worker adopted after a restart has only its reported +// capacity to go by, so it fits when that is what wp would give it on its +// node now; one whose node's allocatable changed while fake-workersync was +// down is replaced too. A held pool's Workers fit, since they are left as +// they are. +func fits(w *fakeWorker, wp *atev1alpha1.WorkerPool, allocatable map[string]corev1.ResourceList) bool { + if w.sandboxClass != string(wp.Spec.SandboxClass) { + return false + } + t, err := templateOf(wp) + if err != nil { + return true + } + if w.tmpl != nil { + return *w.tmpl == t + } + alloc, ok := allocatable[w.node] + if w.capacity == nil || !ok { + return true + } + return proto.Equal(w.capacity, t.capacity(alloc)) +} + +// maxActors is the pool's actor capacity per Worker: ateom's default, unless +// the pool's annotation overrides it. +func maxActors(wp *atev1alpha1.WorkerPool) (int, error) { + v, ok := wp.Annotations[fakeworker.MaxActorsAnnotation] + if !ok { + return fakeworker.DefaultMaxActors, nil + } + // Bounded to 32 bits, the width of a capacity report's actor count. + n, err := strconv.ParseInt(v, 10, 32) + if err != nil || n < 1 { + return 0, fmt.Errorf("WorkerPool %s/%s: %s must be an integer from 1 to %d, got %q", wp.Namespace, wp.Name, fakeworker.MaxActorsAnnotation, math.MaxInt32, v) + } + return int(n), nil +} diff --git a/cmd/benchmarking/isolate/fake-workersync/controller_test.go b/cmd/benchmarking/isolate/fake-workersync/controller_test.go index b5e27ecacb..933ccbea9e 100644 --- a/cmd/benchmarking/isolate/fake-workersync/controller_test.go +++ b/cmd/benchmarking/isolate/fake-workersync/controller_test.go @@ -16,35 +16,48 @@ package main import ( "context" + "fmt" "maps" "slices" "sync" + "sync/atomic" "testing" + "time" "github.com/agent-substrate/substrate/cmd/benchmarking/isolate/internal/fakeworker" + "github.com/agent-substrate/substrate/internal/hardware" atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "google.golang.org/protobuf/proto" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -const testRun = "r1" +const ( + testRun = "r1" + testDrainGrace = time.Minute +) // fakeControl is an in-memory Worker registry. type fakeControl struct { - mu sync.Mutex - workers map[string]*ateapipb.Worker - creates int + mu sync.Mutex + workers map[string]*ateapipb.Worker + assignments map[string]int // actors assigned, by Worker name + creates int + updates int // lostReply, when set, is returned by CreateWorker after the Worker is // stored, as when the server commits a create whose reply never arrives. lostReply error + // drainErr, when set, fails every DrainWorker. + drainErr error } func newFakeControl() *fakeControl { - return &fakeControl{workers: map[string]*ateapipb.Worker{}} + return &fakeControl{workers: map[string]*ateapipb.Worker{}, assignments: map[string]int{}} } func (f *fakeControl) CreateWorker(_ context.Context, in *ateapipb.CreateWorkerRequest, _ ...grpc.CallOption) (*ateapipb.Worker, error) { @@ -73,6 +86,53 @@ func (f *fakeControl) DeleteWorker(_ context.Context, in *ateapipb.DeleteWorkerR return nil, status.Errorf(codes.NotFound, "Worker %s not found", name) } delete(f.workers, name) + // Like ate-api-server: deleting a Worker releases its Actors. + delete(f.assignments, name) + return w, nil +} + +func (f *fakeControl) DrainWorker(_ context.Context, in *ateapipb.DrainWorkerRequest, _ ...grpc.CallOption) (*ateapipb.Worker, error) { + f.mu.Lock() + defer f.mu.Unlock() + if f.drainErr != nil { + return nil, f.drainErr + } + w, ok := f.workers[in.GetWorker().GetName()] + if !ok { + return nil, status.Error(codes.NotFound, "no such Worker") + } + w.Status.State = ateapipb.WorkerState_WORKER_STATE_DRAINING + return w, nil +} + +func (f *fakeControl) GetWorker(_ context.Context, in *ateapipb.GetWorkerRequest, _ ...grpc.CallOption) (*ateapipb.Worker, error) { + f.mu.Lock() + defer f.mu.Unlock() + w, ok := f.workers[in.GetWorker().GetName()] + if !ok { + return nil, status.Error(codes.NotFound, "no such Worker") + } + return proto.CloneOf(w), nil +} + +// UpdateWorker allows only the labels to change, as ate-api-server's +// immutable-field validation does for the fields the controller sets. +func (f *fakeControl) UpdateWorker(_ context.Context, in *ateapipb.UpdateWorkerRequest, _ ...grpc.CallOption) (*ateapipb.Worker, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.updates++ + name := in.GetWorker().GetMetadata().GetName() + old, ok := f.workers[name] + if !ok { + return nil, status.Error(codes.NotFound, "no such Worker") + } + w := proto.CloneOf(in.GetWorker()) + cmpOld, cmpNew := proto.CloneOf(old), proto.CloneOf(w) + cmpOld.Labels, cmpNew.Labels = nil, nil + if !proto.Equal(cmpOld, cmpNew) { + return nil, status.Errorf(codes.InvalidArgument, "Worker %s: only labels may change", name) + } + f.workers[name] = w return w, nil } @@ -82,16 +142,117 @@ func (f *fakeControl) ListWorkers(_ context.Context, _ *ateapipb.ListWorkersRequ return &ateapipb.ListWorkersResponse{Workers: slices.Collect(maps.Values(f.workers))}, nil } +func (f *fakeControl) ListWorkerActorAssignments(_ context.Context, in *ateapipb.ListWorkerActorAssignmentsRequest, _ ...grpc.CallOption) (*ateapipb.ListWorkerActorAssignmentsResponse, error) { + f.mu.Lock() + defer f.mu.Unlock() + name := in.GetWorker().GetName() + if _, ok := f.workers[name]; !ok { + return nil, status.Error(codes.NotFound, "no such Worker") + } + resp := &ateapipb.ListWorkerActorAssignmentsResponse{} + for range f.assignments[name] { + resp.ActorAssignments = append(resp.ActorAssignments, &ateapipb.ActorAssignment{}) + } + return resp, nil +} + +// names returns the pod names of the registered Workers, sorted, once per +// Worker. func (f *fakeControl) names() []string { f.mu.Lock() defer f.mu.Unlock() - return slices.Sorted(maps.Keys(f.workers)) + var out []string + for _, w := range f.workers { + out = append(out, w.GetWorkerPod()) + } + slices.Sort(out) + return out +} + +// worker returns the Worker registered for pod: the one not draining, if +// there is one. +func (f *fakeControl) worker(pod string) *ateapipb.Worker { + f.mu.Lock() + defer f.mu.Unlock() + var found *ateapipb.Worker + for _, w := range f.workers { + if w.GetWorkerPod() != pod { + continue + } + if found == nil || w.GetStatus().GetState() != ateapipb.WorkerState_WORKER_STATE_DRAINING { + found = w + } + } + return found +} + +// all returns every registered Worker. +func (f *fakeControl) all() []*ateapipb.Worker { + f.mu.Lock() + defer f.mu.Unlock() + return slices.Collect(maps.Values(f.workers)) +} + +// draining returns the draining Worker registered for pod, if any. +func (f *fakeControl) draining(pod string) *ateapipb.Worker { + f.mu.Lock() + defer f.mu.Unlock() + for _, w := range f.workers { + if w.GetWorkerPod() == pod && w.GetStatus().GetState() == ateapipb.WorkerState_WORKER_STATE_DRAINING { + return w + } + } + return nil +} + +// uid returns the name of the Worker registered for pod, as worker picks it. +func (f *fakeControl) uid(pod string) string { + return f.worker(pod).GetMetadata().GetName() +} + +// fakeRelay records accepted reports, as ate-api-server's view of capacity, +// and refuses Workers listed in reject with their code. An accepted report is +// also written to control's Worker, as RegisterWorker records it. +type fakeRelay struct { + control *fakeControl + mu sync.Mutex + reported map[string]*ateapipb.WorkerResources // by Worker name + hardware map[string]*ateapipb.HardwareIdentity // by Worker name + via map[string]string // relay address, by Worker name + reject map[string]codes.Code + calls int + live map[string]string // as last pruned to +} + +func newFakeRelay() *fakeRelay { + return &fakeRelay{reported: map[string]*ateapipb.WorkerResources{}, hardware: map[string]*ateapipb.HardwareIdentity{}, via: map[string]string{}, reject: map[string]codes.Code{}} } -func (f *fakeControl) worker(name string) *ateapipb.Worker { +func (f *fakeRelay) Report(_ context.Context, addr string, req *ateapipb.RegisterWorkerRequest) error { f.mu.Lock() defer f.mu.Unlock() - return f.workers[name] + f.calls++ + name := req.GetWorker().GetName() + if code, ok := f.reject[name]; ok { + return status.Error(code, "rejected") + } + f.reported[name] = req.GetCapacity() + f.hardware[name] = req.GetHardware() + f.via[name] = addr + if f.control != nil { + f.control.mu.Lock() + if w, ok := f.control.workers[name]; ok { + w.Status.Capacity, w.Status.Hardware = req.GetCapacity(), req.GetHardware() + } + f.control.mu.Unlock() + } + return nil +} + +func (f *fakeRelay) Prune(live map[string]string) { + f.mu.Lock() + defer f.mu.Unlock() + f.live = maps.Clone(live) } // fakeCluster is the controller's Kubernetes, in memory. @@ -99,6 +260,7 @@ type fakeCluster struct { mu sync.Mutex pools []*atev1alpha1.WorkerPool nodes []node + relays map[string]string statuses map[string]atev1alpha1.WorkerPoolStatus // by pool name updates int } @@ -111,6 +273,12 @@ func (f *fakeCluster) Pools() ([]*atev1alpha1.WorkerPool, error) { func (f *fakeCluster) Nodes(context.Context) ([]node, error) { return f.nodes, nil } +func (f *fakeCluster) Relays(context.Context) (map[string]string, error) { + f.mu.Lock() + defer f.mu.Unlock() + return maps.Clone(f.relays), nil +} + func (f *fakeCluster) UpdateStatus(_ context.Context, wp *atev1alpha1.WorkerPool) error { f.mu.Lock() defer f.mu.Unlock() @@ -128,6 +296,19 @@ func (f *fakeCluster) UpdateStatus(_ context.Context, wp *atev1alpha1.WorkerPool return nil } +// editPool replaces the named pool with a copy edit has changed. +func (f *fakeCluster) editPool(name string, edit func(*atev1alpha1.WorkerPool)) { + f.mu.Lock() + defer f.mu.Unlock() + for i, p := range f.pools { + if p.Name == name { + c := p.DeepCopy() + edit(c) + f.pools[i] = c + } + } +} + func (f *fakeCluster) setReplicas(pool string, n int32) { f.mu.Lock() defer f.mu.Unlock() @@ -140,21 +321,40 @@ func (f *fakeCluster) setReplicas(pool string, n int32) { } } -func pool(name string, replicas int32) *atev1alpha1.WorkerPool { - return &atev1alpha1.WorkerPool{ +func pool(name string, replicas int32, limits corev1.ResourceList) *atev1alpha1.WorkerPool { + wp := &atev1alpha1.WorkerPool{ ObjectMeta: metav1.ObjectMeta{Namespace: "benchmark-workloads", Name: name, Labels: map[string]string{"workload": name}}, Spec: atev1alpha1.WorkerPoolSpec{Replicas: replicas, SandboxClass: atev1alpha1.SandboxClass("gvisor")}, } + if limits != nil { + wp.Spec.Template = &atev1alpha1.WorkerPoolPodTemplate{Resources: &corev1.ResourceRequirements{Limits: limits}} + } + return wp } func testNodes() []node { - return []node{{name: "node-a"}, {name: "node-b"}} + alloc := func(cpu, mem string) corev1.ResourceList { + return corev1.ResourceList{corev1.ResourceCPU: resource.MustParse(cpu), corev1.ResourceMemory: resource.MustParse(mem)} + } + return []node{{name: "node-a", allocatable: alloc("86", "160Gi")}, {name: "node-b", allocatable: alloc("44", "80Gi")}} +} + +func newTestController(pools ...*atev1alpha1.WorkerPool) (*controller, *fakeControl, *fakeRelay, *fakeCluster) { + ctl, rel := newFakeControl(), newFakeRelay() + rel.control = ctl + cl := &fakeCluster{pools: pools, nodes: testNodes(), relays: map[string]string{"node-a": "10.0.0.1:8086", "node-b": "10.0.0.2:8086"}} + c := newController(ctl, rel, cl, testRun, 4, testDrainGrace) + start := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + c.now = func() time.Time { return start } + var uids atomic.Int64 + c.newUID = func() string { return fmt.Sprintf("worker-%d", uids.Add(1)) } + return c, ctl, rel, cl } -func newTestController(pools ...*atev1alpha1.WorkerPool) (*controller, *fakeControl, *fakeCluster) { - ctl := newFakeControl() - cl := &fakeCluster{pools: pools, nodes: testNodes()} - return newController(ctl, cl, testRun, 4), ctl, cl +// advance moves the controller's clock forward by d. +func advance(c *controller, d time.Duration) { + t := c.now().Add(d) + c.now = func() time.Time { return t } } func reconcile(t *testing.T, c *controller) error { @@ -171,8 +371,15 @@ func poolNames(name string, n int) []string { return out } -func TestReconcileHonorsReplicas(t *testing.T) { - c, ctl, cl := newTestController(pool("bench", 3)) +func wantCapacity(actors int32, cpu, mem string) *ateapipb.WorkerResources { + return &ateapipb.WorkerResources{Actors: actors, Resources: &ateapipb.Resources{Limits: []*ateapipb.Limits{ + {Name: "cpu", Quantity: cpu}, {Name: "memory", Quantity: mem}, + }}} +} + +func TestReconcileHonorsReplicasAndLimits(t *testing.T) { + limits := corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1500m"), corev1.ResourceMemory: resource.MustParse("4Gi")} + c, ctl, rel, cl := newTestController(pool("bench", 3, limits)) if err := reconcile(t, c); err != nil { t.Fatalf("reconcile: %v", err) } @@ -191,52 +398,396 @@ func TestReconcileHonorsReplicas(t *testing.T) { t.Errorf("SandboxClass = %q", w.GetSandboxClass()) case w.GetLabels()["workload"] != "bench": t.Errorf("Labels = %v, want the pool's, which template selectors match", w.GetLabels()) - case w.GetWorkerPodUid() != fakeworker.PodUID(name) || w.GetEpoch() != 0: + case w.GetWorkerPodUid() != w.GetMetadata().GetName() || w.GetEpoch() != 0: t.Errorf("WorkerPodUid/Epoch = %q/%d", w.GetWorkerPodUid(), w.GetEpoch()) } + if got := rel.via[ctl.uid(name)]; got != "10.0.0.2:8086" { + t.Errorf("capacity for %s sent via %q, want node-b's fake-atelet", name, got) + } + if got, want := rel.reported[ctl.uid(name)], wantCapacity(1000, "1500m", "4Gi"); !proto.Equal(got, want) { + t.Errorf("reported %v, want %v from the pool's limits", got, want) + } + if got, want := rel.hardware[ctl.uid(name)], hardware.ProbeHost(); !proto.Equal(got, want) { + t.Errorf("hardware %v, want %v, which ate-api-server requires", got, want) + } if got, want := cl.statuses["bench"], (atev1alpha1.WorkerPoolStatus{Replicas: 3, ReadyReplicas: 3, Selector: "ate.dev/worker-pool=bench"}); got != want { t.Errorf("status = %+v, want %+v", got, want) } } +// With no limit set, a real worker reports its node's allocatable, which is +// what the downward API projects for an unset limit. +func TestReconcileReportsNodeAllocatableWithoutLimits(t *testing.T) { + c, ctl, rel, _ := newTestController(pool("bench", 2, nil)) + if err := reconcile(t, c); err != nil { + t.Fatalf("reconcile: %v", err) + } + onA, onB := fakeworker.Name(testRun, "benchmark-workloads", "bench", 0), fakeworker.Name(testRun, "benchmark-workloads", "bench", 1) + if got, want := rel.reported[ctl.uid(onA)], wantCapacity(1000, "86", "160Gi"); !proto.Equal(got, want) { + t.Errorf("node-a Worker reported %v, want %v", got, want) + } + if got, want := rel.reported[ctl.uid(onB)], wantCapacity(1000, "44", "80Gi"); !proto.Equal(got, want) { + t.Errorf("node-b Worker reported %v, want %v", got, want) + } +} + +func TestReconcileHonorsMaxActorsAnnotation(t *testing.T) { + wp := pool("bench", 1, nil) + wp.Annotations = map[string]string{fakeworker.MaxActorsAnnotation: "5"} + bad := pool("broken", 1, nil) + bad.Annotations = map[string]string{fakeworker.MaxActorsAnnotation: "lots"} + c, ctl, rel, _ := newTestController(wp, bad) + if err := reconcile(t, c); err == nil { + t.Error("reconcile succeeded with an unparseable annotation") + } + name := fakeworker.Name(testRun, "benchmark-workloads", "bench", 0) + if got := rel.reported[ctl.uid(name)].GetActors(); got != 5 { + t.Errorf("actors = %d, want 5 from the annotation", got) + } + if slices.Contains(ctl.names(), fakeworker.Name(testRun, "benchmark-workloads", "broken", 0)) { + t.Error("registered a Worker for the pool with a bad annotation") + } +} + +// An actor count past 32 bits would wrap in the capacity report, so it holds +// the pool like any other bad annotation. +func TestReconcileRejectsAMaxActorsPast32Bits(t *testing.T) { + wp := pool("bench", 1, nil) + wp.Annotations = map[string]string{fakeworker.MaxActorsAnnotation: "3000000000"} + c, ctl, rel, _ := newTestController(wp) + if err := reconcile(t, c); err == nil { + t.Error("reconcile succeeded with an actor count past 32 bits") + } + if len(ctl.names()) != 0 || len(rel.reported) != 0 { + t.Errorf("registered %v and reported %v for the pool with a bad annotation", ctl.names(), rel.reported) + } +} + +// Each pass lets go of relays the pod list no longer has, as after a +// fake-atelet restarts under a new IP. +func TestReconcilePrunesGoneRelays(t *testing.T) { + c, _, rel, cl := newTestController(pool("bench", 2, nil)) + delete(cl.relays, "node-b") + _ = reconcile(t, c) + if want := map[string]string{"node-a": "10.0.0.1:8086"}; !maps.Equal(rel.live, want) { + t.Errorf("pruned to %v, want %v", rel.live, want) + } +} + +// An annotation typo on a running pool reports an error and leaves the pool's +// Workers as they are, rather than retiring them all. +func TestBadAnnotationLeavesThePoolsWorkers(t *testing.T) { + c, ctl, _, cl := newTestController(pool("bench", 2, nil)) + if err := reconcile(t, c); err != nil { + t.Fatalf("reconcile: %v", err) + } + bad := cl.pools[0].DeepCopy() + bad.Annotations = map[string]string{fakeworker.MaxActorsAnnotation: "abc"} + cl.pools[0] = bad + if err := reconcile(t, c); err == nil { + t.Error("reconcile succeeded with an unparseable annotation") + } + if got, want := ctl.names(), poolNames("bench", 2); !slices.Equal(got, want) { + t.Errorf("registered %v, want the pool's Workers kept, %v", got, want) + } +} + +// A Worker whose node has left the benchmark set is replaced on a benchmark +// node, as a pod is when its node goes away. The old one drains with the +// capacity it last reported, rather than a report that drops its cpu and +// memory for want of allocatable. +func TestWorkerOnADepartedNodeIsReplaced(t *testing.T) { + c, ctl, rel, cl := newTestController(pool("bench", 2, nil)) + if err := reconcile(t, c); err != nil { + t.Fatalf("reconcile: %v", err) + } + pod := fakeworker.Name(testRun, "benchmark-workloads", "bench", 1) + old := ctl.uid(pod) + before := rel.reported[old] + ctl.assignments[old] = 1 + cl.nodes = cl.nodes[:1] + if err := reconcile(t, c); err != nil { + t.Fatalf("reconcile after node-b left: %v", err) + } + w := ctl.worker(pod) + if w.GetMetadata().GetName() == old || w.GetNodeName() != "node-a" { + t.Errorf("Worker for %s = %s on %s, want a new one on node-a", pod, w.GetMetadata().GetName(), w.GetNodeName()) + } + if got, want := rel.reported[w.GetMetadata().GetName()], wantCapacity(1000, "86", "160Gi"); !proto.Equal(got, want) { + t.Errorf("replacement reported %v, want node-a's %v", got, want) + } + if d := ctl.draining(pod); d.GetMetadata().GetName() != old || !proto.Equal(rel.reported[old], before) { + t.Errorf("old Worker = %v reporting %v, want %s draining with %v kept", d.GetMetadata().GetName(), rel.reported[old], old, before) + } + if got := cl.statuses["bench"]; got.Replicas != 2 || got.ReadyReplicas != 2 { + t.Errorf("status = %+v, want 2 replicas, 2 ready", got) + } +} + +// An empty node list replaces nothing: there is nowhere to put a replacement. +func TestNoBenchmarkNodeLeavesWorkersInPlace(t *testing.T) { + c, ctl, _, cl := newTestController(pool("bench", 2, nil)) + if err := reconcile(t, c); err != nil { + t.Fatalf("reconcile: %v", err) + } + cl.nodes = nil + _ = reconcile(t, c) + if got, want := ctl.names(), poolNames("bench", 2); !slices.Equal(got, want) || ctl.draining(want[0]) != nil || ctl.draining(want[1]) != nil { + t.Errorf("registered %v with drains, want %v left as they are", got, want) + } +} + +// A Worker on a node whose fake-atelet is not up yet is registered but not +// ready; the next pass reports it once the relay is there. +func TestReconcileWaitsForTheNodesFakeAtelet(t *testing.T) { + c, ctl, rel, cl := newTestController(pool("bench", 2, nil)) + delete(cl.relays, "node-b") + if err := reconcile(t, c); err == nil { + t.Error("reconcile reported success with a Worker left unreported") + } + if got := len(ctl.names()); got != 2 { + t.Errorf("registered %d Workers, want 2", got) + } + if got := cl.statuses["bench"]; got.Replicas != 2 || got.ReadyReplicas != 1 { + t.Errorf("status = %+v, want 2 replicas, 1 ready", got) + } + cl.relays["node-b"] = "10.0.0.2:8086" + if err := reconcile(t, c); err != nil { + t.Fatalf("second reconcile: %v", err) + } + if len(rel.reported) != 2 || cl.statuses["bench"].ReadyReplicas != 2 { + t.Errorf("after the relay came up: %d reported, status %+v", len(rel.reported), cl.statuses["bench"]) + } +} + +func TestReconcileRetriesARejectedReport(t *testing.T) { + c, ctl, rel, cl := newTestController(pool("bench", 1, nil)) + name := fakeworker.Name(testRun, "benchmark-workloads", "bench", 0) + const first = "worker-1" // the first name newTestController's controller mints + rel.reject[first] = codes.NotFound + if err := reconcile(t, c); err == nil { + t.Error("reconcile succeeded with a rejected report") + } + delete(rel.reject, first) + if err := reconcile(t, c); err != nil { + t.Fatalf("second reconcile: %v", err) + } + if rel.reported[ctl.uid(name)] == nil || cl.statuses["bench"].ReadyReplicas != 1 { + t.Error("rejected report not retried") + } +} + // A steady fleet costs ate-api-server nothing: a second pass with nothing -// changed creates nothing and rewrites no status. +// changed creates and reports nothing, and rewrites no status. func TestReconcileIsIdempotent(t *testing.T) { - c, ctl, cl := newTestController(pool("bench", 3)) + c, ctl, rel, cl := newTestController(pool("bench", 3, nil)) if err := reconcile(t, c); err != nil { t.Fatalf("reconcile: %v", err) } - creates, updates := ctl.creates, cl.updates + creates, reports, updates := ctl.creates, rel.calls, cl.updates if err := reconcile(t, c); err != nil { t.Fatalf("second reconcile: %v", err) } - if ctl.creates != creates || cl.updates != updates { - t.Errorf("second pass: %d creates, %d status writes; want none", ctl.creates-creates, cl.updates-updates) + if ctl.creates != creates || rel.calls != reports || cl.updates != updates { + t.Errorf("second pass: %d creates, %d reports, %d status writes; want none", ctl.creates-creates, rel.calls-reports, cl.updates-updates) } } -func TestScaleDownDeletes(t *testing.T) { - c, ctl, cl := newTestController(pool("bench", 4)) +// A limits edit replaces the pool's Workers, as it rolls a real pool's pods: +// ateom reads its limits once at startup, so a Worker's capacity never +// changes. The old Worker is not reported again. +func TestPoolLimitsChangeReplacesWorkers(t *testing.T) { + limits := corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("2"), corev1.ResourceMemory: resource.MustParse("4Gi")} + c, ctl, rel, cl := newTestController(pool("bench", 1, limits)) if err := reconcile(t, c); err != nil { t.Fatalf("reconcile: %v", err) } + pod := fakeworker.Name(testRun, "benchmark-workloads", "bench", 0) + old := ctl.uid(pod) + ctl.assignments[old] = 1 + reports := rel.calls + + cl.editPool("bench", func(wp *atev1alpha1.WorkerPool) { + wp.Spec.Template.Resources.Limits = corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("4"), corev1.ResourceMemory: resource.MustParse("8Gi")} + }) + if err := reconcile(t, c); err != nil { + t.Fatalf("reconcile after the limits changed: %v", err) + } + w := ctl.worker(pod) + if w.GetMetadata().GetName() == old { + t.Fatalf("Worker for %s kept after the limits changed, want a new one", pod) + } + if got, want := rel.reported[w.GetMetadata().GetName()], wantCapacity(1000, "4", "8Gi"); !proto.Equal(got, want) { + t.Errorf("replacement reported %v, want %v", got, want) + } + if got, want := rel.reported[old], wantCapacity(1000, "2", "4Gi"); !proto.Equal(got, want) { + t.Errorf("old Worker's capacity = %v, want %v kept while it drains", got, want) + } + if got := rel.calls - reports; got != 1 { + t.Errorf("%d reports after the edit, want 1, for the replacement only", got) + } + if d := ctl.draining(pod); d.GetMetadata().GetName() != old { + t.Errorf("draining Worker = %v, want the old one, %s", d.GetMetadata().GetName(), old) + } + if got := cl.statuses["bench"]; got.Replicas != 1 || got.ReadyReplicas != 1 { + t.Errorf("status = %+v, want only the replacement counted", got) + } +} + +// ateom's actor count is a flag, so changing it rolls a real pool's pods too. +func TestMaxActorsAnnotationChangeReplacesWorkers(t *testing.T) { + c, ctl, rel, cl := newTestController(pool("bench", 1, nil)) + if err := reconcile(t, c); err != nil { + t.Fatalf("reconcile: %v", err) + } + pod := fakeworker.Name(testRun, "benchmark-workloads", "bench", 0) + old := ctl.uid(pod) + cl.editPool("bench", func(wp *atev1alpha1.WorkerPool) { + wp.Annotations = map[string]string{fakeworker.MaxActorsAnnotation: "5"} + }) + if err := reconcile(t, c); err != nil { + t.Fatalf("reconcile after the annotation changed: %v", err) + } + w := ctl.worker(pod) + if w.GetMetadata().GetName() == old { + t.Fatalf("Worker for %s kept after the actor count changed, want a new one", pod) + } + if got := rel.reported[w.GetMetadata().GetName()].GetActors(); got != 5 { + t.Errorf("replacement reported %d actors, want 5", got) + } + if got, want := ctl.names(), poolNames("bench", 1); !slices.Equal(got, want) { + t.Errorf("registered %v, want the idle old Worker deleted, %v", got, want) + } +} + +// A node's allocatable reaches ateom once, at startup, so a later change +// leaves a pool with no limits as it is: no new Worker and no report. +func TestAllocatableChangeKeepsWorkerCapacity(t *testing.T) { + c, ctl, rel, cl := newTestController(pool("bench", 2, nil)) + if err := reconcile(t, c); err != nil { + t.Fatalf("reconcile: %v", err) + } + creates, reports := ctl.creates, rel.calls + cl.nodes[0].allocatable = corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("64"), corev1.ResourceMemory: resource.MustParse("128Gi")} + if err := reconcile(t, c); err != nil { + t.Fatalf("reconcile after allocatable changed: %v", err) + } + if ctl.creates != creates || rel.calls != reports { + t.Errorf("allocatable change cost %d creates and %d reports, want none", ctl.creates-creates, rel.calls-reports) + } + if got := cl.statuses["bench"]; got.Replicas != 2 || got.ReadyReplicas != 2 { + t.Errorf("status = %+v, want 2 replicas, 2 ready", got) + } +} + +// Scaling down mirrors a real worker pod going away: drained first so nothing +// new lands on it, deleted only once no Actor is assigned to it. +func TestScaleDownDrainsThenDeletes(t *testing.T) { + c, ctl, _, cl := newTestController(pool("bench", 4, nil)) + if err := reconcile(t, c); err != nil { + t.Fatalf("reconcile: %v", err) + } + busy := fakeworker.Name(testRun, "benchmark-workloads", "bench", 3) + idle := fakeworker.Name(testRun, "benchmark-workloads", "bench", 2) + ctl.assignments[ctl.uid(busy)] = 1 + cl.setReplicas("bench", 2) if err := reconcile(t, c); err != nil { t.Fatalf("scale-down reconcile: %v", err) } + if slices.Contains(ctl.names(), idle) { + t.Errorf("idle Worker %s not deleted", idle) + } + if w := ctl.worker(busy); w == nil || w.GetStatus().GetState() != ateapipb.WorkerState_WORKER_STATE_DRAINING { + t.Fatalf("busy Worker %s = %v, want kept and draining", busy, w) + } + if got := cl.statuses["bench"]; got.Replicas != 2 || got.ReadyReplicas != 2 { + t.Errorf("status = %+v, want 2 replicas: a draining Worker is no replica", got) + } + + ctl.assignments[ctl.uid(busy)] = 0 + if err := reconcile(t, c); err != nil { + t.Fatalf("reconcile after the Actor left: %v", err) + } if got, want := ctl.names(), poolNames("bench", 2); !slices.Equal(got, want) { t.Errorf("registered %v, want %v", got, want) } +} + +// Nothing stops a fake Worker's Actors the way ateom stops a terminating +// pod's, so a Worker still held after the drain grace is deleted anyway, +// which releases its Actors, rather than left draining for good. +func TestDrainGraceDeletesAWorkerStillHeld(t *testing.T) { + c, ctl, _, cl := newTestController(pool("bench", 2, nil)) + if err := reconcile(t, c); err != nil { + t.Fatalf("reconcile: %v", err) + } + busy := fakeworker.Name(testRun, "benchmark-workloads", "bench", 1) + held := ctl.uid(busy) + ctl.assignments[held] = 1 + cl.setReplicas("bench", 1) + if err := reconcile(t, c); err != nil { + t.Fatalf("scale-down reconcile: %v", err) + } + advance(c, testDrainGrace-time.Second) + if err := reconcile(t, c); err != nil { + t.Fatalf("reconcile within the grace: %v", err) + } + if ctl.worker(busy) == nil { + t.Fatalf("Worker %s deleted within the drain grace", busy) + } + advance(c, time.Second) + if err := reconcile(t, c); err != nil { + t.Fatalf("reconcile after the grace: %v", err) + } + if got, want := ctl.names(), poolNames("bench", 1); !slices.Equal(got, want) { + t.Errorf("registered %v after the drain grace, want %v", got, want) + } + if n := ctl.assignments[held]; n != 0 { + t.Errorf("%d Actors still assigned to the deleted Worker", n) + } +} + +// A pod scaled down while an Actor holds its Worker, then wanted again, gets +// a new Worker at once rather than the draining one back, and the old one is +// deleted once the Actor leaves. +func TestScaleDownThenUpReplacesTheDrainingWorker(t *testing.T) { + c, ctl, _, cl := newTestController(pool("bench", 2, nil)) + if err := reconcile(t, c); err != nil { + t.Fatalf("reconcile: %v", err) + } + busy := fakeworker.Name(testRun, "benchmark-workloads", "bench", 1) + old := ctl.uid(busy) + ctl.assignments[old] = 1 + cl.setReplicas("bench", 1) + if err := reconcile(t, c); err != nil { + t.Fatalf("scale-down reconcile: %v", err) + } + cl.setReplicas("bench", 2) + if err := reconcile(t, c); err != nil { + t.Fatalf("scale-up reconcile: %v", err) + } + if w := ctl.worker(busy); w.GetMetadata().GetName() == old || w.GetStatus().GetState() != ateapipb.WorkerState_WORKER_STATE_ACTIVE { + t.Errorf("Worker for %s = %s/%v, want a new active one", busy, w.GetMetadata().GetName(), w.GetStatus().GetState()) + } if got := cl.statuses["bench"]; got.Replicas != 2 || got.ReadyReplicas != 2 { - t.Errorf("status = %+v, want 2 replicas", got) + t.Errorf("status = %+v while the old Worker drains, want 2 replicas, 2 ready", got) + } + + ctl.assignments[old] = 0 + if err := reconcile(t, c); err != nil { + t.Fatalf("reconcile after the Actor left: %v", err) + } + if got, want := ctl.names(), poolNames("bench", 2); !slices.Equal(got, want) { + t.Errorf("registered %v, want %v", got, want) } } // A create the server commits but whose reply is lost, as on a deadline or a // SIGTERM mid-pass, must not strand the Worker: shutdown still deletes it. func TestDeleteAllAfterALostCreateReply(t *testing.T) { - c, ctl, cl := newTestController(pool("bench", 2)) + c, ctl, _, cl := newTestController(pool("bench", 2, nil)) ctl.lostReply = status.Error(codes.Unavailable, "connection reset") if err := reconcile(t, c); err == nil { t.Fatal("reconcile succeeded with every create reply lost") @@ -255,7 +806,7 @@ func TestDeleteAllAfterALostCreateReply(t *testing.T) { // The same Worker, unwanted by the next pass, is retired rather than left // registered; wanted, its create is retried and confirmed. func TestLostCreateReplyIsRetiredOrConfirmed(t *testing.T) { - c, ctl, cl := newTestController(pool("bench", 2)) + c, ctl, _, cl := newTestController(pool("bench", 2, nil)) ctl.lostReply = status.Error(codes.DeadlineExceeded, "deadline exceeded") if err := reconcile(t, c); err == nil { t.Fatal("reconcile succeeded with every create reply lost") @@ -273,8 +824,132 @@ func TestLostCreateReplyIsRetiredOrConfirmed(t *testing.T) { } } +// A label edit on the pool reaches its Workers in place, as the worker syncer +// applies it, including Workers adopted after a restart. +func TestPoolLabelChangeUpdatesWorkers(t *testing.T) { + c, ctl, rel, cl := newTestController(pool("bench", 2, nil)) + if err := reconcile(t, c); err != nil { + t.Fatalf("reconcile: %v", err) + } + relabel := func(v string) { + cl.editPool("bench", func(wp *atev1alpha1.WorkerPool) { wp.Labels = map[string]string{"workload": v} }) + } + + relabel("v2") + creates := ctl.creates + if err := reconcile(t, c); err != nil { + t.Fatalf("reconcile after relabel: %v", err) + } + for _, name := range poolNames("bench", 2) { + if got := ctl.worker(name).GetLabels()["workload"]; got != "v2" { + t.Errorf("Worker %s workload label = %q, want v2", name, got) + } + } + if ctl.creates != creates || ctl.updates != 2 { + t.Errorf("relabel cost %d creates and %d updates, want 0 and 2", ctl.creates-creates, ctl.updates) + } + + restarted := newController(ctl, rel, cl, testRun, 4, testDrainGrace) + if err := restarted.adopt(context.Background()); err != nil { + t.Fatalf("adopt: %v", err) + } + relabel("v3") + if err := reconcile(t, restarted); err != nil { + t.Fatalf("reconcile after restart and relabel: %v", err) + } + for _, name := range poolNames("bench", 2) { + if got := ctl.worker(name).GetLabels()["workload"]; got != "v3" { + t.Errorf("after restart, Worker %s workload label = %q, want v3", name, got) + } + } +} + +// sandbox_class is immutable on a Worker, so a sandboxClass edit replaces the +// pool's Workers the way it replaces real worker pods: each pod gets a new +// Worker, under a new name and pod UID, at once; an idle old Worker is deleted +// at once, a held one once its Actors leave. +func TestPoolSandboxClassChangeReplacesWorkers(t *testing.T) { + c, ctl, _, cl := newTestController(pool("bench", 2, nil)) + if err := reconcile(t, c); err != nil { + t.Fatalf("reconcile: %v", err) + } + idle := fakeworker.Name(testRun, "benchmark-workloads", "bench", 0) + busy := fakeworker.Name(testRun, "benchmark-workloads", "bench", 1) + oldBusy := ctl.uid(busy) + ctl.assignments[oldBusy] = 1 + + cl.editPool("bench", func(wp *atev1alpha1.WorkerPool) { wp.Spec.SandboxClass = "microvm" }) + if err := reconcile(t, c); err != nil { + t.Fatalf("reconcile after the class change: %v", err) + } + for _, pod := range []string{idle, busy} { + w := ctl.worker(pod) + if w.GetSandboxClass() != "microvm" || w.GetStatus().GetState() != ateapipb.WorkerState_WORKER_STATE_ACTIVE { + t.Errorf("Worker for %s = %q/%v, want microvm and active", pod, w.GetSandboxClass(), w.GetStatus().GetState()) + } + if w.GetWorkerPodUid() != w.GetMetadata().GetName() { + t.Errorf("Worker %s has pod UID %s, want its name", w.GetMetadata().GetName(), w.GetWorkerPodUid()) + } + } + if w := ctl.draining(busy); w.GetMetadata().GetName() != oldBusy || w.GetSandboxClass() != "gvisor" { + t.Errorf("draining Worker for %s = %s/%q, want the old gvisor one, %s", busy, w.GetMetadata().GetName(), w.GetSandboxClass(), oldBusy) + } + if got := cl.statuses["bench"]; got.Replicas != 2 || got.ReadyReplicas != 2 { + t.Errorf("status = %+v, want 2 replicas, 2 ready: the draining Worker is no replica", got) + } + + ctl.assignments[oldBusy] = 0 + if err := reconcile(t, c); err != nil { + t.Fatalf("reconcile after the Actor left: %v", err) + } + if got, want := ctl.names(), poolNames("bench", 2); !slices.Equal(got, want) { + t.Errorf("registered %v, want %v", got, want) + } +} + +// A Worker of the old sandbox class whose drain fails is retried, and until +// then is no replica and is left alone: no label write, no capacity report. +func TestFailedDrainOfAReplacedWorkerIsNoReplica(t *testing.T) { + c, ctl, rel, cl := newTestController(pool("bench", 1, nil)) + if err := reconcile(t, c); err != nil { + t.Fatalf("reconcile: %v", err) + } + pod := fakeworker.Name(testRun, "benchmark-workloads", "bench", 0) + old := ctl.worker(pod) + ctl.drainErr = status.Error(codes.Unavailable, "connection reset") + cl.editPool("bench", func(wp *atev1alpha1.WorkerPool) { + wp.Spec.SandboxClass = "microvm" + wp.Labels = map[string]string{"workload": "v2"} + }) + if err := reconcile(t, c); err == nil { + t.Error("reconcile succeeded with a failed drain") + } + var replacement *ateapipb.Worker + for _, w := range ctl.all() { + if w.GetSandboxClass() == "microvm" { + replacement = w + } + } + if replacement == nil || rel.reported[replacement.GetMetadata().GetName()] == nil { + t.Fatalf("no reported microvm replacement for %s", pod) + } + if got := ctl.workers[old.GetMetadata().GetName()].GetLabels()["workload"]; got != "bench" { + t.Errorf("old Worker's workload label = %q, want it left at bench", got) + } + if got := cl.statuses["bench"]; got.Replicas != 1 || got.ReadyReplicas != 1 { + t.Errorf("status = %+v, want only the replacement counted", got) + } + + ctl.drainErr = nil + if err := reconcile(t, c); err != nil { + t.Fatalf("reconcile after the drain recovered: %v", err) + } + if _, ok := ctl.workers[old.GetMetadata().GetName()]; ok { + t.Error("old Worker not retired once its drain succeeded") + } +} func TestDeletedPoolRetiresItsWorkers(t *testing.T) { - c, ctl, cl := newTestController(pool("bench", 2), pool("other", 1)) + c, ctl, _, cl := newTestController(pool("bench", 2, nil), pool("other", 1, nil)) if err := reconcile(t, c); err != nil { t.Fatalf("reconcile: %v", err) } @@ -290,41 +965,101 @@ func TestDeletedPoolRetiresItsWorkers(t *testing.T) { // After a restart the controller adopts its run's Workers instead of creating // them again, and leaves every other Worker alone. func TestAdoptAfterRestart(t *testing.T) { - c, ctl, cl := newTestController(pool("bench", 2)) + c, ctl, rel, cl := newTestController(pool("bench", 2, nil)) if err := reconcile(t, c); err != nil { t.Fatalf("reconcile: %v", err) } - real := &ateapipb.Worker{Metadata: &ateapipb.ResourceMetadata{Name: "0c4f6c2e-real-pod-uid"}, WorkerNamespace: "benchmark-workloads", WorkerPool: "bench", Status: &ateapipb.WorkerStatus{}} - otherRun := &ateapipb.Worker{Metadata: &ateapipb.ResourceMetadata{Name: fakeworker.Name("r2", "benchmark-workloads", "bench", 0)}, WorkerNamespace: "benchmark-workloads", WorkerPool: "bench", Status: &ateapipb.WorkerStatus{}} + real := &ateapipb.Worker{Metadata: &ateapipb.ResourceMetadata{Name: "0c4f6c2e-real-pod-uid"}, WorkerNamespace: "benchmark-workloads", WorkerPool: "bench", WorkerPod: "bench-7d9f8c-x2x4k", Status: &ateapipb.WorkerStatus{}} + otherRun := &ateapipb.Worker{Metadata: &ateapipb.ResourceMetadata{Name: "9a1e0d3b-other-run"}, WorkerNamespace: "benchmark-workloads", WorkerPool: "bench", WorkerPod: fakeworker.Name("r2", "benchmark-workloads", "bench", 0), Status: &ateapipb.WorkerStatus{}} ctl.workers[real.Metadata.Name] = real ctl.workers[otherRun.Metadata.Name] = otherRun - restarted := newController(ctl, cl, testRun, 4) + restarted := newController(ctl, rel, cl, testRun, 4, testDrainGrace) if err := restarted.adopt(context.Background()); err != nil { t.Fatalf("adopt: %v", err) } if got := len(restarted.workers); got != 2 { t.Fatalf("adopted %d Workers, want this run's 2", got) } - creates := ctl.creates + creates, reports := ctl.creates, rel.calls if err := reconcile(t, restarted); err != nil { t.Fatalf("reconcile after restart: %v", err) } - if ctl.creates != creates { - t.Errorf("created %d Workers again after adopting them", ctl.creates-creates) + if ctl.creates != creates || rel.calls != reports { + t.Errorf("after adopting: %d creates and %d reports, want none", ctl.creates-creates, rel.calls-reports) } - if !slices.Contains(ctl.names(), real.Metadata.Name) || !slices.Contains(ctl.names(), otherRun.Metadata.Name) { + if ctl.workers[real.Metadata.Name] == nil || ctl.workers[otherRun.Metadata.Name] == nil { t.Error("a Worker that is not this run's was retired") } } +// A limits edit made while fake-workersync was down still replaces the +// pool's Workers: an adopted Worker whose reported capacity is not what the +// pool now gives is from an older template. +func TestAdoptReplacesWorkersOfAPoolEditedWhileDown(t *testing.T) { + limits := corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("2"), corev1.ResourceMemory: resource.MustParse("4Gi")} + c, ctl, rel, cl := newTestController(pool("bench", 1, limits)) + if err := reconcile(t, c); err != nil { + t.Fatalf("reconcile: %v", err) + } + pod := fakeworker.Name(testRun, "benchmark-workloads", "bench", 0) + old := ctl.uid(pod) + cl.editPool("bench", func(wp *atev1alpha1.WorkerPool) { + wp.Spec.Template.Resources.Limits = corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("4"), corev1.ResourceMemory: resource.MustParse("8Gi")} + }) + + restarted := newController(ctl, rel, cl, testRun, 4, testDrainGrace) + if err := restarted.adopt(context.Background()); err != nil { + t.Fatalf("adopt: %v", err) + } + if err := reconcile(t, restarted); err != nil { + t.Fatalf("reconcile after restart: %v", err) + } + w := ctl.worker(pod) + if w.GetMetadata().GetName() == old { + t.Fatalf("Worker for %s kept after a limits edit made while down, want a new one", pod) + } + if got, want := rel.reported[w.GetMetadata().GetName()], wantCapacity(1000, "4", "8Gi"); !proto.Equal(got, want) { + t.Errorf("replacement reported %v, want %v", got, want) + } + if got, want := ctl.names(), poolNames("bench", 1); !slices.Equal(got, want) { + t.Errorf("registered %v, want the idle old Worker deleted, %v", got, want) + } +} + +// An adopted Worker whose capacity ate-api-server never accepted is reported +// after the restart, as ateom keeps retrying its one report. +func TestAdoptReportsAWorkerNeverAccepted(t *testing.T) { + c, ctl, rel, cl := newTestController(pool("bench", 1, nil)) + const first = "worker-1" // the first name newTestController's controller mints + rel.reject[first] = codes.Unavailable + if err := reconcile(t, c); err == nil { + t.Fatal("reconcile succeeded with a rejected report") + } + delete(rel.reject, first) + + restarted := newController(ctl, rel, cl, testRun, 4, testDrainGrace) + if err := restarted.adopt(context.Background()); err != nil { + t.Fatalf("adopt: %v", err) + } + if err := reconcile(t, restarted); err != nil { + t.Fatalf("reconcile after restart: %v", err) + } + if got, want := rel.reported[first], wantCapacity(1000, "86", "160Gi"); !proto.Equal(got, want) { + t.Errorf("adopted Worker reported %v, want %v", got, want) + } + if got := cl.statuses["bench"]; got.Replicas != 1 || got.ReadyReplicas != 1 { + t.Errorf("status = %+v, want 1 replica, 1 ready", got) + } +} + func TestDeleteAll(t *testing.T) { - c, ctl, _ := newTestController(pool("bench", 3)) + c, ctl, _, _ := newTestController(pool("bench", 3, nil)) if err := reconcile(t, c); err != nil { t.Fatalf("reconcile: %v", err) } // One already gone, as after a cut-short shutdown. - delete(ctl.workers, fakeworker.Name(testRun, "benchmark-workloads", "bench", 0)) + delete(ctl.workers, ctl.uid(fakeworker.Name(testRun, "benchmark-workloads", "bench", 0))) if err := c.deleteAll(context.Background()); err != nil { t.Fatalf("deleteAll: %v", err) } @@ -332,3 +1067,23 @@ func TestDeleteAll(t *testing.T) { t.Errorf("left %v", got) } } + +func TestCapacityFormatsLikeAteom(t *testing.T) { + capacity := func(wp *atev1alpha1.WorkerPool) *ateapipb.WorkerResources { + t.Helper() + tmpl, err := templateOf(wp) + if err != nil { + t.Fatal(err) + } + return tmpl.capacity(nil) + } + got := capacity(pool("p", 1, corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("0.5"), corev1.ResourceMemory: resource.MustParse("1G")})) + // ateom spells cpu as decimal milli-cores and memory as a binary-SI byte + // count; 1G is not a power of two, so 1e9. + if want := wantCapacity(1000, "500m", "1e9"); !proto.Equal(got, want) { + t.Errorf("capacity = %v, want %v", got, want) + } + if none := capacity(pool("p", 1, nil)); none.GetResources() != nil || none.GetActors() != 1000 { + t.Errorf("no limits and no allocatable: %v; want 1000 actors and no resources", none) + } +} diff --git a/cmd/benchmarking/isolate/fake-workersync/main.go b/cmd/benchmarking/isolate/fake-workersync/main.go index 4037fead57..fa4a8ca762 100644 --- a/cmd/benchmarking/isolate/fake-workersync/main.go +++ b/cmd/benchmarking/isolate/fake-workersync/main.go @@ -15,14 +15,10 @@ // Command fake-workersync stands in for atecontroller's WorkerPool // controller and worker syncer in control-plane benchmarks. It honors every // WorkerPool's spec.replicas with fake Workers that no pod backs, placed -// round robin on the nodes carrying --node-label. It writes each pool's -// status, so the scale subresource and `kubectl get workerpool` work, and -// deletes its Workers when it shuts down. -// -// No capacity is reported for the fake Workers yet. ate-api-server registers -// each one ACTIVE with no capacity, which the scheduler treats as no room, so -// no actor can be placed on one and every resume fails with no_capacity, even -// though the pool's status counts the Worker ready. +// round robin on the nodes carrying --node-label, and has each Worker's +// capacity reported through the fake-atelet on its node, as ateom reports +// through atelet. It writes each pool's status, so the scale subresource and +// `kubectl get workerpool` work, and deletes its Workers when it shuts down. // // atecontroller must not run alongside it: its WorkerPool controller would // create real worker pods for the same pools, and its worker syncer deletes @@ -34,13 +30,16 @@ import ( "context" "fmt" "log/slog" + "net" "os/signal" "slices" + "strconv" "syscall" "time" "github.com/agent-substrate/substrate/cmd/benchmarking/isolate/internal/fakeworker" "github.com/agent-substrate/substrate/internal/benchmarking/boomer/boomerutil" + "github.com/agent-substrate/substrate/internal/installdefaults" "github.com/agent-substrate/substrate/internal/serverboot" "github.com/agent-substrate/substrate/internal/version" atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" @@ -48,6 +47,7 @@ import ( ateinformers "github.com/agent-substrate/substrate/pkg/client/informers/externalversions" atelisters "github.com/agent-substrate/substrate/pkg/client/listers/api/v1alpha1" "github.com/spf13/pflag" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/kubernetes" @@ -57,11 +57,18 @@ import ( var ( apiEndpoint = pflag.String("api-endpoint", "k8s:///api.ate-system.svc.cluster.local:443", "ate-api-server gRPC dial target.") - run = pflag.String("run", "bench", "Run prefix of the fake Worker names.") + run = pflag.String("run", "bench", "Run prefix of the fake Worker pod names.") nodeLabel = pflag.String("node-label", "ate.dev/fake-data-plane=true", "Label selector for the benchmark nodes that run fake-atelet.") + ateletNS = pflag.String("atelet-namespace", "ate-system", "Namespace of the fake-atelet pods.") + ateletLabel = pflag.String("fake-atelet-label", "ate.dev/fake-atelet=true", "Label selector for the fake-atelet pods.") + relayPort = pflag.Int("relay-port", fakeworker.DefaultRelayPort, "Port fake-atelet serves capacity reports on.") + ateletID = pflag.String("atelet-spiffe-id", installdefaults.SPIFFEID("ate-system", "atelet"), "SPIFFE ID the fake-atelet relay must present.") + clientBundle = pflag.String("client-cred-bundle", "/run/podidentity.podcert.ate.dev/credential-bundle.pem", "Credential bundle presented to the relay.") + trustBundle = pflag.String("trust-bundle", "/run/podidentity.podcert.ate.dev/trust-bundle.pem", "Pod-identity trust bundle the relay's certificate must chain to.") resync = pflag.Duration("resync", 10*time.Second, "Reconcile at least this often, besides on every WorkerPool change.") concurrency = pflag.Int("concurrency", 32, "Worker calls in flight.") cleanup = pflag.Bool("cleanup", false, "Delete this run's registered fake Workers, then exit.") + drainGrace = pflag.Duration("drain-grace", time.Minute, "How long a draining fake Worker waits for its Actors to leave before it is deleted, releasing them. Stands in for the time ateom takes to stop a terminating pod's Actors.") deleteTimeout = pflag.Duration("delete-timeout", 5*time.Minute, "How long shutdown spends deleting the fake Workers.") showVersion = pflag.Bool("version", false, "Print version and exit.") @@ -105,6 +112,11 @@ func main() { if err != nil { serverboot.Fatal(ctx, "Failed to create ate client", err) } + relay, err := newGRPCRelay(*clientBundle, *trustBundle, *ateletID) + if err != nil { + serverboot.Fatal(ctx, "Failed to build the relay client", err) + } + defer relay.Close() factory := ateinformers.NewSharedInformerFactory(ac, 0) poolInformer := factory.Api().V1alpha1().WorkerPools() @@ -123,12 +135,15 @@ func main() { serverboot.Fatal(ctx, "Failed to watch WorkerPools", err) } cl := &kubeCluster{ - kc: kc, - ac: ac, - pools: poolInformer.Lister(), - nodeLabel: *nodeLabel, - } - c := newController(control, cl, *run, *concurrency) + kc: kc, + ac: ac, + pools: poolInformer.Lister(), + nodeLabel: *nodeLabel, + ateletNS: *ateletNS, + ateletLabel: *ateletLabel, + relayPort: *relayPort, + } + c := newController(control, relay, cl, *run, *concurrency, *drainGrace) if err := c.adopt(ctx); err != nil { serverboot.Fatal(ctx, "Failed to load registered fake Workers", err) @@ -166,10 +181,13 @@ func main() { // kubeCluster is the controller's view of Kubernetes. type kubeCluster struct { - kc kubernetes.Interface - ac versioned.Interface - pools atelisters.WorkerPoolLister - nodeLabel string + kc kubernetes.Interface + ac versioned.Interface + pools atelisters.WorkerPoolLister + nodeLabel string + ateletNS string + ateletLabel string + relayPort int } func (k *kubeCluster) Pools() ([]*atev1alpha1.WorkerPool, error) { @@ -183,12 +201,27 @@ func (k *kubeCluster) Nodes(ctx context.Context) ([]node, error) { } nodes := make([]node, 0, len(list.Items)) for _, n := range list.Items { - nodes = append(nodes, node{name: n.Name}) + nodes = append(nodes, node{name: n.Name, allocatable: n.Status.Allocatable}) } slices.SortFunc(nodes, func(a, b node) int { return cmp.Compare(a.name, b.name) }) return nodes, nil } +func (k *kubeCluster) Relays(ctx context.Context) (map[string]string, error) { + pods, err := k.kc.CoreV1().Pods(k.ateletNS).List(ctx, metav1.ListOptions{LabelSelector: k.ateletLabel}) + if err != nil { + return nil, err + } + out := map[string]string{} + for _, p := range pods.Items { + if p.Status.Phase != corev1.PodRunning || p.Status.PodIP == "" || p.DeletionTimestamp != nil { + continue + } + out[p.Spec.NodeName] = net.JoinHostPort(p.Status.PodIP, strconv.Itoa(k.relayPort)) + } + return out, nil +} + func (k *kubeCluster) UpdateStatus(ctx context.Context, wp *atev1alpha1.WorkerPool) error { _, err := k.ac.ApiV1alpha1().WorkerPools(wp.Namespace).UpdateStatus(ctx, wp, metav1.UpdateOptions{}) return err diff --git a/cmd/benchmarking/isolate/fake-workersync/relayclient.go b/cmd/benchmarking/isolate/fake-workersync/relayclient.go new file mode 100644 index 0000000000..b53703bfa1 --- /dev/null +++ b/cmd/benchmarking/isolate/fake-workersync/relayclient.go @@ -0,0 +1,141 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "sync" + "time" + + "github.com/agent-substrate/substrate/cmd/benchmarking/isolate/internal/fakeworker" + "github.com/agent-substrate/substrate/internal/credbundle" + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" +) + +// reportTimeout bounds one capacity report, relay hop included. +const reportTimeout = 10 * time.Second + +// grpcRelay sends capacity reports to the fake-atelet relay on a Worker's +// node, as ateom sends them to the atelet on its own node. A rejected report +// carries ate-api-server's status unchanged. +type grpcRelay struct { + creds credentials.TransportCredentials + + mu sync.Mutex + // conns holds one connection per relay address. There is one relay per + // fake node, so the map stays small; a connection to a relay that is gone + // goes idle and holds nothing open. + conns map[string]*grpc.ClientConn +} + +// newGRPCRelay presents this pod's certificate and accepts only a server +// whose certificate chains to the pod-identity trust bundle and carries +// atelet's SPIFFE ID. The server is dialed by pod IP and its certificate has +// no DNS or IP SAN, so the chain is verified here rather than by hostname. +// The trust bundle is read at each handshake, so a CA rotation is picked up +// without a restart. +func newGRPCRelay(clientBundlePath, trustBundlePath, ateletID string) (*grpcRelay, error) { + loadRoots := credbundle.PoolLoader(trustBundlePath) + if _, err := loadRoots(); err != nil { + return nil, fmt.Errorf("load trust bundle %s: %w", trustBundlePath, err) + } + cfg := &tls.Config{ + MinVersion: tls.VersionTLS13, + GetClientCertificate: credbundle.ClientLoader(clientBundlePath), + InsecureSkipVerify: true, + VerifyConnection: func(cs tls.ConnectionState) error { + if len(cs.PeerCertificates) == 0 { + return errors.New("relay presented no certificate") + } + roots, err := loadRoots() + if err != nil { + return err + } + intermediates := x509.NewCertPool() + for _, c := range cs.PeerCertificates[1:] { + intermediates.AddCert(c) + } + if _, err := cs.PeerCertificates[0].Verify(x509.VerifyOptions{ + Roots: roots, + Intermediates: intermediates, + KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageAny}, + }); err != nil { + return fmt.Errorf("verifying relay certificate: %w", err) + } + return fakeworker.VerifyPeerID(cs, ateletID) + }, + } + return &grpcRelay{creds: credentials.NewTLS(cfg), conns: map[string]*grpc.ClientConn{}}, nil +} + +// Report sends req to the relay at addr (host:port). +func (g *grpcRelay) Report(ctx context.Context, addr string, req *ateapipb.RegisterWorkerRequest) error { + conn, err := g.conn(addr) + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(ctx, reportTimeout) + defer cancel() + _, err = ateapipb.NewWorkerServiceClient(conn).RegisterWorker(ctx, req) + return err +} + +func (g *grpcRelay) conn(addr string) (*grpc.ClientConn, error) { + g.mu.Lock() + defer g.mu.Unlock() + if c, ok := g.conns[addr]; ok { + return c, nil + } + c, err := grpc.NewClient("passthrough:///"+addr, grpc.WithTransportCredentials(g.creds)) + if err != nil { + return nil, fmt.Errorf("dialing relay %s: %w", addr, err) + } + g.conns[addr] = c + return c, nil +} + +// Prune closes the connections to relays not in live, the relay addresses by +// node from the latest pod list, so a fake-atelet that restarted under a new +// IP leaves no connection behind. +func (g *grpcRelay) Prune(live map[string]string) { + keep := map[string]bool{} + for _, addr := range live { + keep[addr] = true + } + g.mu.Lock() + defer g.mu.Unlock() + for addr, c := range g.conns { + if !keep[addr] { + _ = c.Close() + delete(g.conns, addr) + } + } +} + +// Close closes every relay connection. +func (g *grpcRelay) Close() { + g.mu.Lock() + defer g.mu.Unlock() + for addr, c := range g.conns { + _ = c.Close() + delete(g.conns, addr) + } +} diff --git a/cmd/benchmarking/isolate/fake-workersync/relayclient_test.go b/cmd/benchmarking/isolate/fake-workersync/relayclient_test.go new file mode 100644 index 0000000000..72c17a79e0 --- /dev/null +++ b/cmd/benchmarking/isolate/fake-workersync/relayclient_test.go @@ -0,0 +1,199 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "crypto/tls" + "crypto/x509" + "maps" + "net" + "os" + "slices" + "strings" + "sync" + "testing" + + "github.com/agent-substrate/substrate/cmd/benchmarking/isolate/internal/fakeworker" + "github.com/agent-substrate/substrate/cmd/benchmarking/isolate/internal/fakeworker/fakeworkertest" + "github.com/agent-substrate/substrate/internal/credbundle" + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" +) + +const ( + testAteletID = "spiffe://cluster.local/ns/ate-system/sa/atelet" + testWorkerSyncID = "spiffe://cluster.local/ns/benchmark-workloads/sa/fake-workersync" +) + +// stubRelay is a stand-in fake-atelet relay: it records each report and +// answers with err. +type stubRelay struct { + ateapipb.UnimplementedWorkerServiceServer + + mu sync.Mutex + got []*ateapipb.RegisterWorkerRequest + err error +} + +func (s *stubRelay) RegisterWorker(_ context.Context, in *ateapipb.RegisterWorkerRequest) (*ateapipb.RegisterWorkerResponse, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.got = append(s.got, in) + if s.err != nil { + return nil, s.err + } + return &ateapipb.RegisterWorkerResponse{}, nil +} + +// relayServer serves svc with a certificate for serverID from ca, requiring +// fake-workersync's certificate from ca, and returns its address. +func relayServer(t *testing.T, ca *fakeworkertest.CA, serverID string, svc ateapipb.WorkerServiceServer) string { + t.Helper() + return relayServerTrusting(t, ca, ca, serverID, svc) +} + +// relayServerTrusting is relayServer with its certificate from ca and +// fake-workersync's required to chain to clientCA. +func relayServerTrusting(t *testing.T, ca, clientCA *fakeworkertest.CA, serverID string, svc ateapipb.WorkerServiceServer) string { + t.Helper() + pem, err := os.ReadFile(clientCA.TrustBundle) + if err != nil { + t.Fatal(err) + } + clientCAs := x509.NewCertPool() + clientCAs.AppendCertsFromPEM(pem) + lis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + srv := grpc.NewServer(grpc.Creds(credentials.NewTLS(&tls.Config{ + GetCertificate: credbundle.Loader(ca.Issue(t, serverID)), + ClientAuth: tls.RequireAndVerifyClientCert, + ClientCAs: clientCAs, + VerifyConnection: func(cs tls.ConnectionState) error { + return fakeworker.VerifyPeerID(cs, testWorkerSyncID) + }, + }))) + ateapipb.RegisterWorkerServiceServer(srv, svc) + go func() { _ = srv.Serve(lis) }() + t.Cleanup(srv.Stop) + return lis.Addr().String() +} + +func newTestRelay(t *testing.T, ca *fakeworkertest.CA) *grpcRelay { + t.Helper() + relay, err := newGRPCRelay(ca.Issue(t, testWorkerSyncID), ca.TrustBundle, testAteletID) + if err != nil { + t.Fatal(err) + } + t.Cleanup(relay.Close) + return relay +} + +func capacityReport() *ateapipb.RegisterWorkerRequest { + return &ateapipb.RegisterWorkerRequest{ + Worker: &ateapipb.ObjectRef{Name: "fake-r1-c52b9bd3-0"}, + Capacity: &ateapipb.WorkerResources{Actors: 1000}, + } +} + +func TestRelayClientSendsTheReport(t *testing.T) { + ca := fakeworkertest.NewCA(t) + svc := &stubRelay{} + addr := relayServer(t, ca, testAteletID, svc) + relay := newTestRelay(t, ca) + for range 2 { + if err := relay.Report(context.Background(), addr, capacityReport()); err != nil { + t.Fatalf("Report: %v", err) + } + } + if len(svc.got) != 2 || !proto.Equal(svc.got[0], capacityReport()) { + t.Errorf("relay received %v, want %v twice", svc.got, capacityReport()) + } + if len(relay.conns) != 1 { + t.Errorf("%d connections to one relay, want 1", len(relay.conns)) + } +} + +// fake-workersync reports only to the atelet identity: a report sent anywhere +// else would hand a Worker's capacity to a workload that is not its atelet. +func TestRelayClientRefusesAServerThatIsNotAtelet(t *testing.T) { + ca := fakeworkertest.NewCA(t) + relay := newTestRelay(t, ca) + impostor := &stubRelay{} + if err := relay.Report(context.Background(), relayServer(t, ca, "spiffe://cluster.local/ns/benchmarking/sa/default", impostor), capacityReport()); err == nil { + t.Error("reported to a server with another workload's identity") + } + other := fakeworkertest.NewCA(t) + untrusted := &stubRelay{} + if err := relay.Report(context.Background(), relayServer(t, other, testAteletID, untrusted), capacityReport()); err == nil { + t.Error("reported to a server whose certificate chains to an untrusted signer") + } + if len(impostor.got)+len(untrusted.got) != 0 { + t.Error("a refused server received the report") + } +} + +// A pod-identity CA rotation during a long run must not cut the relays off: +// the client reads the trust bundle at each handshake. +func TestRelayClientFollowsATrustBundleRotation(t *testing.T) { + ca := fakeworkertest.NewCA(t) + relay := newTestRelay(t, ca) + next := fakeworkertest.NewCA(t) + // The relay is certified by the new CA and still admits fake-workersync's + // certificate from the old one. + addr := relayServerTrusting(t, next, ca, testAteletID, &stubRelay{}) + fakeworkertest.Rotate(t, ca.TrustBundle, next) + if err := relay.Report(context.Background(), addr, capacityReport()); err != nil { + t.Errorf("Report to a relay certified by the rotated-in CA: %v", err) + } +} + +// A relay gone from the pod list, as when its fake-atelet restarts under a new +// IP, has its connection closed; the next report to it dials afresh. +func TestRelayClientPrunesGoneRelays(t *testing.T) { + ca := fakeworkertest.NewCA(t) + kept := relayServer(t, ca, testAteletID, &stubRelay{}) + gone := relayServer(t, ca, testAteletID, &stubRelay{}) + relay := newTestRelay(t, ca) + for _, addr := range []string{kept, gone} { + if err := relay.Report(context.Background(), addr, capacityReport()); err != nil { + t.Fatalf("Report to %s: %v", addr, err) + } + } + relay.Prune(map[string]string{"node-a": kept}) + if _, ok := relay.conns[gone]; ok || len(relay.conns) != 1 { + t.Errorf("connections after pruning = %v, want only %s", slices.Collect(maps.Keys(relay.conns)), kept) + } + if err := relay.Report(context.Background(), gone, capacityReport()); err != nil { + t.Errorf("Report after pruning: %v", err) + } +} + +// The controller tells a Worker ate-api-server has not seen yet from a +// rejected report by the status code, so the relay's must arrive intact. +func TestRelayClientCarriesAteAPIServersStatus(t *testing.T) { + ca := fakeworkertest.NewCA(t) + addr := relayServer(t, ca, testAteletID, &stubRelay{err: status.Error(codes.NotFound, "Worker fake-r1-c52b9bd3-0 not found")}) + err := newTestRelay(t, ca).Report(context.Background(), addr, capacityReport()) + if status.Code(err) != codes.NotFound || !strings.Contains(status.Convert(err).Message(), "not found") { + t.Errorf("Report = %v, want NotFound with the server's message", err) + } +} diff --git a/cmd/benchmarking/isolate/internal/fakeworker/fakeworker.go b/cmd/benchmarking/isolate/internal/fakeworker/fakeworker.go index 54b5432128..c1244940de 100644 --- a/cmd/benchmarking/isolate/internal/fakeworker/fakeworker.go +++ b/cmd/benchmarking/isolate/internal/fakeworker/fakeworker.go @@ -13,33 +13,42 @@ // limitations under the License. // Package fakeworker holds what the two programs of the benchmark fake data -// plane share: how a fake Worker is named and placed. +// plane share: how a fake Worker's stand-in pod is named and placed, and how each checks the +// other's identity on the capacity relay between them. package fakeworker import ( "crypto/sha256" + "crypto/tls" "encoding/hex" + "errors" "fmt" "regexp" "strconv" "strings" - - "github.com/google/uuid" ) // MaxRunLength bounds the run prefix so every generated name stays well inside -// the 63-character limit on Worker names. +// the 63-character limit on pod names. const MaxRunLength = 8 +// DefaultMaxActors is how many actors a fake Worker holds, the default of +// ateom's --max-actors, which no WorkerPool field sets for a real worker. +const DefaultMaxActors = 1000 + +// MaxActorsAnnotation, on a WorkerPool, overrides DefaultMaxActors for that +// pool's fake Workers. +const MaxActorsAnnotation = "ate.dev/fake-max-actors" + // WorkerPoolLabel is the label key the WorkerPool controller selects a pool's // worker pods by; a fake pool's status.selector names it the same way. const WorkerPoolLabel = "ate.dev/worker-pool" -var runPattern = regexp.MustCompile(`^[a-z0-9]+$`) +// DefaultRelayPort is the default of --relay-port on both fake-atelet, which +// serves capacity reports on it, and fake-workersync, which sends them there. +const DefaultRelayPort = 8086 -// podUIDNamespace is the UUIDv5 namespace fake WorkerPodUids are derived in. -// Changing it renames every fake Worker's pod UID. -var podUIDNamespace = uuid.MustParse("6f1c2a5e-8b3d-4e7a-9c01-5d2b7e4f8a90") +var runPattern = regexp.MustCompile(`^[a-z0-9]+$`) // ValidateRun reports whether run is usable as a run prefix. func ValidateRun(run string) error { @@ -49,20 +58,21 @@ func ValidateRun(run string) error { return nil } -// Prefix is the name prefix every fake Worker of run shares. +// Prefix is the pod name prefix every fake Worker of run shares. func Prefix(run string) string { return "fake-" + run + "-" } -// Name returns the name of the index-th fake Worker of a WorkerPool. The pool -// enters as a hash because its namespace and name together can exceed the -// 63-character limit on Worker names. +// Name returns the pod name of the index-th fake Worker of a WorkerPool, the +// same for every Worker that fills that index. The pool enters as a hash +// because its namespace and name together can exceed the 63-character limit +// on pod names. func Name(run, namespace, pool string, index int) string { return poolPrefix(run, namespace, pool) + strconv.Itoa(index) } -// Index returns the index of the fake Worker named name in a WorkerPool, and -// false when the name is not one of that pool's fake Workers. +// Index returns the index of the fake Worker whose pod is named name in a +// WorkerPool, and false when the name is not one of that pool's fake pods. func Index(run, namespace, pool, name string) (int, bool) { rest, ok := strings.CutPrefix(name, poolPrefix(run, namespace, pool)) if !ok { @@ -86,14 +96,23 @@ func Node(index int, nodes []string) string { return nodes[index%len(nodes)] } -// PodUID returns the WorkerPodUid recorded for the Worker named name. -func PodUID(name string) string { - return uuid.NewSHA1(podUIDNamespace, []byte(name)).String() -} - // IP returns the address recorded for the index-th fake Worker of a pool: a // documentation-range address that nothing answers on. Addresses repeat past // 254 Workers, which ate-api-server allows. func IP(index int) string { return "192.0.2." + strconv.Itoa(index%254+1) } + +// VerifyPeerID checks that the peer's leaf certificate carries the SPIFFE ID +// want. It complements the chain verification done against the pod-identity +// trust bundle: the chain proves the signer, this proves which workload. +func VerifyPeerID(cs tls.ConnectionState, want string) error { + if len(cs.PeerCertificates) == 0 { + return errors.New("peer presented no certificate") + } + leaf := cs.PeerCertificates[0] + if len(leaf.URIs) == 0 || leaf.URIs[0].String() != want { + return fmt.Errorf("peer SPIFFE ID %v is not %q", leaf.URIs, want) + } + return nil +} diff --git a/cmd/benchmarking/isolate/internal/fakeworker/fakeworker_test.go b/cmd/benchmarking/isolate/internal/fakeworker/fakeworker_test.go index 717b5bd4cf..bc00b4a3f6 100644 --- a/cmd/benchmarking/isolate/internal/fakeworker/fakeworker_test.go +++ b/cmd/benchmarking/isolate/internal/fakeworker/fakeworker_test.go @@ -15,27 +15,24 @@ package fakeworker import ( + "crypto/tls" + "crypto/x509" "net/netip" + "net/url" "regexp" "strings" "testing" - - "github.com/google/uuid" ) -// Names are pinned: a change renames every fake Worker, which strands the -// ones a previous fake-workersync registered. +// Names are pinned: a change renames every fake Worker's pod, which strands +// the Workers a previous fake-workersync registered. func TestNamePinned(t *testing.T) { if got, want := Name("r1", "benchmark-workloads", "benchmark-ateom", 0), "fake-r1-c52b9bd3-0"; got != want { t.Errorf("Name = %q, want %q", got, want) } - if got, want := PodUID("fake-r1-c52b9bd3-0"), "a44150bd-557f-5e1f-a1fa-7d6adbf63611"; got != want { - t.Errorf("PodUID = %q, want %q", got, want) - } } -// k8sShortName is the k8s-short-name format Worker names are validated -// against. +// k8sShortName is the k8s-short-name format pod names follow. var k8sShortName = regexp.MustCompile(`^[a-z0-9]([-a-z0-9]*[a-z0-9])?$`) func TestNameIsAShortNameAtTheBounds(t *testing.T) { @@ -85,19 +82,6 @@ func TestNodeRoundRobin(t *testing.T) { } } -func TestPodUIDIsAStableUUID(t *testing.T) { - a, b := PodUID("fake-r1-00000000-0"), PodUID("fake-r1-00000000-0") - if a != b { - t.Errorf("PodUID is not deterministic: %q then %q", a, b) - } - if _, err := uuid.Parse(a); err != nil { - t.Errorf("PodUID = %q, not a UUID: %v", a, err) - } - if PodUID("fake-r1-00000000-1") == a { - t.Error("two names share a pod UID") - } -} - func TestIPInDocumentationRange(t *testing.T) { doc := netip.MustParsePrefix("192.0.2.0/24") for _, i := range []int{0, 253, 254, 1000} { @@ -123,3 +107,26 @@ func TestValidateRun(t *testing.T) { } } } + +func TestVerifyPeerID(t *testing.T) { + const want = "spiffe://cluster.local/ns/benchmark-workloads/sa/fake-workersync" + state := func(path string) tls.ConnectionState { + cert := &x509.Certificate{} + if path != "" { + cert.URIs = []*url.URL{{Scheme: "spiffe", Host: "cluster.local", Path: path}} + } + return tls.ConnectionState{PeerCertificates: []*x509.Certificate{cert}} + } + if err := VerifyPeerID(state("/ns/benchmark-workloads/sa/fake-workersync"), want); err != nil { + t.Errorf("VerifyPeerID(matching) = %v", err) + } + if err := VerifyPeerID(state("/ns/ate-system/sa/atelet"), want); err == nil { + t.Error("VerifyPeerID accepted another workload") + } + if err := VerifyPeerID(state(""), want); err == nil { + t.Error("VerifyPeerID accepted a certificate with no SPIFFE ID") + } + if err := VerifyPeerID(tls.ConnectionState{}, want); err == nil { + t.Error("VerifyPeerID accepted no certificate") + } +} diff --git a/cmd/benchmarking/isolate/internal/fakeworker/fakeworkertest/fakeworkertest.go b/cmd/benchmarking/isolate/internal/fakeworker/fakeworkertest/fakeworkertest.go new file mode 100644 index 0000000000..6b1365bdd5 --- /dev/null +++ b/cmd/benchmarking/isolate/internal/fakeworker/fakeworkertest/fakeworkertest.go @@ -0,0 +1,121 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package fakeworkertest issues pod-identity-shaped certificates for tests of +// the fake data plane's capacity relay: a throwaway CA, and leaf certificates +// that carry a SPIFFE ID as their only SAN, as the pod-identity signer's do. +package fakeworkertest + +// TODO: Move this to internal/credbundle/credbundletest. Nothing here is +// specific to fake Workers: it writes the files internal/credbundle loads. + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "encoding/pem" + "math/big" + "net/url" + "os" + "path/filepath" + "testing" + "time" + + "github.com/agent-substrate/substrate/internal/localca" +) + +// CA is a throwaway signer whose files live in a test's temp dir. +type CA struct { + pool *localca.ConcretePool + dir string + // TrustBundle is the path of the CA certificate as a PEM trust bundle. + TrustBundle string +} + +// NewCA creates a CA and writes its trust bundle. +func NewCA(t *testing.T) *CA { + t.Helper() + root, err := localca.GenerateCA("fakeworkertest", localca.KeyTypeECDSAP256, time.Hour) + if err != nil { + t.Fatal(err) + } + dir := t.TempDir() + ca := &CA{pool: &localca.ConcretePool{CAs: []*localca.CA{root}}, dir: dir, TrustBundle: filepath.Join(dir, "trust-bundle.pem")} + writePEM(t, ca.TrustBundle, pem.Block{Type: "CERTIFICATE", Bytes: root.RootCertificate.Raw}) + return ca +} + +// Issue writes a credential bundle (leaf certificate, then its PKCS #8 key) +// for spiffeID and returns its path. +func (ca *CA) Issue(t *testing.T, spiffeID string) string { + t.Helper() + id, err := url.Parse(spiffeID) + if err != nil { + t.Fatal(err) + } + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + serial, err := rand.Int(rand.Reader, big.NewInt(1<<62)) + if err != nil { + t.Fatal(err) + } + tmpl := &x509.Certificate{ + SerialNumber: serial, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + URIs: []*url.URL{id}, + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, + } + chain, err := ca.pool.CreateCertificate(tmpl, &key.PublicKey) + if err != nil { + t.Fatal(err) + } + pkcs8, err := x509.MarshalPKCS8PrivateKey(key) + if err != nil { + t.Fatal(err) + } + path := filepath.Join(ca.dir, serial.String()+"-bundle.pem") + writePEM(t, path, pem.Block{Type: "CERTIFICATE", Bytes: chain[0]}, pem.Block{Type: "PRIVATE KEY", Bytes: pkcs8}) + return path +} + +// Rotate replaces the trust bundle at path with the certificates of cas, as +// kubelet rotates a projected trust bundle: a new file renamed over the old. +func Rotate(t *testing.T, path string, cas ...*CA) { + t.Helper() + var blocks []pem.Block + for _, ca := range cas { + blocks = append(blocks, pem.Block{Type: "CERTIFICATE", Bytes: ca.pool.CAs[0].RootCertificate.Raw}) + } + tmp := path + ".new" + writePEM(t, tmp, blocks...) + if err := os.Rename(tmp, path); err != nil { + t.Fatal(err) + } +} + +func writePEM(t *testing.T, path string, blocks ...pem.Block) { + t.Helper() + var out []byte + for _, b := range blocks { + out = append(out, pem.EncodeToMemory(&b)...) + } + if err := os.WriteFile(path, out, 0o600); err != nil { + t.Fatal(err) + } +} From 760e577a38e66c0de51d05b3dad9a4db7dd0ff51 Mon Sep 17 00:00:00 2001 From: Tim Bai Date: Thu, 8 Oct 2026 15:31:52 +0000 Subject: [PATCH 04/44] ateom-gvisor: emit actor usage records (#2194) Part of #1748, split out of #1984: the gVisor ateom emits the actor usage records, using #2005's building blocks. #1984 keeps the atelet side. - **Sampling.** The ateom samples its actors every `--usage-sample-interval` (default 1m) and writes one `ate.actor.usage_sampled` record per actor per sample. - **Order.** Per activation: `initial` when it starts, then `periodic`, then `final` after a checkpoint or terminate succeeds. The final reading is taken before the snapshot, because a Full checkpoint stops the sandbox's cgroup. - **Discovery read.** `GetActiveWorkloadStats` serves the latest sample, so atelet's poll reads no cgroup. - **Re-host.** A read goes through the activation's own state, so a re-host during it cannot mix two activations' numbers. **Until #1984 lands**, atelet still writes its own stdout usage record, so each gVisor actor has two usage records, with different messages and keys. **Tested:** unit tests. The combined change was tested on ate-dev earlier, with `initial`, `periodic`, and `final` per epoch for both checkpoint and terminate. --- cmd/ateom-gvisor/hosted.go | 8 +- cmd/ateom-gvisor/main.go | 52 ++++++- cmd/ateom-gvisor/stats.go | 122 +++++++++++++---- cmd/ateom-gvisor/stats_test.go | 100 ++++++++------ cmd/ateom-gvisor/usage_test.go | 241 +++++++++++++++++++++++++++++++++ 5 files changed, 448 insertions(+), 75 deletions(-) create mode 100644 cmd/ateom-gvisor/usage_test.go diff --git a/cmd/ateom-gvisor/hosted.go b/cmd/ateom-gvisor/hosted.go index 43c5edb958..890763ab9d 100644 --- a/cmd/ateom-gvisor/hosted.go +++ b/cmd/ateom-gvisor/hosted.go @@ -21,9 +21,11 @@ import ( "fmt" "log/slog" "net" + "time" "github.com/agent-substrate/substrate/internal/apierror" "github.com/agent-substrate/substrate/internal/ateomnet" + "github.com/agent-substrate/substrate/internal/ateomstats" "github.com/agent-substrate/substrate/internal/atunnel" "github.com/agent-substrate/substrate/internal/proto/ateompb" "github.com/agent-substrate/substrate/internal/resources" @@ -38,6 +40,8 @@ type hostedActor struct { resolvConf string // Set once the containers exist. session *workloadSession + // usage is this activation's epoch, CPU baseline, and latest sample. + usage *ateomstats.Activation } // admitActor reserves capacity before network setup. An actor that is already @@ -51,7 +55,9 @@ func (s *AteomService) admitActor(attribution resources.ActorAttribution) (*host } else if len(s.actors)+s.draining >= s.maxActors { return nil, nil, apierror.ResourceExhausted("worker is full: %d actors", s.maxActors) } - hosted := &hostedActor{attribution: attribution} + // The sandbox cgroup is new with every activation, so its counters start + // at zero. + hosted := &hostedActor{attribution: attribution, usage: ateomstats.NewActivation(time.Now(), false)} s.actors[attribution.UID] = hosted return hosted, stale, nil } diff --git a/cmd/ateom-gvisor/main.go b/cmd/ateom-gvisor/main.go index 6101c34b57..ca92ec8374 100644 --- a/cmd/ateom-gvisor/main.go +++ b/cmd/ateom-gvisor/main.go @@ -67,6 +67,7 @@ var ( readinessListenAddress = pflag.String("readiness-listen-address", "0.0.0.0:8080", "Address for HTTP readiness checks") maxActors = pflag.Int("max-actors", 1000, "How many actors this worker will host at once") + usageSampleInterval = pflag.Duration("usage-sample-interval", time.Minute, "How often the ateom samples the resource usage of its actors. Each sample writes an ate.actor.usage_sampled record per actor, and GetActiveWorkloadStats serves the latest.") showVersion = pflag.Bool("version", false, "Print version and exit.") logLevelFlag = pflag.String("log-level", "info", "Minimum log level: debug, info, warn, or error.") @@ -78,6 +79,10 @@ var ( reaper = childreap.New() ) +// minUsageSampleInterval is the floor of --usage-sample-interval, the same as +// atelet's poll interval floor. +const minUsageSampleInterval = 50 * time.Second + // workloadGracePeriod is the whole budget for draining the worker on shutdown. // It needs to stay significantly less than the K8s termination grace period // for the ateom, so the escalation to SIGKILL happens here rather than as a @@ -116,6 +121,9 @@ func do(ctx context.Context) error { if *maxActors < 0 { return fmt.Errorf("--max-actors must not be negative, got %d", *maxActors) } + if *usageSampleInterval < minUsageSampleInterval { + return fmt.Errorf("--usage-sample-interval must be at least %v, got %v", minUsageSampleInterval, *usageSampleInterval) + } const serviceName = "ateom-gvisor" // Export through atelet's node-local relay when it is there, so telemetry @@ -211,6 +219,15 @@ func do(ctx context.Context) error { return err } ateomService := NewService(tunnel, actorLogger, *maxActors) + // The controller sets both from the downward API. + pool := ateomstats.Pool{Namespace: os.Getenv("POD_NAMESPACE"), Name: os.Getenv("WORKER_POOL_NAME")} + if pool.Namespace == "" || pool.Name == "" { + slog.WarnContext(ctx, "Worker pool unknown; usage records will name no pool", slog.Any("pool", pool)) + } + usageStdout := ateomstats.NewStdoutHandler(syncedWriter) + defer usageStdout.Close() + ateomService.usage = ateomstats.NewUsageEmitter(lp, usageStdout, pool) + defer ateomstats.StartSampler(ctx, *usageSampleInterval, func(ctx context.Context) { ateomService.sweepUsage(ctx) })() svr := grpc.NewServer( grpc.StatsHandler(otelgrpc.NewServerHandler()), @@ -296,6 +313,8 @@ type AteomService struct { actorLogger *actorlog.ActorLogger tunnel *ateomtunnel.Tunnel + // usage writes the usage records. Nil writes none. + usage *ateomstats.UsageEmitter // shuttingDown is set once SIGTERM has been received. While true, new // workload RPCs are rejected with codes.Unavailable. @@ -385,6 +404,13 @@ func (s *AteomService) gracefulShutdown(ctx context.Context) { return } + // The kill below ends every activation still hosted. Read each while its + // sandbox runs, and write its final record once the containers are gone. + hosted := s.hostedActors() + for _, h := range hosted { + s.readFinal(ctx, h) + } + var wg sync.WaitGroup for _, session := range sessions { for _, name := range session.containers { @@ -398,6 +424,9 @@ func (s *AteomService) gracefulShutdown(ctx context.Context) { } } wg.Wait() + for _, h := range hosted { + s.recordFinal(ctx, h) + } slog.InfoContext(ctx, "Shutting down") } @@ -554,7 +583,8 @@ func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkload return nil, err } // Publish attribution before boot so stats can include startup usage. - if _, err := s.hostActor(ctx, attribution, req.GetActorDirs()); err != nil { + hosted, err := s.hostActor(ctx, attribution, req.GetActorDirs()) + if err != nil { return nil, err } rcmd := &runsc{ @@ -632,6 +662,7 @@ func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkload s.actorLogger.EmitLifecycleLog(ctx, "Actor started", attribution) s.setSession(req.GetActorUid(), &workloadSession{rcmd: rcmd, containers: containerNames(req.GetSpec().GetContainers())}) + s.recordInitial(ctx, hosted) return &ateompb.RunWorkloadResponse{}, nil } @@ -661,6 +692,12 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec attribution := ateomstats.ActorAttributionFromRequest(req) s.actorLogger.EmitLifecycleLog(ctx, "Actor checkpointing", attribution) + // Read before the snapshot: a Full checkpoint stops the sandbox, and its + // cgroup with it. The final record waits for the teardown. + hosted := s.lookupActor(req.GetActorUid()) + if hosted != nil { + s.readFinal(ctx, hosted) + } // Contract with atelet: // @@ -730,6 +767,7 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec slog.String("actorUID", attribution.UID), slog.Any("err", err)) } + s.recordFinalIfEnded(ctx, hosted) // Report exactly the files runsc wrote so atelet ships precisely this set // (checkpoint.img plus any pages images), rather than a hardcoded list. @@ -856,7 +894,8 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore if err != nil { return nil, err } - if _, err := s.hostActor(ctx, attribution, req.GetActorDirs()); err != nil { + hosted, err := s.hostActor(ctx, attribution, req.GetActorDirs()) + if err != nil { return nil, err } rcmd := &runsc{ @@ -965,6 +1004,7 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore s.actorLogger.EmitLifecycleLog(ctx, "Actor restored", attribution) s.setSession(req.GetActorUid(), &workloadSession{rcmd: rcmd, containers: containerNames(req.GetSpec().GetContainers())}) + s.recordInitial(ctx, hosted) return &ateompb.RestoreWorkloadResponse{}, nil } @@ -983,7 +1023,13 @@ func (s *AteomService) TerminateWorkload(ctx context.Context, req *ateompb.Termi attribution := ateomstats.ActorAttributionFromRequest(req) - if err := s.terminateWorkload(ctx, attribution.Ref, attribution.UID, req.GetRunscPath(), req.GetActorDirs(), req.GetSpec().GetContainers()); err != nil { + hosted := s.lookupActor(attribution.UID) + if hosted != nil { + s.readFinal(ctx, hosted) + } + err := s.terminateWorkload(ctx, attribution.Ref, attribution.UID, req.GetRunscPath(), req.GetActorDirs(), req.GetSpec().GetContainers()) + s.recordFinalIfEnded(ctx, hosted) + if err != nil { return nil, fmt.Errorf("failed to terminate workload: %w", err) } diff --git a/cmd/ateom-gvisor/stats.go b/cmd/ateom-gvisor/stats.go index a30538f10b..b8c80285ac 100644 --- a/cmd/ateom-gvisor/stats.go +++ b/cmd/ateom-gvisor/stats.go @@ -26,6 +26,7 @@ import ( "github.com/agent-substrate/substrate/cmd/ateom-gvisor/internal/cgroupstats" "github.com/agent-substrate/substrate/internal/apierror" + "github.com/agent-substrate/substrate/internal/ateattr" "github.com/agent-substrate/substrate/internal/ocispec" "github.com/agent-substrate/substrate/internal/proto/ateompb" "github.com/agent-substrate/substrate/internal/resources" @@ -103,9 +104,7 @@ func (s *AteomService) GetWorkloadStats(ctx context.Context, req *ateompb.GetWor if hosted == nil { return nil, apierror.NotFound("ateom is not executing actor %q", req.GetActorUid()) } - active := &hosted.attribution - - sample, err := s.sampleSandbox(active) + sample, err := s.measure(ctx, hosted) if err != nil { // The requested actor is the active one but its cgroup is not there. // Most often that is a poll landing in the boot: the ateom retains the @@ -121,48 +120,105 @@ func (s *AteomService) GetWorkloadStats(ctx context.Context, req *ateompb.GetWor return nil, apierror.Internal("reading sandbox cgroup: %v", err) } - // The read above holds no lock, so a checkpoint plus a fresh run can land - // underneath it. Pointer identity catches that: hostActor stores a new - // record every time. + // The read above holds no lifecycle lock, so a checkpoint plus a fresh run + // can land underneath it. Pointer identity catches that: hostActor stores + // a new record every time. if s.lookupActor(req.GetActorUid()) != hosted { return nil, apierror.NotFound("ateom stopped executing actor %q while the sample was being taken", req.GetActorUid()) } + // The reading advanced the activation's CPU, so the cache keeps up with it. + hosted.usage.Store(sample) return &ateompb.GetWorkloadStatsResponse{Sample: sample}, nil } -// GetActiveWorkloadStats implements -// ateompb.Ateom/GetActiveWorkloadStats: the discovery read, sampling -// whatever is executing with no identity asserted. Same lock discipline as -// GetWorkloadStats above, for the same reasons. +// GetActiveWorkloadStats implements ateompb.Ateom/GetActiveWorkloadStats: the +// discovery read. It serves each actor's latest sample from the sampler, so a +// poll costs no read of its own; an actor not sampled yet is pending. func (s *AteomService) GetActiveWorkloadStats(ctx context.Context, req *ateompb.GetActiveWorkloadStatsRequest) (*ateompb.GetActiveWorkloadStatsResponse, error) { hosted := s.hostedActors() samples := make([]*ateompb.WorkloadStatsSample, 0, len(hosted)) for _, h := range hosted { - // An actor with no numbers is reported as pending: most often it is - // booting, but a failed read for one must not lose the others either. - sample, err := s.sampleSandbox(&h.attribution) + sample := h.usage.Latest() + if sample == nil { + sample = h.usage.WithEpoch(pendingSample(&h.attribution)) + } + samples = append(samples, sample) + } + + // An empty list is "available", per the proto: a normal answer for a + // scraper to get, not an error. + return &ateompb.GetActiveWorkloadStatsResponse{Samples: samples}, nil +} + +// sweepUsage samples every hosted actor between its initial reading and its +// final record, and stores each sample and writes its periodic record. Same +// lock discipline as GetWorkloadStats, for the same reasons. +func (s *AteomService) sweepUsage(ctx context.Context) { + for _, h := range s.hostedActors() { + if !h.usage.Sampling() { + continue + } + // An actor with no numbers is reported as pending, so a failed read for + // one does not lose the others. Past the initial reading, a missing + // cgroup means the sandbox is going away. + sample, err := s.measure(ctx, h) if err != nil { if !errors.Is(err, fs.ErrNotExist) { slog.WarnContext(ctx, "Failed to read sandbox cgroup", slog.String("actorUID", h.attribution.UID), slog.Any("err", err)) } - sample = pendingSample(&h.attribution) + sample = h.usage.WithEpoch(pendingSample(&h.attribution)) } - // The read holds no lock, and the cgroup is found by UID alone. If the - // actor was re-hosted meanwhile, perhaps on another template, the - // numbers are the new activation's: report it as pending instead. - switch latest := s.lookupActor(h.attribution.UID); { - case latest == nil: + // The read holds no lifecycle lock, and the cgroup is found by UID + // alone. If the actor was re-hosted meanwhile, perhaps on another + // template, the numbers are the new activation's: drop them, and let + // the new activation start with its own initial reading. + if s.lookupActor(h.attribution.UID) != h { continue - case latest != h: - sample = pendingSample(&latest.attribution) } - samples = append(samples, sample) + h.usage.Periodic(sample, func() { s.usage.Emit(ctx, ateattr.StatsKindPeriodic, sample) }) } +} - // An empty list is "available", per the proto: a normal answer for a - // scraper to get, not an error. - return &ateompb.GetActiveWorkloadStatsResponse{Samples: samples}, nil +// recordInitial samples a new activation once its sandbox is up and writes its +// initial record. +func (s *AteomService) recordInitial(ctx context.Context, h *hostedActor) { + sample, err := s.measure(ctx, h) + if err != nil { + slog.WarnContext(ctx, "No initial usage sample", slog.String(string(ateattr.ActorUIDKey), h.attribution.UID), slog.Any("err", err)) + sample = nil + } + h.usage.Initial(sample, func() { s.usage.Emit(ctx, ateattr.StatsKindInitial, sample) }) +} + +// readFinal samples an activation while its sandbox still exists, for the final +// record a successful checkpoint or terminate writes. A failed read leaves the +// newest measured sample to stand in. +func (s *AteomService) readFinal(ctx context.Context, h *hostedActor) { + if sample, err := s.measure(ctx, h); err == nil { + h.usage.Store(sample) + } +} + +// recordFinalIfEnded writes h's final record if a checkpoint or terminate tore +// its activation down, which unhosts the actor even when a later step fails. +// The caller holds the actor's lock. +func (s *AteomService) recordFinalIfEnded(ctx context.Context, h *hostedActor) { + if h != nil && s.lookupActor(h.attribution.UID) != h { + s.recordFinal(ctx, h) + } +} + +// recordFinal writes the final record of an activation that a checkpoint or a +// terminate ended, from its newest measured sample. +func (s *AteomService) recordFinal(ctx context.Context, h *hostedActor) { + pending := h.usage.WithEpoch(pendingSample(&h.attribution)) + h.usage.Final(func(measured *ateompb.WorkloadStatsSample) { + if measured == nil { + measured = pending + } + s.usage.Emit(ctx, ateattr.StatsKindFinal, measured) + }) } // pendingSample is a workload with no numbers to give yet, as the discovery @@ -183,12 +239,20 @@ func pendingSample(active *resources.ActorAttribution) *ateompb.WorkloadStatsSam } } +// measure reads h's sandbox cgroup as a reading of its activation. +func (s *AteomService) measure(ctx context.Context, h *hostedActor) (*ateompb.WorkloadStatsSample, error) { + return h.usage.Measure(ctx, func() (*ateompb.WorkloadStatsSample, map[string]uint64, error) { + sample, err := s.sampleSandbox(&h.attribution) + return sample, nil, err + }) +} + // sampleSandbox reads the sandbox cgroup and builds the sample attributed to // active. Errors come back raw -- notably fs.ErrNotExist for a cgroup that is -// not there yet -- because the two RPCs disagree on what that means: an error -// code for the keyed read, a normal EXECUTING answer for the discovery read. -// The read holds no lock, so the keyed caller re-checks the actor record it -// loaded after this returns. +// not there yet -- because the callers disagree on what that means: an error +// code for the keyed read, a pending sample for the sweep. +// The read holds no lifecycle lock, so the keyed caller re-checks the actor +// record it loaded after this returns. func (s *AteomService) sampleSandbox(active *resources.ActorAttribution) (*ateompb.WorkloadStatsSample, error) { read := s.readSandboxCgroup if read == nil { diff --git a/cmd/ateom-gvisor/stats_test.go b/cmd/ateom-gvisor/stats_test.go index 85ab49e86a..4d0452f46a 100644 --- a/cmd/ateom-gvisor/stats_test.go +++ b/cmd/ateom-gvisor/stats_test.go @@ -25,12 +25,12 @@ import ( "time" "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" "google.golang.org/grpc/codes" "google.golang.org/protobuf/testing/protocmp" "github.com/agent-substrate/substrate/cmd/ateom-gvisor/internal/cgroupstats" "github.com/agent-substrate/substrate/internal/apierror" + "github.com/agent-substrate/substrate/internal/ateomstats" "github.com/agent-substrate/substrate/internal/ocispec" "github.com/agent-substrate/substrate/internal/proto/ateompb" "github.com/agent-substrate/substrate/internal/resources" @@ -47,6 +47,9 @@ var testActor = resources.ActorAttribution{ TemplateName: "template-a", } +// healthyCPUUsec is the usage_usec in healthyCgroup. +const healthyCPUUsec = 1234567 + var healthyCgroup = map[string]string{ "memory.current": "157286400\n", "memory.peak": "209715200\n", @@ -92,7 +95,7 @@ func setHostedActor(s *AteomService, attribution *resources.ActorAttribution) { defer s.actorsMu.Unlock() s.actors = map[string]*hostedActor{} if attribution != nil { - s.actors[attribution.UID] = &hostedActor{attribution: *attribution} + s.actors[attribution.UID] = &hostedActor{attribution: *attribution, usage: testActivation()} } } @@ -239,7 +242,7 @@ func TestGetActiveWorkloadStats(t *testing.T) { s := newStatsService(t, healthyCgroup) setHostedActor(s, &testActor) - got, err := s.GetActiveWorkloadStats(context.Background(), &ateompb.GetActiveWorkloadStatsRequest{}) + got, err := sweepAndList(s) if err != nil { t.Fatalf("GetActiveWorkloadStats() error = %v, want nil", err) } @@ -267,7 +270,7 @@ func TestGetActiveWorkloadStats(t *testing.T) { func TestGetActiveWorkloadStatsAvailable(t *testing.T) { s := newStatsService(t, healthyCgroup) - got, err := s.GetActiveWorkloadStats(context.Background(), &ateompb.GetActiveWorkloadStatsRequest{}) + got, err := sweepAndList(s) if err != nil { t.Fatalf("GetActiveWorkloadStats() on an available ateom: error = %v, want nil", err) } @@ -290,21 +293,19 @@ func pendingFor(attr resources.ActorAttribution) *ateompb.WorkloadStatsSample { } } -// TestGetActiveWorkloadStatsBooting: executing but nothing to measure yet is +// TestGetActiveWorkloadStatsNoCgroup: a hosted actor with no cgroup to read is // a pending entry -- attribution without measurements -- not an error, unlike -// the keyed read's FAILED_PRECONDITION. A blind caller finds boots as -// routinely as idle workers, and the entry keeps a workload that dies during -// boot attributable. -func TestGetActiveWorkloadStatsBooting(t *testing.T) { - s := newStatsService(t, nil) // no cgroup directory: a poll landing mid-boot +// the keyed read's FAILED_PRECONDITION. +func TestGetActiveWorkloadStatsNoCgroup(t *testing.T) { + s := newStatsService(t, nil) // no cgroup directory: the sandbox is gone setHostedActor(s, &testActor) - got, err := s.GetActiveWorkloadStats(context.Background(), &ateompb.GetActiveWorkloadStatsRequest{}) + got, err := sweepAndList(s) if err != nil { - t.Fatalf("GetActiveWorkloadStats() mid-boot: error = %v, want nil", err) + t.Fatalf("GetActiveWorkloadStats() with no cgroup: error = %v, want nil", err) } if len(got.GetSamples()) != 1 { - t.Fatalf("GetActiveWorkloadStats() mid-boot = %v, want one pending entry", got) + t.Fatalf("GetActiveWorkloadStats() with no cgroup = %v, want one pending entry", got) } entry := got.GetSamples()[0] if entry.GetObservedAtUnixNano() == 0 { @@ -332,35 +333,34 @@ func TestGetActiveWorkloadStatsTransition(t *testing.T) { tests := []struct { name string to *resources.ActorAttribution - // want is the expected samples, observed_at zeroed. - want []*ateompb.WorkloadStatsSample }{ // Resumed on another template under the same UID: the numbers belong - // to the new activation, so they are withheld and it is pending. - {name: "re-hosted on another template", to: &otherTemplate, want: []*ateompb.WorkloadStatsSample{pendingFor(otherTemplate)}}, + // to the new activation, which starts with its own initial reading. + {name: "re-hosted on another template", to: &otherTemplate}, // Gone, or replaced by an actor this read did not snapshot. - {name: "to another actor", to: &otherActor, want: nil}, - {name: "to available", to: nil, want: nil}, + {name: "to another actor", to: &otherActor}, + {name: "to available", to: nil}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { s := newStatsService(t, healthyCgroup) + rec := withUsageRecorder(s) setHostedActor(s, &testActor) s.readSandboxCgroup = func(dir string) (cgroupstats.Sample, error) { setHostedActor(s, tc.to) return cgroupstats.Read(dir) } - got, err := s.GetActiveWorkloadStats(context.Background(), &ateompb.GetActiveWorkloadStatsRequest{}) - if err != nil { - t.Fatalf("GetActiveWorkloadStats() during transition: error = %v, want nil", err) - } - for _, sample := range got.GetSamples() { - sample.ObservedAtUnixNano = 0 + s.sweepUsage(context.Background()) + if got := rec.Kinds(); len(got) != 0 { + t.Errorf("records during transition = %v, want none", got) } - if diff := cmp.Diff(tc.want, got.GetSamples(), protocmp.Transform(), cmpopts.EquateEmpty()); diff != "" { - t.Errorf("GetActiveWorkloadStats() during transition mismatch (-want +got):\n%s", diff) + // Nothing read for the old activation is served for the new one. + if tc.to != nil { + if latest := s.lookupActor(tc.to.UID).usage.Latest(); latest != nil { + t.Errorf("new activation serves %v, want nothing until its own sample", latest) + } } }) } @@ -393,12 +393,12 @@ func TestGetActiveWorkloadStatsSeveralActors(t *testing.T) { s.actorsMu.Lock() s.actors = map[string]*hostedActor{ - testActor.UID: {attribution: testActor}, - second.UID: {attribution: second}, + testActor.UID: {attribution: testActor, usage: testActivation()}, + second.UID: {attribution: second, usage: testActivation()}, } s.actorsMu.Unlock() - got, err := s.GetActiveWorkloadStats(context.Background(), &ateompb.GetActiveWorkloadStatsRequest{}) + got, err := sweepAndList(s) if err != nil { t.Fatalf("GetActiveWorkloadStats() error = %v, want nil", err) } @@ -416,22 +416,22 @@ func TestGetActiveWorkloadStatsSeveralActors(t *testing.T) { } } -// One actor booting does not stop the rest being measured: it answers as a -// pending entry alongside their samples. -func TestGetActiveWorkloadStatsOneBooting(t *testing.T) { - booting := testActor - booting.UID = "uid-booting" +// One actor with no cgroup does not stop the rest being measured: it answers as +// a pending entry alongside their samples. +func TestGetActiveWorkloadStatsOneNoCgroup(t *testing.T) { + gone := testActor + gone.UID = "uid-gone" s := newStatsService(t, healthyCgroup) s.actorsMu.Lock() s.actors = map[string]*hostedActor{ - testActor.UID: {attribution: testActor}, - // No cgroup leaf: accepted, but runsc has not created it yet. - booting.UID: {attribution: booting}, + testActor.UID: {attribution: testActor, usage: testActivation()}, + // No cgroup leaf: the sandbox is gone. + gone.UID: {attribution: gone, usage: testActivation()}, } s.actorsMu.Unlock() - got, err := s.GetActiveWorkloadStats(context.Background(), &ateompb.GetActiveWorkloadStatsRequest{}) + got, err := sweepAndList(s) if err != nil { t.Fatalf("GetActiveWorkloadStats() error = %v, want nil", err) } @@ -462,12 +462,12 @@ func TestGetActiveWorkloadStatsOneUnreadable(t *testing.T) { } s.actorsMu.Lock() s.actors = map[string]*hostedActor{ - testActor.UID: {attribution: testActor}, - broken.UID: {attribution: broken}, + testActor.UID: {attribution: testActor, usage: testActivation()}, + broken.UID: {attribution: broken, usage: testActivation()}, } s.actorsMu.Unlock() - got, err := s.GetActiveWorkloadStats(context.Background(), &ateompb.GetActiveWorkloadStatsRequest{}) + got, err := sweepAndList(s) if err != nil { t.Fatalf("GetActiveWorkloadStats() error = %v, want nil", err) } @@ -481,3 +481,19 @@ func TestGetActiveWorkloadStatsOneUnreadable(t *testing.T) { } } } + +// testActivation is a running actor's activation: its initial reading is done, +// so the sweep samples it. It starts at the unix epoch, so a sample's epoch is +// zero and the expected samples here need not name it; the epoch has tests of +// its own. +func testActivation() *ateomstats.Activation { + a := ateomstats.NewActivation(time.Unix(0, 0), false) + a.Initial(nil, nil) + return a +} + +// sweepAndList runs one sampler sweep, then the discovery read that serves it. +func sweepAndList(s *AteomService) (*ateompb.GetActiveWorkloadStatsResponse, error) { + s.sweepUsage(context.Background()) + return s.GetActiveWorkloadStats(context.Background(), &ateompb.GetActiveWorkloadStatsRequest{}) +} diff --git a/cmd/ateom-gvisor/usage_test.go b/cmd/ateom-gvisor/usage_test.go new file mode 100644 index 0000000000..3ae9e39aeb --- /dev/null +++ b/cmd/ateom-gvisor/usage_test.go @@ -0,0 +1,241 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "os" + "path/filepath" + "slices" + "testing" + "time" + + "github.com/agent-substrate/substrate/internal/actorlock" + "github.com/agent-substrate/substrate/internal/ateattr" + "github.com/agent-substrate/substrate/internal/ateomstats" + "github.com/agent-substrate/substrate/internal/ateomstats/ateomstatstest" + "github.com/agent-substrate/substrate/internal/ocispec" + "github.com/agent-substrate/substrate/internal/proto/ateompb" +) + +func withUsageRecorder(s *AteomService) *ateomstatstest.Recorder { + e, rec := ateomstatstest.NewEmitter() + s.usage = e + return rec +} + +// hostWithEpoch hosts testActor; running means its initial reading is done. +func hostWithEpoch(s *AteomService, epoch time.Time, running bool) *hostedActor { + h := &hostedActor{attribution: testActor, usage: ateomstats.NewActivation(epoch, false)} + if running { + h.usage.Initial(nil, nil) + } + s.actorsMu.Lock() + s.actors = map[string]*hostedActor{testActor.UID: h} + s.actorsMu.Unlock() + return h +} + +func TestSweepStampsEpochAndDiscoveryServesIt(t *testing.T) { + s := newStatsService(t, healthyCgroup) + epoch := time.Unix(1700, 0) + h := hostWithEpoch(s, epoch, true) + + before, err := s.GetActiveWorkloadStats(context.Background(), &ateompb.GetActiveWorkloadStatsRequest{}) + if err != nil { + t.Fatal(err) + } + if got := before.GetSamples()[0]; got.GetSource() != ateompb.StatsSource_STATS_SOURCE_UNSPECIFIED || got.GetEpochUnixNano() != epoch.UnixNano() { + t.Errorf("before a sweep: source %v epoch %d, want pending with epoch %d", got.GetSource(), got.GetEpochUnixNano(), epoch.UnixNano()) + } + + s.sweepUsage(context.Background()) + after, err := s.GetActiveWorkloadStats(context.Background(), &ateompb.GetActiveWorkloadStatsRequest{}) + if err != nil { + t.Fatal(err) + } + got := after.GetSamples()[0] + if got != h.usage.Latest() { + t.Error("discovery read did not serve the swept sample") + } + if got.GetEpochUnixNano() != epoch.UnixNano() || got.GetCpuUsageUsec() != healthyCPUUsec { + t.Errorf("served epoch %d cpu %d, want epoch %d cpu %d", got.GetEpochUnixNano(), got.GetCpuUsageUsec(), epoch.UnixNano(), healthyCPUUsec) + } +} + +func TestGetWorkloadStatsStampsEpoch(t *testing.T) { + s := newStatsService(t, healthyCgroup) + epoch := time.Unix(1700, 0) + hostWithEpoch(s, epoch, true) + got, err := s.GetWorkloadStats(context.Background(), &ateompb.GetWorkloadStatsRequest{ActorUid: testActor.UID}) + if err != nil { + t.Fatal(err) + } + if got.GetSample().GetEpochUnixNano() != epoch.UnixNano() { + t.Errorf("epoch = %d, want %d", got.GetSample().GetEpochUnixNano(), epoch.UnixNano()) + } +} + +func TestRecordInitialAndFinal(t *testing.T) { + s := newStatsService(t, healthyCgroup) + rec := withUsageRecorder(s) + h := hostWithEpoch(s, time.Now(), false) + + s.recordInitial(context.Background(), h) + if h.usage.Latest() == nil { + t.Error("initial sample was not stored for the discovery read") + } + s.recordFinal(context.Background(), h) + + want := []string{ateattr.StatsKindInitial, ateattr.StatsKindFinal} + if len(rec.Kinds()) != 2 || rec.Kinds()[0] != want[0] || rec.Kinds()[1] != want[1] { + t.Errorf("record kinds = %v, want %v", rec.Kinds(), want) + } + for _, src := range rec.Sources() { + if src != ateattr.StatsSourceCgroup { + t.Errorf("record source = %q, want measured", src) + } + } +} + +// TestRecordFinalIfEnded pins that the final record waits for the actor to be +// unhosted, as a teardown does even when a later step fails. +func TestRecordFinalIfEnded(t *testing.T) { + s := newStatsService(t, healthyCgroup) + rec := withUsageRecorder(s) + h := hostWithEpoch(s, time.Now(), true) + + s.recordFinalIfEnded(context.Background(), h) + if got := rec.Kinds(); len(got) != 0 { + t.Fatalf("records while hosted = %v, want none", got) + } + setHostedActor(s, nil) + s.recordFinalIfEnded(context.Background(), h) + if got := rec.Kinds(); len(got) != 1 || got[0] != ateattr.StatsKindFinal { + t.Errorf("records after unhosting = %v, want one final", got) + } +} + +// TestGracefulShutdownWritesFinal pins that the drain ends each hosted +// activation with a final record, read before its containers are killed. +func TestGracefulShutdownWritesFinal(t *testing.T) { + s := newStatsService(t, healthyCgroup) + s.inFlight = actorlock.NewInFlight() + rec := withUsageRecorder(s) + h := hostWithEpoch(s, time.Now(), true) + s.actorsMu.Lock() + h.session = &workloadSession{} + s.actorsMu.Unlock() + + s.gracefulShutdown(context.Background()) + if got := rec.Kinds(); len(got) != 1 || got[0] != ateattr.StatsKindFinal { + t.Fatalf("records = %v, want one final", got) + } + if got := rec.Sources(); got[0] != ateattr.StatsSourceCgroup { + t.Errorf("final record source = %q, want measured", got[0]) + } +} + +// TestRecordFinalAfterSandboxGone pins that a final read that finds the cgroup +// gone falls back to the latest sample, and to a pending record when there is +// none. +func TestRecordFinalAfterSandboxGone(t *testing.T) { + s := newStatsService(t, healthyCgroup) + rec := withUsageRecorder(s) + h := hostWithEpoch(s, time.Now(), true) + s.sweepUsage(context.Background()) + if err := os.RemoveAll(filepath.Join(s.cgroupRoot, ocispec.GVisorCgroupLeaf(testActor.UID, sandboxCgroupContainer))); err != nil { + t.Fatal(err) + } + s.readFinal(context.Background(), h) + s.recordFinal(context.Background(), h) + + fresh := hostWithEpoch(s, time.Now(), true) + s.readFinal(context.Background(), fresh) + s.recordFinal(context.Background(), fresh) + + var finals []string + for i, kind := range rec.Kinds() { + if kind == ateattr.StatsKindFinal { + finals = append(finals, rec.Sources()[i]) + } + } + if want := []string{ateattr.StatsSourceCgroup, ateattr.StatsSourceUnspecified}; !slices.Equal(finals, want) { + t.Errorf("final record sources = %v, want %v", finals, want) + } +} + +// TestReadFinalWritesNothingUntilRecorded pins the checkpoint's order: the +// reading taken before the snapshot writes no record on its own, and the final +// record, written once the checkpoint succeeds, carries that reading. +func TestReadFinalWritesNothingUntilRecorded(t *testing.T) { + s := newStatsService(t, healthyCgroup) + rec := withUsageRecorder(s) + h := hostWithEpoch(s, time.Now(), true) + + s.readFinal(context.Background(), h) + if got := rec.Kinds(); len(got) != 0 { + t.Fatalf("records after the pre-snapshot read = %v, want none", got) + } + read := h.usage.Latest() + if read == nil || read.GetSource() != ateompb.StatsSource_STATS_SOURCE_CGROUP { + t.Fatalf("pre-snapshot read stored %v, want a measured sample", read) + } + if err := os.RemoveAll(filepath.Join(s.cgroupRoot, ocispec.GVisorCgroupLeaf(testActor.UID, sandboxCgroupContainer))); err != nil { + t.Fatal(err) + } + + s.recordFinal(context.Background(), h) + if got := rec.Kinds(); len(got) != 1 || got[0] != ateattr.StatsKindFinal { + t.Fatalf("records after the checkpoint = %v, want one final", got) + } + if got := rec.Sources(); got[0] != ateattr.StatsSourceCgroup { + t.Errorf("final record source = %q, want the pre-snapshot reading", got[0]) + } +} + +// TestSweepWaitsForInitialAndStopsAtFinal pins that the sweep reads no actor +// before its initial reading or after its final record. +func TestSweepWaitsForInitialAndStopsAtFinal(t *testing.T) { + s := newStatsService(t, healthyCgroup) + rec := withUsageRecorder(s) + h := hostWithEpoch(s, time.Now(), false) + + s.sweepUsage(context.Background()) + s.recordInitial(context.Background(), h) + s.sweepUsage(context.Background()) + s.recordFinal(context.Background(), h) + s.sweepUsage(context.Background()) + want := []string{ateattr.StatsKindInitial, ateattr.StatsKindPeriodic, ateattr.StatsKindFinal} + if got := rec.Kinds(); !slices.Equal(got, want) { + t.Errorf("records = %v, want %v", got, want) + } +} + +// TestGetWorkloadStatsStoresItsSample pins that the keyed read, which advances +// the activation's CPU, also updates the sample the cache serves. +func TestGetWorkloadStatsStoresItsSample(t *testing.T) { + s := newStatsService(t, healthyCgroup) + h := hostWithEpoch(s, time.Now(), true) + got, err := s.GetWorkloadStats(context.Background(), &ateompb.GetWorkloadStatsRequest{ActorUid: testActor.UID}) + if err != nil { + t.Fatal(err) + } + if h.usage.Latest() != got.GetSample() { + t.Error("the keyed read's sample is not the one the cache serves") + } +} From 09af6d2e01a30d44983a5df0875865b19b028702 Mon Sep 17 00:00:00 2001 From: Tim Bai Date: Thu, 8 Oct 2026 17:13:48 +0000 Subject: [PATCH 05/44] ateom-microvm: emit actor usage records (#2195) Part of #1748, split out of #1984: the micro-VM ateom emits the actor usage records, using #2005's building blocks. #1984 keeps the atelet side. - **Sampling.** The ateom samples its actors every `--usage-sample-interval` (default 1m) and writes one `ate.actor.usage_sampled` record per actor per sample. - **Order.** Per activation: `initial` once the guest answers, then `periodic`, then `final` after a checkpoint or terminate succeeds. The final record reuses the newest sample, so the checkpoint waits on no guest read. - **Discovery read.** `GetActiveWorkloadStats` serves the latest sample, so atelet's poll reads no guest. - **Re-host.** Every guest read goes through the activation's own state, so a re-host during it cannot mix two activations' numbers. - **Behaviour change.** A guest-stats target that names another actor now logs and reports the actor pending instead of failing the discovery RPC. - **Initial read.** It runs off the resume path and recovers from a panic, as a sweep does. **Until #1984 lands**, atelet still writes its own stdout usage record, so each micro-VM actor has two usage records, with different messages and keys. **Tested:** unit tests. The combined change was tested on ate-dev earlier: CPU rises within an epoch and restarts at zero for resumed guests. --- cmd/ateom-microvm/checkpoint.go | 38 ++- cmd/ateom-microvm/hosted.go | 28 ++- cmd/ateom-microvm/hosted_test.go | 6 +- cmd/ateom-microvm/main.go | 25 ++ cmd/ateom-microvm/net_linux_test.go | 2 +- cmd/ateom-microvm/restore.go | 21 +- cmd/ateom-microvm/run.go | 17 +- cmd/ateom-microvm/shutdown.go | 8 +- cmd/ateom-microvm/stats.go | 259 +++++++++++++------ cmd/ateom-microvm/stats_test.go | 86 +++++-- cmd/ateom-microvm/usage_test.go | 377 ++++++++++++++++++++++++++++ docs/metrics/registry/metrics.yaml | 2 +- 12 files changed, 741 insertions(+), 128 deletions(-) create mode 100644 cmd/ateom-microvm/usage_test.go diff --git a/cmd/ateom-microvm/checkpoint.go b/cmd/ateom-microvm/checkpoint.go index 93434fc962..9fc10eff10 100644 --- a/cmd/ateom-microvm/checkpoint.go +++ b/cmd/ateom-microvm/checkpoint.go @@ -124,6 +124,10 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec return nil, apierror.InvalidArgument("unsupported snapshot scope: %v", scope) } + // Captured now: the checkpoint unhosts the actor, and the final record + // waits for the teardown. + hosted := s.lookupActor(actorUID) + // The actor's CH was booted by RunWorkload or relaunched by RestoreWorkload; // either way ateom owns it and tracks its api-socket. ra := s.runningVM(actorUID) @@ -135,6 +139,10 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec if _, err := client.WaitReady(ctx, 10*time.Second); err != nil { return nil, fmt.Errorf("while waiting for CH api-socket: %w", err) } + // Read before the pause: a paused guest cannot answer. + if hosted != nil { + s.readFinal(ctx, hosted) + } tPause := time.Now() dPrep = tPause.Sub(tStart) @@ -230,6 +238,7 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec // rootfs_upper), and the tar durations scale with the actor's data. slog.Duration("durable_dir", dDurable), slog.Duration("rootfs_upper", dUpper), slog.Duration("teardown", dTeardown)) + s.recordFinalIfEnded(ctx, hosted) return &ateompb.CheckpointWorkloadResponse{SnapshotFiles: snapshotFiles, DataSnapshotFiles: durableFiles}, nil } @@ -396,7 +405,13 @@ func (s *AteomService) TerminateWorkload(ctx context.Context, req *ateompb.Termi attribution := ateomstats.ActorAttributionFromRequest(req) - if err := s.terminateWorkload(ctx, attribution, req.GetActorDirs()); err != nil { + hosted := s.lookupActor(attribution.UID) + if hosted != nil { + s.readFinal(ctx, hosted) + } + err := s.terminateWorkload(ctx, attribution, req.GetActorDirs()) + s.recordFinalIfEnded(ctx, hosted) + if err != nil { return nil, fmt.Errorf("failed to terminate workload: %w", err) } @@ -405,6 +420,27 @@ func (s *AteomService) TerminateWorkload(ctx context.Context, req *ateompb.Termi return &ateompb.TerminateWorkloadResponse{}, nil } +// endPreviousActivation ends the activation a re-host is about to replace. A VM +// still running for the actor would be dropped from tracking by the re-host and +// left running, so it is stopped, read first for the final record. This happens +// after a checkpoint that failed before its teardown, or on a retried Run or +// Restore. Ending the activation also stops a late recordInitial from writing. +func (s *AteomService) endPreviousActivation(ctx context.Context, actorUID string, actorDirs *ateompb.ActorDirs) error { + old := s.lookupActor(actorUID) + if s.runningVM(actorUID) != nil { + if old != nil { + s.readFinal(ctx, old) + } + if err := s.stopActorVM(ctx, actorUID, actorDirs); err != nil { + return fmt.Errorf("while stopping the actor's previous micro-VM: %w", err) + } + } + if old != nil { + s.recordFinal(ctx, old) + } + return nil +} + // stopActorVM tears down the actor's micro-VM, if any, keeping it hosted. func (s *AteomService) stopActorVM(ctx context.Context, actorUID string, actorDirs *ateompb.ActorDirs) error { ra := s.runningVM(actorUID) diff --git a/cmd/ateom-microvm/hosted.go b/cmd/ateom-microvm/hosted.go index c531af9c77..97b811d5e1 100644 --- a/cmd/ateom-microvm/hosted.go +++ b/cmd/ateom-microvm/hosted.go @@ -28,6 +28,7 @@ import ( "github.com/agent-substrate/substrate/internal/ateomcgroup" "github.com/agent-substrate/substrate/internal/ateomnet" "github.com/agent-substrate/substrate/internal/ateomnet/netns" + "github.com/agent-substrate/substrate/internal/ateomstats" "github.com/agent-substrate/substrate/internal/atunnel" "github.com/agent-substrate/substrate/internal/resources" "github.com/agent-substrate/substrate/internal/sizing" @@ -44,11 +45,17 @@ type hostedActor struct { vm *runningActor // Nil when no guest stats connection is available. guest *guestStatsTarget + + // usage is this activation's epoch, CPU baseline, and latest sample. + // Immutable after admission. + usage *ateomstats.Activation } // admitActor reserves capacity before network setup. An actor that is already // hosted keeps its slot; its old network is returned for the caller to close. -func (s *AteomService) admitActor(attribution resources.ActorAttribution) (*hostedActor, *ateomnet.SandboxSession, error) { +// resumesGuest is true when the activation resumes a guest from a snapshot, +// whose CPU counters come back with it. +func (s *AteomService) admitActor(attribution resources.ActorAttribution, resumesGuest bool) (*hostedActor, *ateomnet.SandboxSession, error) { s.actorsMu.Lock() defer s.actorsMu.Unlock() var stale *ateomnet.SandboxSession @@ -57,19 +64,20 @@ func (s *AteomService) admitActor(attribution resources.ActorAttribution) (*host } else if len(s.actors)+s.draining >= s.maxActors { return nil, nil, apierror.ResourceExhausted("worker is full: %d actors", s.maxActors) } - hosted := &hostedActor{attribution: attribution} + hosted := &hostedActor{attribution: attribution, usage: ateomstats.NewActivation(time.Now(), resumesGuest)} s.actors[attribution.UID] = hosted return hosted, stale, nil } -// hostActor sets up the actor's network, replacing any stale one. -func (s *AteomService) hostActor(ctx context.Context, attribution resources.ActorAttribution) (*hostedActor, error) { +// hostActor sets up the actor's network, replacing any stale one. resumesGuest +// is as for admitActor. +func (s *AteomService) hostActor(ctx context.Context, attribution resources.ActorAttribution, resumesGuest bool) (*hostedActor, error) { uid := attribution.UID if uid == "" { return nil, fmt.Errorf("actor UID is required") } - hosted, stale, err := s.admitActor(attribution) + hosted, stale, err := s.admitActor(attribution, resumesGuest) if err != nil { return nil, err } @@ -203,15 +211,11 @@ func (s *AteomService) setGuestStats(actorUID string, guest *guestStatsTarget) { } } -// guestStatsFor is the stats target for an actor, or nil. -func (s *AteomService) guestStatsFor(actorUID string) *guestStatsTarget { - hosted := s.lookupActor(actorUID) - if hosted == nil { - return nil - } +// guestOf is h's stats target, or nil. +func (s *AteomService) guestOf(h *hostedActor) *guestStatsTarget { s.actorsMu.RLock() defer s.actorsMu.RUnlock() - return hosted.guest + return h.guest } // sandboxNetNS is where an actor's tap and atunnel's sockets live, or -1. diff --git a/cmd/ateom-microvm/hosted_test.go b/cmd/ateom-microvm/hosted_test.go index b5d0498ffc..7acf915d6d 100644 --- a/cmd/ateom-microvm/hosted_test.go +++ b/cmd/ateom-microvm/hosted_test.go @@ -29,11 +29,11 @@ import ( // on a full worker and a retry cannot lose its place to another actor. func TestAdmitActorKeepsAHostedActorsSlot(t *testing.T) { s := &AteomService{actors: map[string]*hostedActor{}, maxActors: 1} - first, _, err := s.admitActor(resources.ActorAttribution{UID: "actor-a"}) + first, _, err := s.admitActor(resources.ActorAttribution{UID: "actor-a"}, false) if err != nil { t.Fatal(err) } - again, _, err := s.admitActor(resources.ActorAttribution{UID: "actor-a"}) + again, _, err := s.admitActor(resources.ActorAttribution{UID: "actor-a"}, false) if err != nil { t.Fatalf("re-admitting a hosted actor on a full worker: %v", err) } @@ -41,7 +41,7 @@ func TestAdmitActorKeepsAHostedActorsSlot(t *testing.T) { t.Error("re-admission kept the old record; readers could not tell the incarnations apart") } // ResourceExhausted, so the control plane treats it as a capacity miss. - if _, _, err := s.admitActor(resources.ActorAttribution{UID: "actor-b"}); apierror.Code(err) != codes.ResourceExhausted { + if _, _, err := s.admitActor(resources.ActorAttribution{UID: "actor-b"}, false); apierror.Code(err) != codes.ResourceExhausted { t.Errorf("admitting past the ceiling: got %v, want ResourceExhausted", err) } if got := len(s.hostedActors()); got != 1 { diff --git a/cmd/ateom-microvm/main.go b/cmd/ateom-microvm/main.go index 3ca162d244..9e46395802 100644 --- a/cmd/ateom-microvm/main.go +++ b/cmd/ateom-microvm/main.go @@ -33,6 +33,7 @@ import ( "sync" "sync/atomic" "syscall" + "time" "cloud.google.com/go/compute/metadata" "github.com/agent-substrate/substrate/cmd/ateom-microvm/internal/reaper" @@ -43,6 +44,7 @@ import ( "github.com/agent-substrate/substrate/internal/ateom" "github.com/agent-substrate/substrate/internal/ateomcgroup" "github.com/agent-substrate/substrate/internal/ateomnet" + "github.com/agent-substrate/substrate/internal/ateomstats" "github.com/agent-substrate/substrate/internal/ateomtunnel" "github.com/agent-substrate/substrate/internal/nodepath" "github.com/agent-substrate/substrate/internal/otlprelay" @@ -73,8 +75,13 @@ var ( readinessListenAddress = pflag.String("readiness-listen-address", "0.0.0.0:8080", "Address for HTTP readiness checks") maxActors = pflag.Int("max-actors", 1000, "How many actors this worker will host at once") + usageSampleInterval = pflag.Duration("usage-sample-interval", time.Minute, "How often the ateom samples the resource usage of its actors. Each sample writes an ate.actor.usage_sampled record per actor, and GetActiveWorkloadStats serves the latest.") ) +// minUsageSampleInterval is the floor of --usage-sample-interval, the same as +// atelet's poll interval floor. It keeps statsSweepBudget inside one interval. +const minUsageSampleInterval = 50 * time.Second + func main() { pflag.Parse() if *showVersion { @@ -105,6 +112,9 @@ func do(ctx context.Context) error { if *maxActors < 0 { return fmt.Errorf("--max-actors must not be negative, got %d", *maxActors) } + if *usageSampleInterval < minUsageSampleInterval { + return fmt.Errorf("--usage-sample-interval must be at least %v, got %v", minUsageSampleInterval, *usageSampleInterval) + } const serviceName = "ateom-microvm" // Export through atelet's node-local relay when it is there, so telemetry @@ -222,6 +232,15 @@ func do(ctx context.Context) error { } ateomService := NewService(*podUID, *chBinary, *guestDebug, *vmmMemReserve, *maxActors, tunnel, actorLogger) ateomService.actorCgroups = actorCgroups + // The controller sets both from the downward API. + pool := ateomstats.Pool{Namespace: os.Getenv("POD_NAMESPACE"), Name: os.Getenv("WORKER_POOL_NAME")} + if pool.Namespace == "" || pool.Name == "" { + slog.WarnContext(ctx, "Worker pool unknown; usage records will name no pool", slog.Any("pool", pool)) + } + usageStdout := ateomstats.NewStdoutHandler(logWriter) + defer usageStdout.Close() + ateomService.usage = ateomstats.NewUsageEmitter(lp, usageStdout, pool) + defer ateomstats.StartSampler(ctx, *usageSampleInterval, func(ctx context.Context) { ateomService.sweepUsage(ctx) })() svr := grpc.NewServer( grpc.StatsHandler(otelgrpc.NewServerHandler()), @@ -343,9 +362,14 @@ type AteomService struct { // with ateom-gvisor). actorLogger *actorlog.ActorLogger tunnel *ateomtunnel.Tunnel + // usage writes the usage records. Nil writes none. + usage *ateomstats.UsageEmitter // Guards actors, draining, and mutable hostedActor fields. actorsMu sync.RWMutex + // guestSlots bounds the guest reads of the sweeps and initial readings + // together to statsFanOut at once. + guestSlots chan struct{} // Keyed by actor UID. actors map[string]*hostedActor // Actors undergoing network cleanup still count against capacity. @@ -364,6 +388,7 @@ func NewService(podUID, chBinary string, guestDebug bool, memReserveMiB, maxActo locks: actorlock.New(), inFlight: actorlock.NewInFlight(), actors: map[string]*hostedActor{}, + guestSlots: make(chan struct{}, statsFanOut), maxActors: maxActors, podUID: podUID, chBinary: chBinary, diff --git a/cmd/ateom-microvm/net_linux_test.go b/cmd/ateom-microvm/net_linux_test.go index 1a8b41f3e5..5a5fc803e1 100644 --- a/cmd/ateom-microvm/net_linux_test.go +++ b/cmd/ateom-microvm/net_linux_test.go @@ -52,7 +52,7 @@ func TestHostActorReplacesSameActor(t *testing.T) { } }) for range 2 { - if _, err := service.hostActor(ctx, resources.ActorAttribution{UID: actorUID}); err != nil { + if _, err := service.hostActor(ctx, resources.ActorAttribution{UID: actorUID}, false); err != nil { t.Fatal(err) } } diff --git a/cmd/ateom-microvm/restore.go b/cmd/ateom-microvm/restore.go index 0eb191b064..4a326c3fba 100644 --- a/cmd/ateom-microvm/restore.go +++ b/cmd/ateom-microvm/restore.go @@ -142,15 +142,15 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore attribution := p.actorAttribution() s.actorLogger.EmitLifecycleLog(ctx, "Actor restoring", attribution) - // A VM still running for this actor would be dropped from tracking by the - // re-host below and left running, so stop it first. - if s.runningVM(attribution.UID) != nil { - if err := s.stopActorVM(ctx, attribution.UID, req.GetActorDirs()); err != nil { - return nil, fmt.Errorf("while stopping the actor's previous micro-VM: %w", err) - } + if err := s.endPreviousActivation(ctx, attribution.UID, req.GetActorDirs()); err != nil { + return nil, err } - // Publish attribution before restore so stats can include startup usage. - if _, err := s.hostActor(ctx, attribution); err != nil { + // Publish attribution before restore, so a stats read during it is + // attributed. A Data scope cold-boots, so its CPU counts from zero; the other + // scopes resume the guest's counters, so theirs counts from the first + // reading. + resumesGuest := req.GetScope() != ateompb.SnapshotScope_SNAPSHOT_SCOPE_DATA + if _, err := s.hostActor(ctx, attribution, resumesGuest); err != nil { return nil, err } defer func() { @@ -489,6 +489,11 @@ func (s *AteomService) restoreFullScope(ctx context.Context, p actorBootParams, // for the same reason as in coldBootActor. Same client the forwarding above // reads over. s.setGuestStats(actorUID, &guestStatsTarget{actorUID: actorUID, agent: guestAC, workloadIDs: ra.workloadIDs}) + // Looked up while the caller holds the actor's lock, so it is this + // activation. + if hosted := s.lookupActor(actorUID); hosted != nil { + go s.recordInitial(context.WithoutCancel(ctx), hosted) + } slog.InfoContext(ctx, "Actor restored (overlay rootfs)", slog.String("id", actorUID), slog.Duration("total", time.Since(tStart))) diff --git a/cmd/ateom-microvm/run.go b/cmd/ateom-microvm/run.go index 1fcb0edf90..50c8356104 100644 --- a/cmd/ateom-microvm/run.go +++ b/cmd/ateom-microvm/run.go @@ -269,15 +269,11 @@ func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkload attribution := p.actorAttribution() s.actorLogger.EmitLifecycleLog(ctx, "Actor starting", attribution) - // A VM still running for this actor would be dropped from tracking by the - // re-host below and left running, so stop it first. - if s.runningVM(attribution.UID) != nil { - if err := s.stopActorVM(ctx, attribution.UID, req.GetActorDirs()); err != nil { - return nil, fmt.Errorf("while stopping the actor's previous micro-VM: %w", err) - } + if err := s.endPreviousActivation(ctx, attribution.UID, req.GetActorDirs()); err != nil { + return nil, err } // Publish attribution before boot so stats can include startup usage. - if _, err := s.hostActor(ctx, attribution); err != nil { + if _, err := s.hostActor(ctx, attribution, false); err != nil { return nil, err } defer func() { @@ -364,7 +360,7 @@ func (s *AteomService) coldBootActorRetrying(ctx context.Context, p actorBootPar slog.String("id", p.actorUID), slog.Int("attempt", attempt), slog.Any("err", err)) // The failed attempt deactivated egress, which retires the listener // bound to it, so the network is rebuilt. The actor keeps its slot. - if _, err := s.hostActor(ctx, p.attribution()); err != nil { + if _, err := s.hostActor(ctx, p.attribution(), false); err != nil { return err } } @@ -615,6 +611,11 @@ func (s *AteomService) coldBootActor(ctx context.Context, p actorBootParams) (re // the handler polling a connection nobody owns. Same client the forwarding // above reads over — ttrpc multiplexes, and teardownActor ends both. s.setGuestStats(actorUID, &guestStatsTarget{actorUID: actorUID, agent: ac, workloadIDs: workloadIDs}) + // Looked up while the caller holds the actor's lock, so it is this + // activation even if the read below outlives the RPC. + if hosted := s.lookupActor(actorUID); hosted != nil { + go s.recordInitial(context.WithoutCancel(ctx), hosted) + } return nil } diff --git a/cmd/ateom-microvm/shutdown.go b/cmd/ateom-microvm/shutdown.go index 71a69d03f2..0094ec567f 100644 --- a/cmd/ateom-microvm/shutdown.go +++ b/cmd/ateom-microvm/shutdown.go @@ -94,7 +94,7 @@ func (s *AteomService) gracefulShutdown(ctx context.Context) { if h.vm == nil { continue } - targets = append(targets, drainTarget{id: uid, agent: h.vm.guestAgent, workloadIDs: h.vm.workloadIDs}) + targets = append(targets, drainTarget{id: uid, agent: h.vm.guestAgent, workloadIDs: h.vm.workloadIDs, hosted: h}) } s.actorsMu.RUnlock() @@ -112,7 +112,11 @@ func (s *AteomService) gracefulShutdown(ctx context.Context) { wg.Add(1) go func(t drainTarget) { defer wg.Done() + // The drain ends the activation: read it while the guest answers, + // and write its final record once the workloads are down. + s.readFinal(ctx, t.hosted) gracefullyStopActor(ctx, t, deadline) + s.recordFinal(ctx, t.hosted) }(t) } wg.Wait() @@ -134,6 +138,8 @@ type drainTarget struct { id string agent *kata.AgentClient workloadIDs []string + // hosted is the activation the drain ends, for its final record. + hosted *hostedActor } // guestAgent is the slice of *kata.AgentClient the drain needs: signal a guest diff --git a/cmd/ateom-microvm/stats.go b/cmd/ateom-microvm/stats.go index 97d23e76dd..f70b24fc4a 100644 --- a/cmd/ateom-microvm/stats.go +++ b/cmd/ateom-microvm/stats.go @@ -20,14 +20,15 @@ import ( "context" "errors" "fmt" - "slices" + "log/slog" + "runtime/debug" "sync" - "sync/atomic" "time" "github.com/agent-substrate/substrate/cmd/ateom-microvm/internal/agentstats" "github.com/agent-substrate/substrate/cmd/ateom-microvm/internal/third_party/kata/agentpb" "github.com/agent-substrate/substrate/internal/apierror" + "github.com/agent-substrate/substrate/internal/ateattr" "github.com/agent-substrate/substrate/internal/proto/ateompb" "github.com/agent-substrate/substrate/internal/resources" ) @@ -39,9 +40,14 @@ import ( // file reads inside the guest, and far short of the lifecycle calls' 20-30s. const statsCallTimeout = 2 * time.Second -// The discovery read asks up to statsFanOut guests at once, and gives up on the -// rest after statsSweepBudget, which stays under atelet's 55s deadline for the -// whole call. A guest not reached in time reports as pending. +// finalReadTimeout bounds the final record's guest reading, which sits on the +// checkpoint path. A healthy reading takes about 2ms; a slower one falls back +// to the newest cached sample. +const finalReadTimeout = 10 * time.Millisecond + +// Sweeps and initial readings ask up to statsFanOut guests at once. A sweep +// gives up on the rest after statsSweepBudget, so it ends inside one sample +// interval. A guest not reached in time reports as pending. const ( statsFanOut = 32 statsSweepBudget = 45 * time.Second @@ -106,9 +112,7 @@ func (s *AteomService) GetWorkloadStats(ctx context.Context, req *ateompb.GetWor if hosted == nil { return nil, apierror.NotFound("ateom is not executing actor %q", req.GetActorUid()) } - active := &hosted.attribution - - sample, err := s.sampleGuest(ctx, active) + sample, err := s.measureGuest(ctx, hosted) if err != nil { if errors.Is(err, errStaleGuestTarget) { return nil, apierror.Internal("%v", err) @@ -129,73 +133,167 @@ func (s *AteomService) GetWorkloadStats(ctx context.Context, req *ateompb.GetWor if s.lookupActor(req.GetActorUid()) != hosted { return nil, apierror.NotFound("ateom stopped executing actor %q while the sample was being taken", req.GetActorUid()) } + // The reading advanced the activation's CPU, so the cache keeps up with it. + hosted.usage.Store(sample) return &ateompb.GetWorkloadStatsResponse{Sample: sample}, nil } -// GetActiveWorkloadStats implements -// ateompb.Ateom/GetActiveWorkloadStats: the discovery read, sampling -// whatever is executing with no identity asserted. Same lock discipline as -// GetWorkloadStats above, for the same reasons. +// GetActiveWorkloadStats implements ateompb.Ateom/GetActiveWorkloadStats: the +// discovery read. It serves each actor's latest sample from the sampler, so a +// poll puts no load of its own on the guests; an actor not sampled yet is +// pending. func (s *AteomService) GetActiveWorkloadStats(ctx context.Context, req *ateompb.GetActiveWorkloadStatsRequest) (*ateompb.GetActiveWorkloadStatsResponse, error) { hosted := s.hostedActors() + samples := make([]*ateompb.WorkloadStatsSample, 0, len(hosted)) + for _, h := range hosted { + sample := h.usage.Latest() + if sample == nil { + sample = h.usage.WithEpoch(pendingSample(&h.attribution)) + } + samples = append(samples, sample) + } + + // An empty list is "available", per the proto: a normal answer for a + // scraper to get, not an error. + return &ateompb.GetActiveWorkloadStatsResponse{Samples: samples}, nil +} + +// sweepUsage samples every hosted actor between its initial reading and its +// final record, and stores each sample and writes its periodic record. Same +// lock discipline as GetWorkloadStats, for the same reasons. +func (s *AteomService) sweepUsage(ctx context.Context) { sweepCtx, cancel := context.WithTimeout(ctx, statsSweepBudget) defer cancel() - // A workload with no numbers yet answers as a pending entry, so it stays - // attributable even if it dies during boot, and one of several booting - // does not stop the rest being reported. - samples := make([]*ateompb.WorkloadStatsSample, len(hosted)) - var stale atomic.Bool - slots := make(chan struct{}, statsFanOut) var wg sync.WaitGroup - for i, h := range hosted { + for _, h := range s.hostedActors() { wg.Add(1) go func() { defer wg.Done() - samples[i] = s.sampleHostedGuest(sweepCtx, h, slots, &stale) + // The reads run here, outside the sampler's own recover. + defer func() { + if r := recover(); r != nil { + slog.ErrorContext(ctx, "Usage sample panicked", slog.String(string(ateattr.ActorUIDKey), h.attribution.UID), + slog.Any("panic", r), slog.String("stack", string(debug.Stack()))) + } + }() + s.sampleHostedGuest(sweepCtx, h) }() } wg.Wait() - if stale.Load() { - return nil, apierror.Internal("%v", errStaleGuestTarget) +} + +// sampleHostedGuest measures one actor for the sweep, and stores the sample and +// writes its periodic record unless the actor is outside its sampling window or +// no longer hosted. A guest not reached before ctx is done, or that does not +// answer, is pending, so it stays attributable. +func (s *AteomService) sampleHostedGuest(ctx context.Context, h *hostedActor) { + if !h.usage.Sampling() { + return + } + sample := h.usage.WithEpoch(pendingSample(&h.attribution)) + measured, err := h.usage.Measure(ctx, func() (*ateompb.WorkloadStatsSample, map[string]uint64, error) { + return s.sampleGuestInSlot(ctx, h) + }) + if errors.Is(err, errStaleGuestTarget) { + slog.ErrorContext(ctx, "Guest stats target belongs to another actor", slog.String(string(ateattr.ActorUIDKey), h.attribution.UID), slog.Any("err", err)) + } + // Otherwise an error is boot, restore, teardown in progress, a guest that + // has stopped answering, or the sweep's budget: all routine. + if err == nil { + sample = measured + } + // The guest is found by UID alone. If the actor was re-hosted meanwhile, + // perhaps on another template, the numbers are the new activation's: drop + // them, and let the new activation start with its own initial reading. + if s.lookupActor(h.attribution.UID) != h { + return } - samples = slices.DeleteFunc(samples, func(s *ateompb.WorkloadStatsSample) bool { return s == nil }) + h.usage.Periodic(sample, func() { s.usage.Emit(ctx, ateattr.StatsKindPeriodic, sample) }) +} - // An empty list is "available", per the proto: a normal answer for a - // scraper to get, not an error. - return &ateompb.GetActiveWorkloadStatsResponse{Samples: samples}, nil +// measureGuest reads h's guest as a reading of its activation. +func (s *AteomService) measureGuest(ctx context.Context, h *hostedActor) (*ateompb.WorkloadStatsSample, error) { + return h.usage.Measure(ctx, func() (*ateompb.WorkloadStatsSample, map[string]uint64, error) { + return s.sampleGuest(ctx, h) + }) } -// sampleHostedGuest measures one actor for the discovery read, or returns nil -// when it is no longer hosted. A guest not reached before ctx is done, or that -// does not answer, is pending. -func (s *AteomService) sampleHostedGuest(ctx context.Context, h *hostedActor, slots chan struct{}, stale *atomic.Bool) *ateompb.WorkloadStatsSample { - sample := pendingSample(&h.attribution) +// sampleGuestInSlot reads h's guest while holding one of s.guestSlots. The slot +// is taken inside the reading, so a sweep waiting for another reading of the +// same actor holds no slot. +func (s *AteomService) sampleGuestInSlot(ctx context.Context, h *hostedActor) (*ateompb.WorkloadStatsSample, map[string]uint64, error) { select { - case slots <- struct{}{}: - measured, err := s.sampleGuest(ctx, &h.attribution) - <-slots - if errors.Is(err, errStaleGuestTarget) { - stale.Store(true) - } - // Otherwise an error is boot, restore, teardown in progress, or a guest - // that has stopped answering: all routine, none of them an error. - if err == nil { - sample = measured - } + case s.guestSlots <- struct{}{}: case <-ctx.Done(): + return nil, nil, ctx.Err() } - // The guest is found by UID alone. If the actor was re-hosted meanwhile, - // perhaps on another template, the numbers are the new activation's: - // report it as pending instead. - switch latest := s.lookupActor(h.attribution.UID); { - case latest == nil: - return nil - case latest != h: - return pendingSample(&latest.attribution) + defer func() { <-s.guestSlots }() + return s.sampleGuest(ctx, h) +} + +// recordInitial samples a new activation and writes its initial record. It +// runs off the resume path, since a guest read can take many seconds, and +// bounds that read like one sweep's. It recovers from panics, as a sweep does: +// it runs on its own goroutine, where a panic would take the ateom down. The +// activation is then sampled without an initial record. +func (s *AteomService) recordInitial(ctx context.Context, h *hostedActor) { + actorUID := h.attribution.UID + defer func() { + if r := recover(); r != nil { + slog.ErrorContext(ctx, "Initial usage sample panicked", slog.String(string(ateattr.ActorUIDKey), actorUID), + slog.Any("panic", r), slog.String("stack", string(debug.Stack()))) + h.usage.Initial(nil, nil) + } + }() + ctx, cancel := context.WithTimeout(ctx, statsSweepBudget) + defer cancel() + sample, err := h.usage.Measure(ctx, func() (*ateompb.WorkloadStatsSample, map[string]uint64, error) { + return s.sampleGuestInSlot(ctx, h) + }) + if err != nil { + slog.WarnContext(ctx, "No initial usage sample", slog.String(string(ateattr.ActorUIDKey), actorUID), slog.Any("err", err)) + sample = nil + } + h.usage.Initial(sample, func() { s.usage.Emit(ctx, ateattr.StatsKindInitial, sample) }) +} + +// recordFinalIfEnded writes h's final record if a checkpoint or terminate tore +// its activation down, which unhosts the actor even when a later step fails. +// The caller holds the actor's lock. +func (s *AteomService) recordFinalIfEnded(ctx context.Context, h *hostedActor) { + if h != nil && s.lookupActor(h.attribution.UID) != h { + s.recordFinal(ctx, h) + } +} + +// readFinal reads h's guest for the final record, bounded by finalReadTimeout. +// A failed read leaves the newest measured sample to stand in. If even that +// bound is too much checkpoint latency, drop the read and the final record +// uses the newest cached sample instead. +func (s *AteomService) readFinal(ctx context.Context, h *hostedActor) { + readCtx, cancel := context.WithTimeout(ctx, finalReadTimeout) + defer cancel() + sample, err := s.measureGuest(readCtx, h) + if err != nil { + slog.LogAttrs(ctx, slog.LevelDebug, "Final usage reading failed; using the newest sample", + append(ateattr.ActorLogAttrs(h.attribution), slog.Any("err", err))...) + return } - return sample + h.usage.Store(sample) +} + +// recordFinal writes the final record of an activation that a checkpoint or a +// terminate ended, from its newest measured sample. +func (s *AteomService) recordFinal(ctx context.Context, h *hostedActor) { + pending := h.usage.WithEpoch(pendingSample(&h.attribution)) + h.usage.Final(func(measured *ateompb.WorkloadStatsSample) { + if measured == nil { + measured = pending + } + s.usage.Emit(ctx, ateattr.StatsKindFinal, measured) + }) } // pendingSample is a workload with no numbers to give yet, as the discovery @@ -218,8 +316,9 @@ func pendingSample(active *resources.ActorAttribution) *ateompb.WorkloadStatsSam // errStaleGuestTarget is the one bug-shaped failure sampleGuest can return: // the published guest target and the attribution disagree, which the lifecycle -// RPCs write together under lock and so should never happen. Both stats reads -// map it to Internal; everything else sampleGuest returns is routine. +// RPCs write together under lock and so should never happen. GetWorkloadStats +// maps it to Internal and the sweep logs it as an error; everything else +// sampleGuest returns is routine. var errStaleGuestTarget = errors.New("guest agent connection belongs to a different actor") // sampleGuest reads the guest's container cgroups through the agent and builds @@ -228,10 +327,11 @@ var errStaleGuestTarget = errors.New("guest agent connection belongs to a differ // is routine here, and unlike the gVisor runtime's local file reads, a vsock // call offers no error type that separates "gone" from "broken". The // exception is errStaleGuestTarget, above. Errors come back raw because the -// two RPCs express the routine ones differently: an error code for the keyed -// read, a pending entry for the discovery read. The read holds no lock, so the -// keyed caller re-checks the actor record it loaded after this returns. -func (s *AteomService) sampleGuest(ctx context.Context, active *resources.ActorAttribution) (*ateompb.WorkloadStatsSample, error) { +// callers express the routine ones differently: an error code for the keyed +// read, a pending sample for the sweep. The read holds no lifecycle lock, so +// the keyed caller re-checks the actor record it loaded after this returns. +func (s *AteomService) sampleGuest(ctx context.Context, h *hostedActor) (*ateompb.WorkloadStatsSample, map[string]uint64, error) { + active := &h.attribution // The actor is the one here, but there is no guest to ask yet. Usually that // is a poll landing in the boot or the restore: the ateom retains the // attribution from the moment it accepts the actor, and the target is only @@ -239,22 +339,24 @@ func (s *AteomService) sampleGuest(ctx context.Context, active *resources.ActorA // like from here, since teardownActor clears the target before it closes // the connection, and what a restore whose post-restore agent dial failed // looks like for the rest of that activation. - target := s.guestStatsFor(active.UID) + // The target is read from h, not looked up by UID, so a re-host during + // this read cannot hand over the next activation's guest. + target := s.guestOf(h) if target == nil { - return nil, errors.New("no guest agent connection to measure yet") + return nil, nil, errors.New("no guest agent connection to measure yet") } // Belt and braces against the one thing that must never happen. The target // is published and cleared under lock alongside the attribution, so this // should be unreachable; if the two ever disagree, decline rather than // report a stale guest's numbers under the requested actor's name. if target.actorUID != active.UID { - return nil, fmt.Errorf("%w: %q, not %q", errStaleGuestTarget, target.actorUID, active.UID) + return nil, nil, fmt.Errorf("%w: %q, not %q", errStaleGuestTarget, target.actorUID, active.UID) } observedAt := time.Now() - sample, err := sumContainerStats(ctx, target) + sample, cpuByContainer, err := sumContainerStats(ctx, target) if err != nil { - return nil, fmt.Errorf("no container stats from the guest agent: %w", err) + return nil, nil, fmt.Errorf("no container stats from the guest agent: %w", err) } return &ateompb.WorkloadStatsSample{ @@ -273,10 +375,12 @@ func (s *AteomService) sampleGuest(ctx context.Context, active *resources.ActorA CpuUsageUsec: sample.CPUUsageUsec, ObservedAtUnixNano: observedAt.UnixNano(), - }, nil + }, cpuByContainer, nil } -// sumContainerStats reads every container of the actor and adds them up. +// sumContainerStats reads every container of the actor and adds them up. It +// also returns each container's CPU by id, so the activation can tell a +// container it could not read from one whose usage fell. // // A container the agent cannot report contributes nothing instead of failing // the sample. That is not only the "a partial reading beats none" trade the @@ -286,45 +390,52 @@ func (s *AteomService) sampleGuest(ctx context.Context, active *resources.ActorA // on. Failing the actor's telemetry because one sidecar is gone would be the // wrong answer. // -// It fails only when no container could be read at all, which is the guest as a +// It fails when no container could be read at all, which is the guest as a // whole not answering rather than one container being gone, and returns the -// last error so the caller can say why. +// last error so the caller can say why. It also fails when ctx ends before +// every container answered, so a deadline never yields a partial sum. // // The loop as a whole is bounded by ctx — the caller's RPC deadline — with // maxActorContainers * statsCallTimeout as the ceiling when the caller set // none. statsCallTimeout is per call so that one hung container read cannot // eat the budget the remaining containers still need. -func sumContainerStats(ctx context.Context, target *guestStatsTarget) (agentstats.Sample, error) { +func sumContainerStats(ctx context.Context, target *guestStatsTarget) (agentstats.Sample, map[string]uint64, error) { var ( total agentstats.Sample - read int + cpu = make(map[string]uint64, len(target.workloadIDs)) lastErr error ) for _, id := range target.workloadIDs { if err := ctx.Err(); err != nil { // The caller is gone; the per-call ctx below would fail instantly // anyway, so stop burning through the remaining containers. - return agentstats.Sample{}, err + return agentstats.Sample{}, nil, err } callCtx, cancel := context.WithTimeout(ctx, statsCallTimeout) cs, err := target.agent.StatsContainer(callCtx, id) cancel() if err != nil { + if ctxErr := ctx.Err(); ctxErr != nil { + // ctx ended during this call: report that rather than a sum + // missing this container. + return agentstats.Sample{}, nil, ctxErr + } lastErr = err continue } - read++ - total = total.Plus(agentstats.FromCgroupStats(cs)) + one := agentstats.FromCgroupStats(cs) + cpu[id] = one.CPUUsageUsec + total = total.Plus(one) } - if read == 0 { + if len(cpu) == 0 { if lastErr == nil { // No containers to read. The actor is up with an empty container // list, which the boot path does not produce, so treat it as "no // numbers" rather than reporting a confident zero. lastErr = errors.New("no containers to measure") } - return agentstats.Sample{}, lastErr + return agentstats.Sample{}, nil, lastErr } - return total, nil + return total, cpu, nil } diff --git a/cmd/ateom-microvm/stats_test.go b/cmd/ateom-microvm/stats_test.go index e5462b8b9c..19ad8fdbdb 100644 --- a/cmd/ateom-microvm/stats_test.go +++ b/cmd/ateom-microvm/stats_test.go @@ -152,6 +152,32 @@ func (f *fakeAgent) StatsContainer(ctx context.Context, containerID string) (*ag return f.stats[containerID], nil } +// blockingAgent answers every container but block, which it holds until ctx +// ends. +type blockingAgent struct { + fakeAgent + block string +} + +func (b *blockingAgent) StatsContainer(ctx context.Context, containerID string) (*agentpb.CgroupStats, error) { + if containerID == b.block { + <-ctx.Done() + return nil, ctx.Err() + } + return b.fakeAgent.StatsContainer(ctx, containerID) +} + +// TestSumContainerStatsDeadlineMidRead pins that a deadline hitting during the +// last container's call fails the read rather than returning a partial sum. +func TestSumContainerStatsDeadlineMidRead(t *testing.T) { + agent := &blockingAgent{fakeAgent: fakeAgent{stats: map[string]*agentpb.CgroupStats{"a": containerStats(1000, 2000, 100, 5000)}}, block: "b"} + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + if _, _, err := sumContainerStats(ctx, &guestStatsTarget{agent: agent, workloadIDs: []string{"a", "b"}}); !errors.Is(err, context.DeadlineExceeded) { + t.Errorf("sumContainerStats() error = %v, want %v", err, context.DeadlineExceeded) + } +} + // containerStats is one container's guest reading: usage bytes, peak bytes, // reclaimable page cache, and cumulative CPU nanoseconds. func containerStats(usage, peak, inactiveFile, cpuNanos uint64) *agentpb.CgroupStats { @@ -169,7 +195,7 @@ func containerStats(usage, peak, inactiveFile, cpuNanos uint64) *agentpb.CgroupS // does, since it is a pointer with no usable zero value and // TestGetWorkloadStatsDoesNotTakeLock holds it. func newStatsService(agent containerStatsReader, workloadIDs ...string) *AteomService { - s := &AteomService{locks: actorlock.New(), actors: map[string]*hostedActor{}} + s := &AteomService{locks: actorlock.New(), actors: map[string]*hostedActor{}, guestSlots: make(chan struct{}, statsFanOut)} hostTestActor(s, testActor, &guestStatsTarget{actorUID: testActor.UID, agent: agent, workloadIDs: workloadIDs}) return s } @@ -181,7 +207,7 @@ func hostTestActor(s *AteomService, attribution resources.ActorAttribution, targ if s.actors == nil { s.actors = map[string]*hostedActor{} } - hosted := &hostedActor{attribution: attribution, guest: target} + hosted := &hostedActor{attribution: attribution, guest: target, usage: testActivation()} s.actors[attribution.UID] = hosted return hosted } @@ -451,7 +477,7 @@ func TestGetActiveWorkloadStats(t *testing.T) { }} s := newStatsService(agent, "app_ovl") - got, err := s.GetActiveWorkloadStats(context.Background(), &ateompb.GetActiveWorkloadStatsRequest{}) + got, err := sweepAndList(s) if err != nil { t.Fatalf("GetActiveWorkloadStats() error = %v, want nil", err) } @@ -479,7 +505,7 @@ func TestGetActiveWorkloadStats(t *testing.T) { func TestGetActiveWorkloadStatsAvailable(t *testing.T) { s := &AteomService{} - got, err := s.GetActiveWorkloadStats(context.Background(), &ateompb.GetActiveWorkloadStatsRequest{}) + got, err := sweepAndList(s) if err != nil { t.Fatalf("GetActiveWorkloadStats() on an available ateom: error = %v, want nil", err) } @@ -508,10 +534,10 @@ func pendingFor(attr resources.ActorAttribution) *ateompb.WorkloadStatsSample { // routinely as idle workers, and the entry keeps a workload that dies during // boot attributable. func TestGetActiveWorkloadStatsBooting(t *testing.T) { - s := &AteomService{} + s := &AteomService{guestSlots: make(chan struct{}, statsFanOut)} hostTestActor(s, testActor, nil) // attribution retained, target not published - got, err := s.GetActiveWorkloadStats(context.Background(), &ateompb.GetActiveWorkloadStatsRequest{}) + got, err := sweepAndList(s) if err != nil { t.Fatalf("GetActiveWorkloadStats() mid-boot: error = %v, want nil", err) } @@ -540,7 +566,7 @@ func TestGetActiveWorkloadStatsSeveralActors(t *testing.T) { s := newStatsService(agent, "app_ovl") hostTestActor(s, second, &guestStatsTarget{actorUID: second.UID, agent: agent, workloadIDs: []string{"b_ovl"}}) - got, err := s.GetActiveWorkloadStats(context.Background(), &ateompb.GetActiveWorkloadStatsRequest{}) + got, err := sweepAndList(s) if err != nil { t.Fatalf("GetActiveWorkloadStats() error = %v, want nil", err) } @@ -573,7 +599,7 @@ func TestGetActiveWorkloadStatsOneBooting(t *testing.T) { s := newStatsService(agent, "app_ovl") hostTestActor(s, booting, nil) // accepted, no guest to ask yet - got, err := s.GetActiveWorkloadStats(context.Background(), &ateompb.GetActiveWorkloadStatsRequest{}) + got, err := sweepAndList(s) if err != nil { t.Fatalf("GetActiveWorkloadStats() error = %v, want nil", err) } @@ -604,9 +630,9 @@ func TestGetWorkloadStatsTransition(t *testing.T) { } } -// TestGetActiveWorkloadStatsStaleTarget pins that the one bug-shaped failure -// stays an error on the discovery read too: a target/attribution disagreement -// is an invariant violation, not a NOT_MEASURABLE_YET to skip past silently. +// TestGetActiveWorkloadStatsStaleTarget pins that a target/attribution +// disagreement never files one actor's numbers under another's name: the sweep +// logs it as an error and the actor reports pending. func TestGetActiveWorkloadStatsStaleTarget(t *testing.T) { agent := &fakeAgent{stats: map[string]*agentpb.CgroupStats{ "app_ovl": containerStats(1000, 2000, 100, 5000), @@ -614,14 +640,20 @@ func TestGetActiveWorkloadStatsStaleTarget(t *testing.T) { s := newStatsService(agent, "app_ovl") hostTestActor(s, testActor, &guestStatsTarget{actorUID: "uid-b", agent: agent, workloadIDs: []string{"app_ovl"}}) - _, err := s.GetActiveWorkloadStats(context.Background(), &ateompb.GetActiveWorkloadStatsRequest{}) - if got := apierror.Code(err); got != codes.Internal { - t.Errorf("GetActiveWorkloadStats() with stale target: code = %v, want %v (err: %v)", got, codes.Internal, err) + got, err := sweepAndList(s) + if err != nil { + t.Fatalf("GetActiveWorkloadStats() with stale target: error = %v, want nil", err) + } + if n := len(got.GetSamples()); n != 1 { + t.Fatalf("GetActiveWorkloadStats() with stale target returned %d samples, want 1", n) + } + if src := got.GetSamples()[0].GetSource(); src != ateompb.StatsSource_STATS_SOURCE_UNSPECIFIED { + t.Errorf("sample source = %v, want pending", src) } } -// The discovery read asks the guests concurrently, so a worker full of slow -// guests still answers within the caller's deadline. +// A sweep asks the guests concurrently, so a worker full of slow guests is +// still swept inside the sample interval. func TestGetActiveWorkloadStatsSamplesGuestsConcurrently(t *testing.T) { const actors = 4 var inFlight, peak atomic.Int32 @@ -645,14 +677,14 @@ func TestGetActiveWorkloadStatsSamplesGuestsConcurrently(t *testing.T) { } }, } - s := &AteomService{locks: actorlock.New(), actors: map[string]*hostedActor{}} + s := &AteomService{locks: actorlock.New(), actors: map[string]*hostedActor{}, guestSlots: make(chan struct{}, statsFanOut)} for i := range actors { attr := testActor attr.UID = fmt.Sprintf("uid-%d", i) hostTestActor(s, attr, &guestStatsTarget{actorUID: attr.UID, agent: agent, workloadIDs: []string{"app_ovl"}}) } - got, err := s.GetActiveWorkloadStats(context.Background(), &ateompb.GetActiveWorkloadStatsRequest{}) + got, err := sweepAndList(s) if err != nil { t.Fatalf("GetActiveWorkloadStats() error = %v, want nil", err) } @@ -703,7 +735,7 @@ func TestGetActiveWorkloadStatsTransition(t *testing.T) { var once sync.Once agent.onCall = func() { once.Do(func() { tc.during(s) }) } - got, err := s.GetActiveWorkloadStats(context.Background(), &ateompb.GetActiveWorkloadStatsRequest{}) + got, err := sweepAndList(s) if err != nil { t.Fatalf("GetActiveWorkloadStats() during transition: error = %v, want nil", err) } @@ -722,3 +754,19 @@ func TestGetActiveWorkloadStatsTransition(t *testing.T) { }) } } + +// testActivation is a running actor's activation: its initial reading is done, +// so the sweep samples it. It starts at the unix epoch, so a sample's epoch is +// zero and the expected samples here need not name it; the epoch has tests of +// its own. +func testActivation() *ateomstats.Activation { + a := ateomstats.NewActivation(time.Unix(0, 0), false) + a.Initial(nil, nil) + return a +} + +// sweepAndList runs one sampler sweep, then the discovery read that serves it. +func sweepAndList(s *AteomService) (*ateompb.GetActiveWorkloadStatsResponse, error) { + s.sweepUsage(context.Background()) + return s.GetActiveWorkloadStats(context.Background(), &ateompb.GetActiveWorkloadStatsRequest{}) +} diff --git a/cmd/ateom-microvm/usage_test.go b/cmd/ateom-microvm/usage_test.go new file mode 100644 index 0000000000..b62a8f62e6 --- /dev/null +++ b/cmd/ateom-microvm/usage_test.go @@ -0,0 +1,377 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "errors" + "slices" + "testing" + "time" + + "github.com/agent-substrate/substrate/cmd/ateom-microvm/internal/third_party/kata/agentpb" + "github.com/agent-substrate/substrate/internal/ateattr" + "github.com/agent-substrate/substrate/internal/ateomstats" + "github.com/agent-substrate/substrate/internal/ateomstats/ateomstatstest" + "github.com/agent-substrate/substrate/internal/proto/ateompb" +) + +func withUsageRecorder(s *AteomService) *ateomstatstest.Recorder { + e, rec := ateomstatstest.NewEmitter() + s.usage = e + return rec +} + +// running is a after its initial reading, so the sweep samples it. +func running(a *ateomstats.Activation) *ateomstats.Activation { + a.Initial(nil, nil) + return a +} + +// setActivation replaces the test actor's activation. +func setActivation(s *AteomService, a *ateomstats.Activation) *hostedActor { + h := s.lookupActor(testActor.UID) + s.actorsMu.Lock() + h.usage = a + s.actorsMu.Unlock() + return h +} + +func TestSweepStampsEpochAndDiscoveryServesIt(t *testing.T) { + agent := &fakeAgent{stats: map[string]*agentpb.CgroupStats{"app_ovl": containerStats(1000, 2000, 100, 5_000_000)}} + s := newStatsService(agent, "app_ovl") + epoch := time.Unix(1700, 0) + setActivation(s, running(ateomstats.NewActivation(epoch, false))) + + before, err := s.GetActiveWorkloadStats(context.Background(), &ateompb.GetActiveWorkloadStatsRequest{}) + if err != nil { + t.Fatal(err) + } + if got := before.GetSamples()[0]; got.GetSource() != ateompb.StatsSource_STATS_SOURCE_UNSPECIFIED || got.GetEpochUnixNano() != epoch.UnixNano() { + t.Errorf("before a sweep: source %v epoch %d, want pending with epoch %d", got.GetSource(), got.GetEpochUnixNano(), epoch.UnixNano()) + } + if len(agent.calls) != 0 { + t.Errorf("discovery read called the guest %d times, want 0", len(agent.calls)) + } + + s.sweepUsage(context.Background()) + after, err := s.GetActiveWorkloadStats(context.Background(), &ateompb.GetActiveWorkloadStatsRequest{}) + if err != nil { + t.Fatal(err) + } + if got := after.GetSamples()[0]; got.GetEpochUnixNano() != epoch.UnixNano() || got.GetCpuUsageUsec() != 5000 { + t.Errorf("served epoch %d cpu %d, want epoch %d cpu 5000", got.GetEpochUnixNano(), got.GetCpuUsageUsec(), epoch.UnixNano()) + } +} + +// TestResumedGuestCPUIsActivationRelative: a restored guest brings its CPU +// counter back, so the first reading is the baseline. +func TestResumedGuestCPUIsActivationRelative(t *testing.T) { + agent := &fakeAgent{stats: map[string]*agentpb.CgroupStats{"app_ovl": containerStats(1000, 2000, 100, 9_000_000)}} + s := newStatsService(agent, "app_ovl") + h := setActivation(s, running(ateomstats.NewActivation(time.Now(), true))) + + s.sweepUsage(context.Background()) + if got := h.usage.Latest().GetCpuUsageUsec(); got != 0 { + t.Errorf("first reading of a resumed guest = %d, want 0", got) + } + agent.mu.Lock() + agent.stats["app_ovl"] = containerStats(1000, 2000, 100, 9_400_000) + agent.mu.Unlock() + s.sweepUsage(context.Background()) + if got := h.usage.Latest().GetCpuUsageUsec(); got != 400 { + t.Errorf("second reading = %d, want 400", got) + } +} + +func TestRecordInitialAndFinal(t *testing.T) { + agent := &fakeAgent{stats: map[string]*agentpb.CgroupStats{"app_ovl": containerStats(1000, 2000, 100, 5_000_000)}} + s := newStatsService(agent, "app_ovl") + rec := withUsageRecorder(s) + h := setActivation(s, ateomstats.NewActivation(time.Now(), false)) + + s.recordInitial(context.Background(), h) + if h.usage.Latest() == nil { + t.Fatal("initial sample was not stored for the discovery read") + } + calls := len(agent.calls) + s.recordFinal(context.Background(), h) + if len(agent.calls) != calls { + t.Error("final record read the guest; it must reuse the latest sample") + } + + want := []string{ateattr.StatsKindInitial, ateattr.StatsKindFinal} + if len(rec.Kinds()) != 2 || rec.Kinds()[0] != want[0] || rec.Kinds()[1] != want[1] { + t.Errorf("record kinds = %v, want %v", rec.Kinds(), want) + } +} + +// TestRecordFinalIfEnded pins that the final record waits for the actor to be +// unhosted, as a teardown does even when a later step fails. +func TestRecordFinalIfEnded(t *testing.T) { + s := newStatsService(&fakeAgent{}, "app_ovl") + rec := withUsageRecorder(s) + h := setActivation(s, ateomstats.NewActivation(time.Now(), false)) + + s.recordFinalIfEnded(context.Background(), h) + if got := rec.Kinds(); len(got) != 0 { + t.Fatalf("records while hosted = %v, want none", got) + } + unhostTestActor(s, testActor.UID) + s.recordFinalIfEnded(context.Background(), h) + if got := rec.Kinds(); len(got) != 1 || got[0] != ateattr.StatsKindFinal { + t.Errorf("records after unhosting = %v, want one final", got) + } +} + +// TestReadFinal pins that the final record carries a fresh guest reading, and +// the newest measured sample when that reading fails. +func TestReadFinal(t *testing.T) { + agent := &fakeAgent{stats: map[string]*agentpb.CgroupStats{"app_ovl": containerStats(1000, 2000, 100, 5_000_000)}} + s := newStatsService(agent, "app_ovl") + h := setActivation(s, ateomstats.NewActivation(time.Now(), false)) + s.recordInitial(context.Background(), h) + + agent.stats["app_ovl"] = containerStats(1000, 2000, 100, 9_000_000) + s.readFinal(context.Background(), h) + if got := h.usage.Latest().GetCpuUsageUsec(); got != 9000 { + t.Errorf("CPU after the final read = %d, want 9000", got) + } + + agent.errs = map[string]error{"app_ovl": errors.New("guest gone")} + s.readFinal(context.Background(), h) + if got := h.usage.Latest().GetCpuUsageUsec(); got != 9000 { + t.Errorf("CPU after a failed final read = %d, want the last measured 9000", got) + } +} + +// TestEndPreviousActivation pins that a re-host writes the final record of the +// activation it replaces, which also keeps that activation's late initial +// reading from writing. +func TestEndPreviousActivation(t *testing.T) { + agent := &fakeAgent{stats: map[string]*agentpb.CgroupStats{"app_ovl": containerStats(1000, 2000, 100, 5_000_000)}} + s := newStatsService(agent, "app_ovl") + rec := withUsageRecorder(s) + old := setActivation(s, ateomstats.NewActivation(time.Now(), false)) + + if err := s.endPreviousActivation(context.Background(), testActor.UID, nil); err != nil { + t.Fatal(err) + } + s.recordInitial(context.Background(), old) + if got := rec.Kinds(); len(got) != 1 || got[0] != ateattr.StatsKindFinal { + t.Errorf("records = %v, want only the old activation's final", got) + } +} + +func TestRecordFinalWithNoSampleIsPending(t *testing.T) { + s := newStatsService(&fakeAgent{}, "app_ovl") + rec := withUsageRecorder(s) + h := setActivation(s, ateomstats.NewActivation(time.Now(), false)) + s.recordFinal(context.Background(), h) + if len(rec.Sources()) != 1 || rec.Sources()[0] != ateattr.StatsSourceUnspecified { + t.Errorf("final record sources = %v, want one pending", rec.Sources()) + } +} + +// TestRecordInitialAfterFinal pins that an initial read still in flight when +// a checkpoint or terminate writes the final record writes nothing. +func TestRecordInitialAfterFinal(t *testing.T) { + agent := &fakeAgent{stats: map[string]*agentpb.CgroupStats{"app_ovl": containerStats(1000, 2000, 100, 5_000_000)}} + s := newStatsService(agent, "app_ovl") + rec := withUsageRecorder(s) + h := setActivation(s, ateomstats.NewActivation(time.Now(), false)) + agent.onCall = func() { s.recordFinal(context.Background(), h) } + + s.recordInitial(context.Background(), h) + if got := rec.Kinds(); len(got) != 1 || got[0] != ateattr.StatsKindFinal { + t.Errorf("records = %v, want only the final one", got) + } +} + +// TestRecordInitialRecoversFromPanic pins that a panic in the initial read is +// recovered and still opens the sampling window. +func TestRecordInitialRecoversFromPanic(t *testing.T) { + agent := &fakeAgent{stats: map[string]*agentpb.CgroupStats{"app_ovl": containerStats(1000, 2000, 100, 5_000_000)}} + s := newStatsService(agent, "app_ovl") + rec := withUsageRecorder(s) + h := setActivation(s, ateomstats.NewActivation(time.Now(), false)) + agent.onCall = func() { panic("guest read") } + + s.recordInitial(context.Background(), h) + if got := rec.Kinds(); len(got) != 0 { + t.Errorf("records = %v, want none", got) + } + if !h.usage.Sampling() { + t.Error("activation is not sampling after the panic") + } +} + +// TestSweepRecoversFromPanic pins that a panic in one guest read is recovered +// on its fan-out goroutine, where the sampler's recover does not reach. +func TestSweepRecoversFromPanic(t *testing.T) { + agent := &fakeAgent{stats: map[string]*agentpb.CgroupStats{"app_ovl": containerStats(1000, 2000, 100, 5_000_000)}} + s := newStatsService(agent, "app_ovl") + rec := withUsageRecorder(s) + setActivation(s, running(ateomstats.NewActivation(time.Now(), false))) + agent.onCall = func() { panic("guest read") } + + s.sweepUsage(context.Background()) + if got := rec.Kinds(); len(got) != 0 { + t.Errorf("records = %v, want none", got) + } +} + +// TestRecordInitialTakesASlot pins that an initial reading waits for a guest +// slot, so it adds no guest reads beyond statsFanOut. +func TestRecordInitialTakesASlot(t *testing.T) { + agent := &fakeAgent{stats: map[string]*agentpb.CgroupStats{"app_ovl": containerStats(1000, 2000, 100, 5_000_000)}} + s := newStatsService(agent, "app_ovl") + withUsageRecorder(s) + h := setActivation(s, ateomstats.NewActivation(time.Now(), false)) + for range statsFanOut { + s.guestSlots <- struct{}{} + } + + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + s.recordInitial(ctx, h) + if len(agent.calls) != 0 { + t.Errorf("initial reading read the guest %d times with every slot taken, want 0", len(agent.calls)) + } +} + +// TestRecordInitialAfterRehost pins that a re-host during the read leaves the +// new activation untouched: the reading belongs to the old one. +func TestRecordInitialAfterRehost(t *testing.T) { + agent := &fakeAgent{stats: map[string]*agentpb.CgroupStats{"app_ovl": containerStats(1000, 2000, 100, 5_000_000)}} + s := newStatsService(agent, "app_ovl") + withUsageRecorder(s) + old := setActivation(s, ateomstats.NewActivation(time.Now(), false)) + target := s.guestOf(old) + agent.onCall = func() { hostTestActor(s, testActor, target) } + + s.recordInitial(context.Background(), old) + if got := old.usage.Latest(); got == nil || got.GetEpochUnixNano() != old.usage.Epoch() { + t.Errorf("old activation holds %v, want its own initial reading", got) + } + if s.lookupActor(testActor.UID).usage.Latest() != nil { + t.Error("new activation holds the old activation's sample") + } +} + +// TestPartialGuestReadDoesNotDoubleCharge fails one container's read for a +// sweep: its CPU must neither drop out of the total nor be charged again when +// it is read once more. +func TestPartialGuestReadDoesNotDoubleCharge(t *testing.T) { + agent := &fakeAgent{stats: map[string]*agentpb.CgroupStats{ + "a_ovl": containerStats(1000, 2000, 100, 600_000_000), + "b_ovl": containerStats(1000, 2000, 100, 400_000_000), + }} + s := newStatsService(agent, "a_ovl", "b_ovl") + h := setActivation(s, running(ateomstats.NewActivation(time.Now(), true))) + + s.sweepUsage(context.Background()) + agent.mu.Lock() + agent.stats["a_ovl"] = containerStats(1000, 2000, 100, 605_000_000) + agent.errs = map[string]error{"b_ovl": errors.New("deadline exceeded")} + agent.mu.Unlock() + s.sweepUsage(context.Background()) + if got := h.usage.Latest().GetCpuUsageUsec(); got != 5000 { + t.Fatalf("cpu with b unreadable = %d, want 5000", got) + } + + agent.mu.Lock() + agent.stats["a_ovl"] = containerStats(1000, 2000, 100, 606_000_000) + agent.stats["b_ovl"] = containerStats(1000, 2000, 100, 404_000_000) + agent.errs = nil + agent.mu.Unlock() + s.sweepUsage(context.Background()) + if got := h.usage.Latest().GetCpuUsageUsec(); got != 10000 { + t.Errorf("cpu once b reads again = %d, want 10000 (b's 400s charged once)", got) + } +} + +// TestSweepWaitsForInitialAndStopsAtFinal pins that the sweep reads no actor +// before its initial reading or after its final record. +func TestSweepWaitsForInitialAndStopsAtFinal(t *testing.T) { + agent := &fakeAgent{stats: map[string]*agentpb.CgroupStats{"app_ovl": containerStats(1000, 2000, 100, 5_000_000)}} + s := newStatsService(agent, "app_ovl") + rec := withUsageRecorder(s) + h := setActivation(s, ateomstats.NewActivation(time.Now(), false)) + + s.sweepUsage(context.Background()) + if len(agent.calls) != 0 { + t.Fatalf("sweep before the initial reading read the guest %d times, want 0", len(agent.calls)) + } + s.recordInitial(context.Background(), h) + s.sweepUsage(context.Background()) + s.recordFinal(context.Background(), h) + calls := len(agent.calls) + s.sweepUsage(context.Background()) + if len(agent.calls) != calls { + t.Error("sweep read the guest after the final record") + } + want := []string{ateattr.StatsKindInitial, ateattr.StatsKindPeriodic, ateattr.StatsKindFinal} + if got := rec.Kinds(); !slices.Equal(got, want) { + t.Errorf("records = %v, want %v", got, want) + } +} + +// TestRecordInitialReadsItsOwnGuest re-hosts the actor with another guest +// before the old activation's initial read: the read still uses the old +// activation's guest, and the new activation is untouched. +func TestRecordInitialReadsItsOwnGuest(t *testing.T) { + oldAgent := &fakeAgent{stats: map[string]*agentpb.CgroupStats{"app_ovl": containerStats(1000, 2000, 100, 5_000_000)}} + newAgent := &fakeAgent{stats: map[string]*agentpb.CgroupStats{"app_ovl": containerStats(1000, 2000, 100, 9_000_000)}} + s := newStatsService(oldAgent, "app_ovl") + withUsageRecorder(s) + old := setActivation(s, ateomstats.NewActivation(time.Now(), false)) + fresh := hostTestActor(s, testActor, &guestStatsTarget{actorUID: testActor.UID, agent: newAgent, workloadIDs: []string{"app_ovl"}}) + fresh.usage = ateomstats.NewActivation(time.Now(), false) + + s.recordInitial(context.Background(), old) + if got := old.usage.Latest().GetCpuUsageUsec(); got != 5000 { + t.Errorf("old activation cpu = %d, want 5000 from its own guest", got) + } + if len(newAgent.calls) != 0 { + t.Error("the old activation's initial read asked the new guest") + } + if fresh.usage.Sampling() || fresh.usage.Latest() != nil { + t.Error("the old activation's initial read touched the new activation") + } +} + +// TestLateInitialLeavesNewActivationAlone starts the old activation's +// initial read only after its final record and a re-host: it writes nothing, +// and the new activation still waits for its own initial reading. +func TestLateInitialLeavesNewActivationAlone(t *testing.T) { + agent := &fakeAgent{stats: map[string]*agentpb.CgroupStats{"app_ovl": containerStats(1000, 2000, 100, 5_000_000)}} + s := newStatsService(agent, "app_ovl") + rec := withUsageRecorder(s) + old := setActivation(s, ateomstats.NewActivation(time.Now(), false)) + s.recordFinal(context.Background(), old) + fresh := hostTestActor(s, testActor, nil) + fresh.usage = ateomstats.NewActivation(time.Now(), false) + + s.recordInitial(context.Background(), old) + if got := rec.Kinds(); len(got) != 1 || got[0] != ateattr.StatsKindFinal { + t.Errorf("records = %v, want only the old activation's final", got) + } + if fresh.usage.Sampling() { + t.Error("the late read opened the new activation's sampling window") + } +} diff --git a/docs/metrics/registry/metrics.yaml b/docs/metrics/registry/metrics.yaml index b34b2525f5..70d2343141 100644 --- a/docs/metrics/registry/metrics.yaml +++ b/docs/metrics/registry/metrics.yaml @@ -573,7 +573,7 @@ groups: - id: final stability: development value: final - brief: The last sample of an activation, taken in the checkpoint before the workload is torn down. + brief: The last sample of an activation, written when a checkpoint or a terminate tears it down. It is read just before the teardown; when that read fails or times out, it repeats the newest sample. - id: ate.stats.memory.usage stability: development type: int From 25a0b88c35dd5542dad4c792a69c52df1843d0f7 Mon Sep 17 00:00:00 2001 From: Julian Gutierrez Oschmann Date: Thu, 8 Oct 2026 18:00:16 +0000 Subject: [PATCH 06/44] Delete unused `actor_template_ref` from `ActorAssigment` sub-resource (#2322) The field name doesn't follow the convention and ends with "ref", but more importantly, it's not really used. Part of #1378 --- .../internal/controlapi/actor_template.go | 10 - .../controlapi/actor_template_test.go | 16 -- .../controlapi/functionaltest/actor_test.go | 15 +- cmd/ateapi/internal/controlapi/worker_test.go | 5 +- .../internal/controlapi/workflow_resume.go | 1 - .../controlapi/workflow_resume_test.go | 52 ---- .../internal/store/storecontract/contract.go | 7 +- pkg/proto/ateapipb/ateapi.pb.go | 249 ++++++++---------- pkg/proto/ateapipb/ateapi.proto | 7 - 9 files changed, 128 insertions(+), 234 deletions(-) diff --git a/cmd/ateapi/internal/controlapi/actor_template.go b/cmd/ateapi/internal/controlapi/actor_template.go index 7fb07ccd13..25ce835b6f 100644 --- a/cmd/ateapi/internal/controlapi/actor_template.go +++ b/cmd/ateapi/internal/controlapi/actor_template.go @@ -165,13 +165,3 @@ func resolveActorTemplate(ctx context.Context, st actorTemplateGetter, actor *at } return template, nil } - -// actorTemplateObjectRef returns a fresh copy of the actor's template -// reference — fresh so records built from it never alias the actor message. -func actorTemplateObjectRef(actor *ateapipb.Actor) *ateapipb.ObjectRef { - ref := actor.GetActorTemplate() - if ref == nil { - return nil - } - return &ateapipb.ObjectRef{Atespace: ref.GetAtespace(), Name: ref.GetName()} -} diff --git a/cmd/ateapi/internal/controlapi/actor_template_test.go b/cmd/ateapi/internal/controlapi/actor_template_test.go index ef7d27ee20..2889e13d67 100644 --- a/cmd/ateapi/internal/controlapi/actor_template_test.go +++ b/cmd/ateapi/internal/controlapi/actor_template_test.go @@ -469,19 +469,3 @@ func TestUpdateActorTemplateMetadata(t *testing.T) { t.Errorf("uid after update = %q, want %q", got, want) } } - -// TestActorTemplateObjectRef pins that snapshot and assignment records get a -// fresh copy of the reference, never the actor's own message. -func TestActorTemplateObjectRef(t *testing.T) { - if got := actorTemplateObjectRef(&ateapipb.Actor{}); got != nil { - t.Errorf("actorTemplateObjectRef(no ref) = %v, want nil", got) - } - actor := &ateapipb.Actor{ActorTemplate: &ateapipb.ObjectRef{Atespace: "team-a", Name: "tmpl1"}} - got := actorTemplateObjectRef(actor) - if got == actor.GetActorTemplate() { - t.Error("actorTemplateObjectRef aliases the actor's reference") - } - if got.GetAtespace() != "team-a" || got.GetName() != "tmpl1" { - t.Errorf("actorTemplateObjectRef = %v, want team-a/tmpl1", got) - } -} diff --git a/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go b/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go index 7ff77a7986..513f8a6a9c 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go @@ -4279,9 +4279,8 @@ func TestResumeActor_CrashesIfAssignedWorkerIsDraining(t *testing.T) { t.Fatalf("GetActor failed: %v", err) } if err := tc.persistence.BindActorToWorker(context.Background(), podA, &ateapipb.ActorAssignment{ - Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: id}, - ActorUid: suspended.GetMetadata().GetUid(), - ActorTemplateRef: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"}, + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: id}, + ActorUid: suspended.GetMetadata().GetUid(), }, nil); err != nil { t.Fatalf("BindActorToWorker failed: %v", err) } @@ -4531,9 +4530,8 @@ func TestResumeActor_DanglingWorker(t *testing.T) { t.Fatalf("GetActor failed: %v", err) } if err := tc.persistence.BindActorToWorker(context.Background(), podA, &ateapipb.ActorAssignment{ - Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: name}, - ActorUid: suspended.GetMetadata().GetUid(), - ActorTemplateRef: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"}, + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: name}, + ActorUid: suspended.GetMetadata().GetUid(), }, nil); err != nil { t.Fatalf("BindActorToWorker failed: %v", err) } @@ -5133,9 +5131,8 @@ func TestDeleteActor_ReleasesAnAssignmentTheActorDoesNotReference(t *testing.T) // Bind straight through the store, leaving the Actor's backlink unset: // exactly the state a crash between the two writes leaves behind. if err := tc.persistence.BindActorToWorker(ctx, podUID, &ateapipb.ActorAssignment{ - Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "orphaned"}, - ActorUid: actorUID, - ActorTemplateRef: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"}, + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "orphaned"}, + ActorUid: actorUID, }, nil); err != nil { t.Fatalf("BindActorToWorker failed: %v", err) } diff --git a/cmd/ateapi/internal/controlapi/worker_test.go b/cmd/ateapi/internal/controlapi/worker_test.go index c0b482e996..68c0f6d1fe 100644 --- a/cmd/ateapi/internal/controlapi/worker_test.go +++ b/cmd/ateapi/internal/controlapi/worker_test.go @@ -57,9 +57,8 @@ func validWorker(name string, mods ...func(*ateapipb.Worker)) *ateapipb.Worker { func newAPIAssignment(actorUID string) *ateapipb.ActorAssignment { return &ateapipb.ActorAssignment{ - ActorTemplateRef: &ateapipb.ObjectRef{Atespace: "ate-system", Name: "tmpl"}, - Actor: &ateapipb.ObjectRef{Atespace: "team-a", Name: "actor-1"}, - ActorUid: actorUID, + Actor: &ateapipb.ObjectRef{Atespace: "team-a", Name: "actor-1"}, + ActorUid: actorUID, } } diff --git a/cmd/ateapi/internal/controlapi/workflow_resume.go b/cmd/ateapi/internal/controlapi/workflow_resume.go index c9e5238cad..ee12bd5af1 100644 --- a/cmd/ateapi/internal/controlapi/workflow_resume.go +++ b/cmd/ateapi/internal/controlapi/workflow_resume.go @@ -510,7 +510,6 @@ func (w *ActorWorkflow) assignWorkerAttempt(ctx context.Context, actorRef resour // Record what this claim reserves so release returns the same amount. Resources: admittedResources(constraints), } - assignment.ActorTemplateRef = actorTemplateObjectRef(actor) // The candidate came from a watch-fed cache, so it may already be full or no // longer eligible. The store re-asks under the Worker's row lock, where the diff --git a/cmd/ateapi/internal/controlapi/workflow_resume_test.go b/cmd/ateapi/internal/controlapi/workflow_resume_test.go index 12fcf684d0..859a03862f 100644 --- a/cmd/ateapi/internal/controlapi/workflow_resume_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_resume_test.go @@ -178,58 +178,6 @@ func TestEnsureWorkerAssigned_ConflictExhaustionIsRetryable(t *testing.T) { } } -// TestAssignWorkerAttempt_StampsSubstrateTemplateRef verifies a ref-mode -// actor's worker claim names the substrate template via actor_template_ref -// and leaves the legacy kube reference unset. -func TestAssignWorkerAttempt_StampsSubstrateTemplateRef(t *testing.T) { - ctx := context.Background() - persistence := newTestPersistence(t) - - worker := &ateapipb.Worker{ - Metadata: &ateapipb.ResourceMetadata{Name: testWorkerUID("pod-free")}, - WorkerNamespace: "worker-ns", - WorkerPool: "pool", - WorkerPod: "pod-free", - WorkerPodUid: testWorkerUID("pod-free"), - SandboxClass: "gvisor", - Status: &ateapipb.WorkerStatus{State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, Capacity: &ateapipb.WorkerResources{Actors: 1}}, - } - if _, err := persistence.CreateWorker(ctx, worker); err != nil { - t.Fatalf("CreateWorker: %v", err) - } - - actor := storetest.MustCreateActor(t, ctx, persistence, &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{Atespace: "team-a", Name: "id1"}, - ActorTemplate: &ateapipb.ObjectRef{Atespace: "team-a", Name: "sub-tmpl"}, - Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED}, - }) - - cacheCtx, cancel := context.WithCancel(ctx) - defer cancel() - wc := workercache.New(persistence, time.Minute) - if err := wc.Start(cacheCtx); err != nil { - t.Fatalf("workercache.Start: %v", err) - } - - w := &ActorWorkflow{store: persistence, workerCache: wc, scheduler: scheduling.New(wc)} - tmpl := &ateapipb.ActorTemplate{ - Metadata: &ateapipb.ResourceMetadata{Atespace: "team-a", Name: "sub-tmpl"}, - SandboxConfig: &ateapipb.SandboxConfig{SandboxClass: ateapipb.SandboxClass_SANDBOX_CLASS_GVISOR}, - } - _, assigned, err := w.assignWorkerAttempt(ctx, resources.ActorRef{Atespace: "team-a", Name: "id1"}, actor, tmpl) - if err != nil { - t.Fatalf("assignWorkerAttempt: %v", err) - } - - assignment, err := persistence.GetWorkerAssignment(ctx, assigned.GetMetadata().GetName(), actor.GetMetadata().GetUid()) - if err != nil { - t.Fatalf("GetWorkerAssignment: %v", err) - } - if assignment.GetActorTemplateRef().GetAtespace() != "team-a" || assignment.GetActorTemplateRef().GetName() != "sub-tmpl" { - t.Errorf("assignment ActorTemplateRef = %v, want team-a/sub-tmpl", assignment.GetActorTemplateRef()) - } -} - func TestAssignWorkerAttempt_SkipsWorkerAssignedInOtherAtespace(t *testing.T) { ctx := context.Background() persistence := newTestPersistence(t) diff --git a/cmd/ateapi/internal/store/storecontract/contract.go b/cmd/ateapi/internal/store/storecontract/contract.go index df6b5d502f..5a82a3b411 100644 --- a/cmd/ateapi/internal/store/storecontract/contract.go +++ b/cmd/ateapi/internal/store/storecontract/contract.go @@ -2077,10 +2077,9 @@ func runWorkerAssignmentContractTests(t *testing.T, setup func(t *testing.T) sto // over several of them is unambiguous about which were counted. newTestAssignment := func(actorUID string, cpuMilli, memoryBytes int64) *ateapipb.ActorAssignment { return &ateapipb.ActorAssignment{ - ActorTemplateRef: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "test-template"}, - Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "actor-" + actorUID}, - ActorUid: actorUID, - Resources: resources.CPUMemory(cpuMilli, memoryBytes), + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "actor-" + actorUID}, + ActorUid: actorUID, + Resources: resources.CPUMemory(cpuMilli, memoryBytes), } } diff --git a/pkg/proto/ateapipb/ateapi.pb.go b/pkg/proto/ateapipb/ateapi.pb.go index 8bb90f2770..c73e198052 100644 --- a/pkg/proto/ateapipb/ateapi.pb.go +++ b/pkg/proto/ateapipb/ateapi.pb.go @@ -7075,12 +7075,6 @@ type ActorAssignment struct { // +k8s:required // +k8s:format=k8s-uuid ActorUid string `protobuf:"bytes,3,opt,name=actor_uid,json=actorUid,proto3" json:"actor_uid,omitempty"` - // actor_template_ref names the ActorTemplate resource the assigned Actor - // uses. - // - // +k8s:required - // +k8s:subfield(atespace)=+k8s:required - ActorTemplateRef *ObjectRef `protobuf:"bytes,4,opt,name=actor_template_ref,json=actorTemplateRef,proto3" json:"actor_template_ref,omitempty"` // The resources allocated to the Actor when it was assigned. // // +k8s:optional @@ -7145,13 +7139,6 @@ func (x *ActorAssignment) GetActorUid() string { return "" } -func (x *ActorAssignment) GetActorTemplateRef() *ObjectRef { - if x != nil { - return x.ActorTemplateRef - } - return nil -} - func (x *ActorAssignment) GetResources() *Resources { if x != nil { return x.Resources @@ -8409,12 +8396,11 @@ const file_ateapi_proto_rawDesc = "" + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"Z\n" + "\x0fWorkerResources\x12/\n" + "\tresources\x18\x01 \x01(\v2\x11.ateapi.ResourcesR\tresources\x12\x16\n" + - "\x06actors\x18\x02 \x01(\x05R\x06actors\"\xa2\x02\n" + + "\x06actors\x18\x02 \x01(\x05R\x06actors\"\xe1\x01\n" + "\x0fActorAssignment\x124\n" + "\bmetadata\x18\x06 \x01(\v2\x18.ateapi.ResourceMetadataR\bmetadata\x12'\n" + "\x05actor\x18\x02 \x01(\v2\x11.ateapi.ObjectRefR\x05actor\x12\x1b\n" + - "\tactor_uid\x18\x03 \x01(\tR\bactorUid\x12?\n" + - "\x12actor_template_ref\x18\x04 \x01(\v2\x11.ateapi.ObjectRefR\x10actorTemplateRef\x12/\n" + + "\tactor_uid\x18\x03 \x01(\tR\bactorUid\x12/\n" + "\tresources\x18\x05 \x01(\v2\x11.ateapi.ResourcesR\tresources\x12!\n" + "\fworker_epoch\x18\a \x01(\x03R\vworkerEpoch\"\xad\x01\n" + "\x15RegisterWorkerRequest\x12)\n" + @@ -8812,122 +8798,121 @@ var file_ateapi_proto_depIdxs = []int32{ 31, // 126: ateapi.WorkerResources.resources:type_name -> ateapi.Resources 10, // 127: ateapi.ActorAssignment.metadata:type_name -> ateapi.ResourceMetadata 29, // 128: ateapi.ActorAssignment.actor:type_name -> ateapi.ObjectRef - 29, // 129: ateapi.ActorAssignment.actor_template_ref:type_name -> ateapi.ObjectRef - 31, // 130: ateapi.ActorAssignment.resources:type_name -> ateapi.Resources - 29, // 131: ateapi.RegisterWorkerRequest.worker:type_name -> ateapi.ObjectRef - 104, // 132: ateapi.RegisterWorkerRequest.capacity:type_name -> ateapi.WorkerResources - 103, // 133: ateapi.RegisterWorkerRequest.hardware:type_name -> ateapi.HardwareIdentity - 101, // 134: ateapi.RegisterWorkerResponse.worker:type_name -> ateapi.Worker - 29, // 135: ateapi.MintAteomActorCertificateRequest.actor:type_name -> ateapi.ObjectRef - 29, // 136: ateapi.RequestActorSuspendRequest.worker:type_name -> ateapi.ObjectRef - 29, // 137: ateapi.RequestActorSuspendRequest.actor:type_name -> ateapi.ObjectRef - 12, // 138: ateapi.RequestActorSuspendResponse.actor:type_name -> ateapi.Actor - 10, // 139: ateapi.AccessPolicy.metadata:type_name -> ateapi.ResourceMetadata - 113, // 140: ateapi.AccessPolicy.bindings:type_name -> ateapi.Binding - 112, // 141: ateapi.CreateGlobalAccessPolicyRequest.access_policy:type_name -> ateapi.AccessPolicy - 112, // 142: ateapi.UpdateGlobalAccessPolicyRequest.access_policy:type_name -> ateapi.AccessPolicy - 29, // 143: ateapi.GetAtespaceAccessPolicyRequest.atespace:type_name -> ateapi.ObjectRef - 29, // 144: ateapi.CreateAtespaceAccessPolicyRequest.atespace:type_name -> ateapi.ObjectRef - 112, // 145: ateapi.CreateAtespaceAccessPolicyRequest.access_policy:type_name -> ateapi.AccessPolicy - 29, // 146: ateapi.UpdateAtespaceAccessPolicyRequest.atespace:type_name -> ateapi.ObjectRef - 112, // 147: ateapi.UpdateAtespaceAccessPolicyRequest.access_policy:type_name -> ateapi.AccessPolicy - 29, // 148: ateapi.DeleteAtespaceAccessPolicyRequest.atespace:type_name -> ateapi.ObjectRef - 89, // 149: ateapi.DeleteAtespaceAccessPolicyRequest.options:type_name -> ateapi.DeleteOptions - 63, // 150: ateapi.Control.GetActor:input_type -> ateapi.GetActorRequest - 64, // 151: ateapi.Control.CreateActor:input_type -> ateapi.CreateActorRequest - 65, // 152: ateapi.Control.UpdateActor:input_type -> ateapi.UpdateActorRequest - 66, // 153: ateapi.Control.SuspendActor:input_type -> ateapi.SuspendActorRequest - 68, // 154: ateapi.Control.PauseActor:input_type -> ateapi.PauseActorRequest - 70, // 155: ateapi.Control.ResumeActor:input_type -> ateapi.ResumeActorRequest - 72, // 156: ateapi.Control.RevertActor:input_type -> ateapi.RevertActorRequest - 74, // 157: ateapi.Control.DeleteActor:input_type -> ateapi.DeleteActorRequest - 75, // 158: ateapi.Control.GetActorEgressPolicy:input_type -> ateapi.GetActorEgressPolicyRequest - 76, // 159: ateapi.Control.CreateActorEgressPolicy:input_type -> ateapi.CreateActorEgressPolicyRequest - 77, // 160: ateapi.Control.UpdateActorEgressPolicy:input_type -> ateapi.UpdateActorEgressPolicyRequest - 78, // 161: ateapi.Control.DeleteActorEgressPolicy:input_type -> ateapi.DeleteActorEgressPolicyRequest - 80, // 162: ateapi.Control.MintActorJWT:input_type -> ateapi.MintActorJWTRequest - 82, // 163: ateapi.Control.MintActorCertificate:input_type -> ateapi.MintActorCertificateRequest - 86, // 164: ateapi.Control.CreateTag:input_type -> ateapi.CreateTagRequest - 79, // 165: ateapi.Control.GetTag:input_type -> ateapi.GetTagRequest - 84, // 166: ateapi.Control.ListTags:input_type -> ateapi.ListTagsRequest - 87, // 167: ateapi.Control.UpdateTag:input_type -> ateapi.UpdateTagRequest - 88, // 168: ateapi.Control.DeleteTag:input_type -> ateapi.DeleteTagRequest - 92, // 169: ateapi.Control.ListWorkers:input_type -> ateapi.ListWorkersRequest - 94, // 170: ateapi.Control.GetWorker:input_type -> ateapi.GetWorkerRequest - 95, // 171: ateapi.Control.CreateWorker:input_type -> ateapi.CreateWorkerRequest - 96, // 172: ateapi.Control.UpdateWorker:input_type -> ateapi.UpdateWorkerRequest - 97, // 173: ateapi.Control.DeleteWorker:input_type -> ateapi.DeleteWorkerRequest - 98, // 174: ateapi.Control.DrainWorker:input_type -> ateapi.DrainWorkerRequest - 90, // 175: ateapi.Control.ListWorkerActorAssignments:input_type -> ateapi.ListWorkerActorAssignmentsRequest - 99, // 176: ateapi.Control.ListActors:input_type -> ateapi.ListActorsRequest - 53, // 177: ateapi.Control.CreateAtespace:input_type -> ateapi.CreateAtespaceRequest - 54, // 178: ateapi.Control.GetAtespace:input_type -> ateapi.GetAtespaceRequest - 55, // 179: ateapi.Control.ListAtespaces:input_type -> ateapi.ListAtespacesRequest - 57, // 180: ateapi.Control.DeleteAtespace:input_type -> ateapi.DeleteAtespaceRequest - 58, // 181: ateapi.Control.CreateActorTemplate:input_type -> ateapi.CreateActorTemplateRequest - 59, // 182: ateapi.Control.GetActorTemplate:input_type -> ateapi.GetActorTemplateRequest - 60, // 183: ateapi.Control.ListActorTemplates:input_type -> ateapi.ListActorTemplatesRequest - 62, // 184: ateapi.Control.DeleteActorTemplate:input_type -> ateapi.DeleteActorTemplateRequest - 114, // 185: ateapi.Control.GetGlobalAccessPolicy:input_type -> ateapi.GetGlobalAccessPolicyRequest - 115, // 186: ateapi.Control.CreateGlobalAccessPolicy:input_type -> ateapi.CreateGlobalAccessPolicyRequest - 116, // 187: ateapi.Control.UpdateGlobalAccessPolicy:input_type -> ateapi.UpdateGlobalAccessPolicyRequest - 117, // 188: ateapi.Control.GetAtespaceAccessPolicy:input_type -> ateapi.GetAtespaceAccessPolicyRequest - 118, // 189: ateapi.Control.CreateAtespaceAccessPolicy:input_type -> ateapi.CreateAtespaceAccessPolicyRequest - 119, // 190: ateapi.Control.UpdateAtespaceAccessPolicy:input_type -> ateapi.UpdateAtespaceAccessPolicyRequest - 120, // 191: ateapi.Control.DeleteAtespaceAccessPolicy:input_type -> ateapi.DeleteAtespaceAccessPolicyRequest - 106, // 192: ateapi.WorkerService.RegisterWorker:input_type -> ateapi.RegisterWorkerRequest - 108, // 193: ateapi.WorkerService.MintAteomActorCertificate:input_type -> ateapi.MintAteomActorCertificateRequest - 110, // 194: ateapi.WorkerService.RequestActorSuspend:input_type -> ateapi.RequestActorSuspendRequest - 12, // 195: ateapi.Control.GetActor:output_type -> ateapi.Actor - 12, // 196: ateapi.Control.CreateActor:output_type -> ateapi.Actor - 12, // 197: ateapi.Control.UpdateActor:output_type -> ateapi.Actor - 67, // 198: ateapi.Control.SuspendActor:output_type -> ateapi.SuspendActorResponse - 69, // 199: ateapi.Control.PauseActor:output_type -> ateapi.PauseActorResponse - 71, // 200: ateapi.Control.ResumeActor:output_type -> ateapi.ResumeActorResponse - 73, // 201: ateapi.Control.RevertActor:output_type -> ateapi.RevertActorResponse - 12, // 202: ateapi.Control.DeleteActor:output_type -> ateapi.Actor - 13, // 203: ateapi.Control.GetActorEgressPolicy:output_type -> ateapi.EgressPolicy - 13, // 204: ateapi.Control.CreateActorEgressPolicy:output_type -> ateapi.EgressPolicy - 13, // 205: ateapi.Control.UpdateActorEgressPolicy:output_type -> ateapi.EgressPolicy - 13, // 206: ateapi.Control.DeleteActorEgressPolicy:output_type -> ateapi.EgressPolicy - 81, // 207: ateapi.Control.MintActorJWT:output_type -> ateapi.MintActorJWTResponse - 83, // 208: ateapi.Control.MintActorCertificate:output_type -> ateapi.MintActorCertificateResponse - 27, // 209: ateapi.Control.CreateTag:output_type -> ateapi.Tag - 27, // 210: ateapi.Control.GetTag:output_type -> ateapi.Tag - 85, // 211: ateapi.Control.ListTags:output_type -> ateapi.ListTagsResponse - 27, // 212: ateapi.Control.UpdateTag:output_type -> ateapi.Tag - 27, // 213: ateapi.Control.DeleteTag:output_type -> ateapi.Tag - 93, // 214: ateapi.Control.ListWorkers:output_type -> ateapi.ListWorkersResponse - 101, // 215: ateapi.Control.GetWorker:output_type -> ateapi.Worker - 101, // 216: ateapi.Control.CreateWorker:output_type -> ateapi.Worker - 101, // 217: ateapi.Control.UpdateWorker:output_type -> ateapi.Worker - 101, // 218: ateapi.Control.DeleteWorker:output_type -> ateapi.Worker - 101, // 219: ateapi.Control.DrainWorker:output_type -> ateapi.Worker - 91, // 220: ateapi.Control.ListWorkerActorAssignments:output_type -> ateapi.ListWorkerActorAssignmentsResponse - 100, // 221: ateapi.Control.ListActors:output_type -> ateapi.ListActorsResponse - 28, // 222: ateapi.Control.CreateAtespace:output_type -> ateapi.Atespace - 28, // 223: ateapi.Control.GetAtespace:output_type -> ateapi.Atespace - 56, // 224: ateapi.Control.ListAtespaces:output_type -> ateapi.ListAtespacesResponse - 28, // 225: ateapi.Control.DeleteAtespace:output_type -> ateapi.Atespace - 30, // 226: ateapi.Control.CreateActorTemplate:output_type -> ateapi.ActorTemplate - 30, // 227: ateapi.Control.GetActorTemplate:output_type -> ateapi.ActorTemplate - 61, // 228: ateapi.Control.ListActorTemplates:output_type -> ateapi.ListActorTemplatesResponse - 30, // 229: ateapi.Control.DeleteActorTemplate:output_type -> ateapi.ActorTemplate - 112, // 230: ateapi.Control.GetGlobalAccessPolicy:output_type -> ateapi.AccessPolicy - 112, // 231: ateapi.Control.CreateGlobalAccessPolicy:output_type -> ateapi.AccessPolicy - 112, // 232: ateapi.Control.UpdateGlobalAccessPolicy:output_type -> ateapi.AccessPolicy - 112, // 233: ateapi.Control.GetAtespaceAccessPolicy:output_type -> ateapi.AccessPolicy - 112, // 234: ateapi.Control.CreateAtespaceAccessPolicy:output_type -> ateapi.AccessPolicy - 112, // 235: ateapi.Control.UpdateAtespaceAccessPolicy:output_type -> ateapi.AccessPolicy - 112, // 236: ateapi.Control.DeleteAtespaceAccessPolicy:output_type -> ateapi.AccessPolicy - 107, // 237: ateapi.WorkerService.RegisterWorker:output_type -> ateapi.RegisterWorkerResponse - 109, // 238: ateapi.WorkerService.MintAteomActorCertificate:output_type -> ateapi.MintAteomActorCertificateResponse - 111, // 239: ateapi.WorkerService.RequestActorSuspend:output_type -> ateapi.RequestActorSuspendResponse - 195, // [195:240] is the sub-list for method output_type - 150, // [150:195] is the sub-list for method input_type - 150, // [150:150] is the sub-list for extension type_name - 150, // [150:150] is the sub-list for extension extendee - 0, // [0:150] is the sub-list for field type_name + 31, // 129: ateapi.ActorAssignment.resources:type_name -> ateapi.Resources + 29, // 130: ateapi.RegisterWorkerRequest.worker:type_name -> ateapi.ObjectRef + 104, // 131: ateapi.RegisterWorkerRequest.capacity:type_name -> ateapi.WorkerResources + 103, // 132: ateapi.RegisterWorkerRequest.hardware:type_name -> ateapi.HardwareIdentity + 101, // 133: ateapi.RegisterWorkerResponse.worker:type_name -> ateapi.Worker + 29, // 134: ateapi.MintAteomActorCertificateRequest.actor:type_name -> ateapi.ObjectRef + 29, // 135: ateapi.RequestActorSuspendRequest.worker:type_name -> ateapi.ObjectRef + 29, // 136: ateapi.RequestActorSuspendRequest.actor:type_name -> ateapi.ObjectRef + 12, // 137: ateapi.RequestActorSuspendResponse.actor:type_name -> ateapi.Actor + 10, // 138: ateapi.AccessPolicy.metadata:type_name -> ateapi.ResourceMetadata + 113, // 139: ateapi.AccessPolicy.bindings:type_name -> ateapi.Binding + 112, // 140: ateapi.CreateGlobalAccessPolicyRequest.access_policy:type_name -> ateapi.AccessPolicy + 112, // 141: ateapi.UpdateGlobalAccessPolicyRequest.access_policy:type_name -> ateapi.AccessPolicy + 29, // 142: ateapi.GetAtespaceAccessPolicyRequest.atespace:type_name -> ateapi.ObjectRef + 29, // 143: ateapi.CreateAtespaceAccessPolicyRequest.atespace:type_name -> ateapi.ObjectRef + 112, // 144: ateapi.CreateAtespaceAccessPolicyRequest.access_policy:type_name -> ateapi.AccessPolicy + 29, // 145: ateapi.UpdateAtespaceAccessPolicyRequest.atespace:type_name -> ateapi.ObjectRef + 112, // 146: ateapi.UpdateAtespaceAccessPolicyRequest.access_policy:type_name -> ateapi.AccessPolicy + 29, // 147: ateapi.DeleteAtespaceAccessPolicyRequest.atespace:type_name -> ateapi.ObjectRef + 89, // 148: ateapi.DeleteAtespaceAccessPolicyRequest.options:type_name -> ateapi.DeleteOptions + 63, // 149: ateapi.Control.GetActor:input_type -> ateapi.GetActorRequest + 64, // 150: ateapi.Control.CreateActor:input_type -> ateapi.CreateActorRequest + 65, // 151: ateapi.Control.UpdateActor:input_type -> ateapi.UpdateActorRequest + 66, // 152: ateapi.Control.SuspendActor:input_type -> ateapi.SuspendActorRequest + 68, // 153: ateapi.Control.PauseActor:input_type -> ateapi.PauseActorRequest + 70, // 154: ateapi.Control.ResumeActor:input_type -> ateapi.ResumeActorRequest + 72, // 155: ateapi.Control.RevertActor:input_type -> ateapi.RevertActorRequest + 74, // 156: ateapi.Control.DeleteActor:input_type -> ateapi.DeleteActorRequest + 75, // 157: ateapi.Control.GetActorEgressPolicy:input_type -> ateapi.GetActorEgressPolicyRequest + 76, // 158: ateapi.Control.CreateActorEgressPolicy:input_type -> ateapi.CreateActorEgressPolicyRequest + 77, // 159: ateapi.Control.UpdateActorEgressPolicy:input_type -> ateapi.UpdateActorEgressPolicyRequest + 78, // 160: ateapi.Control.DeleteActorEgressPolicy:input_type -> ateapi.DeleteActorEgressPolicyRequest + 80, // 161: ateapi.Control.MintActorJWT:input_type -> ateapi.MintActorJWTRequest + 82, // 162: ateapi.Control.MintActorCertificate:input_type -> ateapi.MintActorCertificateRequest + 86, // 163: ateapi.Control.CreateTag:input_type -> ateapi.CreateTagRequest + 79, // 164: ateapi.Control.GetTag:input_type -> ateapi.GetTagRequest + 84, // 165: ateapi.Control.ListTags:input_type -> ateapi.ListTagsRequest + 87, // 166: ateapi.Control.UpdateTag:input_type -> ateapi.UpdateTagRequest + 88, // 167: ateapi.Control.DeleteTag:input_type -> ateapi.DeleteTagRequest + 92, // 168: ateapi.Control.ListWorkers:input_type -> ateapi.ListWorkersRequest + 94, // 169: ateapi.Control.GetWorker:input_type -> ateapi.GetWorkerRequest + 95, // 170: ateapi.Control.CreateWorker:input_type -> ateapi.CreateWorkerRequest + 96, // 171: ateapi.Control.UpdateWorker:input_type -> ateapi.UpdateWorkerRequest + 97, // 172: ateapi.Control.DeleteWorker:input_type -> ateapi.DeleteWorkerRequest + 98, // 173: ateapi.Control.DrainWorker:input_type -> ateapi.DrainWorkerRequest + 90, // 174: ateapi.Control.ListWorkerActorAssignments:input_type -> ateapi.ListWorkerActorAssignmentsRequest + 99, // 175: ateapi.Control.ListActors:input_type -> ateapi.ListActorsRequest + 53, // 176: ateapi.Control.CreateAtespace:input_type -> ateapi.CreateAtespaceRequest + 54, // 177: ateapi.Control.GetAtespace:input_type -> ateapi.GetAtespaceRequest + 55, // 178: ateapi.Control.ListAtespaces:input_type -> ateapi.ListAtespacesRequest + 57, // 179: ateapi.Control.DeleteAtespace:input_type -> ateapi.DeleteAtespaceRequest + 58, // 180: ateapi.Control.CreateActorTemplate:input_type -> ateapi.CreateActorTemplateRequest + 59, // 181: ateapi.Control.GetActorTemplate:input_type -> ateapi.GetActorTemplateRequest + 60, // 182: ateapi.Control.ListActorTemplates:input_type -> ateapi.ListActorTemplatesRequest + 62, // 183: ateapi.Control.DeleteActorTemplate:input_type -> ateapi.DeleteActorTemplateRequest + 114, // 184: ateapi.Control.GetGlobalAccessPolicy:input_type -> ateapi.GetGlobalAccessPolicyRequest + 115, // 185: ateapi.Control.CreateGlobalAccessPolicy:input_type -> ateapi.CreateGlobalAccessPolicyRequest + 116, // 186: ateapi.Control.UpdateGlobalAccessPolicy:input_type -> ateapi.UpdateGlobalAccessPolicyRequest + 117, // 187: ateapi.Control.GetAtespaceAccessPolicy:input_type -> ateapi.GetAtespaceAccessPolicyRequest + 118, // 188: ateapi.Control.CreateAtespaceAccessPolicy:input_type -> ateapi.CreateAtespaceAccessPolicyRequest + 119, // 189: ateapi.Control.UpdateAtespaceAccessPolicy:input_type -> ateapi.UpdateAtespaceAccessPolicyRequest + 120, // 190: ateapi.Control.DeleteAtespaceAccessPolicy:input_type -> ateapi.DeleteAtespaceAccessPolicyRequest + 106, // 191: ateapi.WorkerService.RegisterWorker:input_type -> ateapi.RegisterWorkerRequest + 108, // 192: ateapi.WorkerService.MintAteomActorCertificate:input_type -> ateapi.MintAteomActorCertificateRequest + 110, // 193: ateapi.WorkerService.RequestActorSuspend:input_type -> ateapi.RequestActorSuspendRequest + 12, // 194: ateapi.Control.GetActor:output_type -> ateapi.Actor + 12, // 195: ateapi.Control.CreateActor:output_type -> ateapi.Actor + 12, // 196: ateapi.Control.UpdateActor:output_type -> ateapi.Actor + 67, // 197: ateapi.Control.SuspendActor:output_type -> ateapi.SuspendActorResponse + 69, // 198: ateapi.Control.PauseActor:output_type -> ateapi.PauseActorResponse + 71, // 199: ateapi.Control.ResumeActor:output_type -> ateapi.ResumeActorResponse + 73, // 200: ateapi.Control.RevertActor:output_type -> ateapi.RevertActorResponse + 12, // 201: ateapi.Control.DeleteActor:output_type -> ateapi.Actor + 13, // 202: ateapi.Control.GetActorEgressPolicy:output_type -> ateapi.EgressPolicy + 13, // 203: ateapi.Control.CreateActorEgressPolicy:output_type -> ateapi.EgressPolicy + 13, // 204: ateapi.Control.UpdateActorEgressPolicy:output_type -> ateapi.EgressPolicy + 13, // 205: ateapi.Control.DeleteActorEgressPolicy:output_type -> ateapi.EgressPolicy + 81, // 206: ateapi.Control.MintActorJWT:output_type -> ateapi.MintActorJWTResponse + 83, // 207: ateapi.Control.MintActorCertificate:output_type -> ateapi.MintActorCertificateResponse + 27, // 208: ateapi.Control.CreateTag:output_type -> ateapi.Tag + 27, // 209: ateapi.Control.GetTag:output_type -> ateapi.Tag + 85, // 210: ateapi.Control.ListTags:output_type -> ateapi.ListTagsResponse + 27, // 211: ateapi.Control.UpdateTag:output_type -> ateapi.Tag + 27, // 212: ateapi.Control.DeleteTag:output_type -> ateapi.Tag + 93, // 213: ateapi.Control.ListWorkers:output_type -> ateapi.ListWorkersResponse + 101, // 214: ateapi.Control.GetWorker:output_type -> ateapi.Worker + 101, // 215: ateapi.Control.CreateWorker:output_type -> ateapi.Worker + 101, // 216: ateapi.Control.UpdateWorker:output_type -> ateapi.Worker + 101, // 217: ateapi.Control.DeleteWorker:output_type -> ateapi.Worker + 101, // 218: ateapi.Control.DrainWorker:output_type -> ateapi.Worker + 91, // 219: ateapi.Control.ListWorkerActorAssignments:output_type -> ateapi.ListWorkerActorAssignmentsResponse + 100, // 220: ateapi.Control.ListActors:output_type -> ateapi.ListActorsResponse + 28, // 221: ateapi.Control.CreateAtespace:output_type -> ateapi.Atespace + 28, // 222: ateapi.Control.GetAtespace:output_type -> ateapi.Atespace + 56, // 223: ateapi.Control.ListAtespaces:output_type -> ateapi.ListAtespacesResponse + 28, // 224: ateapi.Control.DeleteAtespace:output_type -> ateapi.Atespace + 30, // 225: ateapi.Control.CreateActorTemplate:output_type -> ateapi.ActorTemplate + 30, // 226: ateapi.Control.GetActorTemplate:output_type -> ateapi.ActorTemplate + 61, // 227: ateapi.Control.ListActorTemplates:output_type -> ateapi.ListActorTemplatesResponse + 30, // 228: ateapi.Control.DeleteActorTemplate:output_type -> ateapi.ActorTemplate + 112, // 229: ateapi.Control.GetGlobalAccessPolicy:output_type -> ateapi.AccessPolicy + 112, // 230: ateapi.Control.CreateGlobalAccessPolicy:output_type -> ateapi.AccessPolicy + 112, // 231: ateapi.Control.UpdateGlobalAccessPolicy:output_type -> ateapi.AccessPolicy + 112, // 232: ateapi.Control.GetAtespaceAccessPolicy:output_type -> ateapi.AccessPolicy + 112, // 233: ateapi.Control.CreateAtespaceAccessPolicy:output_type -> ateapi.AccessPolicy + 112, // 234: ateapi.Control.UpdateAtespaceAccessPolicy:output_type -> ateapi.AccessPolicy + 112, // 235: ateapi.Control.DeleteAtespaceAccessPolicy:output_type -> ateapi.AccessPolicy + 107, // 236: ateapi.WorkerService.RegisterWorker:output_type -> ateapi.RegisterWorkerResponse + 109, // 237: ateapi.WorkerService.MintAteomActorCertificate:output_type -> ateapi.MintAteomActorCertificateResponse + 111, // 238: ateapi.WorkerService.RequestActorSuspend:output_type -> ateapi.RequestActorSuspendResponse + 194, // [194:239] is the sub-list for method output_type + 149, // [149:194] is the sub-list for method input_type + 149, // [149:149] is the sub-list for extension type_name + 149, // [149:149] is the sub-list for extension extendee + 0, // [0:149] is the sub-list for field type_name } func init() { file_ateapi_proto_init() } diff --git a/pkg/proto/ateapipb/ateapi.proto b/pkg/proto/ateapipb/ateapi.proto index 79124939c9..c552b21614 100644 --- a/pkg/proto/ateapipb/ateapi.proto +++ b/pkg/proto/ateapipb/ateapi.proto @@ -2203,13 +2203,6 @@ message ActorAssignment { // +k8s:format=k8s-uuid string actor_uid = 3; - // actor_template_ref names the ActorTemplate resource the assigned Actor - // uses. - // - // +k8s:required - // +k8s:subfield(atespace)=+k8s:required - ObjectRef actor_template_ref = 4; - // The resources allocated to the Actor when it was assigned. // // +k8s:optional From 890234f686f3b7470c7f35619ab1c4f64962289e Mon Sep 17 00:00:00 2001 From: Eren Akbulut Date: Thu, 8 Oct 2026 18:19:20 +0000 Subject: [PATCH 07/44] Return the caller's context error when an atelet call fails (#2236) When the caller's context ends during an atelet call, for example because its deadline passes during a slow restore, the control API keeps the actor assigned but returns `Internal` to the caller. So `ResumeActor` can leave an actor `RESUMING` on a worker whose restore already succeeded while the caller receives a server-failure status. When the workflow context has ended, return its error, so the caller gets `Canceled` or `DeadlineExceeded` (the `apierror` convention for context errors). The actor keeps its worker assignment, so a retry can finish the operation. A `Canceled` or `DeadlineExceeded` from atelet with a live context still returns `Internal`, as on main, so the caller is not told that it cancelled. `Unavailable` keeps the mapping from #2196. Permanent errors keep their crash handling. This follows the shape that @laoj2 suggested in review. The unit tests cover a cancelled and an expired workflow context, atelet `Canceled` and `DeadlineExceeded` with a live context, and the existing permanent failures. The gRPC functional test checks `Unavailable`, atelet `Canceled` and `DeadlineExceeded`, and a caller deadline that passes during a slow restore. In each case the actor stays `RESUMING` with its worker assignment, and a retry reaches `RUNNING` on that worker. A permanent local restore failure still returns `Internal`, records a crash, clears the assignment, and releases worker capacity. ### Validation On main `0b91488d79637052458a29a6ba6c757bcf7a9292`, on 2026-10-08, the `controlapi` (51.4 s) and `functionaltest` (64.9 s, `REQUIRE_DOCKER=true`) race suites pass. Linux-targeted `go vet` and `gofmt` are clean. Without the context-error branch, three `TestHandleAteletError` subtests fail. ### Earlier lab evidence These trials used the first version of this PR, which also mapped atelet's own `Canceled` and `DeadlineExceeded` with a live context. With the current version, the second and third rows return `Internal` again. The actor still stays assigned and the retry still works. A separate two-worker ARM64 lab ran three trials per error code. A diagnostic interceptor replaced the reply only after the real atelet `Restore` handler succeeded. It tested selected error replies, rather than physical packet loss. | Injected reply code | Before: client code | With this mapping: client code | Fresh trials with the fix | | --- | --- | --- | ---: | | `Unavailable` | `Unavailable` | `Unavailable` | 3 passed | | `Canceled` | `Internal` | `Canceled` | 3 passed | | `DeadlineExceeded` | `Internal` | `DeadlineExceeded` | 3 passed | All nine trials passed 72 assertions. The actor retained its worker assignment and one observed live guest; explicit retry reached `RUNNING` with the same actor UID, guest execution ID, worker Pod UID, and checkpoint from reply loss through retry. A local AX outcome-reporting candidate consequently retained an uncertain result instead of reporting a definitive failure. Those trials used the unchanged AX candidate `55602feda4accf71d17705d5e6c48bc2c1a5ff1e` on `be7b1a8`, and Substrate mapping commit `438291462203f253366b1e5f8fc4f5b3fd63af9f` on an earlier lab base containing the availability candidate from #2055. This PR contains only the three-file mapping/test patch, freshly checked on current upstream main; the earlier trials are supporting evidence, not a live run of this rebased branch. The richer retry contract discussed in #1562 and automatic worker recovery remain separate work. --- cmd/ateapi/internal/controlapi/crash.go | 5 + cmd/ateapi/internal/controlapi/crash_test.go | 30 +++- .../controlapi/functionaltest/actor_test.go | 129 +++++++++++------- 3 files changed, 112 insertions(+), 52 deletions(-) diff --git a/cmd/ateapi/internal/controlapi/crash.go b/cmd/ateapi/internal/controlapi/crash.go index 0700de5710..e32b5f8353 100644 --- a/cmd/ateapi/internal/controlapi/crash.go +++ b/cmd/ateapi/internal/controlapi/crash.go @@ -93,6 +93,11 @@ func handleAteletError(ctx context.Context, st crashActorStore, actorRef resourc return fmt.Errorf("actor %s crashed: %w", actorRef, err) } + // The caller went away or its deadline passed: report that, not atelet's + // error. The actor keeps its assignment, so a retry can finish the operation. + if ctx.Err() != nil { + return fmt.Errorf("while calling atelet %s: %w: %w", rpc, err, ctx.Err()) + } if status.Code(err) == codes.Unavailable { return apierror.Unavailable("while calling atelet %s: %w", rpc, err) } diff --git a/cmd/ateapi/internal/controlapi/crash_test.go b/cmd/ateapi/internal/controlapi/crash_test.go index 3c919f584b..54b34af65f 100644 --- a/cmd/ateapi/internal/controlapi/crash_test.go +++ b/cmd/ateapi/internal/controlapi/crash_test.go @@ -352,6 +352,8 @@ func TestAteletCrashMessage(t *testing.T) { func TestHandleAteletError(t *testing.T) { ended, cancel := context.WithCancel(context.Background()) cancel() + expired, cancelExpired := context.WithDeadline(context.Background(), time.Unix(0, 0)) + defer cancelExpired() tests := []struct { name string @@ -396,12 +398,28 @@ func TestHandleAteletError(t *testing.T) { wantCode: codes.Unavailable, wantState: ateapipb.ActorState_ACTOR_STATE_RUNNING, }, + { + name: "cancelled caller gets Canceled and leaves the actor as it was", + ctx: ended, + rpc: "Restore", + err: fmt.Errorf("while restoring actor: %w", status.Error(codes.Canceled, "connection closing")), + wantCode: codes.Canceled, + wantState: ateapipb.ActorState_ACTOR_STATE_RUNNING, + }, + { + name: "expired caller deadline gets DeadlineExceeded and leaves the actor as it was", + ctx: expired, + rpc: "Restore", + err: fmt.Errorf("while restoring actor: %w", status.Error(codes.DeadlineExceeded, "restore reply timed out")), + wantCode: codes.DeadlineExceeded, + wantState: ateapipb.ActorState_ACTOR_STATE_RUNNING, + }, { name: "ended workflow context leaves the actor as it was", ctx: ended, rpc: "Restore", err: status.Error(codes.Internal, "context canceled"), - wantCode: codes.Internal, + wantCode: codes.Canceled, wantState: ateapipb.ActorState_ACTOR_STATE_RUNNING, }, { @@ -452,6 +470,16 @@ func TestHandleAteletError(t *testing.T) { t.Errorf("apierror.Code(handleAteletError()) = %v, want %v (err: %v)", got, tt.wantCode, err) } + if !errors.Is(err, tt.err) { + t.Errorf("original atelet error was lost: %v", err) + } + if tt.wantCode == codes.Canceled && !errors.Is(err, context.Canceled) { + t.Errorf("error does not wrap context.Canceled: %v", err) + } + if tt.wantCode == codes.DeadlineExceeded && !errors.Is(err, context.DeadlineExceeded) { + t.Errorf("error does not wrap context.DeadlineExceeded: %v", err) + } + actor, err := st.GetActor(ctx, actorRef) if err != nil { t.Fatalf("GetActor: %v", err) diff --git a/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go b/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go index 513f8a6a9c..60656acb1e 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go @@ -3400,60 +3400,87 @@ func TestResumeActor_AteletFailureCrashesActor(t *testing.T) { } } -// TestResumeActor_AteletUnavailableLeavesActorResuming: an Unavailable from -// atelet, such as atelet restarting, does not crash the actor. It stays -// RESUMING on its worker, and the next resume finishes. -func TestResumeActor_AteletUnavailableLeavesActorResuming(t *testing.T) { - ns := namespaceForTest("ns-resume-atelet-unavailable") - tc := setupTest(t, ns) - defer tc.cleanup() +// A transient atelet failure, or a caller whose deadline passes during the +// restore, leaves the actor RESUMING on its worker, and the next resume finishes. +// Only Unavailable and the caller's own deadline reach the caller with their +// code. Atelet's own Canceled or DeadlineExceeded reach it as Internal. +func TestResumeActor_AteletTransportErrorLeavesActorResuming(t *testing.T) { + for _, tt := range []struct { + name string + fail error + delay time.Duration + deadline time.Duration + wantCode codes.Code + }{ + {name: "Unavailable", fail: status.Error(codes.Unavailable, "restore reply lost"), wantCode: codes.Unavailable}, + {name: "atelet Canceled", fail: status.Error(codes.Canceled, "restore reply lost"), wantCode: codes.Internal}, + {name: "atelet DeadlineExceeded", fail: status.Error(codes.DeadlineExceeded, "restore reply lost"), wantCode: codes.Internal}, + {name: "caller deadline", delay: time.Second, deadline: 200 * time.Millisecond, wantCode: codes.DeadlineExceeded}, + } { + t.Run(tt.name, func(t *testing.T) { + ns := namespaceForTest("ns-resume-atelet-" + strings.ReplaceAll(strings.ToLower(tt.name), " ", "-")) + tc := setupTest(t, ns) + defer tc.cleanup() - createTemplate(t, tc, ns) - podUID := createWorkerPod(t, tc, ns, "worker-1", "node1", "pool1") + createTemplate(t, tc, ns) + podUID := createWorkerPod(t, tc, ns, "worker-1", "node1", "pool1") - name := "id1" - ref := &ateapipb.ObjectRef{Atespace: testAtespace, Name: name} - actorRef := resources.ActorRef{Atespace: testAtespace, Name: name} - if _, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: name}, - ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"}, - }}); err != nil { - t.Fatalf("CreateActor failed: %v", err) - } + name := "id1" + ref := &ateapipb.ObjectRef{Atespace: testAtespace, Name: name} + actorRef := resources.ActorRef{Atespace: testAtespace, Name: name} + if _, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: name}, + ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"}, + }}); err != nil { + t.Fatalf("CreateActor failed: %v", err) + } - tc.fakeAtelet.FailRestore = status.Error(codes.Unavailable, "connection refused") - _, err := tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{Actor: ref}) - if got := status.Code(err); got != codes.Unavailable { - t.Fatalf("ResumeActor status code = %v, want %v (err: %v)", got, codes.Unavailable, err) - } + tc.fakeAtelet.FailRestore = tt.fail + tc.fakeAtelet.RestoreDelay = tt.delay + ctx := context.Background() + if tt.deadline > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, tt.deadline) + defer cancel() + } + _, err := tc.client.ResumeActor(ctx, &ateapipb.ResumeActorRequest{Actor: ref}) + if got := status.Code(err); got != tt.wantCode { + t.Fatalf("ResumeActor status code = %v, want %v (err: %v)", got, tt.wantCode, err) + } - actor, err := tc.persistence.GetActor(context.Background(), actorRef) - if err != nil { - t.Fatalf("failed to get actor from store: %v", err) - } - if got := actor.GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_RESUMING { - t.Fatalf("state after Unavailable = %v, want RESUMING", got) - } - if crash := actor.GetStatus().GetCrash(); crash != nil { - t.Errorf("crash recorded after Unavailable: %v", crash) - } - if got := actor.GetStatus().GetWorkerAssignment().GetWorker().GetName(); got != podUID { - t.Errorf("assigned worker = %q, want %q kept for the retry", got, podUID) - } + actor, err := tc.persistence.GetActor(context.Background(), actorRef) + if err != nil { + t.Fatalf("failed to get actor from store: %v", err) + } + if got := actor.GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_RESUMING { + t.Fatalf("state after %s = %v, want RESUMING", tt.name, got) + } + if crash := actor.GetStatus().GetCrash(); crash != nil { + t.Errorf("crash recorded after %s: %v", tt.name, crash) + } + if got := actor.GetStatus().GetWorkerAssignment().GetWorker().GetName(); got != podUID { + t.Errorf("assigned worker = %q, want %q kept for the retry", got, podUID) + } - tc.fakeAtelet.FailRestore = nil - if _, err := tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{Actor: ref}); err != nil { - t.Fatalf("retried ResumeActor failed: %v", err) - } - actor, err = tc.persistence.GetActor(context.Background(), actorRef) - if err != nil { - t.Fatalf("failed to get actor from store: %v", err) - } - if got := actor.GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_RUNNING { - t.Errorf("state after retry = %v, want RUNNING", got) - } - if got := actor.GetStatus().GetWorkerAssignment().GetWorker().GetName(); got != podUID { - t.Errorf("assigned worker after retry = %q, want %q", got, podUID) + // The fake still holds its lock while a delayed restore sleeps. + tc.fakeAtelet.Lock.Lock() + tc.fakeAtelet.FailRestore = nil + tc.fakeAtelet.RestoreDelay = 0 + tc.fakeAtelet.Lock.Unlock() + if _, err := tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{Actor: ref}); err != nil { + t.Fatalf("retried ResumeActor failed: %v", err) + } + actor, err = tc.persistence.GetActor(context.Background(), actorRef) + if err != nil { + t.Fatalf("failed to get actor from store: %v", err) + } + if got := actor.GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_RUNNING { + t.Errorf("state after retry = %v, want RUNNING", got) + } + if got := actor.GetStatus().GetWorkerAssignment().GetWorker().GetName(); got != podUID { + t.Errorf("assigned worker after retry = %q, want %q", got, podUID) + } + }) } } @@ -3484,8 +3511,8 @@ func TestResumeActor_LocalRestoreFailureCrashesActor(t *testing.T) { tc.fakeAtelet.Reset() tc.fakeAtelet.FailRestore = status.Error(codes.Internal, "injected restore failure") - if _, err := tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{Actor: ref}); err == nil { - t.Fatal("ResumeActor succeeded despite failing restore") + if _, err := tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{Actor: ref}); status.Code(err) != codes.Internal { + t.Fatalf("ResumeActor status code = %v, want Internal (err: %v)", status.Code(err), err) } if got := tc.fakeAtelet.RestoreRequest.GetType(); got != ateletpb.CheckpointType_CHECKPOINT_TYPE_LOCAL { t.Fatalf("restore type = %v, want LOCAL", got) From 130d74636146a5f23616c10e4b130b58ec07bc59 Mon Sep 17 00:00:00 2001 From: Lucky Abolorunke <62015433+Oneimu@users.noreply.github.com> Date: Thu, 8 Oct 2026 18:24:31 +0000 Subject: [PATCH 08/44] =?UTF-8?q?benchmark(ateom-gvisor):=20log=20per-acto?= =?UTF-8?q?r=20checkpoint=20and=20restore=20phase=20b=E2=80=A6=20(#2302)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### ateom-gvisor: log per-actor checkpoint and restore phase breakdowns Follow-up to #1941, which gave ateom-microvm the per-phase `Checkpoint timing breakdown` / `Restore timing breakdown` records. This PR gives ateom-gvisor the same, so the report in #2009 covers both sandbox classes. #### Why it need On a gVisor pool the breakdown stops at atelet's `ateom_restore` / `ateom_checkpoint` phase. That phase is most of a gVisor wake — roughly 165 ms of a 190 ms resume in local measurements — and nothing inside it is timed, so there is no way to say whether a slow resume was the network setup, the pause container, the app container's `runsc restore`, or the wakeup probe. gVisor is the default sandbox class, so this is the larger blind spot of the two. #### What - **`cmd/ateom-gvisor`**: `CheckpointWorkload` and `RestoreWorkload` emit one joinable record per operation from a `defer`, with the actor identity, snapshot scope, `error.type` on failure, and one float-seconds field per phase under `ateom.actor.{checkpoint,restore}.duration.`. Both records are written on failure too, carrying the phases that completed and the failing step's elapsed time. - restore phases: `prep`, `egress_prepare`, `egress_join`, `net_setup`, `durable_dir`, `pause_rootfs`, `pause_create`, `pause_restore`, `app_rootfs`, `app_create`, `app_restore`, `wakeup_probe`, `activate`, `total` — sequential, they partition the total. The restore record also carries the container count. - checkpoint phases: `pause`, `checkpoint`, `resume`, `teardown`, `total`. A small `restoreTiming` / `checkpointTiming` pair and a `lap()` helper stamp the phases as the handler advances; `phaselog_test.go` covers them. - **`internal/ateomphaselog`** (new): the record rendering both ateoms now share — `Phase`, `SnapshotPhaseAttrs`, `LogSnapshotPhases`, `ScopeLogValue` — moved out of `cmd/ateom-microvm/phaselog.go` with its tests. Phase *names* stay private to each binary, as in #1941: they are implementation details of one runtime, not values of the `ate.snapshot.phase` metric enum, and this PR adds nothing to the registry. - **`cmd/ateom-microvm`**: a mechanical refactor onto the shared package; the micro-VM records are unchanged. - **`docs/observability.md`**: the ateom paragraph now describes both runtimes' records and their failure semantics. No new instruments, no registry change, no behavior change. #### Example ```json {"msg":"Restore timing breakdown","ate.actor.uid":"8f2a…","ate.snapshot.scope":"full", "ateom.actor.restore.duration.prep":0.004, "ateom.actor.restore.duration.net_setup":0.011, "ateom.actor.restore.duration.pause_restore":0.038, "ateom.actor.restore.duration.app_restore":0.097, "ateom.actor.restore.duration.wakeup_probe":0.021, "ateom.actor.restore.duration.total":0.183, "container_count":2} ``` Joined with atelet's record for the same actor (which carries `ate.sandbox.class=gvisor`), `phase_report.py` from #2009 lists the gVisor rows alongside the micro-VM ones. The analyzer needs a small follow-up to stop assuming every ateom record is micro-VM (class from the paired atelet record, residual rule by class); that goes with #2148. ### Testing - `go test ./internal/ateomphaselog/...` — 4 tests (record shape, seconds not nanoseconds, scope mapping, `error.type` for gRPC / context / plain errors). --- cmd/ateom-gvisor/main.go | 147 +++++++++++---- cmd/ateom-gvisor/phaselog.go | 115 ++++++++++++ cmd/ateom-gvisor/phaselog_test.go | 168 ++++++++++++++++++ cmd/ateom-microvm/checkpoint.go | 19 +- cmd/ateom-microvm/phaselog.go | 92 +--------- cmd/ateom-microvm/restore.go | 29 +-- docs/observability.md | 2 +- internal/ateomphaselog/phaselog.go | 106 +++++++++++ .../ateomphaselog}/phaselog_test.go | 64 +++++-- 9 files changed, 576 insertions(+), 166 deletions(-) create mode 100644 cmd/ateom-gvisor/phaselog.go create mode 100644 cmd/ateom-gvisor/phaselog_test.go create mode 100644 internal/ateomphaselog/phaselog.go rename {cmd/ateom-microvm => internal/ateomphaselog}/phaselog_test.go (72%) diff --git a/cmd/ateom-gvisor/main.go b/cmd/ateom-gvisor/main.go index ca92ec8374..a74f3cc02f 100644 --- a/cmd/ateom-gvisor/main.go +++ b/cmd/ateom-gvisor/main.go @@ -40,6 +40,7 @@ import ( "github.com/agent-substrate/substrate/internal/ateom" "github.com/agent-substrate/substrate/internal/ateomcgroup" "github.com/agent-substrate/substrate/internal/ateomnet" + "github.com/agent-substrate/substrate/internal/ateomphaselog" "github.com/agent-substrate/substrate/internal/ateomstats" "github.com/agent-substrate/substrate/internal/ateomtunnel" "github.com/agent-substrate/substrate/internal/childreap" @@ -669,7 +670,9 @@ func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkload // Allow checkpointing even if the pod is shutting down. This will allow actors // (or the harness) to suspend on shutdown. -func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.CheckpointWorkloadRequest) (*ateompb.CheckpointWorkloadResponse, error) { +func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.CheckpointWorkloadRequest) (_ *ateompb.CheckpointWorkloadResponse, err error) { + // Taken before the lock, so prep covers the wait for it. + tStart := time.Now() if err := validateActorDirs(req.GetActorDirs()); err != nil { return nil, err } @@ -686,11 +689,22 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec // Not cancelable: a checkpoint is saving the actor's state. defer s.inFlight.Add(req.GetActorUid(), rpcCheckpointWorkload, nil)() - if err := s.tunnel.Deactivate(ctx, ateomstats.ActorAttributionFromRequest(req)); err != nil { + // Per-phase timing, recorded on the way out so a failed checkpoint still + // reports the phases it completed, and the failing step its elapsed time. + attribution := ateomstats.ActorAttributionFromRequest(req) + scope := req.GetScope() + var timing checkpointTiming + tLast := tStart + defer func() { + timing.total = time.Since(tStart) + ateomphaselog.LogSnapshotPhases(ctx, "Checkpoint timing breakdown", attribution, scope, + ateomphaselog.CheckpointDurationKey, err, timing.phases()) + }() + + if err := s.tunnel.Deactivate(ctx, attribution); err != nil { return nil, err } - attribution := ateomstats.ActorAttributionFromRequest(req) s.actorLogger.EmitLifecycleLog(ctx, "Actor checkpointing", attribution) // Read before the snapshot: a Full checkpoint stops the sandbox, and its // cgroup with it. The final record waits for the teardown. @@ -712,7 +726,9 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec } checkpointPath := req.GetActorDirs().GetCheckpointDir() - if err := os.MkdirAll(checkpointPath, 0o700); err != nil { + err = os.MkdirAll(checkpointPath, 0o700) + timing.prep = lap(&tLast) + if err != nil { return nil, fmt.Errorf("while creating checkpoint directory: %w", err) } @@ -720,22 +736,27 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec var durableFiles []string // Always take durable-dir snapshot if at least one container has a durable-dir volume mount. // TODO(dberkov): this is a temporary workaround until gVisor supports taking durable-dir snapshots in a single request with the process snapshot. - switch req.GetScope() { + switch scope { case ateompb.SnapshotScope_SNAPSHOT_SCOPE_DATA: if !hasDurableVolumes(req.GetSpec().GetContainers()) { return nil, fmt.Errorf("no durable-dir volumes found for DATA snapshot") } - if err := rcmd.cmdPause(ctx, ocispec.PauseContainer); err != nil { + err := rcmd.cmdPause(ctx, ocispec.PauseContainer) + timing.pause = lap(&tLast) + if err != nil { return nil, fmt.Errorf("while pausing pause container: %w", err) } var tarErr error durableFiles, tarErr = tarDurableVolumes(ctx, req.GetActorDirs().GetDurableDirVolumeMountsDir(), checkpointPath, durableVolumeNames(req.GetSpec())) + timing.durableDir = lap(&tLast) // Undoing our own pause must not depend on the caller's context: // tarutil does not check ctx, so a deadline expiring mid-tar would // fail the resume instantly and leave the sandbox paused forever. resumeCtx, cancelResume := context.WithTimeout(context.WithoutCancel(ctx), resumeTimeout) defer cancelResume() - if err := rcmd.cmdResume(resumeCtx, ocispec.PauseContainer); err != nil { + err = rcmd.cmdResume(resumeCtx, ocispec.PauseContainer) + timing.resume = lap(&tLast) + if err != nil { return nil, fmt.Errorf("while resuming pause container: %w", err) } if tarErr != nil { @@ -744,18 +765,21 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec case ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL: // Checkpoint pause container (root of the sandbox) // TODO: Consider pause -> tar -> resume -> checkpoint order for better failure handling. - if err := rcmd.cmdCheckpoint(ctx, ocispec.PauseContainer, checkpointPath); err != nil { + err := rcmd.cmdCheckpoint(ctx, ocispec.PauseContainer, checkpointPath) + timing.checkpoint = lap(&tLast) + if err != nil { return nil, fmt.Errorf("while checkpointing pause: %w", err) } if hasDurableVolumes(req.GetSpec().GetContainers()) { var err error durableFiles, err = tarDurableVolumes(ctx, req.GetActorDirs().GetDurableDirVolumeMountsDir(), checkpointPath, durableVolumeNames(req.GetSpec())) + timing.durableDir = lap(&tLast) if err != nil { return nil, fmt.Errorf("while archiving durable-dir volumes: %w", err) } } default: - return nil, fmt.Errorf("unsupported snapshot scope: %v", req.GetScope()) + return nil, fmt.Errorf("unsupported snapshot scope: %v", scope) } // Cleanup the containers after checkpointing. This also unhosts the actor, @@ -767,6 +791,7 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec slog.String("actorUID", attribution.UID), slog.Any("err", err)) } + timing.teardown = lap(&tLast) s.recordFinalIfEnded(ctx, hosted) // Report exactly the files runsc wrote so atelet ships precisely this set @@ -856,6 +881,8 @@ func isContainerAlreadyGone(ctx context.Context, rcmd containerRuntime, name str } func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.RestoreWorkloadRequest) (resp *ateompb.RestoreWorkloadResponse, retErr error) { + // Taken before the lock, so prep covers the wait for it. + tStart := time.Now() if err := validateActorDirs(req.GetActorDirs()); err != nil { return nil, err } @@ -868,6 +895,23 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore defer s.locks.Unlock(req.GetActorUid()) ctx, cancel := context.WithCancel(ctx) defer cancel() + + // Per-phase timing, recorded on the way out so a failed restore still + // reports the phases it completed, and the failing step its elapsed time. + // Registered ahead of the failure cleanup below, so on failure the total + // includes that cleanup; the record is marked by error.type either way. + attribution := ateomstats.ActorAttributionFromRequest(req) + scope := req.GetScope() + containers := req.GetSpec().GetContainers() + var timing restoreTiming + tLast := tStart + defer func() { + timing.total = time.Since(tStart) + attrs := ateomphaselog.SnapshotPhaseAttrs(attribution, scope, ateomphaselog.RestoreDurationKey, retErr, timing.phases()) + attrs = append(attrs, slog.Int(containerCountKey, len(containers))) + slog.LogAttrs(ctx, slog.LevelInfo, "Restore timing breakdown", attrs...) + }() + release, err := s.beginRPC(req.GetActorUid(), rpcRestoreWorkload, cancel) if err != nil { return nil, err @@ -877,11 +921,12 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore return nil, err } - if err := s.tunnel.Deactivate(ctx, ateomstats.ActorAttributionFromRequest(req)); err != nil { + err = s.tunnel.Deactivate(ctx, attribution) + timing.prep = lap(&tLast) + if err != nil { return nil, err } - attribution := ateomstats.ActorAttributionFromRequest(req) s.actorLogger.EmitLifecycleLog(ctx, "Actor restoring", attribution) // Contract with atelet: @@ -890,11 +935,13 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore // * All OCI bundles are set up, including for the pause container. // * Checkpoint downloaded and placed on disk - egress, err := s.tunnel.PrepareEgress(ctx, ateomstats.ActorAttributionFromRequest(req), req.GetEgressGateway()) + egress, err := s.tunnel.PrepareEgress(ctx, attribution, req.GetEgressGateway()) + timing.egressPrepare = lap(&tLast) if err != nil { return nil, err } hosted, err := s.hostActor(ctx, attribution, req.GetActorDirs()) + timing.netSetup = lap(&tLast) if err != nil { return nil, err } @@ -910,7 +957,7 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore if retErr != nil { cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) defer cancel() - if err := s.tunnel.Deactivate(cleanupCtx, ateomstats.ActorAttributionFromRequest(req)); err != nil { + if err := s.tunnel.Deactivate(cleanupCtx, attribution); err != nil { slog.WarnContext(cleanupCtx, "Failed to deactivate actor networking after Restore failure", slog.Any("err", err)) } deleteContainers(cleanupCtx, rcmd, containersToDelete, "Restore") @@ -926,84 +973,106 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore }() checkpointDir := req.GetActorDirs().GetRestoreDir() - if hasDurableVolumes(req.GetSpec().GetContainers()) { - if err := untarDurableVolumes(req.GetActorDirs().GetDurableDirVolumeMountsDir(), checkpointDir, durableVolumeNames(req.GetSpec())); err != nil { + if hasDurableVolumes(containers) { + err := untarDurableVolumes(req.GetActorDirs().GetDurableDirVolumeMountsDir(), checkpointDir, durableVolumeNames(req.GetSpec())) + timing.durableDir = lap(&tLast) + if err != nil { return nil, fmt.Errorf("while restoring durable-dir volumes: %w", err) } } // Compose the pause rootfs before create (see RunWorkload). runsc restore // only needs the rootfs to hold the correct content; whether it came from // an untar or an overlay of cached layers is transparent to it. - if err := imagecache.SetupBundleRootfs(ociBundlePath(req.GetActorDirs(), ocispec.PauseContainer)); err != nil { + err = imagecache.SetupBundleRootfs(ociBundlePath(req.GetActorDirs(), ocispec.PauseContainer)) + timing.pauseRootfs = lap(&tLast) + if err != nil { return nil, fmt.Errorf("while composing pause rootfs: %w", err) } - switch req.GetScope() { + switch scope { case ateompb.SnapshotScope_SNAPSHOT_SCOPE_DATA: // Create and start pause container (cold boot with durable-dir volumes restored) containersToDelete = append(containersToDelete, ocispec.PauseContainer) - if err := rcmd.cmdCreate(ctx, os.Stdout, ocispec.PauseContainer, nil); err != nil { + err := rcmd.cmdCreate(ctx, os.Stdout, ocispec.PauseContainer, nil) + timing.pauseCreate = lap(&tLast) + if err != nil { return nil, fmt.Errorf("while creating pause container: %w", err) } - if err := rcmd.cmdStart(ctx, os.Stdout, ocispec.PauseContainer); err != nil { + err = rcmd.cmdStart(ctx, os.Stdout, ocispec.PauseContainer) + timing.pauseRestore = lap(&tLast) + if err != nil { return nil, fmt.Errorf("while starting pause container: %w", err) } case ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL: // Create and restore pause container containersToDelete = append(containersToDelete, ocispec.PauseContainer) - if err := rcmd.cmdCreate(ctx, os.Stdout, ocispec.PauseContainer, nil); err != nil { + err := rcmd.cmdCreate(ctx, os.Stdout, ocispec.PauseContainer, nil) + timing.pauseCreate = lap(&tLast) + if err != nil { return nil, fmt.Errorf("while creating pause container: %w", err) } - if err := rcmd.cmdRestore(ctx, os.Stdout, ocispec.PauseContainer, checkpointDir); err != nil { + err = rcmd.cmdRestore(ctx, os.Stdout, ocispec.PauseContainer, checkpointDir) + timing.pauseRestore = lap(&tLast) + if err != nil { return nil, fmt.Errorf("while restoring pause container: %w", err) } default: - return nil, fmt.Errorf("unexpected snapshot scope: %v", req.GetScope()) + return nil, fmt.Errorf("unexpected snapshot scope: %v", scope) } // Create and restore each application container, each with its own log pipe so // every line is tagged with the originating container (ate.actor.container.name). - for _, ac := range req.GetSpec().GetContainers() { + // app_create includes starting that pipe. + for _, ac := range containers { pw, err := s.actorLogger.StartJSONLogPipe(attribution, ac.GetName()) + timing.appCreate += lap(&tLast) if err != nil { return nil, fmt.Errorf("while starting json log pipe for %q: %w", ac.GetName(), err) } defer pw.Close() - if err := imagecache.SetupBundleRootfs(ociBundlePath(req.GetActorDirs(), ac.GetName())); err != nil { + err = imagecache.SetupBundleRootfs(ociBundlePath(req.GetActorDirs(), ac.GetName())) + timing.appRootfs += lap(&tLast) + if err != nil { return nil, fmt.Errorf("while composing %q rootfs: %w", ac.GetName(), err) } - switch req.GetScope() { + containersToDelete = append(containersToDelete, ac.GetName()) + err = rcmd.cmdCreate(ctx, pw, ac.GetName(), nil) + timing.appCreate += lap(&tLast) + if err != nil { + return nil, fmt.Errorf("while creating %q application container: %w", ac.GetName(), err) + } + switch scope { case ateompb.SnapshotScope_SNAPSHOT_SCOPE_DATA: - containersToDelete = append(containersToDelete, ac.GetName()) - if err := rcmd.cmdCreate(ctx, pw, ac.GetName(), nil); err != nil { - return nil, fmt.Errorf("while creating %q application container: %w", ac.GetName(), err) - } - if err := rcmd.cmdStart(ctx, pw, ac.GetName()); err != nil { + err = rcmd.cmdStart(ctx, pw, ac.GetName()) + timing.appRestore += lap(&tLast) + if err != nil { return nil, fmt.Errorf("while starting %q application container: %w", ac.GetName(), err) } case ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL: - containersToDelete = append(containersToDelete, ac.GetName()) - if err := rcmd.cmdCreate(ctx, pw, ac.GetName(), nil); err != nil { - return nil, fmt.Errorf("while creating %q application container: %w", ac.GetName(), err) - } - if err := rcmd.cmdRestore(ctx, pw, ac.GetName(), checkpointDir); err != nil { + err = rcmd.cmdRestore(ctx, pw, ac.GetName(), checkpointDir) + timing.appRestore += lap(&tLast) + if err != nil { return nil, fmt.Errorf("while restoring %q application container: %w", ac.GetName(), err) } default: - return nil, fmt.Errorf("unexpected snapshot scope: %v", req.GetScope()) + return nil, fmt.Errorf("unexpected snapshot scope: %v", scope) } } // Block until every wakeup-probe-enabled container reports 200. - if err := wakeupprobe.WaitAll(ctx, req.GetSpec().GetContainers(), ateomnet.ActorVethIP, wakeupprobe.DialFunc(s.sandboxDialer(req.GetActorUid()))); err != nil { + err = wakeupprobe.WaitAll(ctx, containers, ateomnet.ActorVethIP, wakeupprobe.DialFunc(s.sandboxDialer(req.GetActorUid()))) + timing.wakeupProbe = lap(&tLast) + if err != nil { return nil, fmt.Errorf("while waiting for container wakeup probe: %w", err) } - if err := s.tunnel.Activate(ateomstats.ActorAttributionFromRequest(req), s.sandboxDialer(req.GetActorUid()), egress); err != nil { + err = s.tunnel.Activate(attribution, s.sandboxDialer(req.GetActorUid()), egress) + timing.activate = lap(&tLast) + if err != nil { return nil, err } s.actorLogger.EmitLifecycleLog(ctx, "Actor restored", attribution) - s.setSession(req.GetActorUid(), &workloadSession{rcmd: rcmd, containers: containerNames(req.GetSpec().GetContainers())}) + s.setSession(req.GetActorUid(), &workloadSession{rcmd: rcmd, containers: containerNames(containers)}) s.recordInitial(ctx, hosted) return &ateompb.RestoreWorkloadResponse{}, nil diff --git a/cmd/ateom-gvisor/phaselog.go b/cmd/ateom-gvisor/phaselog.go new file mode 100644 index 0000000000..1509545447 --- /dev/null +++ b/cmd/ateom-gvisor/phaselog.go @@ -0,0 +1,115 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "time" + + "github.com/agent-substrate/substrate/internal/ateattr" + "github.com/agent-substrate/substrate/internal/ateomphaselog" +) + +// The phase names, suffixed onto the ateomphaselog duration keys. Kept out of +// ateattr on purpose: that package's SnapshotPhase* values are the +// ate.snapshot.phase metric enum, and these are not values of it. "total" is +// shared so the two layers' records agree on the denominator. +// +// Every phase of both operations is sequential, so the phases partition the +// total up to the few microseconds between them. The app_* phases are summed +// over the application containers, which are restored one after another; the +// container count rides along on the record so a reader can divide. +const ( + phasePrep = "prep" + phaseEgressPrepare = "egress_prepare" + phaseNetSetup = "net_setup" + phaseDurableDir = "durable_dir" + phasePauseRootfs = "pause_rootfs" + phasePauseCreate = "pause_create" + phasePauseRestore = "pause_restore" + phaseAppRootfs = "app_rootfs" + phaseAppCreate = "app_create" + phaseAppRestore = "app_restore" + phaseWakeupProbe = "wakeup_probe" + phaseActivate = "activate" + + phasePause = "pause" + phaseCheckpoint = "checkpoint" + phaseResume = "resume" + phaseTeardown = "teardown" + + phaseTotal = ateattr.SnapshotPhaseTotal +) + +// containerCountKey carries the number of application containers on the +// restore record, since the app_* phases are sums over them. +const containerCountKey = "ate.actor.container.count" + +// restoreTiming is the elapsed time of each RestoreWorkload phase. A phase +// left at zero never ran. Under a Data scope pauseRestore and appRestore time +// the cold start that stands in for the restore. +type restoreTiming struct { + prep, egressPrepare, netSetup, durableDir time.Duration + pauseRootfs, pauseCreate, pauseRestore time.Duration + appRootfs, appCreate, appRestore time.Duration + wakeupProbe, activate, total time.Duration +} + +func (t restoreTiming) phases() []ateomphaselog.Phase { + return []ateomphaselog.Phase{ + {Name: phasePrep, D: t.prep}, + {Name: phaseEgressPrepare, D: t.egressPrepare}, + {Name: phaseNetSetup, D: t.netSetup}, + {Name: phaseDurableDir, D: t.durableDir}, + {Name: phasePauseRootfs, D: t.pauseRootfs}, + {Name: phasePauseCreate, D: t.pauseCreate}, + {Name: phasePauseRestore, D: t.pauseRestore}, + {Name: phaseAppRootfs, D: t.appRootfs}, + {Name: phaseAppCreate, D: t.appCreate}, + {Name: phaseAppRestore, D: t.appRestore}, + {Name: phaseWakeupProbe, D: t.wakeupProbe}, + {Name: phaseActivate, D: t.activate}, + {Name: phaseTotal, D: t.total}, + } +} + +// checkpointTiming is the elapsed time of each CheckpointWorkload phase. A +// Full scope runs checkpoint; a Data scope runs pause and resume around the +// durable_dir capture instead. +type checkpointTiming struct { + prep, pause, checkpoint, durableDir, resume, teardown, total time.Duration +} + +func (t checkpointTiming) phases() []ateomphaselog.Phase { + return []ateomphaselog.Phase{ + {Name: phasePrep, D: t.prep}, + {Name: phasePause, D: t.pause}, + {Name: phaseCheckpoint, D: t.checkpoint}, + {Name: phaseDurableDir, D: t.durableDir}, + {Name: phaseResume, D: t.resume}, + {Name: phaseTeardown, D: t.teardown}, + {Name: phaseTotal, D: t.total}, + } +} + +// lap returns the time since *last and advances it, so consecutive calls +// measure consecutive phases without a timestamp per boundary. +func lap(last *time.Time) time.Duration { + now := time.Now() + d := now.Sub(*last) + *last = now + return d +} diff --git a/cmd/ateom-gvisor/phaselog_test.go b/cmd/ateom-gvisor/phaselog_test.go new file mode 100644 index 0000000000..1173c1021f --- /dev/null +++ b/cmd/ateom-gvisor/phaselog_test.go @@ -0,0 +1,168 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "context" + "encoding/json" + "log/slog" + "testing" + "time" + + "github.com/agent-substrate/substrate/internal/ateomphaselog" + "github.com/agent-substrate/substrate/internal/proto/ateompb" + "github.com/agent-substrate/substrate/internal/resources" +) + +// renderPhaseRecord logs attrs through a JSON handler, so the test checks the +// record a collector would parse rather than the slice that built it. +func renderPhaseRecord(t *testing.T, attrs []slog.Attr) map[string]any { + t.Helper() + var buf bytes.Buffer + slog.New(slog.NewJSONHandler(&buf, nil)).LogAttrs(context.Background(), slog.LevelInfo, "Restore timing breakdown", attrs...) + var rec map[string]any + if err := json.Unmarshal(buf.Bytes(), &rec); err != nil { + t.Fatalf("unmarshal record %q: %v", buf.String(), err) + } + return rec +} + +// TestRestoreTimingPhases pins the keys a Full restore record carries, in +// the layout benchmarking tooling joins with atelet's record on, and that a +// phase which never ran stays off it. +func TestRestoreTimingPhases(t *testing.T) { + t.Parallel() + + timing := restoreTiming{ + prep: 10 * time.Millisecond, + egressPrepare: 120 * time.Millisecond, + netSetup: 30 * time.Millisecond, + // No durable volumes: the phase never ran. + durableDir: 0, + pauseRootfs: 5 * time.Millisecond, + pauseCreate: 200 * time.Millisecond, + pauseRestore: 900 * time.Millisecond, + appRootfs: 15 * time.Millisecond, + appCreate: 50 * time.Millisecond, + appRestore: 400 * time.Millisecond, + wakeupProbe: 250 * time.Millisecond, + activate: 2 * time.Millisecond, + total: 2 * time.Second, + } + attrs := ateomphaselog.SnapshotPhaseAttrs(resources.ActorAttribution{UID: "uid-abc"}, + ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL, ateomphaselog.RestoreDurationKey, nil, timing.phases()) + attrs = append(attrs, slog.Int(containerCountKey, 2)) + + seen := make(map[string]bool, len(attrs)) + for _, attr := range attrs { + if seen[attr.Key] { + t.Errorf("key %s emitted twice", attr.Key) + } + seen[attr.Key] = true + } + + rec := renderPhaseRecord(t, attrs) + for k, want := range map[string]float64{ + "ateom.actor.restore.duration.prep": 0.01, + "ateom.actor.restore.duration.egress_prepare": 0.12, + "ateom.actor.restore.duration.net_setup": 0.03, + "ateom.actor.restore.duration.pause_rootfs": 0.005, + "ateom.actor.restore.duration.pause_create": 0.2, + "ateom.actor.restore.duration.pause_restore": 0.9, + "ateom.actor.restore.duration.app_rootfs": 0.015, + "ateom.actor.restore.duration.app_create": 0.05, + "ateom.actor.restore.duration.app_restore": 0.4, + "ateom.actor.restore.duration.wakeup_probe": 0.25, + "ateom.actor.restore.duration.activate": 0.002, + "ateom.actor.restore.duration.total": 2, + "ate.actor.container.count": 2, + } { + got, ok := rec[k].(float64) + if !ok { + t.Errorf("%s = %v (%T), want a number", k, rec[k], rec[k]) + } else if got != want { + t.Errorf("%s = %v, want %v", k, got, want) + } + } + if v, ok := rec["ateom.actor.restore.duration.durable_dir"]; ok { + t.Errorf("durable_dir present with %v, want absent", v) + } + if rec["ate.actor.uid"] != "uid-abc" { + t.Errorf("ate.actor.uid = %v, want uid-abc", rec["ate.actor.uid"]) + } + if rec["ate.snapshot.scope"] != "full" { + t.Errorf("ate.snapshot.scope = %v, want full", rec["ate.snapshot.scope"]) + } +} + +// TestCheckpointTimingPhases: a Data checkpoint pauses and resumes around the +// durable capture and never runs the runsc checkpoint, and the record says so. +func TestCheckpointTimingPhases(t *testing.T) { + t.Parallel() + + timing := checkpointTiming{ + prep: 20 * time.Millisecond, + pause: 3 * time.Millisecond, + durableDir: 300 * time.Millisecond, + resume: 4 * time.Millisecond, + teardown: 150 * time.Millisecond, + total: 500 * time.Millisecond, + } + rec := renderPhaseRecord(t, ateomphaselog.SnapshotPhaseAttrs(resources.ActorAttribution{UID: "uid-abc"}, + ateompb.SnapshotScope_SNAPSHOT_SCOPE_DATA, ateomphaselog.CheckpointDurationKey, nil, timing.phases())) + + for k, want := range map[string]float64{ + "ateom.actor.checkpoint.duration.prep": 0.02, + "ateom.actor.checkpoint.duration.pause": 0.003, + "ateom.actor.checkpoint.duration.durable_dir": 0.3, + "ateom.actor.checkpoint.duration.resume": 0.004, + "ateom.actor.checkpoint.duration.teardown": 0.15, + "ateom.actor.checkpoint.duration.total": 0.5, + } { + got, ok := rec[k].(float64) + if !ok { + t.Errorf("%s = %v (%T), want a number", k, rec[k], rec[k]) + } else if got != want { + t.Errorf("%s = %v, want %v", k, got, want) + } + } + if v, ok := rec["ateom.actor.checkpoint.duration.checkpoint"]; ok { + t.Errorf("checkpoint present with %v, want absent", v) + } + if rec["ate.snapshot.scope"] != "data" { + t.Errorf("ate.snapshot.scope = %v, want data", rec["ate.snapshot.scope"]) + } +} + +// TestLap: consecutive laps measure consecutive intervals from one cursor. +func TestLap(t *testing.T) { + t.Parallel() + + last := time.Now().Add(-time.Second) + before := last + d := lap(&last) + if d < time.Second { + t.Errorf("lap = %v, want >= 1s", d) + } + if !last.After(before) { + t.Error("lap did not advance the cursor") + } + if d2 := lap(&last); d2 >= time.Second { + t.Errorf("second lap = %v, want far below 1s", d2) + } +} diff --git a/cmd/ateom-microvm/checkpoint.go b/cmd/ateom-microvm/checkpoint.go index 9fc10eff10..fe873c30c9 100644 --- a/cmd/ateom-microvm/checkpoint.go +++ b/cmd/ateom-microvm/checkpoint.go @@ -30,6 +30,7 @@ import ( "github.com/agent-substrate/substrate/cmd/ateom-microvm/internal/ch" "github.com/agent-substrate/substrate/cmd/ateom-microvm/internal/kata" + "github.com/agent-substrate/substrate/internal/ateomphaselog" "github.com/agent-substrate/substrate/internal/ateomstats" "github.com/agent-substrate/substrate/internal/imagecache" "github.com/agent-substrate/substrate/internal/proto/ateompb" @@ -82,15 +83,15 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec attribution := ateomstats.ActorAttributionFromRequest(req) scope := req.GetScope() defer func() { - logSnapshotPhases(ctx, "Checkpoint timing breakdown", attribution, scope, - checkpointDurationKey, err, []phase{ - {phasePrep, dPrep}, - {phasePause, dPause}, - {phaseSnapshot, dSnapshot}, - {phaseDurableDir, dDurable}, - {phaseRootfsUpper, dUpper}, - {phaseTeardown, dTeardown}, - {phaseTotal, time.Since(tStart)}, + ateomphaselog.LogSnapshotPhases(ctx, "Checkpoint timing breakdown", attribution, scope, + ateomphaselog.CheckpointDurationKey, err, []ateomphaselog.Phase{ + {Name: phasePrep, D: dPrep}, + {Name: phasePause, D: dPause}, + {Name: phaseSnapshot, D: dSnapshot}, + {Name: phaseDurableDir, D: dDurable}, + {Name: phaseRootfsUpper, D: dUpper}, + {Name: phaseTeardown, D: dTeardown}, + {Name: phaseTotal, D: time.Since(tStart)}, }) }() diff --git a/cmd/ateom-microvm/phaselog.go b/cmd/ateom-microvm/phaselog.go index bd41cfc728..bde1ee32ea 100644 --- a/cmd/ateom-microvm/phaselog.go +++ b/cmd/ateom-microvm/phaselog.go @@ -16,33 +16,12 @@ package main -import ( - "context" - "log/slog" - "time" +import "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/proto/ateompb" - "github.com/agent-substrate/substrate/internal/resources" - "google.golang.org/grpc/status" -) - -// The keys the per-phase durations are logged under. They are named like -// instruments but deliberately are not ones: the phases are implementation -// details of this binary, so they stay a developer-facing log record rather -// than becoming metric API. They decompose the ateom_restore / -// ateom_checkpoint phases of atelet's ate.actor.*.duration histograms, whose -// names they extend, and are joined per actor by benchmarking tooling. -const ( - restoreDurationKey = "ateom.actor.restore.duration" - checkpointDurationKey = "ateom.actor.checkpoint.duration" -) - -// The phase names, suffixed onto the duration keys. Kept out of ateattr on -// purpose: that package's SnapshotPhase* values are the ate.snapshot.phase -// metric enum, and these are not values of it. "total" is shared so the two -// layers' records agree on the denominator. +// The phase names, suffixed onto the ateomphaselog duration keys. Kept out of +// ateattr on purpose: that package's SnapshotPhase* values are the +// ate.snapshot.phase metric enum, and these are not values of it. "total" is +// shared so the two layers' records agree on the denominator. // // Checkpoint: snapshot, durable_dir and rootfs_upper run concurrently on the // paused guest, so the paused window costs their max, not their sum; prep, @@ -67,64 +46,3 @@ const ( phaseTotal = ateattr.SnapshotPhaseTotal ) - -// phase is one timed step of a snapshot operation. A zero duration means the -// phase never ran (a Data-scope checkpoint captures no guest) and is skipped -// rather than logged as instant. -type phase struct { - name string - d time.Duration -} - -// scopeLogValue maps the ateom wire enum onto the shared scope label values, -// the same way ateattr.SnapshotScopeValue does for the atelet enum. An -// unrecognized scope reports as unknown rather than stringified, so no wire -// value can widen the value set readers key on. -func scopeLogValue(scope ateompb.SnapshotScope) string { - switch scope { - case ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL: - return ateattr.SnapshotScopeFull - case ateompb.SnapshotScope_SNAPSHOT_SCOPE_DATA: - return ateattr.SnapshotScopeData - default: - return ateattr.SnapshotScopeUnknown - } -} - -// snapshotPhaseAttrs renders one joinable per-actor record for a checkpoint -// or restore, mirroring atelet's snapshotLogAttrs: full actor identity, the -// scope, and one float-seconds attr per non-zero phase under -// durationKey.. One record carries every phase of the operation, so a -// reader never joins two half-records that disagree about the same actor. -// -// err is the operation's outcome. A failed operation still records the phases -// it completed, marked with error.type (the gRPC code, context errors as -// DeadlineExceeded / Canceled) so a reader can leave it out of a latency -// distribution. Absence means success. -func snapshotPhaseAttrs(a resources.ActorAttribution, scope ateompb.SnapshotScope, durationKey string, err error, phases []phase) []slog.Attr { - attrs := ateattr.ActorLogAttrs(a) - attrs = append(attrs, slog.String(string(ateattr.SnapshotScopeKey), scopeLogValue(scope))) - if err != nil { - s, ok := apierror.FromError(err) - code := s.Code() - if !ok { - code = status.Code(err) - } - attrs = append(attrs, slog.String(string(ateattr.ErrorTypeKey), code.String())) - } - for _, p := range phases { - if p.d == 0 { - continue - } - // Seconds and not slog.Duration's nanoseconds: the values sit under - // the atelet histograms' unit (s), so readers compare them without a - // per-key unit table. - attrs = append(attrs, slog.Float64(durationKey+"."+p.name, p.d.Seconds())) - } - return attrs -} - -// logSnapshotPhases emits the snapshotPhaseAttrs record under msg. -func logSnapshotPhases(ctx context.Context, msg string, a resources.ActorAttribution, scope ateompb.SnapshotScope, durationKey string, err error, phases []phase) { - slog.LogAttrs(ctx, slog.LevelInfo, msg, snapshotPhaseAttrs(a, scope, durationKey, err, phases)...) -} diff --git a/cmd/ateom-microvm/restore.go b/cmd/ateom-microvm/restore.go index 4a326c3fba..561710101f 100644 --- a/cmd/ateom-microvm/restore.go +++ b/cmd/ateom-microvm/restore.go @@ -28,6 +28,7 @@ import ( "time" "github.com/agent-substrate/substrate/internal/apierror" + "github.com/agent-substrate/substrate/internal/ateomphaselog" "github.com/agent-substrate/substrate/internal/ateomstats" "github.com/agent-substrate/substrate/internal/ateomnet" @@ -188,8 +189,8 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore slog.String("id", p.actorUID), slog.Duration("total", dTotal)) // A cold boot has none of the full-scope phases, so the total is the // only observation on its record. - logSnapshotPhases(ctx, "Restore timing breakdown", attribution, scope, - restoreDurationKey, nil, []phase{{phaseTotal, dTotal}}) + ateomphaselog.LogSnapshotPhases(ctx, "Restore timing breakdown", attribution, scope, + ateomphaselog.RestoreDurationKey, nil, []ateomphaselog.Phase{{Name: phaseTotal, D: dTotal}}) default: return nil, apierror.InvalidArgument("unsupported snapshot scope: %v", scope) } @@ -444,18 +445,18 @@ func (s *AteomService) restoreFullScope(ctx context.Context, p actorBootParams, // The joinable per-actor record the benchmarking tooling aggregates. The // durable delta is not carried: tDurable is pinned to tLowers today, so it // would always be the zero a record skips. - logSnapshotPhases(ctx, "Restore timing breakdown", p.actorAttribution(), scope, - restoreDurationKey, nil, []phase{ - {phasePrep, tPrep.Sub(tStart)}, - {phaseBundles, tBundles.Sub(tPrep)}, - {phaseUpperJoin, tUpper.Sub(tBundles)}, - {phaseLowers, tLowers.Sub(tUpper)}, - {phaseTap, tTap.Sub(tDurable)}, - {phaseVMMLaunch, tLaunch.Sub(tTap)}, - {phaseVMRestore, tVMRestore.Sub(tLaunch)}, - {phaseResume, tResume.Sub(tVMRestore)}, - {phaseWakeupProbe, dWakeupProbe}, - {phaseTotal, dTotal}, + ateomphaselog.LogSnapshotPhases(ctx, "Restore timing breakdown", p.actorAttribution(), scope, + ateomphaselog.RestoreDurationKey, nil, []ateomphaselog.Phase{ + {Name: phasePrep, D: tPrep.Sub(tStart)}, + {Name: phaseBundles, D: tBundles.Sub(tPrep)}, + {Name: phaseUpperJoin, D: tUpper.Sub(tBundles)}, + {Name: phaseLowers, D: tLowers.Sub(tUpper)}, + {Name: phaseTap, D: tTap.Sub(tDurable)}, + {Name: phaseVMMLaunch, D: tLaunch.Sub(tTap)}, + {Name: phaseVMRestore, D: tVMRestore.Sub(tLaunch)}, + {Name: phaseResume, D: tResume.Sub(tVMRestore)}, + {Name: phaseWakeupProbe, D: dWakeupProbe}, + {Name: phaseTotal, D: dTotal}, }) ra := &runningActor{ diff --git a/docs/observability.md b/docs/observability.md index cd1c086760..bc771b3edc 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -141,7 +141,7 @@ The duration keys are the [`ate.actor.restore.duration`](#the-metric-registry) i This is the record to use for a per-actor wake-up distribution. The histogram cannot answer that question at all, because actor identity is barred from metric labels; traces can, but the data plane is head-sampled at 1%. -ateom-microvm writes records with the same two messages, from inside the `ateom_restore` and `ateom_checkpoint` phases, under `ateom.actor.restore.duration.` and `ateom.actor.checkpoint.duration.` (for example `vm_restore`, `wakeup_probe`, `prep`, `pause`, `snapshot`, `teardown`). Those keys are not instruments: the phases are implementation details of one runtime, so they stay a developer-facing record. The same identity keys make the two layers' records joinable per actor. The checkpoint record is written on failure too, with `error.type` and the elapsed time of the step that failed; the restore record is written on success only. The restore phases are sequential and partition the total; a checkpoint's `snapshot`, `durable_dir` and `rootfs_upper` run concurrently on the paused guest, so the paused window costs their maximum, not their sum. +Both ateoms write records with the same two messages from inside the `ateom_restore` and `ateom_checkpoint` phases, under `ateom.actor.restore.duration.` and `ateom.actor.checkpoint.duration.`. The phases are each runtime's own and are listed in `cmd/ateom-gvisor/phaselog.go` and `cmd/ateom-microvm/phaselog.go`; `internal/ateomphaselog` renders the record. They are not instruments. The same identity keys make the two layers' records joinable per actor, and a failed operation's record carries `error.type`. ateapi's `Actor state changed` is written once per committed actor state transition. ateapi owns the state machine, so this is where an actor's state and the time it reached it come from: diff --git a/internal/ateomphaselog/phaselog.go b/internal/ateomphaselog/phaselog.go new file mode 100644 index 0000000000..e74283b7c6 --- /dev/null +++ b/internal/ateomphaselog/phaselog.go @@ -0,0 +1,106 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package ateomphaselog renders the per-actor "timing breakdown" log record an +// ateom emits for each checkpoint and restore. Every ateom binary times its +// own phases, which are implementation details of that runtime; this package +// holds the shape they share so the records of all ateoms carry the same +// identity, scope and key layout, and atelet's own record joins to any of them. +package ateomphaselog + +import ( + "context" + "log/slog" + "time" + + "github.com/agent-substrate/substrate/internal/apierror" + "github.com/agent-substrate/substrate/internal/ateattr" + "github.com/agent-substrate/substrate/internal/proto/ateompb" + "github.com/agent-substrate/substrate/internal/resources" + "google.golang.org/grpc/status" +) + +// The keys the per-phase durations are logged under. They are named like +// instruments but deliberately are not ones: the phases are implementation +// details of each ateom, so they stay a developer-facing log record rather +// than becoming metric API. They decompose the ateom_restore / +// ateom_checkpoint phases of atelet's ate.actor.*.duration histograms, whose +// names they extend, and are joined per actor by benchmarking tooling. +const ( + RestoreDurationKey = "ateom.actor.restore.duration" + CheckpointDurationKey = "ateom.actor.checkpoint.duration" +) + +// Phase is one timed step of a snapshot operation. A zero duration means the +// phase never ran (a Data-scope checkpoint captures no guest) and is skipped +// rather than logged as instant. Name is suffixed onto the duration key; each +// ateom defines its own names, with ateattr.SnapshotPhaseTotal shared so the +// two layers' records agree on the denominator. +type Phase struct { + Name string + D time.Duration +} + +// ScopeLogValue maps the ateom wire enum onto the shared scope label values, +// the same way ateattr.SnapshotScopeValue does for the atelet enum. An +// unrecognized scope reports as unknown rather than stringified, so no wire +// value can widen the value set readers key on. +func ScopeLogValue(scope ateompb.SnapshotScope) string { + switch scope { + case ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL: + return ateattr.SnapshotScopeFull + case ateompb.SnapshotScope_SNAPSHOT_SCOPE_DATA: + return ateattr.SnapshotScopeData + default: + return ateattr.SnapshotScopeUnknown + } +} + +// SnapshotPhaseAttrs renders one joinable per-actor record for a checkpoint +// or restore, mirroring atelet's snapshotLogAttrs: full actor identity, the +// scope, and one float-seconds attr per non-zero phase under +// durationKey.. One record carries every phase of the operation, so a +// reader never joins two half-records that disagree about the same actor. +// +// err is the operation's outcome. A failed operation still records the phases +// it completed, marked with error.type (the gRPC code, context errors as +// DeadlineExceeded / Canceled) so a reader can leave it out of a latency +// distribution. Absence means success. +func SnapshotPhaseAttrs(a resources.ActorAttribution, scope ateompb.SnapshotScope, durationKey string, err error, phases []Phase) []slog.Attr { + attrs := ateattr.ActorLogAttrs(a) + attrs = append(attrs, slog.String(string(ateattr.SnapshotScopeKey), ScopeLogValue(scope))) + if err != nil { + s, ok := apierror.FromError(err) + code := s.Code() + if !ok { + code = status.Code(err) + } + attrs = append(attrs, slog.String(string(ateattr.ErrorTypeKey), code.String())) + } + for _, p := range phases { + if p.D == 0 { + continue + } + // Seconds and not slog.Duration's nanoseconds: the values sit under + // the atelet histograms' unit (s), so readers compare them without a + // per-key unit table. + attrs = append(attrs, slog.Float64(durationKey+"."+p.Name, p.D.Seconds())) + } + return attrs +} + +// LogSnapshotPhases emits the SnapshotPhaseAttrs record under msg. +func LogSnapshotPhases(ctx context.Context, msg string, a resources.ActorAttribution, scope ateompb.SnapshotScope, durationKey string, err error, phases []Phase) { + slog.LogAttrs(ctx, slog.LevelInfo, msg, SnapshotPhaseAttrs(a, scope, durationKey, err, phases)...) +} diff --git a/cmd/ateom-microvm/phaselog_test.go b/internal/ateomphaselog/phaselog_test.go similarity index 72% rename from cmd/ateom-microvm/phaselog_test.go rename to internal/ateomphaselog/phaselog_test.go index cddd1dd54d..6595ab598b 100644 --- a/cmd/ateom-microvm/phaselog_test.go +++ b/internal/ateomphaselog/phaselog_test.go @@ -1,5 +1,3 @@ -//go:build linux - // Copyright 2026 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package ateomphaselog import ( "bytes" @@ -55,15 +53,15 @@ func renderPhaseRecord(t *testing.T, attrs []slog.Attr) map[string]any { func TestSnapshotPhaseAttrs(t *testing.T) { t.Parallel() - attrs := snapshotPhaseAttrs(phaseLogAttribution(), ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL, - checkpointDurationKey, nil, []phase{ - {phasePrep, 40 * time.Millisecond}, - {phasePause, 3 * time.Millisecond}, - {phaseSnapshot, 850 * time.Millisecond}, + attrs := SnapshotPhaseAttrs(phaseLogAttribution(), ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL, + CheckpointDurationKey, nil, []Phase{ + {Name: "prep", D: 40 * time.Millisecond}, + {Name: "pause", D: 3 * time.Millisecond}, + {Name: "snapshot", D: 850 * time.Millisecond}, // A capture that did not run stays off the record. - {phaseDurableDir, 0}, - {phaseTeardown, 230 * time.Millisecond}, - {phaseTotal, 1250 * time.Millisecond}, + {Name: "durable_dir", D: 0}, + {Name: "teardown", D: 230 * time.Millisecond}, + {Name: ateattr.SnapshotPhaseTotal, D: 1250 * time.Millisecond}, }) // json.Unmarshal keeps the last of a repeated key, so a duplicate is @@ -141,9 +139,13 @@ func TestSnapshotPhaseAttrsFailure(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - rec := renderPhaseRecord(t, snapshotPhaseAttrs(phaseLogAttribution(), - ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL, checkpointDurationKey, tt.err, - []phase{{phasePrep, 40 * time.Millisecond}, {phasePause, 3 * time.Millisecond}, {phaseTotal, 30 * time.Second}})) + rec := renderPhaseRecord(t, SnapshotPhaseAttrs(phaseLogAttribution(), + ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL, CheckpointDurationKey, tt.err, + []Phase{ + {Name: "prep", D: 40 * time.Millisecond}, + {Name: "pause", D: 3 * time.Millisecond}, + {Name: ateattr.SnapshotPhaseTotal, D: 30 * time.Second}, + })) if got := rec["error.type"]; got != tt.want { t.Errorf("error.type = %v, want %q", got, tt.want) } @@ -174,8 +176,38 @@ func TestScopeLogValue(t *testing.T) { {ateompb.SnapshotScope(99), ateattr.SnapshotScopeUnknown}, } for _, tt := range tests { - if got := scopeLogValue(tt.scope); got != tt.want { - t.Errorf("scopeLogValue(%v) = %q, want %q", tt.scope, got, tt.want) + if got := ScopeLogValue(tt.scope); got != tt.want { + t.Errorf("ScopeLogValue(%v) = %q, want %q", tt.scope, got, tt.want) } } } + +// TestLogSnapshotPhases: the record goes out under the caller's message and +// at Info, through the default logger the ateoms write to. +func TestLogSnapshotPhases(t *testing.T) { + var buf bytes.Buffer + prev := slog.Default() + slog.SetDefault(slog.New(slog.NewJSONHandler(&buf, nil))) + t.Cleanup(func() { slog.SetDefault(prev) }) + + LogSnapshotPhases(context.Background(), "Restore timing breakdown", phaseLogAttribution(), + ateompb.SnapshotScope_SNAPSHOT_SCOPE_DATA, RestoreDurationKey, nil, + []Phase{{Name: ateattr.SnapshotPhaseTotal, D: 2 * time.Second}}) + + var rec map[string]any + if err := json.Unmarshal(buf.Bytes(), &rec); err != nil { + t.Fatalf("unmarshal record %q: %v", buf.String(), err) + } + if rec["msg"] != "Restore timing breakdown" { + t.Errorf("msg = %v, want %q", rec["msg"], "Restore timing breakdown") + } + if rec["level"] != "INFO" { + t.Errorf("level = %v, want INFO", rec["level"]) + } + if rec["ate.snapshot.scope"] != ateattr.SnapshotScopeData { + t.Errorf("ate.snapshot.scope = %v, want %q", rec["ate.snapshot.scope"], ateattr.SnapshotScopeData) + } + if got := rec["ateom.actor.restore.duration.total"]; got != 2.0 { + t.Errorf("ateom.actor.restore.duration.total = %v, want 2", got) + } +} From e4f6cca568505c8dc1f75a3de5ef9001c8485067 Mon Sep 17 00:00:00 2001 From: Julian Gutierrez Oschmann Date: Thu, 8 Oct 2026 19:37:02 +0000 Subject: [PATCH 09/44] Don't default name in store layer. (#2359) Be consistent with other sub-resources (e.g. EgressPolicy) and validate in the RPC layer and let store honor what the client specifies. --- .../internal/apivalidation/access_policy.go | 2 +- .../apivalidation/access_policy_test.go | 18 ++++++++++++++++++ .../internal/store/atepg/access_policy.go | 2 -- .../internal/store/atepg/access_policy_test.go | 2 ++ 4 files changed, 21 insertions(+), 3 deletions(-) diff --git a/cmd/ateapi/internal/apivalidation/access_policy.go b/cmd/ateapi/internal/apivalidation/access_policy.go index e263ed157f..1f281ceb27 100644 --- a/cmd/ateapi/internal/apivalidation/access_policy.go +++ b/cmd/ateapi/internal/apivalidation/access_policy.go @@ -95,7 +95,7 @@ func ValidateDeleteAtespaceAccessPolicyRequest(ctx context.Context, req *ateapip } // ValidateCustom_AccessPolicy_Metadata requires an access policy to be the -// unatespaced singleton named "default". An empty name is left to defaulting. +// unatespaced singleton named "default". func ValidateCustom_AccessPolicy_Metadata(_ context.Context, _ operation.Operation, root *field.Path, meta, _ *ateapipb.ResourceMetadata) field.ErrorList { var errs field.ErrorList if meta.GetAtespace() != "" { diff --git a/cmd/ateapi/internal/apivalidation/access_policy_test.go b/cmd/ateapi/internal/apivalidation/access_policy_test.go index 71119037ee..9ac5dcad47 100644 --- a/cmd/ateapi/internal/apivalidation/access_policy_test.go +++ b/cmd/ateapi/internal/apivalidation/access_policy_test.go @@ -89,6 +89,12 @@ func TestValidateCreateGlobalAccessPolicyRequest(t *testing.T) { p.Metadata.Atespace = "team-a" })}, wantError: field.ErrorList{field.Forbidden(accessPolicyPath.Child("metadata", "atespace"), "")}, + }, { + name: "missing name", + req: &ateapipb.CreateGlobalAccessPolicyRequest{AccessPolicy: validAccessPolicy(func(p *ateapipb.AccessPolicy) { + p.Metadata.Name = "" + })}, + wantError: field.ErrorList{field.Required(accessPolicyPath.Child("metadata", "name"), "")}, }, { name: "name not default", req: &ateapipb.CreateGlobalAccessPolicyRequest{AccessPolicy: validAccessPolicy(func(p *ateapipb.AccessPolicy) { @@ -252,6 +258,18 @@ func TestValidateCreateAtespaceAccessPolicyRequest(t *testing.T) { name: "missing access_policy", req: &ateapipb.CreateAtespaceAccessPolicyRequest{Atespace: atespace}, wantError: field.ErrorList{field.Required(accessPolicyPath, "")}, + }, { + name: "missing name", + req: &ateapipb.CreateAtespaceAccessPolicyRequest{Atespace: atespace, AccessPolicy: validAccessPolicy(func(p *ateapipb.AccessPolicy) { + p.Metadata.Name = "" + })}, + wantError: field.ErrorList{field.Required(accessPolicyPath.Child("metadata", "name"), "")}, + }, { + name: "name not default", + req: &ateapipb.CreateAtespaceAccessPolicyRequest{Atespace: atespace, AccessPolicy: validAccessPolicy(func(p *ateapipb.AccessPolicy) { + p.Metadata.Name = "other" + })}, + wantError: field.ErrorList{field.Invalid(accessPolicyPath.Child("metadata", "name"), nil, "").WithOrigin("custom=default")}, }, { name: "unknown role", req: &ateapipb.CreateAtespaceAccessPolicyRequest{Atespace: atespace, AccessPolicy: validAccessPolicy(func(p *ateapipb.AccessPolicy) { diff --git a/cmd/ateapi/internal/store/atepg/access_policy.go b/cmd/ateapi/internal/store/atepg/access_policy.go index 475498ddb3..e09880f33d 100644 --- a/cmd/ateapi/internal/store/atepg/access_policy.go +++ b/cmd/ateapi/internal/store/atepg/access_policy.go @@ -27,7 +27,6 @@ import ( func (p *Persistence) CreateGlobalAccessPolicy(ctx context.Context, policy *ateapipb.AccessPolicy) (*ateapipb.AccessPolicy, error) { dbPolicy := proto.Clone(policy).(*ateapipb.AccessPolicy) - dbPolicy.Metadata = &ateapipb.ResourceMetadata{Name: "default"} setCreateMetadata(dbPolicy.Metadata) protoBytes, err := proto.Marshal(dbPolicy) if err != nil { @@ -111,7 +110,6 @@ func (p *Persistence) UpdateGlobalAccessPolicy(ctx context.Context, precondition func (p *Persistence) CreateAtespaceAccessPolicy(ctx context.Context, name string, policy *ateapipb.AccessPolicy) (*ateapipb.AccessPolicy, error) { dbPolicy := proto.Clone(policy).(*ateapipb.AccessPolicy) - dbPolicy.Metadata = &ateapipb.ResourceMetadata{Name: "default"} setCreateMetadata(dbPolicy.Metadata) protoBytes, err := proto.Marshal(dbPolicy) if err != nil { diff --git a/cmd/ateapi/internal/store/atepg/access_policy_test.go b/cmd/ateapi/internal/store/atepg/access_policy_test.go index 5f646e54f0..48abda0138 100644 --- a/cmd/ateapi/internal/store/atepg/access_policy_test.go +++ b/cmd/ateapi/internal/store/atepg/access_policy_test.go @@ -36,6 +36,7 @@ func TestGlobalAccessPolicy_Lifecycle(t *testing.T) { } policy := &ateapipb.AccessPolicy{ + Metadata: &ateapipb.ResourceMetadata{Name: "default"}, Bindings: []*ateapipb.Binding{ {Role: authz.RoleViewer, Members: []string{"user:bob"}}, {Role: authz.RoleOwner, Members: []string{"user:alice"}}, @@ -87,6 +88,7 @@ func TestAtespaceAccessPolicy_LifecycleAndCascade(t *testing.T) { ctx := t.Context() policy := &ateapipb.AccessPolicy{ + Metadata: &ateapipb.ResourceMetadata{Name: "default"}, Bindings: []*ateapipb.Binding{ {Role: authz.RoleEditor, Members: []string{"user:bob"}}, }, From 6299696642d69c863c1c0c27bfec32b723682c12 Mon Sep 17 00:00:00 2001 From: Max Thompson Date: Thu, 8 Oct 2026 19:39:49 +0000 Subject: [PATCH 10/44] kubectl ate admin can rotate the actor JWT signing key (#2140) Part of #1756. - Adds `kubectl ate admin add-jwt-key`, `activate-jwt-key`, and `remove-jwt-key`, so rotating the actor JWT signing key no longer means editing the pool Secret by hand. A new key is published but signs nothing until activated, which gives relying parties time to fetch it. - Adds `list-jwt-keys`, which prints each key's ID and algorithm and marks the active one, so finding the key to remove doesn't mean decoding a Secret full of private keys. - `ConcretePool` gains `AddAuthority`, `Activate`, `RemoveAuthority`, and `ActiveID`. They refuse a duplicate or empty key ID, activating a key the pool doesn't hold, and removing the key that signs. - Each command writes the Secret back conditional on the resourceVersion it read and retries on a conflict, so a concurrent edit isn't lost. - Generated RS256 keys are now 4096-bit RSA instead of 2048-bit. RS256 stays only for relying parties that require it, such as Azure workload identity federation; ES256 is still the default. - The pool commands share one Kubernetes client helper, and make-jwt-pool's flag variables are renamed so the new commands can reuse them. The commands don't wait for relying parties; the operator decides when to activate and remove. The `--key-id` help describes the PKIX key IDs from #2132, which main doesn't use until that PR merges. Testing: unit tests for the pool methods, and fake-clientset tests for a full rotation, a conflict retry, refused changes, and the listing, plus a test that generated RS256 keys are 4096-bit. Not run against a cluster. - [ ] Tests pass - [ ] Appropriate changes to documentation are included in the PR --- cmd/kubectl-ate/README.md | 15 +- cmd/kubectl-ate/internal/cmd/admin.go | 217 ++++++++++++++++-- cmd/kubectl-ate/internal/cmd/admin_test.go | 187 +++++++++++++++ hack/install-ate.sh | 2 +- .../localjwtauthority/localjwtauthority.go | 57 ++++- .../localjwtauthority_test.go | 120 +++++++++- 6 files changed, 571 insertions(+), 27 deletions(-) diff --git a/cmd/kubectl-ate/README.md b/cmd/kubectl-ate/README.md index 55000782ec..9da5c3ca7a 100644 --- a/cmd/kubectl-ate/README.md +++ b/cmd/kubectl-ate/README.md @@ -313,10 +313,19 @@ kubectl ate admin make-ca-pool \ --ca-id "1" # Generate a new JWT authority pool and push it to a Kubernetes Secret. The -# key is ES256 (--alg RS256 for relying parties that don't support ES256) and -# its ID defaults to the base64url SHA-256 of the public key's PKIX encoding -# (--key-id to override). +# key is ES256 (--alg RS256, a 4096-bit RSA key, for relying parties that don't +# support ES256) and its ID defaults to the base64url SHA-256 of the public +# key's PKIX encoding (--key-id to override). kubectl ate admin make-jwt-pool \ --name actor-id-jwt-pool \ --secret-namespace ate-system + +# Rotate the JWT signing key. The new key is published but signs nothing until +# activated; activate it once relying parties have refetched the key set, and +# remove the old key once every token it signed has expired. list-jwt-keys +# shows each key's ID and algorithm and marks the active one. +kubectl ate admin list-jwt-keys --name actor-id-jwt-pool --secret-namespace ate-system +kubectl ate admin add-jwt-key --name actor-id-jwt-pool --secret-namespace ate-system +kubectl ate admin activate-jwt-key --name actor-id-jwt-pool --secret-namespace ate-system --key-id +kubectl ate admin remove-jwt-key --name actor-id-jwt-pool --secret-namespace ate-system --key-id ``` diff --git a/cmd/kubectl-ate/internal/cmd/admin.go b/cmd/kubectl-ate/internal/cmd/admin.go index 67d26831d8..ab27bcfbb6 100644 --- a/cmd/kubectl-ate/internal/cmd/admin.go +++ b/cmd/kubectl-ate/internal/cmd/admin.go @@ -15,7 +15,10 @@ package cmd import ( + "context" "fmt" + "io" + "text/tabwriter" "time" "github.com/agent-substrate/substrate/internal/ateclient" @@ -25,17 +28,19 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" + typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/client-go/util/retry" ) var ( - // Both pool commands write one secret, so they share the flags naming it. + // Each pool command writes one secret, so they share the flags naming it. poolSecretNamespaceFlag string poolSecretNameFlag string makeCaPoolIDFlag string makeCaPoolKeyTypeFlag string makeCaPoolValidityFlag time.Duration - makeJwtPoolAlgFlag string - makeJwtPoolKeyIDFlag string + jwtAlgFlag string + jwtKeyIDFlag string ) var adminCmd = &cobra.Command{ @@ -49,14 +54,9 @@ var makeCaPoolCmd = &cobra.Command{ RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() - kconfig, err := ateclient.LoadKubeConfig(kubeconfig, k8sContext) + kc, err := newKubeClient() if err != nil { - return fmt.Errorf("while reading kubeconfig: %w", err) - } - - kc, err := kubernetes.NewForConfig(kconfig) - if err != nil { - return fmt.Errorf("while creating Kubernetes client: %w", err) + return err } var keyType localca.KeyType @@ -125,31 +125,187 @@ var makeJwtPoolCmd = &cobra.Command{ RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() - kconfig, err := ateclient.LoadKubeConfig(kubeconfig, k8sContext) + kc, err := newKubeClient() if err != nil { - return fmt.Errorf("while reading kubeconfig: %w", err) + return err } - kc, err := kubernetes.NewForConfig(kconfig) + secret, keyID, err := newJWTPoolSecret(poolSecretNamespaceFlag, poolSecretNameFlag, jwtAlgFlag, jwtKeyIDFlag) if err != nil { - return fmt.Errorf("while creating Kubernetes client: %w", err) + return err } - secret, keyID, err := newJWTPoolSecret(poolSecretNamespaceFlag, poolSecretNameFlag, makeJwtPoolAlgFlag, makeJwtPoolKeyIDFlag) + _, err = kc.CoreV1().Secrets(poolSecretNamespaceFlag).Create(ctx, secret, metav1.CreateOptions{}) + if err != nil { + return fmt.Errorf("while uploading pool state to secret: %w", err) + } + + fmt.Printf("Successfully created JWT authority pool secret %s/%s with %s key %s\n", poolSecretNamespaceFlag, poolSecretNameFlag, jwtAlgFlag, keyID) + return nil + }, +} + +var listJwtKeysCmd = &cobra.Command{ + Use: "list-jwt-keys", + Short: "List the keys in a JWT authority pool secret", + RunE: func(cmd *cobra.Command, args []string) error { + kc, err := newKubeClient() if err != nil { return err } - _, err = kc.CoreV1().Secrets(poolSecretNamespaceFlag).Create(ctx, secret, metav1.CreateOptions{}) + _, pool, err := getJWTPool(cmd.Context(), kc.CoreV1().Secrets(poolSecretNamespaceFlag), poolSecretNameFlag) if err != nil { - return fmt.Errorf("while uploading pool state to secret: %w", err) + return err + } + return printJWTKeys(cmd.OutOrStdout(), pool) + }, +} + +var addJwtKeyCmd = &cobra.Command{ + Use: "add-jwt-key", + Short: "Add an inactive signing key to a JWT authority pool secret", + Long: `Add an inactive signing key to a JWT authority pool secret. + +The key is published to relying parties but signs nothing until activate-jwt-key +makes it the signing key. Activate it only once relying parties have refetched +the key set.`, + RunE: func(cmd *cobra.Command, args []string) error { + kc, err := newKubeClient() + if err != nil { + return err + } + + authority, err := localjwtauthority.GenerateAuthority(jwtAlgFlag, jwtKeyIDFlag) + if err != nil { + return fmt.Errorf("while generating JWT authority: %w", err) + } + if err := updateJWTPool(cmd.Context(), kc.CoreV1().Secrets(poolSecretNamespaceFlag), poolSecretNameFlag, func(pool *localjwtauthority.ConcretePool) error { + return pool.AddAuthority(authority) + }); err != nil { + return err + } + + fmt.Printf("Added inactive %s key %s to JWT authority pool secret %s/%s\n", jwtAlgFlag, authority.ID, poolSecretNamespaceFlag, poolSecretNameFlag) + return nil + }, +} + +var activateJwtKeyCmd = &cobra.Command{ + Use: "activate-jwt-key", + Short: "Make a key in a JWT authority pool secret the one that signs", + RunE: func(cmd *cobra.Command, args []string) error { + kc, err := newKubeClient() + if err != nil { + return err + } + + if err := updateJWTPool(cmd.Context(), kc.CoreV1().Secrets(poolSecretNamespaceFlag), poolSecretNameFlag, func(pool *localjwtauthority.ConcretePool) error { + return pool.Activate(jwtKeyIDFlag) + }); err != nil { + return err + } + + fmt.Printf("Activated key %s in JWT authority pool secret %s/%s\n", jwtKeyIDFlag, poolSecretNamespaceFlag, poolSecretNameFlag) + return nil + }, +} + +var removeJwtKeyCmd = &cobra.Command{ + Use: "remove-jwt-key", + Short: "Remove an inactive key from a JWT authority pool secret", + Long: `Remove an inactive key from a JWT authority pool secret. + +Tokens the key signed stop verifying once relying parties refetch the key set, +so remove it only after the last of them has expired.`, + RunE: func(cmd *cobra.Command, args []string) error { + kc, err := newKubeClient() + if err != nil { + return err + } + + if err := updateJWTPool(cmd.Context(), kc.CoreV1().Secrets(poolSecretNamespaceFlag), poolSecretNameFlag, func(pool *localjwtauthority.ConcretePool) error { + return pool.RemoveAuthority(jwtKeyIDFlag) + }); err != nil { + return err } - fmt.Printf("Successfully created JWT authority pool secret %s/%s with %s key %s\n", poolSecretNamespaceFlag, poolSecretNameFlag, makeJwtPoolAlgFlag, keyID) + fmt.Printf("Removed key %s from JWT authority pool secret %s/%s\n", jwtKeyIDFlag, poolSecretNamespaceFlag, poolSecretNameFlag) return nil }, } +// updateJWTPool applies change to the pool in the named secret and writes it +// back. The write is conditional on the secret's resourceVersion, and a +// conflict reruns change against the current pool. +func updateJWTPool(ctx context.Context, secrets typedcorev1.SecretInterface, name string, change func(*localjwtauthority.ConcretePool) error) error { + return retry.RetryOnConflict(retry.DefaultRetry, func() error { + secret, pool, err := getJWTPool(ctx, secrets, name) + if err != nil { + return err + } + + if err := change(pool); err != nil { + return err + } + + wire, err := localjwtauthority.Marshal(pool) + if err != nil { + return fmt.Errorf("while marshaling pool: %w", err) + } + secret.Data["pool"] = wire + if _, err := secrets.Update(ctx, secret, metav1.UpdateOptions{}); err != nil { + return fmt.Errorf("while writing pool secret: %w", err) + } + return nil + }) +} + +// getJWTPool reads the named secret and parses the pool it holds. +func getJWTPool(ctx context.Context, secrets typedcorev1.SecretInterface, name string) (*corev1.Secret, *localjwtauthority.ConcretePool, error) { + secret, err := secrets.Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return nil, nil, fmt.Errorf("while reading pool secret: %w", err) + } + wire, ok := secret.Data["pool"] + if !ok { + return nil, nil, fmt.Errorf("secret %s/%s has no \"pool\" key", secret.Namespace, secret.Name) + } + pool, err := localjwtauthority.Unmarshal(wire) + if err != nil { + return nil, nil, fmt.Errorf("while parsing pool: %w", err) + } + return secret, pool, nil +} + +// printJWTKeys prints the ID and algorithm of every key in the pool, marking +// the one that signs. It prints nothing about the private keys. +func printJWTKeys(out io.Writer, pool *localjwtauthority.ConcretePool) error { + w := tabwriter.NewWriter(out, 0, 0, 3, ' ', 0) + fmt.Fprintln(w, "ACTIVE\tKEY ID\tALGORITHM") + for _, authority := range pool.Authorities { + active := "" + if authority.ID == pool.ActiveID() { + active = "*" + } + fmt.Fprintf(w, "%s\t%s\t%s\n", active, authority.ID, authority.Algorithm) + } + return w.Flush() +} + +// newKubeClient builds a client from the --kubeconfig and --context flags. +func newKubeClient() (kubernetes.Interface, error) { + kconfig, err := ateclient.LoadKubeConfig(kubeconfig, k8sContext) + if err != nil { + return nil, fmt.Errorf("while reading kubeconfig: %w", err) + } + kc, err := kubernetes.NewForConfig(kconfig) + if err != nil { + return nil, fmt.Errorf("while creating Kubernetes client: %w", err) + } + return kc, nil +} + // newJWTPoolSecret builds a Secret holding a pool with one active authority, // and returns the authority's key ID. func newJWTPoolSecret(namespace, name, algorithm, keyID string) (*corev1.Secret, string, error) { @@ -180,10 +336,31 @@ func init() { _ = makeCaPoolCmd.MarkFlagRequired("name") adminCmd.AddCommand(makeCaPoolCmd) - makeJwtPoolCmd.Flags().StringVar(&makeJwtPoolAlgFlag, "alg", "ES256", "Signing algorithm of the initial key. One of [ES256, RS256]") - makeJwtPoolCmd.Flags().StringVar(&makeJwtPoolKeyIDFlag, "key-id", "", "The ID of the initial JWT signing key in the pool. Defaults to the base64url SHA-256 of the key's PKIX encoding") + makeJwtPoolCmd.Flags().StringVar(&jwtAlgFlag, "alg", "ES256", "Signing algorithm of the initial key. One of [ES256, RS256]; RS256 keys are 4096-bit RSA") + makeJwtPoolCmd.Flags().StringVar(&jwtKeyIDFlag, "key-id", "", "The ID of the initial JWT signing key in the pool. Defaults to the base64url SHA-256 of the key's PKIX encoding") makeJwtPoolCmd.Flags().StringVar(&poolSecretNamespaceFlag, "secret-namespace", "default", "Create the secret in this namespace") makeJwtPoolCmd.Flags().StringVar(&poolSecretNameFlag, "name", "", "Create the secret with this name") _ = makeJwtPoolCmd.MarkFlagRequired("name") adminCmd.AddCommand(makeJwtPoolCmd) + + listJwtKeysCmd.Flags().StringVar(&poolSecretNamespaceFlag, "secret-namespace", "default", "The namespace of the pool secret") + listJwtKeysCmd.Flags().StringVar(&poolSecretNameFlag, "name", "", "The name of the pool secret") + _ = listJwtKeysCmd.MarkFlagRequired("name") + adminCmd.AddCommand(listJwtKeysCmd) + + addJwtKeyCmd.Flags().StringVar(&jwtAlgFlag, "alg", "ES256", "Signing algorithm of the new key. One of [ES256, RS256]; RS256 keys are 4096-bit RSA") + addJwtKeyCmd.Flags().StringVar(&jwtKeyIDFlag, "key-id", "", "The ID of the new key. Defaults to the base64url SHA-256 of the key's PKIX encoding") + addJwtKeyCmd.Flags().StringVar(&poolSecretNamespaceFlag, "secret-namespace", "default", "The namespace of the pool secret") + addJwtKeyCmd.Flags().StringVar(&poolSecretNameFlag, "name", "", "The name of the pool secret") + _ = addJwtKeyCmd.MarkFlagRequired("name") + adminCmd.AddCommand(addJwtKeyCmd) + + for _, cmd := range []*cobra.Command{activateJwtKeyCmd, removeJwtKeyCmd} { + cmd.Flags().StringVar(&jwtKeyIDFlag, "key-id", "", "The ID of the key") + cmd.Flags().StringVar(&poolSecretNamespaceFlag, "secret-namespace", "default", "The namespace of the pool secret") + cmd.Flags().StringVar(&poolSecretNameFlag, "name", "", "The name of the pool secret") + _ = cmd.MarkFlagRequired("key-id") + _ = cmd.MarkFlagRequired("name") + adminCmd.AddCommand(cmd) + } } diff --git a/cmd/kubectl-ate/internal/cmd/admin_test.go b/cmd/kubectl-ate/internal/cmd/admin_test.go index 6060f641f5..143adab735 100644 --- a/cmd/kubectl-ate/internal/cmd/admin_test.go +++ b/cmd/kubectl-ate/internal/cmd/admin_test.go @@ -15,8 +15,19 @@ package cmd import ( + "context" + "strings" "testing" + "github.com/google/go-cmp/cmp" + corev1 "k8s.io/api/core/v1" + k8errors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" + "github.com/agent-substrate/substrate/internal/localjwtauthority" ) @@ -70,3 +81,179 @@ func TestNewJWTPoolSecretRejectsUnsupportedAlgorithm(t *testing.T) { t.Error("newJWTPoolSecret(HS256) returned nil error") } } + +// poolState reads back the key IDs and active key of the pool secret. +func poolState(t *testing.T, kc *fake.Clientset) ([]string, string) { + t.Helper() + secret, err := kc.CoreV1().Secrets("ate-system").Get(context.Background(), "actor-id-jwt-pool", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + pool, err := localjwtauthority.Unmarshal(secret.Data["pool"]) + if err != nil { + t.Fatal(err) + } + var ids []string + for _, a := range pool.Authorities { + ids = append(ids, a.ID) + } + return ids, pool.ActiveForSigning +} + +func newPoolClientset(t *testing.T) *fake.Clientset { + t.Helper() + secret, _, err := newJWTPoolSecret("ate-system", "actor-id-jwt-pool", "ES256", "old") + if err != nil { + t.Fatal(err) + } + return fake.NewClientset(secret) +} + +func TestUpdateJWTPoolRotation(t *testing.T) { + kc := newPoolClientset(t) + secrets := kc.CoreV1().Secrets("ate-system") + ctx := context.Background() + next, err := localjwtauthority.GenerateAuthority("RS256", "next") + if err != nil { + t.Fatal(err) + } + + for _, step := range []struct { + name string + change func(*localjwtauthority.ConcretePool) error + wantIDs []string + wantActive string + }{ + {"add", func(p *localjwtauthority.ConcretePool) error { return p.AddAuthority(next) }, []string{"old", "next"}, "old"}, + {"activate", func(p *localjwtauthority.ConcretePool) error { return p.Activate("next") }, []string{"old", "next"}, "next"}, + {"remove", func(p *localjwtauthority.ConcretePool) error { return p.RemoveAuthority("old") }, []string{"next"}, "next"}, + } { + if err := updateJWTPool(ctx, secrets, "actor-id-jwt-pool", step.change); err != nil { + t.Fatalf("%s: %v", step.name, err) + } + ids, active := poolState(t, kc) + if diff := cmp.Diff(step.wantIDs, ids); diff != "" || active != step.wantActive { + t.Errorf("after %s: active %q, want %q; key IDs (-want +got):\n%s", step.name, active, step.wantActive, diff) + } + } +} + +func TestUpdateJWTPoolRetriesConflict(t *testing.T) { + kc := newPoolClientset(t) + conflicted := false + kc.PrependReactor("update", "secrets", func(k8stesting.Action) (bool, runtime.Object, error) { + if conflicted { + return false, nil, nil + } + conflicted = true + // Another writer added a key between our read and write. + secret, _, err := newJWTPoolSecret("ate-system", "actor-id-jwt-pool", "ES256", "old") + if err != nil { + t.Fatal(err) + } + pool, err := localjwtauthority.Unmarshal(secret.Data["pool"]) + if err != nil { + t.Fatal(err) + } + other, err := localjwtauthority.GenerateAuthority("ES256", "other") + if err != nil { + t.Fatal(err) + } + if err := pool.AddAuthority(other); err != nil { + t.Fatal(err) + } + if secret.Data["pool"], err = localjwtauthority.Marshal(pool); err != nil { + t.Fatal(err) + } + if err := kc.Tracker().Update(schema.GroupVersionResource{Version: "v1", Resource: "secrets"}, secret, "ate-system"); err != nil { + t.Fatal(err) + } + return true, nil, k8errors.NewConflict(schema.GroupResource{Resource: "secrets"}, "actor-id-jwt-pool", nil) + }) + + next, err := localjwtauthority.GenerateAuthority("ES256", "next") + if err != nil { + t.Fatal(err) + } + if err := updateJWTPool(context.Background(), kc.CoreV1().Secrets("ate-system"), "actor-id-jwt-pool", func(p *localjwtauthority.ConcretePool) error { + return p.AddAuthority(next) + }); err != nil { + t.Fatalf("updateJWTPool: %v", err) + } + ids, _ := poolState(t, kc) + if diff := cmp.Diff([]string{"old", "other", "next"}, ids); diff != "" { + t.Errorf("key IDs (-want +got):\n%s", diff) + } +} + +func TestUpdateJWTPoolLeavesSecretOnError(t *testing.T) { + for _, tc := range []struct { + name string + secret *corev1.Secret + }{ + {name: "change refused"}, + {name: "no pool key", secret: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: "ate-system", Name: "actor-id-jwt-pool"}, + Data: map[string][]byte{"other": []byte("x")}, + }}, + } { + t.Run(tc.name, func(t *testing.T) { + kc := newPoolClientset(t) + if tc.secret != nil { + kc = fake.NewClientset(tc.secret) + } + err := updateJWTPool(context.Background(), kc.CoreV1().Secrets("ate-system"), "actor-id-jwt-pool", func(p *localjwtauthority.ConcretePool) error { + return p.RemoveAuthority("old") + }) + if err == nil { + t.Error("got nil error") + } + for _, action := range kc.Actions() { + if action.GetVerb() == "update" { + t.Errorf("secret was written: %v", action) + } + } + }) + } +} + +func TestUpdateJWTPoolMissingSecret(t *testing.T) { + kc := fake.NewClientset() + err := updateJWTPool(context.Background(), kc.CoreV1().Secrets("ate-system"), "actor-id-jwt-pool", func(*localjwtauthority.ConcretePool) error { + t.Error("change ran without a pool") + return nil + }) + if !k8errors.IsNotFound(err) { + t.Errorf("err = %v, want NotFound", err) + } +} + +func TestListJWTKeys(t *testing.T) { + kc := newPoolClientset(t) + secrets := kc.CoreV1().Secrets("ate-system") + next, err := localjwtauthority.GenerateAuthority("RS256", "next") + if err != nil { + t.Fatal(err) + } + if err := updateJWTPool(context.Background(), secrets, "actor-id-jwt-pool", func(p *localjwtauthority.ConcretePool) error { + return p.AddAuthority(next) + }); err != nil { + t.Fatal(err) + } + + _, pool, err := getJWTPool(context.Background(), secrets, "actor-id-jwt-pool") + if err != nil { + t.Fatal(err) + } + var out strings.Builder + if err := printJWTKeys(&out, pool); err != nil { + t.Fatal(err) + } + want := "" + + "ACTIVE KEY ID ALGORITHM\n" + + "* old ES256\n" + + " next RS256\n" + if diff := cmp.Diff(want, out.String()); diff != "" { + t.Errorf("output (-want +got):\n%s", diff) + } +} diff --git a/hack/install-ate.sh b/hack/install-ate.sh index 8d046bce7b..4036be08cb 100755 --- a/hack/install-ate.sh +++ b/hack/install-ate.sh @@ -164,7 +164,7 @@ usage() { echo " (https://container.googleapis.com/v1/projects/.../clusters/...)," echo " else the cluster's OIDC discovery document" echo " ACTOR_JWT_ALGORITHM Signing algorithm of a newly created actor JWT pool: ES256 (default) | RS256." - echo " Use RS256 for relying parties that don't support ES256. Has no effect once the pool exists." + echo " Use RS256 (a 4096-bit RSA key) for relying parties that don't support ES256. Has no effect once the pool exists." echo "" echo "Benchmarks (see benchmarking/README.md for details and customization):" echo "" diff --git a/internal/localjwtauthority/localjwtauthority.go b/internal/localjwtauthority/localjwtauthority.go index 8db9683fbd..6e3fc5adc2 100644 --- a/internal/localjwtauthority/localjwtauthority.go +++ b/internal/localjwtauthority/localjwtauthority.go @@ -27,6 +27,7 @@ import ( "fmt" "hash" "os" + "slices" "sync" "time" @@ -189,6 +190,56 @@ func (p *ConcretePool) VerificationKeys() ([]*VerificationKey, error) { return keys, nil } +// AddAuthority adds authority to the pool without activating it, so relying +// parties can learn its key before it signs anything. +func (p *ConcretePool) AddAuthority(authority *Authority) error { + if authority.ID == "" { + return fmt.Errorf("authority has no ID") + } + if p.index(authority.ID) >= 0 { + return fmt.Errorf("authority %q already present", authority.ID) + } + p.Authorities = append(p.Authorities, authority) + return nil +} + +// Activate makes the authority with id the one that signs. +func (p *ConcretePool) Activate(id string) error { + if p.index(id) < 0 { + return fmt.Errorf("authority %q not present", id) + } + p.ActiveForSigning = id + return nil +} + +// RemoveAuthority removes the authority with id, which must not be the one +// that signs. +func (p *ConcretePool) RemoveAuthority(id string) error { + i := p.index(id) + if i < 0 { + return fmt.Errorf("authority %q not present", id) + } + if id == p.ActiveID() { + return fmt.Errorf("authority %q is active for signing", id) + } + p.Authorities = slices.Delete(p.Authorities, i, i+1) + return nil +} + +// ActiveID returns the ID of the authority that signs: ActiveForSigning, or +// the first authority's when none is designated. +func (p *ConcretePool) ActiveID() string { + if p.ActiveForSigning == "" && len(p.Authorities) > 0 { + return p.Authorities[0].ID + } + return p.ActiveForSigning +} + +// index returns the position of the authority with id, or -1. +func (p *ConcretePool) index(id string) int { + return slices.IndexFunc(p.Authorities, func(a *Authority) bool { return a.ID == id }) +} + type wireHeader struct { Type string `json:"typ,omitempty"` Algorithm string `json:"alg,omitempty"` @@ -338,6 +389,10 @@ func Unmarshal(wireBytes []byte) (*ConcretePool, error) { return pool, nil } +// rsaKeyBits is the size of the RSA keys GenerateAuthority makes. RS256 does +// not fix one, and smaller keys are too weak. +const rsaKeyBits = 4096 + // GenerateAuthority generates a JWT signing key for algorithm, which must be // RS256 or ES256. An empty id defaults to the Thumbprint of the public key. func GenerateAuthority(algorithm, id string) (*Authority, error) { @@ -345,7 +400,7 @@ func GenerateAuthority(algorithm, id string) (*Authority, error) { var err error switch algorithm { case "RS256": - key, err = rsa.GenerateKey(rand.Reader, 2048) + key, err = rsa.GenerateKey(rand.Reader, rsaKeyBits) case "ES256": key, err = ecdsa.GenerateKey(elliptic.P256(), rand.Reader) default: diff --git a/internal/localjwtauthority/localjwtauthority_test.go b/internal/localjwtauthority/localjwtauthority_test.go index ee5cd4fc2c..4b3e8e5243 100644 --- a/internal/localjwtauthority/localjwtauthority_test.go +++ b/internal/localjwtauthority/localjwtauthority_test.go @@ -190,6 +190,122 @@ func TestSignJWTHeader(t *testing.T) { } } +func authorityIDs(p *ConcretePool) []string { + var ids []string + for _, a := range p.Authorities { + ids = append(ids, a.ID) + } + return ids +} + +func testPool(t *testing.T, ids ...string) *ConcretePool { + t.Helper() + pool := &ConcretePool{ActiveForSigning: ids[0]} + for _, id := range ids { + authority, err := GenerateAuthority("ES256", id) + if err != nil { + t.Fatal(err) + } + pool.Authorities = append(pool.Authorities, authority) + } + return pool +} + +func TestPoolRotation(t *testing.T) { + pool := testPool(t, "old") + next, err := GenerateAuthority("RS256", "next") + if err != nil { + t.Fatal(err) + } + + if err := pool.AddAuthority(next); err != nil { + t.Fatalf("AddAuthority: %v", err) + } + if pool.ActiveForSigning != "old" { + t.Errorf("after AddAuthority, active = %q, want old", pool.ActiveForSigning) + } + if err := pool.Activate("next"); err != nil { + t.Fatalf("Activate: %v", err) + } + if err := pool.RemoveAuthority("old"); err != nil { + t.Fatalf("RemoveAuthority: %v", err) + } + + if diff := cmp.Diff([]string{"next"}, authorityIDs(pool)); diff != "" { + t.Errorf("authorities (-want +got):\n%s", diff) + } + jwt, err := pool.SignJWT(&actoridjwt.Claims{Subject: "actor/a/b", Audiences: []string{"aud"}}) + if err != nil { + t.Fatalf("SignJWT: %v", err) + } + header, err := base64.RawURLEncoding.DecodeString(strings.Split(jwt, ".")[0]) + if err != nil { + t.Fatal(err) + } + if want := `{"typ":"JWT","alg":"RS256","kid":"next"}`; string(header) != want { + t.Errorf("header = %s, want %s", header, want) + } +} + +func TestPoolRotationRejects(t *testing.T) { + dup, err := GenerateAuthority("ES256", "1") + if err != nil { + t.Fatal(err) + } + noID := &Authority{Algorithm: "ES256", SigningKey: dup.SigningKey} + + for _, tc := range []struct { + name string + pool *ConcretePool + change func(*ConcretePool) error + }{ + {name: "add duplicate ID", pool: testPool(t, "1", "2"), change: func(p *ConcretePool) error { return p.AddAuthority(dup) }}, + {name: "add without ID", pool: testPool(t, "1"), change: func(p *ConcretePool) error { return p.AddAuthority(noID) }}, + {name: "activate absent", pool: testPool(t, "1"), change: func(p *ConcretePool) error { return p.Activate("2") }}, + {name: "remove absent", pool: testPool(t, "1"), change: func(p *ConcretePool) error { return p.RemoveAuthority("2") }}, + {name: "remove active", pool: testPool(t, "1", "2"), change: func(p *ConcretePool) error { return p.RemoveAuthority("1") }}, + {name: "remove first with none designated", pool: func() *ConcretePool { + p := testPool(t, "1", "2") + p.ActiveForSigning = "" + return p + }(), change: func(p *ConcretePool) error { return p.RemoveAuthority("1") }}, + } { + t.Run(tc.name, func(t *testing.T) { + wantIDs, wantActive := authorityIDs(tc.pool), tc.pool.ActiveForSigning + if err := tc.change(tc.pool); err == nil { + t.Error("got nil error") + } + if diff := cmp.Diff(wantIDs, authorityIDs(tc.pool)); diff != "" { + t.Errorf("authorities changed (-want +got):\n%s", diff) + } + if tc.pool.ActiveForSigning != wantActive { + t.Errorf("active = %q, want %q", tc.pool.ActiveForSigning, wantActive) + } + }) + } +} + +func TestPoolActiveID(t *testing.T) { + designated := testPool(t, "1", "2") + designated.ActiveForSigning = "2" + undesignated := testPool(t, "1", "2") + undesignated.ActiveForSigning = "" + + for _, tc := range []struct { + name string + pool *ConcretePool + want string + }{ + {name: "designated", pool: designated, want: "2"}, + {name: "none designated", pool: undesignated, want: "1"}, + {name: "empty", pool: &ConcretePool{}, want: ""}, + } { + if got := tc.pool.ActiveID(); got != tc.want { + t.Errorf("%s: ActiveID() = %q, want %q", tc.name, got, tc.want) + } + } +} + func TestGenerateAuthority(t *testing.T) { for _, alg := range []string{"RS256", "ES256"} { t.Run(alg, func(t *testing.T) { @@ -209,8 +325,8 @@ func TestGenerateAuthority(t *testing.T) { } switch key := authority.SigningKey.(type) { case *rsa.PrivateKey: - if alg != "RS256" || key.N.BitLen() != 2048 { - t.Errorf("got a %d-bit RSA key for %s, want 2048-bit for RS256", key.N.BitLen(), alg) + if alg != "RS256" || key.N.BitLen() != 4096 { + t.Errorf("got a %d-bit RSA key for %s, want 4096-bit for RS256", key.N.BitLen(), alg) } case *ecdsa.PrivateKey: if alg != "ES256" || key.Curve != elliptic.P256() { From 38a3d45af9d650fdc034cefaa840b7ff35d0435e Mon Sep 17 00:00:00 2001 From: Benjamin Elder Date: Thu, 8 Oct 2026 20:21:59 +0000 Subject: [PATCH 11/44] cdiinject: confine device paths and host mounts to allowed roots (#2315) Validate CDI device node paths under /dev via os.OpenRoot so symlinks cannot escape the device root, and require CDI mount HostPaths (and ELF DT_SONAME reads during SONAME symlink staging) to resolve within an explicit AllowedHostDirs allowlist via os.OpenRoot). Fixes https://github.com/agent-substrate/substrate/issues/2091 This code isn't used at the moment, but we know we want to add back device injection in the future, and we know AI security scans will find it. For now I think that's cheaper than deleting and re-introducing (and possibly bringing back this bug) --- cmd/ateom-gvisor/internal/cdiinject/inject.go | 110 ++++++- .../internal/cdiinject/inject_test.go | 309 +++++++++++++++++- cmd/ateom-gvisor/internal/cdiinject/soname.go | 42 ++- 3 files changed, 434 insertions(+), 27 deletions(-) diff --git a/cmd/ateom-gvisor/internal/cdiinject/inject.go b/cmd/ateom-gvisor/internal/cdiinject/inject.go index fadc35a7b9..a1306d02de 100644 --- a/cmd/ateom-gvisor/internal/cdiinject/inject.go +++ b/cmd/ateom-gvisor/internal/cdiinject/inject.go @@ -24,11 +24,13 @@ package cdiinject import ( "context" "encoding/json" + "errors" "fmt" "log/slog" "os" "path/filepath" "strings" + "syscall" specs "github.com/opencontainers/runtime-spec/specs-go" "golang.org/x/sys/unix" @@ -52,6 +54,11 @@ type Options struct { // binary the caller trusts rather than wherever the generator pointed. HookBinary string + // AllowedHostDirs are the host directories from which CDI mounts may be + // sourced. Every mount's HostPath must resolve inside one of these + // directories without escaping via symlinks. + AllowedHostDirs []string + // LibraryDirs are prepended to the container's LD_LIBRARY_PATH. LibraryDirs []string @@ -112,14 +119,20 @@ func IntoBundle(ctx context.Context, bundleDir string, spec *cdi.Spec, opts Opti }) } + cdiMounts := make([]specs.Mount, 0, len(edits.Mounts)) for _, m := range edits.Mounts { + if err := validateMount(m, opts.AllowedHostDirs); err != nil { + return err + } mType := m.Type if mType == "" { mType = "bind" // CDI omits type for its bind mounts; runsc's gofer needs it. } - ociSpec.Mounts = append(ociSpec.Mounts, specs.Mount{ + mount := specs.Mount{ Source: m.HostPath, Destination: m.ContainerPath, Type: mType, Options: m.Options, - }) + } + ociSpec.Mounts = append(ociSpec.Mounts, mount) + cdiMounts = append(cdiMounts, mount) } if ociSpec.Process != nil { @@ -160,7 +173,7 @@ func IntoBundle(ctx context.Context, bundleDir string, spec *cdi.Spec, opts Opti if !filepath.IsAbs(rootfs) { rootfs = filepath.Join(bundleDir, rootfs) } - if err := StageSonameSymlinks(ctx, rootfs, ociSpec.Mounts); err != nil { + if err := StageSonameSymlinks(ctx, rootfs, cdiMounts, opts.AllowedHostDirs); err != nil { return fmt.Errorf("staging SONAME symlinks: %w", err) } @@ -188,22 +201,101 @@ func hasInjectedDevice(l *specs.Linux, nodes []cdi.Dev) bool { return false } -// resolveDevNumbers fills a device node's major/minor from the host when the CDI -// spec omitted them. CDI delegates that to the OCI runtime, which stats the -// host; the merge happens here instead, so the stat happens here too. Otherwise -// the container gets bogus 0,0 char devices that reach no driver. +// devRoot is the host directory under which CDI device nodes must reside. A var +// so tests can point it at a fixture directory. +var devRoot = "/dev" + +// resolveDevNumbers validates that path is a clean device path confined to +// devRoot and fills its major/minor from the host when the CDI spec omitted +// them. CDI delegates that to the OCI runtime, which stats the host; the merge +// happens here instead, so the stat happens here too. Otherwise the container +// gets bogus 0,0 char devices that reach no driver. func resolveDevNumbers(path string, major, minor int64) (int64, int64, error) { + cleanDevRoot := filepath.Clean(devRoot) + if path == "" || !filepath.IsAbs(path) || filepath.Clean(path) != path { + return 0, 0, fmt.Errorf("CDI device path %q must be a clean absolute path", path) + } + rel, err := filepath.Rel(cleanDevRoot, path) + if err != nil || rel == "." || !filepath.IsLocal(rel) { + return 0, 0, fmt.Errorf("CDI device path %q is outside %s", path, cleanDevRoot) + } + root, err := os.OpenRoot(cleanDevRoot) + if err != nil { + return 0, 0, fmt.Errorf("opening device root %s: %w", cleanDevRoot, err) + } + defer root.Close() + + fi, err := root.Stat(rel) if major != 0 { + if err != nil && !errors.Is(err, os.ErrNotExist) { + return 0, 0, fmt.Errorf("stat device %s: %w", path, err) + } + if err == nil && fi.Mode()&os.ModeDevice == 0 { + return 0, 0, fmt.Errorf("CDI device path %s is not a device node", path) + } return major, minor, nil } - var st unix.Stat_t - if err := unix.Stat(path, &st); err != nil { + if err != nil { return 0, 0, fmt.Errorf("stat device %s: %w", path, err) } + if fi.Mode()&os.ModeDevice == 0 { + return 0, 0, fmt.Errorf("CDI device path %s is not a device node", path) + } + st, ok := fi.Sys().(*syscall.Stat_t) + if !ok { + return 0, 0, fmt.Errorf("unexpected stat type for %s", path) + } rdev := uint64(st.Rdev) return int64(unix.Major(rdev)), int64(unix.Minor(rdev)), nil } +// validateMount checks that a CDI mount has a clean, rootfs-local containerPath +// and a hostPath confined to one of allowedHostDirs. +func validateMount(m cdi.Mount, allowedHostDirs []string) error { + if m.ContainerPath == "" || !filepath.IsAbs(m.ContainerPath) || filepath.Clean(m.ContainerPath) != m.ContainerPath { + return fmt.Errorf("CDI mount containerPath %q must be a clean absolute path", m.ContainerPath) + } + relDst := strings.TrimPrefix(m.ContainerPath, "/") + if relDst == "" || !filepath.IsLocal(relDst) { + return fmt.Errorf("CDI mount containerPath %q escapes the rootfs", m.ContainerPath) + } + _, _, err := resolveAllowedHostPath(m.HostPath, allowedHostDirs) + return err +} + +// resolveAllowedHostPath verifies that hostPath is a clean absolute path inside +// one of allowedDirs and that resolving it through os.Root does not escape that +// directory via symlinks. It returns the matched root directory and relative path. +func resolveAllowedHostPath(hostPath string, allowedDirs []string) (string, string, error) { + if hostPath == "" || !filepath.IsAbs(hostPath) || filepath.Clean(hostPath) != hostPath { + return "", "", fmt.Errorf("CDI mount hostPath %q must be a clean absolute path", hostPath) + } + for _, dir := range allowedDirs { + if dir == "" { + continue + } + cleanDir := filepath.Clean(dir) + if !filepath.IsAbs(cleanDir) { + continue + } + rel, err := filepath.Rel(cleanDir, hostPath) + if err != nil || rel == "." || !filepath.IsLocal(rel) { + continue + } + root, err := os.OpenRoot(cleanDir) + if err != nil { + return "", "", fmt.Errorf("opening allowed host dir %q: %w", cleanDir, err) + } + _, statErr := root.Stat(rel) + root.Close() + if statErr != nil { + return "", "", fmt.Errorf("CDI mount hostPath %q invalid in allowed dir %q: %w", hostPath, cleanDir, statErr) + } + return cleanDir, rel, nil + } + return "", "", fmt.Errorf("CDI mount hostPath %q is not under any allowed host directory", hostPath) +} + // dropEnvVar returns env with every "KEY=..." entry for the given key removed. func dropEnvVar(env []string, key string) []string { prefix := key + "=" diff --git a/cmd/ateom-gvisor/internal/cdiinject/inject_test.go b/cmd/ateom-gvisor/internal/cdiinject/inject_test.go index 038d46fefc..3ce891d3e2 100644 --- a/cmd/ateom-gvisor/internal/cdiinject/inject_test.go +++ b/cmd/ateom-gvisor/internal/cdiinject/inject_test.go @@ -56,11 +56,12 @@ func injectFixture(t *testing.T, bundleDir, specDir string) error { return err } return IntoBundle(context.Background(), bundleDir, spec, Options{ - Devices: []string{"all"}, - AllowedHooks: map[string]bool{"create-symlinks": true, "enable-cuda-compat": true}, - HookBinary: testHookBinary, - LibraryDirs: []string{"/usr/local/nvidia/lib64"}, - DropEnv: []string{"NVIDIA_VISIBLE_DEVICES"}, + Devices: []string{"all"}, + AllowedHooks: map[string]bool{"create-symlinks": true, "enable-cuda-compat": true}, + HookBinary: testHookBinary, + AllowedHostDirs: []string{"/usr/local/nvidia"}, + LibraryDirs: []string{"/usr/local/nvidia/lib64"}, + DropEnv: []string{"NVIDIA_VISIBLE_DEVICES"}, }) } @@ -320,15 +321,24 @@ func TestStageSonameSymlinks_ConfinedToRootfs(t *testing.T) { } } + hostDir := filepath.Join(dir, "host") + if err := os.MkdirAll(hostDir, 0o755); err != nil { + t.Fatal(err) + } + hostLib := filepath.Join(hostDir, "libcuda.so.580.65.06") + if err := os.WriteFile(hostLib, []byte("stub"), 0o644); err != nil { + t.Fatal(err) + } + old := elfSonameFn - elfSonameFn = func(string) (string, error) { return tc.soname, nil } + elfSonameFn = func(string, string) (string, error) { return tc.soname, nil } defer func() { elfSonameFn = old }() // Refusing outright and skipping are both acceptable; writing outside is not. _ = StageSonameSymlinks(context.Background(), rootfs, []specs.Mount{{ - Source: "/host/libcuda.so.580.65.06", + Source: hostLib, Destination: tc.dest, - }}) + }}, []string{hostDir}) if b, err := os.ReadFile(victim); err != nil || string(b) != "do not delete" { t.Fatalf("a file outside the rootfs was modified: err=%v content=%q", err, b) @@ -355,3 +365,286 @@ func TestInjectGPUIntoBundle_MissingSpecFails(t *testing.T) { t.Fatal("expected error when the CDI spec is missing") } } + +func TestIntoBundle_HostMountsAllowlistAndConfinement(t *testing.T) { + dir := t.TempDir() + allowedDir := filepath.Join(dir, "allowed-driver") + outsideDir := filepath.Join(dir, "outside") + if err := os.MkdirAll(filepath.Join(allowedDir, "lib64"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(outsideDir, 0o755); err != nil { + t.Fatal(err) + } + + validLib := filepath.Join(allowedDir, "lib64", "libcuda.so.580.65.06") + if err := os.WriteFile(validLib, []byte("stub-lib"), 0o644); err != nil { + t.Fatal(err) + } + outsideSecret := filepath.Join(outsideDir, "libsecret.so.1.0") + if err := os.WriteFile(outsideSecret, []byte("secret"), 0o644); err != nil { + t.Fatal(err) + } + + escapeFileLink := filepath.Join(allowedDir, "lib64", "libescape.so.1.0") + if err := os.Symlink(outsideSecret, escapeFileLink); err != nil { + t.Fatal(err) + } + escapeRelLink := filepath.Join(allowedDir, "lib64", "librel-escape.so.1.0") + if err := os.Symlink("../../outside/libsecret.so.1.0", escapeRelLink); err != nil { + t.Fatal(err) + } + escapeDirLink := filepath.Join(allowedDir, "escaped-dir") + if err := os.Symlink(outsideDir, escapeDirLink); err != nil { + t.Fatal(err) + } + + old := elfSonameFn + elfSonameFn = func(string, string) (string, error) { return "libcuda.so.1", nil } + defer func() { elfSonameFn = old }() + + newBundle := func(t *testing.T) string { + t.Helper() + b := filepath.Join(t.TempDir(), "bundle") + if err := os.MkdirAll(filepath.Join(b, "rootfs"), 0o755); err != nil { + t.Fatal(err) + } + data, err := json.Marshal(&specs.Spec{Version: "1.0.0", Process: &specs.Process{Args: []string{"true"}}}) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(b, "config.json"), data, 0o644); err != nil { + t.Fatal(err) + } + return b + } + + specForMount := func(m cdi.Mount) *cdi.Spec { + return &cdi.Spec{ + Devices: []cdi.Device{{ + Name: "all", + ContainerEdits: cdi.Edits{ + DeviceNodes: []cdi.Dev{{Path: "/dev/nvidia0", Major: 195, Minor: 0, Type: "c"}}, + Mounts: []cdi.Mount{m}, + }, + }}, + } + } + + for _, tc := range []struct { + name string + mount cdi.Mount + allowedDirs []string + wantErr bool + }{ + { + name: "allowed regular file mount succeeds", + mount: cdi.Mount{ + HostPath: validLib, + ContainerPath: "/usr/local/nvidia/lib64/libcuda.so.580.65.06", + }, + allowedDirs: []string{allowedDir}, + wantErr: false, + }, + { + name: "empty allowlist rejects mount", + mount: cdi.Mount{ + HostPath: validLib, + ContainerPath: "/usr/local/nvidia/lib64/libcuda.so.580.65.06", + }, + allowedDirs: nil, + wantErr: true, + }, + { + name: "hostPath outside allowed directory is rejected", + mount: cdi.Mount{ + HostPath: outsideSecret, + ContainerPath: "/usr/local/nvidia/lib64/libsecret.so.1.0", + }, + allowedDirs: []string{allowedDir}, + wantErr: true, + }, + { + name: "unclean hostPath with dot-dot is rejected", + mount: cdi.Mount{ + HostPath: allowedDir + "/../outside/libsecret.so.1.0", + ContainerPath: "/usr/local/nvidia/lib64/libsecret.so.1.0", + }, + allowedDirs: []string{allowedDir}, + wantErr: true, + }, + { + name: "absolute symlink escaping allowed directory is rejected", + mount: cdi.Mount{ + HostPath: escapeFileLink, + ContainerPath: "/usr/local/nvidia/lib64/libescape.so.1.0", + }, + allowedDirs: []string{allowedDir}, + wantErr: true, + }, + { + name: "relative symlink escaping allowed directory is rejected", + mount: cdi.Mount{ + HostPath: escapeRelLink, + ContainerPath: "/usr/local/nvidia/lib64/librel-escape.so.1.0", + }, + allowedDirs: []string{allowedDir}, + wantErr: true, + }, + { + name: "parent directory symlink escaping allowed directory is rejected", + mount: cdi.Mount{ + HostPath: filepath.Join(escapeDirLink, "libsecret.so.1.0"), + ContainerPath: "/usr/local/nvidia/lib64/libsecret.so.1.0", + }, + allowedDirs: []string{allowedDir}, + wantErr: true, + }, + { + name: "relative containerPath is rejected", + mount: cdi.Mount{ + HostPath: validLib, + ContainerPath: "usr/local/nvidia/lib64/libcuda.so.580.65.06", + }, + allowedDirs: []string{allowedDir}, + wantErr: true, + }, + { + name: "unclean containerPath is rejected", + mount: cdi.Mount{ + HostPath: validLib, + ContainerPath: "/usr/local/../../escaped", + }, + allowedDirs: []string{allowedDir}, + wantErr: true, + }, + } { + t.Run(tc.name, func(t *testing.T) { + bundle := newBundle(t) + err := IntoBundle(context.Background(), bundle, specForMount(tc.mount), Options{ + Devices: []string{"all"}, + AllowedHostDirs: tc.allowedDirs, + }) + if (err != nil) != tc.wantErr { + t.Fatalf("IntoBundle() error = %v, wantErr %v", err, tc.wantErr) + } + if !tc.wantErr { + linkTarget, err := os.Readlink(filepath.Join(bundle, "rootfs/usr/local/nvidia/lib64/libcuda.so.1")) + if err != nil || linkTarget != "libcuda.so.580.65.06" { + t.Fatalf("SONAME symlink target = %q, err = %v; want libcuda.so.580.65.06", linkTarget, err) + } + } + }) + } +} + +func TestIntoBundle_DevicePathConfinement(t *testing.T) { + dir := t.TempDir() + fakeDev := filepath.Join(dir, "dev") + outside := filepath.Join(dir, "outside") + if err := os.MkdirAll(fakeDev, 0o755); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(outside, 0o755); err != nil { + t.Fatal(err) + } + + regularInDev := filepath.Join(fakeDev, "not-a-device") + if err := os.WriteFile(regularInDev, []byte("plain"), 0o644); err != nil { + t.Fatal(err) + } + escapeLink := filepath.Join(fakeDev, "escaped-dev") + if err := os.Symlink("/dev/null", escapeLink); err != nil { + t.Fatal(err) + } + + oldDevRoot := devRoot + devRoot = fakeDev + defer func() { devRoot = oldDevRoot }() + + for _, tc := range []struct { + name string + dev cdi.Dev + }{ + { + name: "device path outside devRoot is rejected", + dev: cdi.Dev{Path: "/etc/shadow", Major: 195, Minor: 0}, + }, + { + name: "unclean device path with dot-dot is rejected", + dev: cdi.Dev{Path: fakeDev + "/../outside/node", Major: 195, Minor: 0}, + }, + { + name: "symlink escaping devRoot is rejected when major is 0", + dev: cdi.Dev{Path: escapeLink}, + }, + { + name: "symlink escaping devRoot is rejected even when major is set", + dev: cdi.Dev{Path: escapeLink, Major: 1, Minor: 3}, + }, + { + name: "regular file inside devRoot is rejected", + dev: cdi.Dev{Path: regularInDev}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + bundle := filepath.Join(t.TempDir(), "bundle") + if err := os.MkdirAll(filepath.Join(bundle, "rootfs"), 0o755); err != nil { + t.Fatal(err) + } + data, _ := json.Marshal(&specs.Spec{Version: "1.0.0", Process: &specs.Process{Args: []string{"true"}}}) + if err := os.WriteFile(filepath.Join(bundle, "config.json"), data, 0o644); err != nil { + t.Fatal(err) + } + spec := &cdi.Spec{ + Devices: []cdi.Device{{ + Name: "all", + ContainerEdits: cdi.Edits{DeviceNodes: []cdi.Dev{tc.dev}}, + }}, + } + if err := IntoBundle(context.Background(), bundle, spec, Options{Devices: []string{"all"}}); err == nil { + t.Fatalf("expected error for device %+v, got nil", tc.dev) + } + }) + } +} + +func TestStageSonameSymlinks_HostConfinement(t *testing.T) { + dir := t.TempDir() + allowedDir := filepath.Join(dir, "allowed") + outsideDir := filepath.Join(dir, "outside") + rootfs := filepath.Join(dir, "rootfs") + for _, d := range []string{allowedDir, outsideDir, rootfs} { + if err := os.MkdirAll(d, 0o755); err != nil { + t.Fatal(err) + } + } + + outsideFile := filepath.Join(outsideDir, "liboutside.so.1.0") + if err := os.WriteFile(outsideFile, []byte("not-allowed"), 0o644); err != nil { + t.Fatal(err) + } + escapeLink := filepath.Join(allowedDir, "libescape.so.1.0") + if err := os.Symlink(outsideFile, escapeLink); err != nil { + t.Fatal(err) + } + + // Unallowed host path must fail StageSonameSymlinks. + if err := StageSonameSymlinks(context.Background(), rootfs, []specs.Mount{{ + Source: outsideFile, + Destination: "/usr/lib/liboutside.so.1.0", + }}, []string{allowedDir}); err == nil { + t.Fatal("expected error for mount source outside allowedHostDirs") + } + + // Symlink inside allowedDir pointing outside must fail both StageSonameSymlinks and elfSoname. + if err := StageSonameSymlinks(context.Background(), rootfs, []specs.Mount{{ + Source: escapeLink, + Destination: "/usr/lib/libescape.so.1.0", + }}, []string{allowedDir}); err == nil { + t.Fatal("expected error for symlink escaping allowedHostDirs") + } + if _, err := elfSoname(allowedDir, "libescape.so.1.0"); err == nil { + t.Fatal("expected elfSoname to reject symlink escaping allowedDir") + } +} diff --git a/cmd/ateom-gvisor/internal/cdiinject/soname.go b/cmd/ateom-gvisor/internal/cdiinject/soname.go index 281f9976d6..ea88c5c368 100644 --- a/cmd/ateom-gvisor/internal/cdiinject/soname.go +++ b/cmd/ateom-gvisor/internal/cdiinject/soname.go @@ -42,7 +42,7 @@ import ( // symlink out of the rootfs would otherwise have the kernel resolve it in ateom's // mount namespace, where the shared image cache and other actors' bundles are // mounted. Same treatment as createExtraDirs in internal/imagecache. -func StageSonameSymlinks(ctx context.Context, rootfs string, mounts []specs.Mount) error { +func StageSonameSymlinks(ctx context.Context, rootfs string, mounts []specs.Mount, allowedHostDirs []string) error { root, err := os.OpenRoot(rootfs) if err != nil { return fmt.Errorf("while opening rootfs %q: %w", rootfs, err) @@ -55,7 +55,11 @@ func StageSonameSymlinks(ctx context.Context, rootfs string, mounts []specs.Moun if !strings.Contains(base, ".so.") || m.Source == "" { continue } - soname, err := elfSonameFn(m.Source) + hostRoot, relSource, err := resolveAllowedHostPath(m.Source, allowedHostDirs) + if err != nil { + return err + } + soname, err := elfSonameFn(hostRoot, relSource) if err != nil { slog.WarnContext(ctx, "Skipping SONAME symlink for driver library", slog.String("library", m.Source), slog.Any("err", err)) @@ -91,18 +95,36 @@ func StageSonameSymlinks(ctx context.Context, rootfs string, mounts []specs.Moun // real ELF file on disk. var elfSonameFn = elfSoname -// elfSoname returns a shared library's DT_SONAME. It returns "" with a nil error for -// a library that simply carries no DT_SONAME entry, and an error when the file could -// not be opened or parsed as ELF. -func elfSoname(path string) (string, error) { - f, err := elf.Open(path) +// elfSoname returns a shared library's DT_SONAME, opening rel inside rootDir via +// os.Root so symlinks cannot escape the allowed host directory. It returns "" with +// a nil error for a library that simply carries no DT_SONAME entry, and an error +// when the file could not be opened or parsed as a regular ELF file. +func elfSoname(rootDir, rel string) (string, error) { + root, err := os.OpenRoot(rootDir) + if err != nil { + return "", fmt.Errorf("opening allowed host dir %s: %w", rootDir, err) + } + defer root.Close() + f, err := root.Open(rel) if err != nil { - return "", fmt.Errorf("opening ELF %s: %w", path, err) + return "", fmt.Errorf("opening ELF %s/%s: %w", rootDir, rel, err) } defer f.Close() - names, err := f.DynString(elf.DT_SONAME) + fi, err := f.Stat() + if err != nil { + return "", fmt.Errorf("stat ELF %s/%s: %w", rootDir, rel, err) + } + if !fi.Mode().IsRegular() { + return "", fmt.Errorf("ELF %s/%s is not a regular file", rootDir, rel) + } + ef, err := elf.NewFile(f) + if err != nil { + return "", fmt.Errorf("parsing ELF %s/%s: %w", rootDir, rel, err) + } + defer ef.Close() + names, err := ef.DynString(elf.DT_SONAME) if err != nil { - return "", fmt.Errorf("reading DT_SONAME from %s: %w", path, err) + return "", fmt.Errorf("reading DT_SONAME from %s/%s: %w", rootDir, rel, err) } if len(names) == 0 { return "", nil From 769fd32630aa47bd6d7050b39e1aa0804a8622dc Mon Sep 17 00:00:00 2001 From: Julian Gutierrez Oschmann Date: Thu, 8 Oct 2026 21:02:39 +0000 Subject: [PATCH 12/44] Add sections about sub-resources and singletons to the API style guide. (#2354) Also add a resource-model.md markdown file that goes over the current resources and reference it from AGENTS.md in API proto directory. --- docs/api-style-guide.md | 34 ++++++++++- docs/resource-model.md | 110 +++++++++++++++++++++++++++++++++++ pkg/proto/ateapipb/AGENTS.md | 1 + 3 files changed, 144 insertions(+), 1 deletion(-) create mode 100644 docs/resource-model.md diff --git a/docs/api-style-guide.md b/docs/api-style-guide.md index 8b8ca0ce69..6bfb643ccf 100644 --- a/docs/api-style-guide.md +++ b/docs/api-style-guide.md @@ -31,7 +31,7 @@ are **atespace-scoped**, whereas `Atespace` resources are naturally **global-sco ### 2.1 Identity and scope * Atespace-scoped resources belong to an atespace. Their identity is `(atespace, name)`, unique within the resource type. -* Global-scoped resources are global across the entire deployment and do not belong to any atespace. For these, the identity is `name` alone. +* Global-scoped resources are global across the entire Substrate installation and do not belong to any atespace. For these, the identity is `name` alone. In both cases, a `metadata` field contains both `atespace` and `name`. For global resources, the `atespace` must always be empty. @@ -119,6 +119,38 @@ Note that this assumes that ActorTemplates are also resources in the substrate g TODO: Decide the convention for cross-references that point to a resource in the *same* atespace as the referrer: whether the caller must fill in `atespace` explicitly, or leaves it empty and the server resolves/validates it against the referrer's atespace. Current leaning is to require it be filled in. +### 2.4 Subresources + +A **subresource** is a resource nested under a parent resource. It exists only in the context of that parent, is reached through the parent, and is deleted with it. Examples are `EgressPolicy` under `Actor`, `AccessPolicy` under `Atespace`, and `ActorAssignment` under `Worker`. + +A subresource's parent may also be **Global**, the implicit root that stands for the whole Substrate installation. The global `AccessPolicy` is a subresource of Global. + +A resource that only *lives in* an atespace is not a subresource of the `Atespace`. `Actor` is atespace-scoped but top-level, with its own standard methods and its own `ObjectRef`. A resource that only *references* another is not its subresource either. A `Tag` references its source `Actor` through `source_actor`, but outlives it. + +Rules: +- Each instance has exactly one parent instance, which never changes. A subresource type may allow more than one parent type, including Global. +- Deleting the parent **must** delete its subresources. +- A subresource has the same scope as its parent. `metadata.atespace` **must** equal the parent's atespace. For an `EgressPolicy` that is the Actor's atespace. For an `AccessPolicy` under an `Atespace` it is empty, because `Atespace` is global-scoped, and it is empty for every subresource of Global. +- `metadata.name` **must** be unique among the parent's subresources of that type. A singleton is named `default` (see section #2.5). +- The message **must** be named for the subresource itself, without the parent type. Use `EgressPolicy`, not `ActorEgressPolicy`. A subresource that allows more than one parent type uses the same message under each, and the RPC names tell the parents apart (`AccessPolicy` under Global and under `Atespace`). +- RPC names **must** be the standard verb, then the parent type, then the subresource type, as in `GetActorEgressPolicy`, `CreateAtespaceAccessPolicy` and `ListWorkerActorAssignments`. For a subresource of Global, the parent type is `Global`, as in `GetGlobalAccessPolicy`. +- The request **must** identify the parent with an `ObjectRef` field named after the parent type (`actor`, `atespace`, `worker`). Get and Delete requests **must** also carry a `name` field holding the subresource's name, and for a singleton `name` **must** be `default`. A request for a subresource of Global carries no parent reference, since Global has no identity. +- On Create and Update, the subresource is embedded in a field named after its snake_case type (`egress_policy`, `access_policy`). Its `metadata` **must** agree with the parent, and a mismatch returns `INVALID_ARGUMENT`. +- A subresource may be managed by the system, such as `ActorAssignment`. The system creates and deletes it as part of other operations, and the API exposes only List and, optionally, Get. +- Subresource's methods follow section #3. Update replaces the whole object with `uid` and `version` preconditions, Delete takes `DeleteOptions`, and List is paginated. + +TODO: The existing Get and Delete requests for singleton subresources do not carry `name` yet. They identify the subresource by its parent alone. + +### 2.5 Singletons + +A **singleton** is a resource type that allows at most one instance per parent. For example, each `Actor` has at most one `EgressPolicy`, and each `Atespace` has at most one `AccessPolicy`, as does Global. + +Rules: +- Clients **must** always set `metadata.name` to `default`. Any other value, including an empty one, fails with `INVALID_ARGUMENT`. +- A singleton **must not** have a List method. +- Get **must** return `NOT_FOUND` until the singleton has been created. +- Create **must** return `ALREADY_EXISTS` if the singleton exists. It is never an upsert. + --- ## 3. Standard Methods diff --git a/docs/resource-model.md b/docs/resource-model.md new file mode 100644 index 0000000000..b4a90adf0b --- /dev/null +++ b/docs/resource-model.md @@ -0,0 +1,110 @@ +# Substrate Resource Model + +Substrate exposes resources through two APIs: + +- a **gRPC API** (`pkg/proto/ateapipb/ateapi.proto`), served by the + Substrate API server. Its resources follow the + [API Style Guide](api-style-guide.md). +- The **Kubernetes API**, through CRDs in `pkg/api/v1alpha1`, for + managing compute infrastructure. Its resources follow the + [Kubernetes API conventions](https://github.com/kubernetes/community/blob/main/contributors/devel/sig-architecture/api-conventions.md) + +## Scope and parent + +Every resource belongs either to the whole Substrate installation or to a +single atespace. A **global** resource, such as an `Atespace` or a `Worker`, +belongs to the whole Substrate installation. An **atespace-scoped** resource, +such as an `Actor` or a `Tag`, belongs to one atespace, and atespaces keep +their resources apart from each other. + +Some resources exist only as part of another resource. These are +**subresources**, and the resource they belong to is their **parent**. For +example, an `EgressPolicy` is part of an `Actor`. A subresource is always +reached through its parent and goes away when its parent does. + +A subresource's parent can also be **Global**, which stands for the whole +Substrate installation. Global is not a resource. It is the root of the +hierarchy below. + +## Hierarchy + +``` +Global +├── AccessPolicy "default" subresource (singleton) +├── Atespace +│ ├── AccessPolicy "default" subresource (singleton) +│ ├── ActorTemplate +│ ├── Actor +│ │ └── EgressPolicy "default" subresource (singleton) +│ └── Tag +└── Worker + └── ActorAssignment subresource (one per hosted Actor, managed by Substrate) +``` + +## Top-level resources + +| Resource | Scope | Cardinality | Written by | +| :--- | :--- | :--- | :--- | +| `Atespace` | Global | 0..N | Clients | +| `ActorTemplate` | Atespace | 0..N | Clients | +| `Actor` | Atespace | 0..N | Clients | +| `Tag` | Atespace | 0..N | Clients | +| `Worker` | Global | 0..N | Substrate | + +## Subresources + +| Subresource | Parent | Cardinality | Written by | +| :--- | :--- | :--- | :--- | +| `EgressPolicy` | `Actor` | 0..1, named `default` | Clients | +| `AccessPolicy` | Global | 0..1, named `default` | Clients | +| `AccessPolicy` | `Atespace` | 0..1, named `default` | Clients | +| `ActorAssignment` | `Worker` | 0..N | Substrate | + +## Kubernetes resources + +| CRD | Scope | +| :--- | :--- | +| `WorkerPool` | Namespaced | +| `SandboxConfig` | Cluster | +| `CSIDriverConfig` | Cluster | + +## Relationships + +```mermaid +classDiagram + namespace kube-apiserver { + class WorkerPool { + <> + } + class SandboxConfig { + <> + } + class WorkerPod + } + + namespace substrate-api-server { + class Atespace + class AtespaceAccessPolicy + class ActorTemplate + class Actor + class EgressPolicy + class Tag + class Worker + class ActorAssignment + } + + Atespace "1" *-- "0..1" AtespaceAccessPolicy + Atespace "1" o-- "*" ActorTemplate + Atespace "1" o-- "*" Actor + Atespace "1" o-- "*" Tag + Actor "1" *-- "0..1" EgressPolicy + Worker "1" *-- "*" ActorAssignment + Actor "*" --> "1" ActorTemplate : actor_template + Actor "*" --> "0..1" Tag : source_tag + Tag "*" --> "1" Actor : source_actor + ActorAssignment "0..1" --> "1" Actor : actor + ActorTemplate ..> WorkerPool : worker_selector + ActorTemplate ..> SandboxConfig : sandbox_config + WorkerPool "1" *-- "*" WorkerPod + Worker "1" --> "1" WorkerPod +``` diff --git a/pkg/proto/ateapipb/AGENTS.md b/pkg/proto/ateapipb/AGENTS.md index e34cbbcdef..3da3ae6db7 100644 --- a/pkg/proto/ateapipb/AGENTS.md +++ b/pkg/proto/ateapipb/AGENTS.md @@ -3,5 +3,6 @@ `ateapi.proto` defines Substrate's public API. - See `docs/api-style-guide.md` for the API style guide. +- See `docs/resource-model.md` for the resources and subresources the API exposes. Update it when you add, remove, or re-parent a resource. - See `docs/api-validation.md` for how to use declarative validation (DV) tags. - `*.pb.go` files are generated. Never edit them by hand. After changing `ateapi.proto`, run `hack/update/codegen.sh`, which also regenerates the DV code in `cmd/ateapi/internal/apivalidation`. From 1033909d899134ad88fabde5fa3b762e6282a206 Mon Sep 17 00:00:00 2001 From: Eric Curtin Date: Thu, 8 Oct 2026 22:06:10 +0000 Subject: [PATCH 13/44] ate-setup: choose the StorageClass of the bundled PostgreSQL (#2314) Adds `--postgres-storage-class` / `ATE_INSTALL_POSTGRES_STORAGE_CLASS`. Unset keeps the cluster default, since the claim template is immutable on existing installs. Fixes #2298 - [x] Tests pass - [x] Appropriate changes to documentation are included in the PR --- cmd/ate-setup/commands.md | 1 + cmd/ate-setup/internal/config/config.go | 3 + cmd/ate-setup/internal/config/load.go | 1 + cmd/ate-setup/internal/config/setting.go | 6 ++ cmd/ate-setup/internal/steps/postgres.go | 69 ++++++++++++--- cmd/ate-setup/internal/steps/postgres_test.go | 86 +++++++++++++++++++ docs/operator-install.md | 1 + 7 files changed, 156 insertions(+), 11 deletions(-) diff --git a/cmd/ate-setup/commands.md b/cmd/ate-setup/commands.md index 160dc5d6b1..d6a05c830e 100644 --- a/cmd/ate-setup/commands.md +++ b/cmd/ate-setup/commands.md @@ -28,6 +28,7 @@ a pre-scan pass, so they may appear anywhere on its command line. | `--rollout-timeout DURATION` | `--rollout-timeout DURATION` | Readiness timeout for workloads (default `60s`). Unlike the shell flag it also governs the podcertificate-controller and CSI waits, which stay at their 120s default until it is passed | | `--podcert-workers-per-signer N` | `--podcert-workers-per-signer N` | Concurrent workers per podcertificate-controller signer | | `--cluster-size size0\|size10` | `--cluster-size size0\|size10` | Footprint profile (default `size0`). `size10` assumes a dedicated PostgreSQL node: it resizes the bundled StatefulSet and its `postgresql.conf`, pins the apiserver's connection pool, and raises the podcertificate-controller's API rate limits. `ATE_INSTALL_CLUSTER_SIZE` when the flag is absent | +| `--ateapi-postgres-storage-class NAME` | `ATE_API_POSTGRES_STORAGE_CLASS=NAME` | StorageClass for the bundled PostgreSQL volume instead of the cluster default, such as `dynamic-rwo` on GKE so C4 and N4 nodes can attach it. An existing install must delete the postgres StatefulSet and its claim first | | `--cordon-control-plane` | `--cordon-control-plane` | Keep the control plane off the worker nodes. Assumes a small shared pool labeled and tainted `ate.dev/workloadType=ate-control-plane:NoSchedule`, across which each workload's replicas are spread, and a one-node pool labeled and tainted `ate.dev/workloadType=ate-postgres:NoSchedule` for postgres alone. `ATE_INSTALL_CORDON_CONTROL_PLANE=true` when the flag is absent | | `--experimental-additional-egress-extproc-service NS/SVC:PORT` | `--experimental-additional-egress-extproc-service NS/SVC:PORT` | External processor authorization filter | | `--credential-provider JSON` | `--credential-provider JSON`, or `ATE_CREDENTIAL_PROVIDER` | Required by `deploy ate-system` and `deploy atenet`. A JSON object: `{"name":"k8s.io"}` deploys and uses the bundled Kubernetes Secrets provider, with a NetworkPolicy that admits only the egress gateway; `{"enabled":false}` turns egress credential injection off; `{"name":"","address":":"}` uses a provider you deploy yourself. See [`docs/egress-credential-injection.md`](../../docs/egress-credential-injection.md) | diff --git a/cmd/ate-setup/internal/config/config.go b/cmd/ate-setup/internal/config/config.go index 7e7042398a..0042ac184d 100644 --- a/cmd/ate-setup/internal/config/config.go +++ b/cmd/ate-setup/internal/config/config.go @@ -160,6 +160,9 @@ type Config struct { // published as the postgres-server-ca Secret, which ate-api-server mounts // at /run/postgres-server-ca/server-ca.pem for sslmode=verify-ca DSNs. PostgresServerCAFile string + // PostgresStorageClass is the StorageClass of the bundled PostgreSQL volume + // (ATE_API_POSTGRES_STORAGE_CLASS). Empty leaves the cluster default. + PostgresStorageClass string // CloudSQL points the apiserver at a Cloud SQL instance through the Auth // Proxy sidecar instead of a directly reachable PostgreSQL. CloudSQL CloudSQLConfig diff --git a/cmd/ate-setup/internal/config/load.go b/cmd/ate-setup/internal/config/load.go index 0d6bec4461..0fee8fb34e 100644 --- a/cmd/ate-setup/internal/config/load.go +++ b/cmd/ate-setup/internal/config/load.go @@ -220,6 +220,7 @@ func buildConfig(root string, env map[string]string, r *Resolved) (*Config, erro PostgresSchema: r.String("ateapi.postgres.schema"), PostgresPoolMaxConns: r.String("ateapi.postgres.poolMaxConns"), PostgresServerCAFile: r.String("ateapi.postgres.serverCAFile"), + PostgresStorageClass: r.String("ateapi.postgres.storageClass"), PostgresReadWriteConnectionString: readWriteConnectionString, PostgresOwnerConnectionString: ownerConnectionString, diff --git a/cmd/ate-setup/internal/config/setting.go b/cmd/ate-setup/internal/config/setting.go index e52d783a4e..6f102b6c01 100644 --- a/cmd/ate-setup/internal/config/setting.go +++ b/cmd/ate-setup/internal/config/setting.go @@ -211,6 +211,12 @@ var Registry = []Setting{ Flag: "ateapi-postgres-server-ca-file", Kind: KindString, Usage: "Local PEM file holding an external PostgreSQL server CA", }, + { + Key: "ateapi.postgres.storageClass", Env: "ATE_API_POSTGRES_STORAGE_CLASS", + Flag: "ateapi-postgres-storage-class", Kind: KindString, + Usage: "StorageClass for the bundled PostgreSQL volume, instead of the cluster default. " + + "An existing install must delete the postgres StatefulSet and its claim first", + }, { Key: "ateapi.postgres.cloudsql.instance", Env: "ATE_API_POSTGRES_CLOUDSQL_INSTANCE", Flag: "ateapi-postgres-cloudsql-instance", Kind: KindString, diff --git a/cmd/ate-setup/internal/steps/postgres.go b/cmd/ate-setup/internal/steps/postgres.go index 33d7179326..dea4883edf 100644 --- a/cmd/ate-setup/internal/steps/postgres.go +++ b/cmd/ate-setup/internal/steps/postgres.go @@ -152,35 +152,82 @@ func (e *Env) postgresManifestPath() string { // applyPostgres renders and applies the bundled PostgreSQL StatefulSet at the // selected cluster size. -// -// The size10 changes are made to the rendered objects rather than patched onto -// the cluster after a base apply: one apply means one rollout, and the pod the -// rollout wait sees is the resized one. It also means a later server-side -// apply of the same objects cannot half-revert them, which a post-apply patch -// under a different field manager would be exposed to. func (e *Env) applyPostgres(ctx context.Context) error { if err := e.requirePostgresPool(ctx); err != nil { return err } - manifest, err := e.render(e.postgresManifestPath()) + objs, err := e.postgresObjects() if err != nil { return err } + return e.Kube.Apply(ctx, objs) +} + +// postgresObjects renders the bundled PostgreSQL objects with the configured +// StorageClass and cluster size applied. +// +// The changes are made to the rendered objects rather than patched onto the +// cluster after a base apply: one apply means one rollout, and the pod the +// rollout wait sees is the resized one. It also means a later server-side +// apply of the same objects cannot half-revert them, which a post-apply patch +// under a different field manager would be exposed to. +func (e *Env) postgresObjects() ([]*unstructured.Unstructured, error) { + manifest, err := e.render(e.postgresManifestPath()) + if err != nil { + return nil, err + } objs, err := kube.DecodeManifestBytes(manifest) if err != nil { - return err + return nil, err + } + if class := e.Cfg.PostgresStorageClass; class != "" { + log.Stepf("Using StorageClass %s for the bundled PostgreSQL volume", class) + if err := setPostgresStorageClass(objs, class); err != nil { + return nil, err + } } if e.Cfg.Size10() { log.Step("apply_postgres_size10_overrides") conf, err := os.ReadFile(e.Cfg.Manifest("postgres-size10", "postgres-config-patch.yaml")) if err != nil { - return fmt.Errorf("while reading the size10 postgres config: %w", err) + return nil, fmt.Errorf("while reading the size10 postgres config: %w", err) } if err := applyPostgresSize10Overrides(objs, conf); err != nil { - return err + return nil, err } } - return e.Kube.Apply(ctx, objs) + return objs, nil +} + +// setPostgresStorageClass sets the StorageClass of the StatefulSet's data +// volume claim. A claim without one gets the cluster default, which some +// machine series cannot attach. +func setPostgresStorageClass(objs []*unstructured.Unstructured, class string) error { + for _, obj := range objs { + if obj.GetKind() != "StatefulSet" || obj.GetName() != "postgres" { + continue + } + claims, _, err := unstructured.NestedSlice(obj.Object, "spec", "volumeClaimTemplates") + if err != nil { + return fmt.Errorf("while reading the volume claims of %s: %w", kube.Describe(obj), err) + } + for i, c := range claims { + claim, ok := c.(map[string]any) + if !ok { + continue + } + if name, _, _ := unstructured.NestedString(claim, "metadata", "name"); name != "data" { + continue + } + if err := unstructured.SetNestedField(claim, class, "spec", "storageClassName"); err != nil { + return fmt.Errorf("while setting the StorageClass on %s: %w", kube.Describe(obj), err) + } + claims[i] = claim + return unstructured.SetNestedSlice(obj.Object, claims, "spec", "volumeClaimTemplates") + } + return fmt.Errorf("%s has no data volume claim to set the StorageClass on", kube.Describe(obj)) + } + return fmt.Errorf("the postgres manifest has no statefulset/postgres to set the StorageClass on") } // requirePostgresPool refuses to apply the bundled PostgreSQL under diff --git a/cmd/ate-setup/internal/steps/postgres_test.go b/cmd/ate-setup/internal/steps/postgres_test.go index 6caf28ed8c..7b326f92fb 100644 --- a/cmd/ate-setup/internal/steps/postgres_test.go +++ b/cmd/ate-setup/internal/steps/postgres_test.go @@ -152,6 +152,92 @@ func TestApplyPostgresSize10OverridesRejectsMissingObjects(t *testing.T) { } } +// dataClaimStorageClass returns the StorageClass of the postgres StatefulSet's +// data claim, and whether it is set at all. +func dataClaimStorageClass(t *testing.T, objs []*unstructured.Unstructured) (string, bool) { + t.Helper() + ss := findObject(objs, "StatefulSet", "postgres") + if ss == nil { + t.Fatal("no statefulset/postgres") + } + claims, _, _ := unstructured.NestedSlice(ss.Object, "spec", "volumeClaimTemplates") + for _, c := range claims { + claim := c.(map[string]any) + if name, _, _ := unstructured.NestedString(claim, "metadata", "name"); name == "data" { + class, found, _ := unstructured.NestedString(claim, "spec", "storageClassName") + return class, found + } + } + t.Fatal("statefulset/postgres has no data claim") + return "", false +} + +// Runs over the real manifests, so a renamed claim fails here rather than on +// an install that asked for a StorageClass. +func TestSetPostgresStorageClass(t *testing.T) { + for _, kind := range []bool{false, true} { + objs := postgresObjects(t, kind) + if class, found := dataClaimStorageClass(t, objs); found { + t.Fatalf("kind=%v: manifest already names StorageClass %q", kind, class) + } + if err := setPostgresStorageClass(objs, "dynamic-rwo"); err != nil { + t.Fatalf("kind=%v: setPostgresStorageClass: %v", kind, err) + } + if class, _ := dataClaimStorageClass(t, objs); class != "dynamic-rwo" { + t.Errorf("kind=%v: StorageClass = %q, want dynamic-rwo", kind, class) + } + } +} + +func TestSetPostgresStorageClassRejectsMissingObjects(t *testing.T) { + objs := postgresObjects(t, false) + + var withoutStatefulSet []*unstructured.Unstructured + for _, obj := range objs { + if obj.GetKind() != "StatefulSet" { + withoutStatefulSet = append(withoutStatefulSet, obj) + } + } + if err := setPostgresStorageClass(withoutStatefulSet, "x"); err == nil { + t.Error("setPostgresStorageClass succeeded without a StatefulSet, want an error") + } + + ss := findObject(objs, "StatefulSet", "postgres") + unstructured.RemoveNestedField(ss.Object, "spec", "volumeClaimTemplates") + if err := setPostgresStorageClass(objs, "x"); err == nil { + t.Error("setPostgresStorageClass succeeded without a data claim, want an error") + } +} + +func TestPostgresObjectsStorageClass(t *testing.T) { + for _, tc := range []struct { + name string + class string + size10 bool + wantClass string + wantSet bool + }{ + {name: "unset keeps the cluster default"}, + {name: "set", class: "dynamic-rwo", wantClass: "dynamic-rwo", wantSet: true}, + {name: "set with size10", class: "dynamic-rwo", size10: true, wantClass: "dynamic-rwo", wantSet: true}, + } { + t.Run(tc.name, func(t *testing.T) { + cfg := &config.Config{Root: repoRoot(t), PostgresStorageClass: tc.class} + if tc.size10 { + cfg.ClusterSize = config.ClusterSizeSize10 + } + objs, err := (&Env{Cfg: cfg}).postgresObjects() + if err != nil { + t.Fatalf("postgresObjects: %v", err) + } + class, set := dataClaimStorageClass(t, objs) + if class != tc.wantClass || set != tc.wantSet { + t.Errorf("StorageClass = %q (set %v), want %q (set %v)", class, set, tc.wantClass, tc.wantSet) + } + }) + } +} + func TestPlanPostgres(t *testing.T) { for _, tc := range []struct { name string diff --git a/docs/operator-install.md b/docs/operator-install.md index a8b0b319ba..da9b2d7fe6 100644 --- a/docs/operator-install.md +++ b/docs/operator-install.md @@ -279,6 +279,7 @@ command. A setting with no default is unset unless something supplies it. | `ateapi.postgres.readWrite.role` | `ATE_API_POSTGRES_READ_WRITE_ROLE` | `--ateapi-postgres-read-write-role` | `substrate_readwrite` | (global) | | `ateapi.postgres.schema` | `ATE_API_POSTGRES_SCHEMA` | `--ateapi-postgres-schema` | — | (global) | | `ateapi.postgres.serverCAFile` | `ATE_API_POSTGRES_SERVER_CA_FILE` | `--ateapi-postgres-server-ca-file` | — | (global) | +| `ateapi.postgres.storageClass` | `ATE_API_POSTGRES_STORAGE_CLASS` | `--ateapi-postgres-storage-class` | — | (global) | | `atenet.dataplane` | `ATE_ATENET_DATAPLANE` | `--atenet-dataplane` | `envoy` | (global) | | `atenet.egress.additionalExtprocService` | `ATE_ADDITIONAL_EGRESS_EXTPROC_SERVICE` | `--experimental-additional-egress-extproc-service` | — | (global) | | `atenet.egress.credentialProvider` | `ATE_CREDENTIAL_PROVIDER` | `--credential-provider` | — | (global) | From fa9875f791dbc445dc753a97fa99309f1c8fe524 Mon Sep 17 00:00:00 2001 From: ILLUM1N0X Date: Thu, 8 Oct 2026 22:10:57 +0000 Subject: [PATCH 14/44] Move snapshot transfers behind the object-store snapshot plugin (#2224) Part of #1827 Move external snapshot transfers in atelet and ate-api-server behind the `objectstoresnapshot.v1` plugin API, and ship a reference plugin for GCS and S3 that runs as a sidecar next to each. ## What changes | Commit | Change | |---|---| | objectstoreplugin: add object-store plugin and snapshot-plugin binary | `internal/objectstoreplugin` implements `NodeProvider` and `ControlProvider` on GCS/S3. `cmd/snapshot-plugin` serves one of them on a Unix socket (`node` or `control` mode). A `healthcheck` mode checks a running plugin's gRPC health service, for the sidecar's startup probe. | | atelet: transfer snapshots through NodeProvider | atelet fetches and uploads external snapshots through the node plugin instead of calling object storage. | | ateapi: clean up and copy snapshots through ControlProvider | ate-api-server deletes and copies external snapshots through the control plugin instead of its own storage client. | | manifests: run snapshot-plugin as atelet and ate-api-server sidecars | Native sidecars next to atelet (node mode) and ate-api-server (control mode), each sharing a pod-local emptyDir for the socket. Each sidecar has an exec `startupProbe` running `snapshot-plugin healthcheck` against its socket, so the main container starts only once the plugin serves. | ## Reviewing - **Moved code.** The node plugin's per-file upload and download are atelet's code moved as is, so it can be found in the old location on `main`. The control plugin wraps the existing `objectstore.DeletePrefix` and `CopyPrefix`. New code is limited to the gRPC glue, request validation, the startup health check, the binary, and the manifests. - **Unchanged object layout.** Files are stored as `.zstd` with an uncompressed `manifest.json`, so snapshots written before and after this change are interchangeable. The manifest still goes last, in its own call, and remains the commit marker. - **Validation added at the plugin boundary.** The plugin now serves a socket API, so it checks every request: the snapshot URI, that the local directory is inside `--root` (`/var/lib/ate`), and that file names are base names with no duplicates. Every directory atelet passes today is under `/var/lib/ate`, and the file-name checks are the ones atelet already applied before transferring. - **A plugin that never comes up fails fast.** Clients dial with `WaitForReady`, so without a check the first transfer would hang. atelet and ate-api- server run `objectstoreplugin.WaitReady` (a gRPC health check) at startup with a one-minute bound and exit if the plugin is not serving. The sidecar's `startupProbe` runs that same health check by executing `snapshot-plugin healthcheck` inside the container, so the kubelet does not start atelet or ate-api-server until the plugin serves. Transfer RPCs get no fixed deadlines, since one would be wrong for large snapshots - **Shared backend selection.** atelet's `ATE_STORAGE_BACKEND` selection moves to `objectstorage.NewFromEnv` in `pkg/objectstorage`, along with `UsesS3` and `NewS3ClientFromEnv`, so atelet and both plugin modes select the backend the same way. These are new exported functions in `pkg/`. - **Package location.** The plugin lives in `internal/objectstoreplugin`, not `internal/plugins/`, because atelet and ate-api-server tests import its in-process clients (`objectstoreplugintest`) and it imports `internal/objectstore`. ## Not in this PR - `actor_jwt` is not set yet. Per-actor authorization is #777. ## Breaking change The object storage settings (`ATE_STORAGE_BACKEND`, `AWS_*`) move from the `ate-api-server` container to its snapshot-plugin` sidecar, which is now the only process in that pod that touches object storage for snapshots. ate-api-server exits at startup if `ATE_STORAGE_BACKEND` is still set on it. An install that patches these settings onto the `ate-api-server` container must patch the sidecar instead; see the kind overlay for an S3 example. ## Testing - `make test` and `make verify`. - New unit tests for the plugins (`internal/objectstoreplugin`), backend selection (`pkg/objectstorage`), and the shared validation helpers (`internal/resources`). - E2E on kind runs in CI (gVisor and micro-VM, S3 via rustfs). - E2E on GKE (GCS) --- Makefile | 3 +- cmd/ate-setup/internal/images/images.go | 1 + .../controlapi/actor_template_test.go | 5 +- .../controlapi/functionaltest/common_test.go | 3 +- cmd/ateapi/internal/controlapi/service.go | 16 +- cmd/ateapi/internal/controlapi/workflow.go | 28 +- .../internal/controlapi/workflow_delete.go | 5 +- .../controlapi/workflow_delete_test.go | 10 +- .../internal/controlapi/workflow_revert.go | 5 +- .../internal/controlapi/workflow_suspend.go | 5 +- .../controlapi/workflow_suspend_test.go | 11 +- .../internal/controlapi/workflow_tag.go | 9 +- .../internal/controlapi/workflow_tag_test.go | 8 +- .../controlapi/workflow_testutil_test.go | 9 +- cmd/ateapi/main.go | 68 ++- cmd/ateapi/main_test.go | 31 ++ cmd/atelet/internal/ateletpath/ateletpath.go | 5 + cmd/atelet/main.go | 170 +++---- cmd/atelet/main_test.go | 20 +- cmd/atelet/sandbox_assets.go | 12 +- cmd/atelet/snapshot_transfer.go | 94 ++++ cmd/atelet/snapshot_transfer_test.go | 138 ++++++ cmd/snapshot-plugin/main.go | 160 ++++++ docs/architecture.md | 2 +- internal/objectstore/objectstore.go | 5 +- internal/objectstoreplugin/control.go | 65 +++ internal/objectstoreplugin/node.go | 216 +++++++++ .../objectstoreplugintest.go | 61 +++ internal/objectstoreplugin/plugin_test.go | 459 ++++++++++++++++++ internal/objectstoreplugin/socket.go | 83 ++++ internal/objectstoreplugin/status.go | 41 ++ internal/resources/snapshot.go | 17 + internal/resources/snapshot_test.go | 24 + internal/resources/snapshotfiles.go | 37 ++ internal/resources/snapshotfiles_test.go | 42 ++ manifests/ate-install/ate-api-server.yaml | 51 +- manifests/ate-install/atelet.yaml | 57 +++ .../kind/atelet/kustomization.yaml | 19 + manifests/ate-install/kind/kustomization.yaml | 9 +- pkg/objectstorage/env.go | 65 +++ pkg/objectstorage/env_test.go | 69 +++ 41 files changed, 1913 insertions(+), 225 deletions(-) create mode 100644 cmd/atelet/snapshot_transfer.go create mode 100644 cmd/atelet/snapshot_transfer_test.go create mode 100644 cmd/snapshot-plugin/main.go create mode 100644 internal/objectstoreplugin/control.go create mode 100644 internal/objectstoreplugin/node.go create mode 100644 internal/objectstoreplugin/objectstoreplugintest/objectstoreplugintest.go create mode 100644 internal/objectstoreplugin/plugin_test.go create mode 100644 internal/objectstoreplugin/socket.go create mode 100644 internal/objectstoreplugin/status.go create mode 100644 internal/resources/snapshotfiles.go create mode 100644 internal/resources/snapshotfiles_test.go create mode 100644 pkg/objectstorage/env.go create mode 100644 pkg/objectstorage/env_test.go diff --git a/Makefile b/Makefile index d353d93072..f01760e765 100644 --- a/Makefile +++ b/Makefile @@ -58,7 +58,8 @@ CONTROL_PLANE_IMAGES := ./cmd/ateapi \ ./cmd/atelet \ ./cmd/atenet \ ./cmd/credential-provider/kubernetes-secrets \ - ./cmd/podcertcontroller + ./cmd/podcertcontroller \ + ./cmd/snapshot-plugin WORKER_IMAGES := ./cmd/ateom-gvisor \ ./cmd/ateom-microvm DEMO_IMAGES := ./demos/counter \ diff --git a/cmd/ate-setup/internal/images/images.go b/cmd/ate-setup/internal/images/images.go index 461ebef1b9..f106359947 100644 --- a/cmd/ate-setup/internal/images/images.go +++ b/cmd/ate-setup/internal/images/images.go @@ -48,6 +48,7 @@ var Components = []string{ "cmd/ateom-microvm", "cmd/credential-provider/kubernetes-secrets", "cmd/podcertcontroller", + "cmd/snapshot-plugin", "demos/counter", "demos/egress", "demos/multi-template/fspersist", diff --git a/cmd/ateapi/internal/controlapi/actor_template_test.go b/cmd/ateapi/internal/controlapi/actor_template_test.go index 2889e13d67..7fe9c23f70 100644 --- a/cmd/ateapi/internal/controlapi/actor_template_test.go +++ b/cmd/ateapi/internal/controlapi/actor_template_test.go @@ -24,6 +24,7 @@ import ( "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest" "github.com/agent-substrate/substrate/internal/apierror" + "github.com/agent-substrate/substrate/internal/objectstoreplugin/objectstoreplugintest" "github.com/agent-substrate/substrate/internal/resources" atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" listersv1alpha1 "github.com/agent-substrate/substrate/pkg/client/listers/api/v1alpha1" @@ -244,7 +245,7 @@ func TestDeleteActorTemplate(t *testing.T) { tagRef := resources.TagRefFromTag(tag) tagURI := mustReservedTagSnapshotURI(t, tag) objects.PutSnapshot(t, tagURI, "manifest.json") - svc := &RPCService{impl: newServiceImpl(persistence, nil), actorWorkflow: workflow, objectStore: objects} + svc := &RPCService{impl: newServiceImpl(persistence, nil), actorWorkflow: workflow, snapshotPlugin: objectstoreplugintest.ControlClient(objects)} // The handler must request AnyState to clean up an active golden actor. mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapipb.ActorStatus) { s.State = ateapipb.ActorState_ACTOR_STATE_RUNNING @@ -286,7 +287,7 @@ func TestDeleteActorTemplate(t *testing.T) { req := &ateapipb.DeleteActorTemplateRequest{ActorTemplate: templateRef.ToObjectRef()} deleted, err := svc.DeleteActorTemplate(ctx, req) if tt.failPrefix != "" { - if !errors.Is(err, errObjectStore) { + if !isObjectStoreErr(err) { t.Fatalf("DeleteActorTemplate = %v, want object storage error", err) } if _, err := persistence.GetActorTemplate(ctx, templateRef); err != nil { diff --git a/cmd/ateapi/internal/controlapi/functionaltest/common_test.go b/cmd/ateapi/internal/controlapi/functionaltest/common_test.go index 45e66cef04..7814bea8bb 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/common_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/common_test.go @@ -31,6 +31,7 @@ import ( "github.com/agent-substrate/substrate/internal/localca" "github.com/agent-substrate/substrate/internal/localjwtauthority" "github.com/agent-substrate/substrate/internal/objectstore/objectstoretest" + "github.com/agent-substrate/substrate/internal/objectstoreplugin/objectstoreplugintest" "github.com/agent-substrate/substrate/internal/resources" "github.com/agent-substrate/substrate/internal/volume" atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" @@ -222,7 +223,7 @@ func setupTestWithVolumePlugins(t *testing.T, ns string, plugins map[string]volu instruments, "", volPlugins, - objectStore, + objectstoreplugintest.ControlClient(objectStore), testActorJWTIssuer, actorJWTAuthorityPool, actorCAPool, diff --git a/cmd/ateapi/internal/controlapi/service.go b/cmd/ateapi/internal/controlapi/service.go index 3c85d971f7..f40caa3bb1 100644 --- a/cmd/ateapi/internal/controlapi/service.go +++ b/cmd/ateapi/internal/controlapi/service.go @@ -22,12 +22,12 @@ import ( "github.com/agent-substrate/substrate/cmd/ateapi/internal/workercache" "github.com/agent-substrate/substrate/internal/localca" "github.com/agent-substrate/substrate/internal/localjwtauthority" - "github.com/agent-substrate/substrate/internal/objectstore" "github.com/agent-substrate/substrate/internal/resources" "github.com/agent-substrate/substrate/internal/volume" "github.com/agent-substrate/substrate/internal/volume/csi" listersv1alpha1 "github.com/agent-substrate/substrate/pkg/client/listers/api/v1alpha1" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + objectstoresnapshotv1 "github.com/agent-substrate/substrate/pkg/proto/objectstoresnapshotpb/v1" storagev1listers "k8s.io/client-go/listers/storage/v1" ) @@ -49,7 +49,7 @@ type RPCService struct { instruments *Instruments mu sync.RWMutex volumePlugins map[string]volume.VolumePluginControlPlane - objectStore objectstore.Store + snapshotPlugin objectstoresnapshotv1.ControlProviderClient actorJWTIssuer string actorIDJWTPool localjwtauthority.Pool @@ -68,9 +68,9 @@ type VolumePluginRegistry interface { // // instruments may be nil; the record helpers no-op. // -// objectStore may be nil, which leaves external snapshots in place instead of -// copying and releasing them. Only tests that never reach those steps pass nil; -// ate-api always builds one. +// snapshotPlugin may be nil, which leaves external snapshots in place instead +// of copying and releasing them. Only tests that never reach those steps pass +// nil; ate-api always builds one. // // actorJWTIssuer is copied verbatim into the iss claim of every actor JWT. func NewRPCService( @@ -83,7 +83,7 @@ func NewRPCService( instruments *Instruments, egressGatewayAddress string, volumePlugins map[string]volume.VolumePluginControlPlane, - objectStore objectstore.Store, + snapshotPlugin objectstoresnapshotv1.ControlProviderClient, actorJWTIssuer string, actorIDJWTPool localjwtauthority.Pool, actorIDCAPool localca.Pool, @@ -98,12 +98,12 @@ func NewRPCService( dialer: dialer, instruments: instruments, volumePlugins: volumePlugins, - objectStore: objectStore, + snapshotPlugin: snapshotPlugin, actorJWTIssuer: actorJWTIssuer, actorIDJWTPool: actorIDJWTPool, actorIDCAPool: actorIDCAPool, } - s.actorWorkflow = NewActorWorkflow(impl, workerCache, dialer, sandboxConfigLister, storageClassLister, instruments, egressGatewayAddress, s, objectStore) + s.actorWorkflow = NewActorWorkflow(impl, workerCache, dialer, sandboxConfigLister, storageClassLister, instruments, egressGatewayAddress, s, snapshotPlugin) s.workerWorkflow = NewWorkerWorkflow(impl) return s } diff --git a/cmd/ateapi/internal/controlapi/workflow.go b/cmd/ateapi/internal/controlapi/workflow.go index f7612ef8c1..810eabe24f 100644 --- a/cmd/ateapi/internal/controlapi/workflow.go +++ b/cmd/ateapi/internal/controlapi/workflow.go @@ -26,10 +26,10 @@ import ( "github.com/agent-substrate/substrate/internal/actorevent" "github.com/agent-substrate/substrate/internal/apierror" "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/objectstore" "github.com/agent-substrate/substrate/internal/resources" listersv1alpha1 "github.com/agent-substrate/substrate/pkg/client/listers/api/v1alpha1" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + objectstoresnapshotv1 "github.com/agent-substrate/substrate/pkg/proto/objectstoresnapshotpb/v1" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/codes" @@ -109,14 +109,14 @@ type ActorWorkflow struct { instruments *Instruments egressGatewayAddress string pluginRegistry VolumePluginRegistry - objectStore objectstore.Store + snapshotPlugin objectstoresnapshotv1.ControlProviderClient } // NewActorWorkflow creates a new ActorWorkflow. instruments may be nil. // -// objectStore may be nil, which leaves external snapshots in place instead of -// copying and releasing them. Only tests that never reach those steps pass nil; -// ate-api always builds one. +// snapshotPlugin may be nil, which leaves external snapshots in place instead +// of copying and releasing them. Only tests that never reach those steps pass +// nil; ate-api always builds one. func NewActorWorkflow( store actorWorkflowStore, workerCache *workercache.Cache, @@ -126,7 +126,7 @@ func NewActorWorkflow( instruments *Instruments, egressGatewayAddress string, pluginRegistry VolumePluginRegistry, - objectStore objectstore.Store, + snapshotPlugin objectstoresnapshotv1.ControlProviderClient, ) *ActorWorkflow { return &ActorWorkflow{ store: store, @@ -138,10 +138,24 @@ func NewActorWorkflow( instruments: instruments, egressGatewayAddress: egressGatewayAddress, pluginRegistry: pluginRegistry, - objectStore: objectStore, + snapshotPlugin: snapshotPlugin, } } +// cleanupSnapshot deletes every object under prefix through the control +// snapshot plugin. +func (w *ActorWorkflow) cleanupSnapshot(ctx context.Context, prefix resources.StoragePrefix) error { + _, err := w.snapshotPlugin.CleanupSnapshot(ctx, &objectstoresnapshotv1.CleanupSnapshotRequest{SnapshotUri: prefix.String()}) + return err +} + +// copySnapshot copies every object under src to dst through the control +// snapshot plugin. +func (w *ActorWorkflow) copySnapshot(ctx context.Context, src, dst resources.StoragePrefix) error { + _, err := w.snapshotPlugin.CopySnapshot(ctx, &objectstoresnapshotv1.CopySnapshotRequest{SrcUri: src.String(), DstUri: dst.String()}) + return err +} + // actorWorkflowStore enumerates the exact storage methods needed by // ActorWorkflow and nothing more. type actorWorkflowStore interface { diff --git a/cmd/ateapi/internal/controlapi/workflow_delete.go b/cmd/ateapi/internal/controlapi/workflow_delete.go index 4a7e811ea5..ef0a671375 100644 --- a/cmd/ateapi/internal/controlapi/workflow_delete.go +++ b/cmd/ateapi/internal/controlapi/workflow_delete.go @@ -23,7 +23,6 @@ import ( "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" "github.com/agent-substrate/substrate/internal/apierror" "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/objectstore" "github.com/agent-substrate/substrate/internal/proto/ateletpb" "github.com/agent-substrate/substrate/internal/resources" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" @@ -376,7 +375,7 @@ func (w *ActorWorkflow) ensureExternalSnapshotsReleased(ctx context.Context, act ctx, done := stepSpan(ctx, "ReleaseExternalSnapshots") defer func() { err = done(err) }() - if w.objectStore == nil { + if w.snapshotPlugin == nil { markSkipped(ctx, "no object store configured") return nil } @@ -389,7 +388,7 @@ func (w *ActorWorkflow) ensureExternalSnapshotsReleased(ctx context.Context, act markSkipped(ctx, "the actor owns no external snapshot") return nil } - return objectstore.DeletePrefix(ctx, w.objectStore, prefix) + return w.cleanupSnapshot(ctx, prefix) } // actorSnapshotStoragePrefix returns the prefix holding every object the actor wrote: diff --git a/cmd/ateapi/internal/controlapi/workflow_delete_test.go b/cmd/ateapi/internal/controlapi/workflow_delete_test.go index e7120cc548..f391973870 100644 --- a/cmd/ateapi/internal/controlapi/workflow_delete_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_delete_test.go @@ -17,12 +17,14 @@ package controlapi import ( "context" "errors" + "strings" "testing" "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest" "github.com/agent-substrate/substrate/internal/apierror" "github.com/agent-substrate/substrate/internal/objectstore/objectstoretest" + "github.com/agent-substrate/substrate/internal/objectstoreplugin/objectstoreplugintest" "github.com/agent-substrate/substrate/internal/resources" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "google.golang.org/grpc/codes" @@ -337,7 +339,7 @@ func TestDeleteActor_CollectsInFlightSnapshotWithoutTemplate(t *testing.T) { ctx := context.Background() persistence := newTestPersistence(t) objects := objectstoretest.New() - w := NewActorWorkflow(persistence, nil, nil, nil, nil, nil, "", nil, objects) + w := NewActorWorkflow(persistence, nil, nil, nil, nil, nil, "", nil, objectstoreplugintest.ControlClient(objects)) actorRef := resources.ActorRef{Atespace: "team-a", Name: "actor-1"} actor := storetest.MustCreateActor(t, ctx, persistence, &ateapipb.Actor{ @@ -392,8 +394,8 @@ func TestEnsureExternalSnapshotsReleased_DeletePrefixFailure(t *testing.T) { } err := w.ensureExternalSnapshotsReleased(ctx, actor) - if !errors.Is(err, errTransient) { - t.Fatalf("ensureExternalSnapshotsReleased error = %v, want error wrapping %v", err, errTransient) + if err == nil || !strings.Contains(err.Error(), errTransient.Error()) { + t.Fatalf("ensureExternalSnapshotsReleased error = %v, want it to report %v", err, errTransient) } // Objects should not have been deleted @@ -481,7 +483,7 @@ func TestDeleteActor_CollectsSnapshotsAfterWorkerDelete(t *testing.T) { }) } - actorWorkflow := NewActorWorkflow(persistence, nil, nil, nil, nil, nil, "", nil, objects) + actorWorkflow := NewActorWorkflow(persistence, nil, nil, nil, nil, nil, "", nil, objectstoreplugintest.ControlClient(objects)) // Suspend the actor as far as it gets: MarkSuspending mints the // in-progress URI, and the checkpoint writes under it actor, err := actorWorkflow.ensureMarkedSuspending(ctx, actorRef, actor, template) diff --git a/cmd/ateapi/internal/controlapi/workflow_revert.go b/cmd/ateapi/internal/controlapi/workflow_revert.go index b8154eaeb7..26695ee975 100644 --- a/cmd/ateapi/internal/controlapi/workflow_revert.go +++ b/cmd/ateapi/internal/controlapi/workflow_revert.go @@ -25,7 +25,6 @@ import ( "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" "github.com/agent-substrate/substrate/internal/apierror" "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/objectstore" "github.com/agent-substrate/substrate/internal/resources" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "go.opentelemetry.io/otel/attribute" @@ -202,7 +201,7 @@ func (w *ActorWorkflow) ensureInProgressSnapshotDiscarded(ctx context.Context, a inProgress := actor.GetStatus().GetInProgressSnapshotUri() switch { - case w.objectStore == nil: + case w.snapshotPlugin == nil: markSkipped(ctx, "no object store configured") return nil case inProgress == "": @@ -223,7 +222,7 @@ func (w *ActorWorkflow) ensureInProgressSnapshotDiscarded(ctx context.Context, a } // Only the abandoned snapshot goes, not the actor's whole prefix: the // external snapshot the revert returns the actor to lives under it too. - return objectstore.DeletePrefix(ctx, w.objectStore, uri.Prefix()) + return w.cleanupSnapshot(ctx, uri.Prefix()) } // ensureRevertedFinalized commits SUSPENDED and drops every pointer to the diff --git a/cmd/ateapi/internal/controlapi/workflow_suspend.go b/cmd/ateapi/internal/controlapi/workflow_suspend.go index e39a8ce4ee..a0892d40a1 100644 --- a/cmd/ateapi/internal/controlapi/workflow_suspend.go +++ b/cmd/ateapi/internal/controlapi/workflow_suspend.go @@ -24,7 +24,6 @@ import ( "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" "github.com/agent-substrate/substrate/internal/apierror" "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/objectstore" "github.com/agent-substrate/substrate/internal/proto/ateletpb" "github.com/agent-substrate/substrate/internal/resources" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" @@ -426,7 +425,7 @@ func (w *ActorWorkflow) releaseReplacedSnapshot(ctx context.Context, actor *atea previous := actor.GetStatus().GetExternalSnapshot().GetSnapshotUri() switch { - case w.objectStore == nil: + case w.snapshotPlugin == nil: markSkipped(ctx, "no object store configured") return nil case previous == "": @@ -444,7 +443,7 @@ func (w *ActorWorkflow) releaseReplacedSnapshot(ctx context.Context, actor *atea markSkipped(ctx, "the replaced external snapshot is owned by another resource") return nil } - return objectstore.DeletePrefix(ctx, w.objectStore, uri.Prefix()) + return w.cleanupSnapshot(ctx, uri.Prefix()) } func actorSnapshotOwner(actor *ateapipb.Actor) resources.SnapshotOwner { diff --git a/cmd/ateapi/internal/controlapi/workflow_suspend_test.go b/cmd/ateapi/internal/controlapi/workflow_suspend_test.go index 8a1f7a1b9a..4ee20fdad1 100644 --- a/cmd/ateapi/internal/controlapi/workflow_suspend_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_suspend_test.go @@ -17,6 +17,7 @@ package controlapi import ( "context" "errors" + "strings" "testing" "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" @@ -24,6 +25,7 @@ import ( "github.com/agent-substrate/substrate/internal/apierror" "github.com/agent-substrate/substrate/internal/installdefaults" "github.com/agent-substrate/substrate/internal/objectstore/objectstoretest" + "github.com/agent-substrate/substrate/internal/objectstoreplugin/objectstoreplugintest" "github.com/agent-substrate/substrate/internal/resources" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "google.golang.org/grpc/codes" @@ -409,6 +411,13 @@ func TestEnsureSuspendedFinalized_ReleasesReplacedSnapshot(t *testing.T) { // errObjectStore stands in for object storage being unreachable. var errObjectStore = errors.New("object storage is unavailable") +// isObjectStoreErr reports whether err came from errObjectStore. The object +// store sits behind the snapshot plugin's gRPC boundary, which carries the +// message but not the error value, so errors.Is cannot match it. +func isObjectStoreErr(err error) bool { + return err != nil && strings.Contains(err.Error(), errObjectStore.Error()) +} + // TestEnsureSuspendedFinalized_CommitsDespiteObjectStoreFailure verifies that // failing to collect the snapshot a suspend replaced does not fail the // suspend. The worker is already released by then, so aborting would leave the @@ -486,7 +495,7 @@ func TestEnsureSuspendedFinalized_KeepsReplacedSnapshotOnConflict(t *testing.T) s.ExternalSnapshot = &ateapipb.ExternalSnapshot{SnapshotUri: previous.String()} }) - w := &ActorWorkflow{store: &conflictingUpdateStore{Interface: persistence}, objectStore: objects} + w := &ActorWorkflow{store: &conflictingUpdateStore{Interface: persistence}, snapshotPlugin: objectstoreplugintest.ControlClient(objects)} if _, err := w.ensureSuspendedFinalized(ctx, actorRef, template); apierror.Code(err) != codes.Aborted { t.Fatalf("ensureSuspendedFinalized = %v, want code Aborted", err) } diff --git a/cmd/ateapi/internal/controlapi/workflow_tag.go b/cmd/ateapi/internal/controlapi/workflow_tag.go index 8b6fa949f4..4f1d296d6f 100644 --- a/cmd/ateapi/internal/controlapi/workflow_tag.go +++ b/cmd/ateapi/internal/controlapi/workflow_tag.go @@ -21,7 +21,6 @@ import ( "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/objectstore" "github.com/agent-substrate/substrate/internal/resources" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" ) @@ -155,7 +154,7 @@ func (w *ActorWorkflow) ensureTagSnapshotReleased(ctx context.Context, tag *atea ctx, done := stepSpan(ctx, "ReleaseTagSnapshot") defer func() { err = done(err) }() - if w.objectStore == nil { + if w.snapshotPlugin == nil { markSkipped(ctx, "no object store configured") return nil } @@ -164,7 +163,7 @@ func (w *ActorWorkflow) ensureTagSnapshotReleased(ctx context.Context, tag *atea if err != nil { return fmt.Errorf("while resolving the external snapshot of tag %s: %w", tagRef, err) } - if err := objectstore.DeletePrefix(ctx, w.objectStore, uri.Prefix()); err != nil { + if err := w.cleanupSnapshot(ctx, uri.Prefix()); err != nil { return fmt.Errorf("while releasing the external snapshot %q of tag %s: %w", uri, tagRef, err) } return nil @@ -274,7 +273,7 @@ func (w *ActorWorkflow) ensureTagSnapshotCopied(ctx context.Context, tag *ateapi ctx, done := stepSpan(ctx, "CopyTagSnapshot") defer func() { err = done(err) }() - if w.objectStore == nil { + if w.snapshotPlugin == nil { markSkipped(ctx, "no object store configured") return nil } @@ -283,7 +282,7 @@ func (w *ActorWorkflow) ensureTagSnapshotCopied(ctx context.Context, tag *ateapi if err != nil { return fmt.Errorf("while parsing the external snapshot %q of the source actor: %w", snapshot.GetSnapshotUri(), err) } - if err := objectstore.CopyPrefix(ctx, w.objectStore, src.Prefix(), dst.Prefix()); err != nil { + if err := w.copySnapshot(ctx, src.Prefix(), dst.Prefix()); err != nil { return fmt.Errorf("while copying the external snapshot for tag %s: %w", tagRef, err) } return nil diff --git a/cmd/ateapi/internal/controlapi/workflow_tag_test.go b/cmd/ateapi/internal/controlapi/workflow_tag_test.go index a3a15e0fa3..6a8c6ca03d 100644 --- a/cmd/ateapi/internal/controlapi/workflow_tag_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_tag_test.go @@ -239,7 +239,7 @@ func TestTagActorSnapshot_RecreateAfterCopyFailure(t *testing.T) { } return nil } - if _, err := w.TagActorSnapshot(ctx, tagToCreate(actorRef, "v1")); !errors.Is(err, errObjectStore) { + if _, err := w.TagActorSnapshot(ctx, tagToCreate(actorRef, "v1")); !isObjectStoreErr(err) { t.Fatalf("TagActorSnapshot = %v, want an error wrapping %v", err, errObjectStore) } objects.OnCopy = nil @@ -382,7 +382,7 @@ func TestTagActorSnapshot_NameTakenByAnotherActor(t *testing.T) { } return nil } - if _, err := w.TagActorSnapshot(ctx, tagToCreate(resources.ActorRefFromActor(first), "v1")); !errors.Is(err, errObjectStore) { + if _, err := w.TagActorSnapshot(ctx, tagToCreate(resources.ActorRefFromActor(first), "v1")); !isObjectStoreErr(err) { t.Fatalf("TagActorSnapshot(actor-1) = %v, want an error wrapping %v", err, errObjectStore) } pending, err := persistence.GetTag(ctx, tagRef) @@ -433,7 +433,7 @@ func TestDeleteTag_ReleasesExternalSnapshot(t *testing.T) { // A delete that cannot reach object storage must not drop the row: it is // the only handle left on the snapshot. objects.OnDelete = func(string, string) error { return errObjectStore } - if _, err := w.DeleteTag(ctx, tagRef, store.DeletePreconditions{}); !errors.Is(err, errObjectStore) { + if _, err := w.DeleteTag(ctx, tagRef, store.DeletePreconditions{}); !isObjectStoreErr(err) { t.Fatalf("DeleteTag = %v, want an error wrapping %v", err, errObjectStore) } if _, err := persistence.GetTag(ctx, tagRef); err != nil { @@ -480,7 +480,7 @@ func TestDeleteTag_ReleasesPendingSnapshot(t *testing.T) { t.Fatalf("DeleteActor: %v", err) } objects.OnDelete = func(string, string) error { return errObjectStore } - if _, err := w.DeleteTag(ctx, tagRef, store.DeletePreconditions{}); !errors.Is(err, errObjectStore) { + if _, err := w.DeleteTag(ctx, tagRef, store.DeletePreconditions{}); !isObjectStoreErr(err) { t.Fatalf("DeleteTag = %v, want an error wrapping %v", err, errObjectStore) } if _, err := persistence.GetTag(ctx, tagRef); err != nil { diff --git a/cmd/ateapi/internal/controlapi/workflow_testutil_test.go b/cmd/ateapi/internal/controlapi/workflow_testutil_test.go index 5265972539..d2e78bd3cb 100644 --- a/cmd/ateapi/internal/controlapi/workflow_testutil_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_testutil_test.go @@ -24,6 +24,7 @@ import ( "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest" "github.com/agent-substrate/substrate/internal/apierror" "github.com/agent-substrate/substrate/internal/objectstore/objectstoretest" + "github.com/agent-substrate/substrate/internal/objectstoreplugin/objectstoreplugintest" "github.com/agent-substrate/substrate/internal/resources" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/google/uuid" @@ -40,8 +41,8 @@ func testWorkerUID(podName string) string { // with one minimal ActorTemplate stored in tmplAtespace. Dependencies the // unit tests never reach (worker cache, atelet dialer, k8s clients) are nil, // so a step that unexpectedly executes against them fails the test loudly. -// External snapshots go to an in-memory object store, reachable from a test as -// w.objectStore.(*objectstoretest.Fake). +// External snapshots go to an in-memory object store through the control +// snapshot plugin. func newTestActorWorkflow(t *testing.T, st store.Interface, tmplAtespace, tmplName string) *ActorWorkflow { t.Helper() storetest.MustCreateAtespace(t, context.Background(), st, tmplAtespace) @@ -57,7 +58,7 @@ func newTestActorWorkflow(t *testing.T, st store.Interface, tmplAtespace, tmplNa }); err != nil && !errors.Is(err, store.ErrAlreadyExists) { t.Fatalf("create test ActorTemplate: %v", err) } - return NewActorWorkflow(st, nil, nil, nil, nil, nil, "", nil, objectstoretest.New()) + return NewActorWorkflow(st, nil, nil, nil, nil, nil, "", nil, objectstoreplugintest.ControlClient(objectstoretest.New())) } // newFinalizeWorkflow builds an ActorWorkflow over persistence with an @@ -65,7 +66,7 @@ func newTestActorWorkflow(t *testing.T, st store.Interface, tmplAtespace, tmplNa // directly rather than going through newTestActorWorkflow. func newFinalizeWorkflow(persistence store.Interface) (*ActorWorkflow, *objectstoretest.Fake) { objects := objectstoretest.New() - return &ActorWorkflow{store: persistence, objectStore: objects}, objects + return &ActorWorkflow{store: persistence, snapshotPlugin: objectstoreplugintest.ControlClient(objects)}, objects } // mustActorSnapshotURI builds the URI of a snapshot the actor took under diff --git a/cmd/ateapi/main.go b/cmd/ateapi/main.go index b25c27330a..4bd1553336 100644 --- a/cmd/ateapi/main.go +++ b/cmd/ateapi/main.go @@ -27,7 +27,6 @@ import ( "syscall" "time" - "cloud.google.com/go/storage" "github.com/agent-substrate/substrate/cmd/ateapi/internal/apiauthn" "github.com/agent-substrate/substrate/cmd/ateapi/internal/authz" "github.com/agent-substrate/substrate/cmd/ateapi/internal/controlapi" @@ -40,7 +39,7 @@ import ( "github.com/agent-substrate/substrate/internal/installdefaults" "github.com/agent-substrate/substrate/internal/localca" "github.com/agent-substrate/substrate/internal/localjwtauthority" - "github.com/agent-substrate/substrate/internal/objectstore" + "github.com/agent-substrate/substrate/internal/objectstoreplugin" "github.com/agent-substrate/substrate/internal/oidcdiscovery" "github.com/agent-substrate/substrate/internal/serverboot" "github.com/agent-substrate/substrate/internal/version" @@ -48,8 +47,7 @@ import ( "github.com/agent-substrate/substrate/pkg/client/clientset/versioned" "github.com/agent-substrate/substrate/pkg/client/informers/externalversions" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" - "github.com/aws/aws-sdk-go-v2/config" - "github.com/aws/aws-sdk-go-v2/service/s3" + objectstoresnapshotv1 "github.com/agent-substrate/substrate/pkg/proto/objectstoresnapshotpb/v1" "github.com/jackc/pgx/v5/pgconn" "github.com/spf13/pflag" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" @@ -98,6 +96,8 @@ var ( drainDelay = pflag.Duration("drain-delay", 13*time.Second, "How long to keep accepting new work after SIGTERM, before starting the gRPC drain.") drainTimeout = pflag.Duration("drain-timeout", 15*time.Second, "Deadline for the graceful gRPC drain on shutdown. In-flight RPCs still running past it are forcefully cancelled.") + snapshotPluginSocket = pflag.String("snapshot-plugin-socket", "/run/snapshot-plugin/control.sock", "Unix socket of the control snapshot plugin that external snapshots are deleted and copied through.") + templateResyncInterval = pflag.Duration("template-resync-interval", 20*time.Second, fmt.Sprintf("Interval between actor template resyncs. Must be at least %s.", minResyncInterval)) showVersion = pflag.Bool("version", false, "Print version and exit.") @@ -118,6 +118,9 @@ func main() { if err := loadFlagsFromEnv(); err != nil { serverboot.Fatal(ctx, "Invalid PostgreSQL configuration", err) } + if err := rejectStorageEnv(); err != nil { + serverboot.Fatal(ctx, "Storage settings moved to the snapshot-plugin sidecar", err) + } slog.InfoContext(ctx, "ateapi starting", slog.String("version", version.Version)) if *templateResyncInterval < minResyncInterval { serverboot.Fatal(ctx, "Invalid --template-resync-interval", fmt.Errorf("must be at least %s", minResyncInterval)) @@ -258,9 +261,18 @@ func main() { serverboot.Fatal(ctx, "Failed to create metric instruments", err) } - objectStore, err := newObjectStore(ctx) + snapshotPluginConn, err := objectstoreplugin.Dial(*snapshotPluginSocket) if err != nil { - serverboot.Fatal(ctx, "Failed to set up the object storage backend", err) + serverboot.Fatal(ctx, "Failed to set up the snapshot plugin client", err) + } + defer snapshotPluginConn.Close() + // Plugin calls wait for the connection to become ready, so a plugin + // that never serves would hang them. Fail at startup instead. + readyCtx, cancelReady := context.WithTimeout(ctx, time.Minute) + err = objectstoreplugin.WaitReady(readyCtx, snapshotPluginConn) + cancelReady() + if err != nil { + serverboot.Fatal(ctx, "Snapshot plugin is not serving", err) } volPlugins := make(map[string]volume.VolumePluginControlPlane) @@ -286,7 +298,7 @@ func main() { instruments, *defaultEgressGatewayAddress, volPlugins, - objectStore, + objectstoresnapshotv1.NewControlProviderClient(snapshotPluginConn), resolvedActorJWTIssuer, actorIDJWTAuthorityPool, actorIDCAPool, @@ -413,6 +425,19 @@ func loadFlagsFromEnv() error { return nil } +// rejectStorageEnv fails if ATE_STORAGE_BACKEND is set on ate-api-server, +// which no longer reads it; objectstorage.UsesS3 reads it in the sidecar. An +// install that still patches it onto this container would otherwise come up +// healthy with a sidecar on the default backend, and fail at the first +// snapshot cleanup or copy. Only this variable is a signal: EKS pod identity +// sets AWS_* in every container. +func rejectStorageEnv() error { + if v, ok := os.LookupEnv("ATE_STORAGE_BACKEND"); ok { + return fmt.Errorf("ATE_STORAGE_BACKEND=%q is set on ate-api-server, which no longer reads it; set the storage backend on its snapshot-plugin sidecar", v) + } + return nil +} + func logFlagValues(ctx context.Context) { slog.InfoContext(ctx, "Final flag values", slog.String("grpc-listen-addr", *listenAddr), @@ -436,35 +461,6 @@ func logFlagValues(ctx context.Context) { ) } -// newObjectStore builds the client ate-api manages external snapshots with. -// The backend is selected the same way atelet selects the one it reads and -// writes snapshots through, so both ends of a snapshot's life agree on where -// it lives. -func newObjectStore(ctx context.Context) (objectstore.Store, error) { - switch backend := os.Getenv("ATE_STORAGE_BACKEND"); backend { - case "s3": - slog.InfoContext(ctx, "Using S3 storage backend") - // Depends on the standard AWS environment variables, which have to be - // set on the ate-api pod. - cfg, err := config.LoadDefaultConfig(ctx) - if err != nil { - return nil, fmt.Errorf("loading S3 config: %w", err) - } - return objectstore.NewS3(s3.NewFromConfig(cfg, func(o *s3.Options) { - if os.Getenv("AWS_S3_USE_PATH_STYLE") == "true" { - o.UsePathStyle = true - } - })), nil - // GCS is currently the default, TODO: we assume workload identity / ADC - default: - client, err := storage.NewClient(ctx) - if err != nil { - return nil, fmt.Errorf("creating GCS client: %w", err) - } - return objectstore.NewGCS(client), nil - } -} - // postgresConnectionAttr describes the connection string for the startup log // without echoing it. The bundled and Cloud SQL IAM setups use passwordless // strings, but an external database DSN can carry a password, and the raw diff --git a/cmd/ateapi/main_test.go b/cmd/ateapi/main_test.go index 89d269dde8..58522fc209 100644 --- a/cmd/ateapi/main_test.go +++ b/cmd/ateapi/main_test.go @@ -304,3 +304,34 @@ func writeCA(t *testing.T, path, cn string) { t.Fatalf("WriteFile(%s) error = %v", path, err) } } + +func TestRejectStorageEnv(t *testing.T) { + for _, tc := range []struct { + name string + value *string // nil leaves the variable unset + want bool // an error naming the sidecar + }{ + {name: "unset", value: nil, want: false}, + {name: "s3", value: ptr("s3"), want: true}, + // The old manifests set the default explicitly, so a stale patch + // can carry it. + {name: "gcs", value: ptr("gcs"), want: true}, + {name: "empty", value: ptr(""), want: true}, + } { + t.Run(tc.name, func(t *testing.T) { + // t.Setenv restores the variable after the subtest, which also + // undoes the Unsetenv. + t.Setenv("ATE_STORAGE_BACKEND", "") + os.Unsetenv("ATE_STORAGE_BACKEND") + if tc.value != nil { + os.Setenv("ATE_STORAGE_BACKEND", *tc.value) + } + err := rejectStorageEnv() + if got := err != nil && strings.Contains(err.Error(), "snapshot-plugin sidecar"); got != tc.want { + t.Errorf("rejectStorageEnv() = %v, want error naming the sidecar: %t", err, tc.want) + } + }) + } +} + +func ptr(s string) *string { return &s } diff --git a/cmd/atelet/internal/ateletpath/ateletpath.go b/cmd/atelet/internal/ateletpath/ateletpath.go index 9c2a9ebef8..f7590e81d6 100644 --- a/cmd/atelet/internal/ateletpath/ateletpath.go +++ b/cmd/atelet/internal/ateletpath/ateletpath.go @@ -29,6 +29,11 @@ var ( // directories are visible at the same path in atelet (which writes them) // and in every ateom pod (which mounts them as overlay lowerdirs). ImageCacheDir = filepath.Join(nodepath.BasePath, "image-cache") + + // SnapshotScratchDir holds short-lived directories for snapshot manifests + // moved through the snapshot plugin. It lives under BasePath, which the + // node plugin shares with atelet. + SnapshotScratchDir = filepath.Join(nodepath.BasePath, "snapshot-scratch") ) func RunSCBinaryPath(sha256 string) string { diff --git a/cmd/atelet/main.go b/cmd/atelet/main.go index 651f78cf04..99bc760a51 100644 --- a/cmd/atelet/main.go +++ b/cmd/atelet/main.go @@ -44,6 +44,7 @@ import ( "github.com/agent-substrate/substrate/internal/credbundle" "github.com/agent-substrate/substrate/internal/imagecache" "github.com/agent-substrate/substrate/internal/nodepath" + "github.com/agent-substrate/substrate/internal/objectstoreplugin" "github.com/agent-substrate/substrate/internal/ocispec" "github.com/agent-substrate/substrate/internal/otlprelay" "github.com/agent-substrate/substrate/internal/proto/ateletpb" @@ -59,8 +60,7 @@ import ( "github.com/agent-substrate/substrate/pkg/client/informers/externalversions" "github.com/agent-substrate/substrate/pkg/objectstorage" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" - "github.com/aws/aws-sdk-go-v2/config" - "github.com/aws/aws-sdk-go-v2/service/s3" + objectstoresnapshotv1 "github.com/agent-substrate/substrate/pkg/proto/objectstoresnapshotpb/v1" "github.com/google/go-containerregistry/pkg/authn" "github.com/spf13/pflag" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" @@ -114,6 +114,8 @@ var ( drainDelay = pflag.Duration("drain-delay", 0, "How long to keep accepting new RPCs after SIGTERM before starting the gRPC drain.") drainTimeout = pflag.Duration("drain-timeout", 5*time.Minute, "Deadline for the graceful gRPC drain on shutdown. In-flight RPCs still running past it are forcefully cancelled.") + + snapshotPluginSocket = pflag.String("snapshot-plugin-socket", "/run/snapshot-plugin/node.sock", "Unix socket of the node snapshot plugin that external snapshots are fetched and uploaded through.") ) var _ imagecache.CandidateKeychain = (*credentialprovider.Keychain)(nil) @@ -238,28 +240,9 @@ func main() { serverboot.Fatal(ctx, "Failed to create anonymous GCS client", err) } - var wrappedGCS objectstorage.ObjectStorage - storageBackend := os.Getenv("ATE_STORAGE_BACKEND") - switch storageBackend { - case "s3": - slog.InfoContext(ctx, "Using S3 storage backend") - // depend on standard AWS environment variables to configure the client - // these will need to be set on the atelet pods - cfg, err := config.LoadDefaultConfig(ctx) - if err != nil { - serverboot.Fatal(ctx, "Failed to load S3 config", err) - } - wrappedGCS = objectstorage.NewS3Client(s3.NewFromConfig(cfg, func(o *s3.Options) { - if usePathStyle := os.Getenv("AWS_S3_USE_PATH_STYLE"); usePathStyle == "true" { - o.UsePathStyle = true - } - })) - // GCS is currently the default, TODO: we assume workload identity / ADC - default: - wrappedGCS, err = objectstorage.NewGCSClient(ctx) - if err != nil { - serverboot.Fatal(ctx, "Failed to create GCS client", err) - } + wrappedGCS, err := objectstorage.NewFromEnv(ctx) + if err != nil { + serverboot.Fatal(ctx, "Failed to set up the object storage backend", err) } volPlugins := make(map[string]volume.VolumePluginWorkerPlane) @@ -307,11 +290,26 @@ func main() { go trustBundles.Informer().Run(stopCh) cache.WaitForCacheSync(stopCh, trustBundles.Informer().HasSynced) + snapshotPluginConn, err := objectstoreplugin.Dial(*snapshotPluginSocket) + if err != nil { + serverboot.Fatal(ctx, "Failed to set up the snapshot plugin client", err) + } + defer snapshotPluginConn.Close() + // Transfers wait for the connection to become ready, so a plugin that + // never serves would hang them. Fail at startup instead. + readyCtx, cancelReady := context.WithTimeout(ctx, time.Minute) + err = objectstoreplugin.WaitReady(readyCtx, snapshotPluginConn) + cancelReady() + if err != nil { + serverboot.Fatal(ctx, "Snapshot plugin is not serving", err) + } + wmService := NewService( ctx, ateomDialer, wrappedAnonGCS, wrappedGCS, + objectstoresnapshotv1.NewNodeProviderClient(snapshotPluginConn), imageCache, instruments, volPlugins, @@ -460,10 +458,17 @@ func (g *directCSIDriverConfigGetter) Get(name string) (*atev1alpha1.CSIDriverCo type AteomHerder struct { ateletpb.UnimplementedAteomHerderServer - ateomDialer *AteomDialer - imageCache *imagecache.Store - anonGCSClient objectstorage.ObjectStorage - gcsClient objectstorage.ObjectStorage + ateomDialer *AteomDialer + imageCache *imagecache.Store + anonGCSClient objectstorage.ObjectStorage + // gcsClient downloads sandbox assets. Snapshot files never use it: they move + // through snapshotPlugin. + gcsClient objectstorage.ObjectStorage + // snapshotPlugin moves external snapshot files between the node and storage. + snapshotPlugin objectstoresnapshotv1.NodeProviderClient + // snapshotScratchDir holds short-lived manifest directories; it must be + // inside the node plugin's root. + snapshotScratchDir string instruments *Instruments mu sync.RWMutex volumePlugins map[string]volume.VolumePluginWorkerPlane @@ -479,6 +484,7 @@ func NewService( ateomDialer *AteomDialer, anonGCSClient objectstorage.ObjectStorage, gcsClient objectstorage.ObjectStorage, + snapshotPlugin objectstoresnapshotv1.NodeProviderClient, imageCache *imagecache.Store, instruments *Instruments, volumePlugins map[string]volume.VolumePluginWorkerPlane, @@ -490,6 +496,8 @@ func NewService( imageCache: imageCache, anonGCSClient: anonGCSClient, gcsClient: gcsClient, + snapshotPlugin: snapshotPlugin, + snapshotScratchDir: ateletpath.SnapshotScratchDir, instruments: instruments, volumePlugins: volumePlugins, csiDriverConfigGetter: csiDriverConfigGetter, @@ -868,12 +876,12 @@ func (s *AteomHerder) uploadExternalCheckpoint(ctx context.Context, req *ateletp return s.uploadSnapshot(ctx, uri, checkpointDir, rec, req.GetActorTemplateAtespace(), req.GetActorTemplateName()) } -// uploadSnapshot uploads rec's snapshot files from srcDir to uri (each -// zstd-compressed, concurrently), then the marshaled manifest. The manifest -// goes last, never in parallel: its presence is the commit marker — readers -// assume every file it lists is already present. A crash mid-upload thus -// leaves only orphaned files, never a manifest pointing at files that never -// landed; retries overwrite the deterministic object names. +// uploadSnapshot uploads rec's snapshot files from srcDir to uri through the +// snapshot plugin, then the marshaled manifest. The manifest goes last, in its +// own call: its presence is the commit marker — readers assume every file it +// lists is already present. A crash mid-upload thus leaves only orphaned +// files, never a manifest pointing at files that never landed; retries +// overwrite the deterministic object names. func (s *AteomHerder) uploadSnapshot(ctx context.Context, uri resources.SnapshotURI, srcDir string, rec *sandboxAssetsRecord, templateAtespace, templateName string) error { root, err := os.OpenRoot(srcDir) if err != nil { @@ -881,46 +889,22 @@ func (s *AteomHerder) uploadSnapshot(ctx context.Context, uri resources.Snapshot } defer root.Close() - g, gCtx := errgroup.WithContext(ctx) for _, fileName := range rec.SnapshotFiles { - g.Go(func() error { - local, err := root.Open(fileName) - if err != nil { - return fmt.Errorf("while opening %s in snapshot directory: %w", fileName, err) - } - defer local.Close() - info, err := local.Stat() - if err != nil { - return fmt.Errorf("while inspecting %s in snapshot directory: %w", fileName, err) - } - if !info.Mode().IsRegular() { - return fmt.Errorf("snapshot file %s is not a regular file", fileName) - } - recordSnapshotSize(ctx, fileName, allocatedBytes(info), templateAtespace, templateName) - - objectURI, err := uri.ObjectURI(fileName + ".zstd") - if err != nil { - return fmt.Errorf("while addressing %s in GCS: %w", fileName, err) - } - if err := objectstorage.SendFileToGCSWithZstd(gCtx, s.gcsClient, objectURI, local); err != nil { - return fmt.Errorf("while uploading %s to GCS: %w", fileName, err) - } - return nil - }) + info, err := root.Lstat(fileName) + if err != nil { + return fmt.Errorf("while inspecting %s in snapshot directory: %w", fileName, err) + } + recordSnapshotSize(ctx, fileName, allocatedBytes(info), templateAtespace, templateName) } - if err := g.Wait(); err != nil { - return err + if err := s.uploadSnapshotFiles(ctx, uri.String(), srcDir, rec.SnapshotFiles); err != nil { + return fmt.Errorf("while uploading snapshot files: %w", err) } manifest, err := json.Marshal(rec) if err != nil { return fmt.Errorf("while marshaling snapshot manifest: %w", err) } - manifestURI, err := uri.ObjectURI(sandboxManifestName) - if err != nil { - return fmt.Errorf("while addressing snapshot manifest in GCS: %w", err) - } - if err := objectstorage.SendBytesToGCS(ctx, s.gcsClient, manifestURI, manifest); err != nil { + if err := s.uploadManifest(ctx, uri.String(), manifest); err != nil { return fmt.Errorf("while uploading snapshot manifest: %w", err) } return nil @@ -979,11 +963,6 @@ func (s *AteomHerder) UploadPausedCheckpoint(ctx context.Context, req *ateletpb. // returns the sandbox class recorded in the snapshot manifest (empty when the // manifest was not read). Parameterized by localDir for tests. func (s *AteomHerder) uploadLocalCheckpointDir(ctx context.Context, req *ateletpb.UploadPausedCheckpointRequest, localDir string, uri resources.SnapshotURI) (string, error) { - manifestURI, err := uri.ObjectURI(sandboxManifestName) - if err != nil { - return "", fmt.Errorf("while addressing snapshot manifest in GCS: %w", err) - } - manifest, err := readSnapshotManifest(localDir) if errors.Is(err, os.ErrNotExist) { // The local snapshot is gone. A previous invocation may have uploaded @@ -991,12 +970,12 @@ func (s *AteomHerder) uploadLocalCheckpointDir(ctx context.Context, req *ateletp // means the whole snapshot is committed and this retry already // succeeded. Absent on both sides, the paused actor's state is // unrecoverable. - _, fetchErr := objectstorage.FetchFromGCS(ctx, s.gcsClient, manifestURI) + _, fetchErr := s.fetchManifest(ctx, uri.String()) if fetchErr == nil { slog.InfoContext(ctx, "Local snapshot already uploaded and pruned; nothing to do", slog.String("snapshot_uri", req.GetDestinationSnapshotUri())) return "", nil } - if errors.Is(fetchErr, objectstorage.ErrObjectNotFound) { + if status.Code(fetchErr) == codes.NotFound { return "", fmt.Errorf("local snapshot %q is gone and no uploaded copy exists: %w", req.GetLocalSnapshotName(), fetchErr) } @@ -1140,15 +1119,7 @@ func (s *AteomHerder) Restore(ctx context.Context, req *ateletpb.RestoreRequest) var sandboxRec *sandboxAssetsRecord switch req.GetType() { case ateletpb.CheckpointType_CHECKPOINT_TYPE_EXTERNAL: - uri, err := resources.ParseSnapshotURI(req.GetExternalConfig().GetSnapshotUri()) - if err != nil { - return nil, err - } - manifestURI, err := uri.ObjectURI(sandboxManifestName) - if err != nil { - return nil, err - } - manifest, err := objectstorage.FetchFromGCS(ctx, s.gcsClient, manifestURI) + manifest, err := s.fetchManifest(ctx, req.GetExternalConfig().GetSnapshotUri()) if err != nil { return nil, fmt.Errorf("while fetching snapshot manifest: %w", err) } @@ -1472,40 +1443,9 @@ func copyRootFile(root *os.Root, src, dst string) (int64, error) { } func (s *AteomHerder) downloadExternalCheckpoint(ctx context.Context, snapshotURI string, dstDir string, files []string) error { - uri, err := resources.ParseSnapshotURI(snapshotURI) - if err != nil { - return err - } - root, err := os.OpenRoot(dstDir) - if err != nil { - return fmt.Errorf("while opening restore directory: %w", err) + if err := s.fetchSnapshotFiles(ctx, snapshotURI, dstDir, files); err != nil { + return fmt.Errorf("while downloading snapshot files: %w", err) } - defer root.Close() - - g, gCtx := errgroup.WithContext(ctx) - for _, fileName := range files { - fileName := fileName - g.Go(func() error { - objectURI, err := uri.ObjectURI(fileName + ".zstd") - if err != nil { - return fmt.Errorf("while addressing %s in GCS: %w", fileName, err) - } - local, err := root.OpenFile(fileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600) - if err != nil { - return fmt.Errorf("while opening %s in restore directory: %w", fileName, err) - } - fetchErr := objectstorage.FetchFileFromGCSWithZstd(gCtx, s.gcsClient, objectURI, local) - closeErr := local.Close() - if err := errors.Join(fetchErr, closeErr); err != nil { - return fmt.Errorf("while downloading %s from GCS: %w", fileName, err) - } - return nil - }) - } - if err := g.Wait(); err != nil { - return err - } - return nil } diff --git a/cmd/atelet/main_test.go b/cmd/atelet/main_test.go index 3507e67604..f9a09dea3c 100644 --- a/cmd/atelet/main_test.go +++ b/cmd/atelet/main_test.go @@ -351,7 +351,7 @@ func TestUploadSnapshotRejectsSymlinkOutsideRoot(t *testing.T) { t.Fatal(err) } store := &recordingObjectStorage{} - err = (&AteomHerder{gcsClient: store}).uploadSnapshot(context.Background(), uri, checkpointDir, + err = newPluginHerder(t, store).uploadSnapshot(context.Background(), uri, checkpointDir, &sandboxAssetsRecord{SnapshotFiles: []string{"checkpoint.img"}}, "test", "test") if err == nil { t.Fatal("uploadSnapshot() followed a symlink outside the checkpoint directory") @@ -383,7 +383,7 @@ func TestDownloadExternalCheckpointRejectsSymlinkOutsideRoot(t *testing.T) { if err := objectstorage.SendLocalFileToGCSWithZstd(context.Background(), store, testSnapshotURI+"/checkpoint.img.zstd", payload); err != nil { t.Fatal(err) } - err := (&AteomHerder{gcsClient: store}).downloadExternalCheckpoint( + err := newPluginHerder(t, store).downloadExternalCheckpoint( context.Background(), testSnapshotURI, restoreDir, []string{"checkpoint.img"}) if err == nil { t.Fatal("downloadExternalCheckpoint() followed a symlink outside the restore directory") @@ -1479,7 +1479,7 @@ func TestUploadLocalCheckpointDir(t *testing.T) { t.Run("matching scope uploads all files", func(t *testing.T) { store := &recordingObjectStorage{} - s := &AteomHerder{gcsClient: store} + s := newPluginHerder(t, store) dir := filepath.Join(t.TempDir(), "pause-snap-1") writeLocalSnapshot(t, dir, fullRec("microvm"), map[string]string{ "config.json": "cfg", "memory-ranges": "mem", "data.tar": "data", @@ -1504,7 +1504,7 @@ func TestUploadLocalCheckpointDir(t *testing.T) { t.Run("full capture uploads the reported data files alone as data", func(t *testing.T) { store := &recordingObjectStorage{} - s := &AteomHerder{gcsClient: store} + s := newPluginHerder(t, store) dir := filepath.Join(t.TempDir(), "pause-snap-1") writeLocalSnapshot(t, dir, fullRec("microvm"), map[string]string{ "config.json": "cfg", "memory-ranges": "mem", "data.tar": "data", @@ -1533,7 +1533,7 @@ func TestUploadLocalCheckpointDir(t *testing.T) { t.Run("full capture listing no data files is rejected", func(t *testing.T) { store := &recordingObjectStorage{} - s := &AteomHerder{gcsClient: store} + s := newPluginHerder(t, store) dir := filepath.Join(t.TempDir(), "pause-snap-1") rec := fullRec("microvm") rec.DataSnapshotFiles = nil @@ -1553,7 +1553,7 @@ func TestUploadLocalCheckpointDir(t *testing.T) { }) t.Run("data capture cannot become full", func(t *testing.T) { - s := &AteomHerder{gcsClient: &recordingObjectStorage{}} + s := newPluginHerder(t, &recordingObjectStorage{}) dir := filepath.Join(t.TempDir(), "pause-snap-1") writeLocalSnapshot(t, dir, sandboxAssetsRecord{ SandboxClass: "microvm", @@ -1570,7 +1570,7 @@ func TestUploadLocalCheckpointDir(t *testing.T) { t.Run("manifest without scope is rejected", func(t *testing.T) { store := &recordingObjectStorage{} - s := &AteomHerder{gcsClient: store} + s := newPluginHerder(t, store) dir := filepath.Join(t.TempDir(), "pause-snap-1") writeLocalSnapshot(t, dir, sandboxAssetsRecord{ SandboxClass: "microvm", @@ -1593,7 +1593,7 @@ func TestUploadLocalCheckpointDir(t *testing.T) { store := &recordingObjectStorage{objects: map[string][]byte{ pausedSnapshotPath + "/manifest.json": []byte(`{"sandboxClass":"microvm"}`), }} - s := &AteomHerder{gcsClient: store} + s := newPluginHerder(t, store) if _, err := s.uploadLocalCheckpointDir(ctx, validUploadPausedCheckpointRequest(), filepath.Join(t.TempDir(), "never-created"), uri); err != nil { t.Fatalf("uploadLocalCheckpointDir: %v", err) @@ -1601,7 +1601,7 @@ func TestUploadLocalCheckpointDir(t *testing.T) { }) t.Run("gone locally and remotely crashes the actor", func(t *testing.T) { - s := &AteomHerder{gcsClient: &recordingObjectStorage{}} + s := newPluginHerder(t, &recordingObjectStorage{}) _, err := s.uploadLocalCheckpointDir(ctx, validUploadPausedCheckpointRequest(), filepath.Join(t.TempDir(), "never-created"), uri) if err == nil { @@ -1613,7 +1613,7 @@ func TestUploadLocalCheckpointDir(t *testing.T) { }) t.Run("upload failure returns the error", func(t *testing.T) { - s := &AteomHerder{gcsClient: &recordingObjectStorage{putErr: errors.New("boom")}} + s := newPluginHerder(t, &recordingObjectStorage{putErr: errors.New("boom")}) dir := filepath.Join(t.TempDir(), "pause-snap-1") writeLocalSnapshot(t, dir, fullRec("microvm"), map[string]string{ "config.json": "cfg", "memory-ranges": "mem", "data.tar": "data", diff --git a/cmd/atelet/sandbox_assets.go b/cmd/atelet/sandbox_assets.go index 6b291371f9..0a41e16a34 100644 --- a/cmd/atelet/sandbox_assets.go +++ b/cmd/atelet/sandbox_assets.go @@ -516,17 +516,13 @@ func unmarshalSandboxRecord(data []byte) (*sandboxAssetsRecord, error) { // Actual file access must still use os.Root so symlinks cannot escape that // directory. func validateSnapshotFiles(files []string) error { - seen := make(map[string]bool, len(files)) + if err := resources.ValidateSnapshotFileNames(files); err != nil { + return err + } for i, name := range files { - switch { - case name != filepath.Base(name) || !filepath.IsLocal(name) || name == ".": - return fmt.Errorf("snapshotFiles[%d] %q is not a file name in the checkpoint directory", i, name) - case name == sandboxManifestName: + if name == sandboxManifestName { return fmt.Errorf("snapshotFiles[%d] %q is reserved for the snapshot manifest", i, name) - case seen[name]: - return fmt.Errorf("snapshotFiles[%d] %q is duplicated", i, name) } - seen[name] = true } return nil } diff --git a/cmd/atelet/snapshot_transfer.go b/cmd/atelet/snapshot_transfer.go new file mode 100644 index 0000000000..f6b2df59d2 --- /dev/null +++ b/cmd/atelet/snapshot_transfer.go @@ -0,0 +1,94 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "fmt" + "os" + "path/filepath" + + objectstoresnapshotv1 "github.com/agent-substrate/substrate/pkg/proto/objectstoresnapshotpb/v1" +) + +// External snapshots move between the node and storage only through the +// snapshot plugin. These helpers adapt the plugin's file-oriented API to the +// manifest bytes atelet works with. + +// fetchSnapshotFiles downloads the named snapshot files into dstDir. +func (s *AteomHerder) fetchSnapshotFiles(ctx context.Context, snapshotURI, dstDir string, files []string) error { + if len(files) == 0 { + return nil + } + _, err := s.snapshotPlugin.FetchSnapshot(ctx, &objectstoresnapshotv1.FetchSnapshotRequest{ + SnapshotUri: snapshotURI, + WritePath: dstDir, + Files: files, + }) + return err +} + +// uploadSnapshotFiles uploads the named files in srcDir to the snapshot. +func (s *AteomHerder) uploadSnapshotFiles(ctx context.Context, snapshotURI, srcDir string, files []string) error { + if len(files) == 0 { + return nil + } + _, err := s.snapshotPlugin.UploadSnapshot(ctx, &objectstoresnapshotv1.UploadSnapshotRequest{ + SnapshotUri: snapshotURI, + LocalPath: srcDir, + Files: files, + }) + return err +} + +// fetchManifest returns a snapshot's manifest. The returned error is the +// plugin's gRPC status, so a missing manifest reports codes.NotFound. +func (s *AteomHerder) fetchManifest(ctx context.Context, snapshotURI string) ([]byte, error) { + dir, err := s.snapshotScratch() + if err != nil { + return nil, err + } + defer func() { _ = os.RemoveAll(dir) }() + if err := s.fetchSnapshotFiles(ctx, snapshotURI, dir, []string{sandboxManifestName}); err != nil { + return nil, err + } + return os.ReadFile(filepath.Join(dir, sandboxManifestName)) +} + +// uploadManifest uploads manifest as the snapshot's manifest. +func (s *AteomHerder) uploadManifest(ctx context.Context, snapshotURI string, manifest []byte) error { + dir, err := s.snapshotScratch() + if err != nil { + return err + } + defer func() { _ = os.RemoveAll(dir) }() + if err := os.WriteFile(filepath.Join(dir, sandboxManifestName), manifest, 0o600); err != nil { + return fmt.Errorf("while staging snapshot manifest: %w", err) + } + return s.uploadSnapshotFiles(ctx, snapshotURI, dir, []string{sandboxManifestName}) +} + +// snapshotScratch creates a fresh directory for one manifest transfer. The +// caller removes it. +func (s *AteomHerder) snapshotScratch() (string, error) { + if err := os.MkdirAll(s.snapshotScratchDir, 0o700); err != nil { + return "", fmt.Errorf("while creating snapshot scratch dir: %w", err) + } + dir, err := os.MkdirTemp(s.snapshotScratchDir, "manifest-") + if err != nil { + return "", fmt.Errorf("while creating snapshot scratch dir: %w", err) + } + return dir, nil +} diff --git a/cmd/atelet/snapshot_transfer_test.go b/cmd/atelet/snapshot_transfer_test.go new file mode 100644 index 0000000000..e9b8a14e76 --- /dev/null +++ b/cmd/atelet/snapshot_transfer_test.go @@ -0,0 +1,138 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "slices" + "strings" + "sync" + "testing" + + "github.com/agent-substrate/substrate/internal/objectstoreplugin" + "github.com/agent-substrate/substrate/internal/objectstoreplugin/objectstoreplugintest" + "github.com/agent-substrate/substrate/internal/resources" + "github.com/agent-substrate/substrate/pkg/objectstorage" +) + +// newPluginHerder returns an AteomHerder whose snapshots go through the +// object-store plugin backed by store. +func newPluginHerder(t *testing.T, store objectstorage.ObjectStorage) *AteomHerder { + t.Helper() + // Test directories come from t.TempDir, so admit any absolute path. + plugin, err := objectstoreplugin.NewNodePlugin(store, "/") + if err != nil { + t.Fatal(err) + } + return &AteomHerder{ + snapshotPlugin: objectstoreplugintest.NodeClient(plugin), + snapshotScratchDir: t.TempDir(), + } +} + +// orderedObjectStorage records the order objects are stored in. +type orderedObjectStorage struct { + mu sync.Mutex + puts []string + data map[string][]byte +} + +func (o *orderedObjectStorage) PutObject(_ context.Context, bucket, object string, r io.Reader) error { + b, err := io.ReadAll(r) + if err != nil { + return err + } + o.mu.Lock() + defer o.mu.Unlock() + o.puts = append(o.puts, object) + if o.data == nil { + o.data = map[string][]byte{} + } + o.data[bucket+"/"+object] = b + return nil +} + +func (o *orderedObjectStorage) GetObject(_ context.Context, bucket, object string) (io.ReadCloser, error) { + o.mu.Lock() + defer o.mu.Unlock() + b, ok := o.data[bucket+"/"+object] + if !ok { + return nil, fmt.Errorf("%w: %s/%s", objectstorage.ErrObjectNotFound, bucket, object) + } + return io.NopCloser(bytes.NewReader(b)), nil +} + +func TestUploadSnapshotManifestLast(t *testing.T) { + store := &orderedObjectStorage{} + s := newPluginHerder(t, store) + uri, err := resources.ParseSnapshotURI(pausedSnapshotURI) + if err != nil { + t.Fatal(err) + } + dir := t.TempDir() + rec := sandboxAssetsRecord{SandboxClass: "gvisor", PauseImage: testPauseImage, SnapshotFiles: []string{"a", "b", "c"}} + for _, f := range rec.SnapshotFiles { + if err := os.WriteFile(dir+"/"+f, []byte(f), 0o600); err != nil { + t.Fatal(err) + } + } + + if err := s.uploadSnapshot(context.Background(), uri, dir, &rec, "ate-demo", "counter"); err != nil { + t.Fatalf("uploadSnapshot: %v", err) + } + if len(store.puts) != 4 { + t.Fatalf("puts = %v, want 3 files then the manifest", store.puts) + } + for i, p := range store.puts { + isManifest := strings.HasSuffix(p, "/"+sandboxManifestName) + if isManifest != (i == len(store.puts)-1) { + t.Fatalf("puts = %v, want the manifest last and only last", store.puts) + } + } + + // The manifest round-trips, and scratch space is cleaned up. + got, err := s.fetchManifest(context.Background(), uri.String()) + if err != nil { + t.Fatalf("fetchManifest: %v", err) + } + back, err := unmarshalSandboxRecord(got) + if err != nil { + t.Fatal(err) + } + if !slices.Equal(back.SnapshotFiles, rec.SnapshotFiles) { + t.Errorf("manifest files = %v, want %v", back.SnapshotFiles, rec.SnapshotFiles) + } + if entries, _ := os.ReadDir(s.snapshotScratchDir); len(entries) != 0 { + t.Errorf("scratch dir not cleaned up: %v", entries) + } +} + +func TestSnapshotTransferSkipsEmptyFileLists(t *testing.T) { + store := &orderedObjectStorage{} + s := newPluginHerder(t, store) + if err := s.downloadExternalCheckpoint(context.Background(), pausedSnapshotURI, t.TempDir(), nil); err != nil { + t.Errorf("downloadExternalCheckpoint(no files) = %v, want nil", err) + } + if err := s.uploadSnapshotFiles(context.Background(), pausedSnapshotURI, t.TempDir(), nil); err != nil { + t.Errorf("uploadSnapshotFiles(no files) = %v, want nil", err) + } + if len(store.puts) != 0 { + t.Errorf("objects stored = %v, want none", store.puts) + } +} diff --git a/cmd/snapshot-plugin/main.go b/cmd/snapshot-plugin/main.go new file mode 100644 index 0000000000..4f8d8c82c5 --- /dev/null +++ b/cmd/snapshot-plugin/main.go @@ -0,0 +1,160 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Command snapshot-plugin serves the snapshot plugin API on GCS or S3 over a +// Unix socket. It runs as a sidecar: "node" next to atelet, "control" next to +// ate-api-server. The backend is chosen by ATE_STORAGE_BACKEND ("s3", or GCS +// by default) with the ambient credentials, as atelet and ate-api-server do. +// "healthcheck" reports whether a running plugin is serving, for the sidecar's +// startup probe. +package main + +import ( + "context" + "fmt" + "log/slog" + "os" + "os/signal" + "syscall" + "time" + + "cloud.google.com/go/storage" + "github.com/agent-substrate/substrate/internal/nodepath" + "github.com/agent-substrate/substrate/internal/objectstore" + "github.com/agent-substrate/substrate/internal/objectstoreplugin" + "github.com/agent-substrate/substrate/internal/serverboot" + "github.com/agent-substrate/substrate/pkg/objectstorage" + objectstoresnapshotv1 "github.com/agent-substrate/substrate/pkg/proto/objectstoresnapshotpb/v1" + "github.com/spf13/pflag" + "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" + "google.golang.org/grpc" + "google.golang.org/grpc/health" + healthpb "google.golang.org/grpc/health/grpc_health_v1" +) + +const usage = `usage: snapshot-plugin [flags]` + +func main() { + if len(os.Args) < 2 { + fmt.Fprintln(os.Stderr, usage) + os.Exit(2) + } + mode := os.Args[1] + flags := pflag.NewFlagSet(mode, pflag.ExitOnError) + socket := flags.String("socket", "", "Unix socket to serve on, or to check in healthcheck mode") + root := flags.String("root", nodepath.BasePath, "node mode: the only directory tree local snapshot files may be read from or written to") + timeout := flags.Duration("timeout", 5*time.Second, "healthcheck mode: how long to wait for the plugin to report that it is serving") + _ = flags.Parse(os.Args[2:]) + + ctx := context.Background() + serverboot.InitLogger() + if *socket == "" { + serverboot.Fatal(ctx, "Missing --socket", fmt.Errorf("--socket is required")) + } + + // healthcheck is what the sidecar's startup probe runs: the image has no + // shell or grpc_health_probe, so the binary checks itself. + if mode == "healthcheck" { + if err := healthcheck(ctx, *socket, *timeout); err != nil { + serverboot.Fatal(ctx, "Snapshot plugin is not serving", err) + } + return + } + + tp, err := serverboot.InitTracing(ctx, serverboot.TracingOptions{ + ServiceName: "snapshot-plugin-" + mode, + Sampling: serverboot.ResolveTraceSampling(ctx, serverboot.ParentRatioSampling(serverboot.ControlPlaneTraceRatio)), + }) + if err != nil { + serverboot.Fatal(ctx, "Failed to initialize tracing", err) + } + defer serverboot.ShutdownProvider("TracerProvider", tp.Shutdown) + + srv := grpc.NewServer(grpc.StatsHandler(otelgrpc.NewServerHandler())) + switch mode { + case "node": + objects, err := objectstorage.NewFromEnv(ctx) + if err != nil { + serverboot.Fatal(ctx, "Failed to set up the object storage backend", err) + } + plugin, err := objectstoreplugin.NewNodePlugin(objects, *root) + if err != nil { + serverboot.Fatal(ctx, "Invalid --root", err) + } + objectstoresnapshotv1.RegisterNodeProviderServer(srv, plugin) + case "control": + store, err := newObjectStore(ctx) + if err != nil { + serverboot.Fatal(ctx, "Failed to set up the object storage backend", err) + } + objectstoresnapshotv1.RegisterControlProviderServer(srv, objectstoreplugin.NewControlPlugin(store)) + default: + fmt.Fprintln(os.Stderr, usage) + os.Exit(2) + } + healthpb.RegisterHealthServer(srv, health.NewServer()) + + lis, err := objectstoreplugin.Listen(*socket) + if err != nil { + serverboot.Fatal(ctx, "Failed to listen", err) + } + + // Finish in-flight transfers on SIGTERM. The pod's grace period bounds + // how long that may take. + sigCtx, stop := signal.NotifyContext(ctx, syscall.SIGTERM, os.Interrupt) + defer stop() + go func() { + <-sigCtx.Done() + slog.InfoContext(ctx, "Draining snapshot plugin") + srv.GracefulStop() + }() + + slog.InfoContext(ctx, "Serving snapshot plugin", slog.String("mode", mode), slog.String("socket", *socket)) + if err := srv.Serve(lis); err != nil { + serverboot.Fatal(ctx, "Serve failed", err) + } +} + +// newObjectStore builds the client the control plugin manages snapshot +// prefixes with, on the same backend objectstorage.NewFromEnv selects for the +// node plugin. It builds its own GCS client because objectstorage's carries a +// retry policy scoped to that package. +func newObjectStore(ctx context.Context) (objectstore.Store, error) { + if objectstorage.UsesS3() { + client, err := objectstorage.NewS3ClientFromEnv(ctx) + if err != nil { + return nil, err + } + return objectstore.NewS3(client), nil + } + // GCS is currently the default, TODO: we assume workload identity / ADC + client, err := storage.NewClient(ctx) + if err != nil { + return nil, fmt.Errorf("creating GCS client: %w", err) + } + return objectstore.NewGCS(client), nil +} + +// healthcheck dials the plugin at socket and waits up to timeout for it to +// report that it is serving. +func healthcheck(ctx context.Context, socket string, timeout time.Duration) error { + conn, err := objectstoreplugin.Dial(socket) + if err != nil { + return err + } + defer conn.Close() + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + return objectstoreplugin.WaitReady(ctx, conn) +} diff --git a/docs/architecture.md b/docs/architecture.md index effb2c23b0..560a4cba7f 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -327,7 +327,7 @@ The node-level subsystem manages the physical execution of sandboxes and the mov * **Lifecycle Management**: The `ateom` process invokes the sandbox runtime to checkpoint or restore processes within the physical pod boundaries — `runsc` for gVisor, or the Kata + Cloud Hypervisor stack for micro-VMs. (Note: the gVisor backend currently requires a `runsc` version with the `--allow-connected-on-save` flag to work around a bug in networking resumption during checkpointing.) - * **Storage Mover**: The `atelet` streams snapshots to and from GCS/S3, ensuring process state is persistent and portable across the cluster. + * **Storage Mover**: External snapshots move through a *snapshot plugin*, a sidecar that serves the `objectstoresnapshot.v1` gRPC API over a Unix socket in its pod. `atelet`'s sidecar (`NodeProvider`) uploads and fetches a snapshot's files; `ate-api-server`'s sidecar (`ControlProvider`) copies and deletes whole snapshots. Neither main process talks to object storage for snapshots. The in-tree plugin, `cmd/snapshot-plugin`, backs the API with GCS or S3; `ATE_STORAGE_BACKEND` on the sidecar (unset or `gcs`, or `s3` with the `AWS_*` variables) selects which, see `manifests/ate-install/atelet.yaml`, `ate-api-server.yaml`, and the kind overlay for an S3 example. `atelet` keeps a storage client of its own only for sandbox asset downloads. ### Sandbox Classes diff --git a/internal/objectstore/objectstore.go b/internal/objectstore/objectstore.go index 671e580647..59675e157b 100644 --- a/internal/objectstore/objectstore.go +++ b/internal/objectstore/objectstore.go @@ -41,6 +41,9 @@ import ( // their round trips rather than about throughput. const prefixConcurrency = 8 +// ErrEmptySource reports that CopyPrefix found no objects to copy. +var ErrEmptySource = errors.New("no objects to copy") + // Store is the object-storage surface the control plane needs. Every method // addresses objects by name only. type Store interface { @@ -120,7 +123,7 @@ func CopyPrefix(ctx context.Context, s Store, src, dst resources.StoragePrefix) // leave the destination naming an external snapshot that cannot be // restored from. if len(objects) == 0 { - return fmt.Errorf("external snapshot %s has no objects to copy", src) + return fmt.Errorf("external snapshot %s: %w", src, ErrEmptySource) } group, ctx := errgroup.WithContext(ctx) group.SetLimit(prefixConcurrency) diff --git a/internal/objectstoreplugin/control.go b/internal/objectstoreplugin/control.go new file mode 100644 index 0000000000..647226fd2e --- /dev/null +++ b/internal/objectstoreplugin/control.go @@ -0,0 +1,65 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package objectstoreplugin + +import ( + "context" + + "github.com/agent-substrate/substrate/internal/objectstore" + "github.com/agent-substrate/substrate/internal/resources" + objectstoresnapshotv1 "github.com/agent-substrate/substrate/pkg/proto/objectstoresnapshotpb/v1" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// ControlPlugin serves ControlProvider on an object store. +type ControlPlugin struct { + objectstoresnapshotv1.UnimplementedControlProviderServer + + store objectstore.Store +} + +// NewControlPlugin returns a ControlPlugin backed by store. +func NewControlPlugin(store objectstore.Store) *ControlPlugin { + return &ControlPlugin{store: store} +} + +// CleanupSnapshot deletes every object under the given prefix. +func (p *ControlPlugin) CleanupSnapshot(ctx context.Context, req *objectstoresnapshotv1.CleanupSnapshotRequest) (*objectstoresnapshotv1.CleanupSnapshotResponse, error) { + prefix, err := resources.ParseStoragePrefix(req.GetSnapshotUri()) + if err != nil { + return nil, status.Error(codes.InvalidArgument, err.Error()) + } + if err := objectstore.DeletePrefix(ctx, p.store, prefix); err != nil { + return nil, toStatus(err) + } + return &objectstoresnapshotv1.CleanupSnapshotResponse{}, nil +} + +// CopySnapshot copies every object of src_uri to dst_uri. +func (p *ControlPlugin) CopySnapshot(ctx context.Context, req *objectstoresnapshotv1.CopySnapshotRequest) (*objectstoresnapshotv1.CopySnapshotResponse, error) { + src, err := resources.ParseStoragePrefix(req.GetSrcUri()) + if err != nil { + return nil, status.Error(codes.InvalidArgument, err.Error()) + } + dst, err := resources.ParseStoragePrefix(req.GetDstUri()) + if err != nil { + return nil, status.Error(codes.InvalidArgument, err.Error()) + } + if err := objectstore.CopyPrefix(ctx, p.store, src, dst); err != nil { + return nil, toStatus(err) + } + return &objectstoresnapshotv1.CopySnapshotResponse{}, nil +} diff --git a/internal/objectstoreplugin/node.go b/internal/objectstoreplugin/node.go new file mode 100644 index 0000000000..8dd9759753 --- /dev/null +++ b/internal/objectstoreplugin/node.go @@ -0,0 +1,216 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package objectstoreplugin implements the snapshot plugin API on GCS or S3 +// object storage, using the plugin's own credentials. +// +// Snapshot files are stored as zstd-compressed objects named .zstd +// under the snapshot URI, except the manifest, which is stored as is. This is +// the layout atelet has always written, so snapshots taken before the plugin +// existed stay readable. +package objectstoreplugin + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + + "github.com/agent-substrate/substrate/internal/resources" + "github.com/agent-substrate/substrate/pkg/objectstorage" + objectstoresnapshotv1 "github.com/agent-substrate/substrate/pkg/proto/objectstoresnapshotpb/v1" + "golang.org/x/sync/errgroup" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// This plugin stores each file as .zstd, except manifestFile, which it +// stores as-is so that snapshots written before atelet used this plugin stay +// readable. +const manifestFile = "manifest.json" + +// NodePlugin serves NodeProvider on an object storage client. +type NodePlugin struct { + objectstoresnapshotv1.UnimplementedNodeProviderServer + + gcsClient objectstorage.ObjectStorage + // root confines every local path a caller names. + root string +} + +// NewNodePlugin returns a NodePlugin that reads and writes local files only +// below root. +func NewNodePlugin(gcsClient objectstorage.ObjectStorage, root string) (*NodePlugin, error) { + if !filepath.IsAbs(root) { + return nil, fmt.Errorf("root %q is not an absolute path", root) + } + return &NodePlugin{gcsClient: gcsClient, root: filepath.Clean(root)}, nil +} + +// FetchSnapshot downloads the requested snapshot files into write_path. +func (p *NodePlugin) FetchSnapshot(ctx context.Context, req *objectstoresnapshotv1.FetchSnapshotRequest) (*objectstoresnapshotv1.FetchSnapshotResponse, error) { + uri, dstDir, err := p.validate(req.GetSnapshotUri(), req.GetWritePath(), req.GetFiles()) + if err != nil { + return nil, err + } + root, err := p.openDir(dstDir) + if err != nil { + return nil, toStatus(fmt.Errorf("while opening restore directory: %w", err)) + } + defer root.Close() + + g, gCtx := errgroup.WithContext(ctx) + for _, fileName := range req.GetFiles() { + fileName := fileName + g.Go(func() error { + if fileName == manifestFile { + return p.fetchManifest(gCtx, uri, root) + } + objectURI, err := uri.ObjectURI(fileName + ".zstd") + if err != nil { + return fmt.Errorf("while addressing %s in GCS: %w", fileName, err) + } + local, err := root.OpenFile(fileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600) + if err != nil { + return fmt.Errorf("while opening %s in restore directory: %w", fileName, err) + } + fetchErr := objectstorage.FetchFileFromGCSWithZstd(gCtx, p.gcsClient, objectURI, local) + closeErr := local.Close() + if err := errors.Join(fetchErr, closeErr); err != nil { + return fmt.Errorf("while downloading %s from GCS: %w", fileName, err) + } + return nil + }) + } + if err := g.Wait(); err != nil { + return nil, toStatus(err) + } + return &objectstoresnapshotv1.FetchSnapshotResponse{}, nil +} + +// UploadSnapshot uploads the requested files from local_path into the +// snapshot. +func (p *NodePlugin) UploadSnapshot(ctx context.Context, req *objectstoresnapshotv1.UploadSnapshotRequest) (*objectstoresnapshotv1.UploadSnapshotResponse, error) { + uri, srcDir, err := p.validate(req.GetSnapshotUri(), req.GetLocalPath(), req.GetFiles()) + if err != nil { + return nil, err + } + root, err := p.openDir(srcDir) + if err != nil { + return nil, toStatus(fmt.Errorf("while opening snapshot directory: %w", err)) + } + defer root.Close() + + g, gCtx := errgroup.WithContext(ctx) + for _, fileName := range req.GetFiles() { + g.Go(func() error { + local, err := root.Open(fileName) + if err != nil { + return fmt.Errorf("while opening %s in snapshot directory: %w", fileName, err) + } + defer local.Close() + info, err := local.Stat() + if err != nil { + return fmt.Errorf("while inspecting %s in snapshot directory: %w", fileName, err) + } + if !info.Mode().IsRegular() { + return fmt.Errorf("snapshot file %s is not a regular file", fileName) + } + if fileName == manifestFile { + return p.uploadManifest(gCtx, uri, local) + } + + objectURI, err := uri.ObjectURI(fileName + ".zstd") + if err != nil { + return fmt.Errorf("while addressing %s in GCS: %w", fileName, err) + } + if err := objectstorage.SendFileToGCSWithZstd(gCtx, p.gcsClient, objectURI, local); err != nil { + return fmt.Errorf("while uploading %s to GCS: %w", fileName, err) + } + return nil + }) + } + if err := g.Wait(); err != nil { + return nil, toStatus(err) + } + return &objectstoresnapshotv1.UploadSnapshotResponse{}, nil +} + +// fetchManifest downloads the uncompressed manifest into root. +func (p *NodePlugin) fetchManifest(ctx context.Context, uri resources.SnapshotURI, root *os.Root) error { + manifestURI, err := uri.ObjectURI(manifestFile) + if err != nil { + return err + } + manifest, err := objectstorage.FetchFromGCS(ctx, p.gcsClient, manifestURI) + if err != nil { + return fmt.Errorf("while fetching snapshot manifest: %w", err) + } + if err := root.WriteFile(manifestFile, manifest, 0o600); err != nil { + return fmt.Errorf("while writing %s in restore directory: %w", manifestFile, err) + } + return nil +} + +// uploadManifest uploads the manifest from local uncompressed. +func (p *NodePlugin) uploadManifest(ctx context.Context, uri resources.SnapshotURI, local *os.File) error { + manifest, err := io.ReadAll(local) + if err != nil { + return fmt.Errorf("while reading %s in snapshot directory: %w", manifestFile, err) + } + manifestURI, err := uri.ObjectURI(manifestFile) + if err != nil { + return fmt.Errorf("while addressing snapshot manifest in GCS: %w", err) + } + if err := objectstorage.SendBytesToGCS(ctx, p.gcsClient, manifestURI, manifest); err != nil { + return fmt.Errorf("while uploading snapshot manifest: %w", err) + } + return nil +} + +// validate checks a request's snapshot URI, local directory and file names, +// returning the parsed URI and the local directory relative to p.root. +func (p *NodePlugin) validate(snapshotURI, dir string, files []string) (resources.SnapshotURI, string, error) { + uri, err := resources.ParseSnapshotURI(snapshotURI) + if err != nil { + return resources.SnapshotURI{}, "", status.Error(codes.InvalidArgument, err.Error()) + } + if !filepath.IsAbs(dir) { + return resources.SnapshotURI{}, "", status.Errorf(codes.InvalidArgument, "local path %q is not absolute", dir) + } + rel, err := filepath.Rel(p.root, dir) + if err != nil || !filepath.IsLocal(rel) { + return resources.SnapshotURI{}, "", status.Errorf(codes.InvalidArgument, "local path %q is outside %q", dir, p.root) + } + if len(files) == 0 { + return resources.SnapshotURI{}, "", status.Error(codes.InvalidArgument, "no files requested") + } + if err := resources.ValidateSnapshotFileNames(files); err != nil { + return resources.SnapshotURI{}, "", status.Error(codes.InvalidArgument, err.Error()) + } + return uri, rel, nil +} + +// openDir opens rel, a directory below p.root, through an os.Root at p.root, +// so a symlink anywhere on the path cannot lead outside p.root. +func (p *NodePlugin) openDir(rel string) (*os.Root, error) { + base, err := os.OpenRoot(p.root) + if err != nil { + return nil, err + } + defer base.Close() + return base.OpenRoot(rel) +} diff --git a/internal/objectstoreplugin/objectstoreplugintest/objectstoreplugintest.go b/internal/objectstoreplugin/objectstoreplugintest/objectstoreplugintest.go new file mode 100644 index 0000000000..8b672da707 --- /dev/null +++ b/internal/objectstoreplugin/objectstoreplugintest/objectstoreplugintest.go @@ -0,0 +1,61 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package objectstoreplugintest provides snapshot plugin clients that call the +// object-store plugins directly, without a socket, for tests that drive atelet +// or ate-api-server through the real plugin. +package objectstoreplugintest + +import ( + "context" + + "github.com/agent-substrate/substrate/internal/objectstore" + "github.com/agent-substrate/substrate/internal/objectstoreplugin" + objectstoresnapshotv1 "github.com/agent-substrate/substrate/pkg/proto/objectstoresnapshotpb/v1" + "google.golang.org/grpc" +) + +// NodeClient returns a client that calls p directly. +func NodeClient(p *objectstoreplugin.NodePlugin) objectstoresnapshotv1.NodeProviderClient { + return nodeClient{p} +} + +type nodeClient struct { + p *objectstoreplugin.NodePlugin +} + +func (c nodeClient) FetchSnapshot(ctx context.Context, in *objectstoresnapshotv1.FetchSnapshotRequest, _ ...grpc.CallOption) (*objectstoresnapshotv1.FetchSnapshotResponse, error) { + return c.p.FetchSnapshot(ctx, in) +} + +func (c nodeClient) UploadSnapshot(ctx context.Context, in *objectstoresnapshotv1.UploadSnapshotRequest, _ ...grpc.CallOption) (*objectstoresnapshotv1.UploadSnapshotResponse, error) { + return c.p.UploadSnapshot(ctx, in) +} + +// ControlClient returns a client that calls a ControlPlugin on store directly. +func ControlClient(store objectstore.Store) objectstoresnapshotv1.ControlProviderClient { + return controlClient{objectstoreplugin.NewControlPlugin(store)} +} + +type controlClient struct { + p *objectstoreplugin.ControlPlugin +} + +func (c controlClient) CleanupSnapshot(ctx context.Context, in *objectstoresnapshotv1.CleanupSnapshotRequest, _ ...grpc.CallOption) (*objectstoresnapshotv1.CleanupSnapshotResponse, error) { + return c.p.CleanupSnapshot(ctx, in) +} + +func (c controlClient) CopySnapshot(ctx context.Context, in *objectstoresnapshotv1.CopySnapshotRequest, _ ...grpc.CallOption) (*objectstoresnapshotv1.CopySnapshotResponse, error) { + return c.p.CopySnapshot(ctx, in) +} diff --git a/internal/objectstoreplugin/plugin_test.go b/internal/objectstoreplugin/plugin_test.go new file mode 100644 index 0000000000..d3e6ac5a43 --- /dev/null +++ b/internal/objectstoreplugin/plugin_test.go @@ -0,0 +1,459 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package objectstoreplugin + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "testing" + "time" + + "github.com/agent-substrate/substrate/pkg/objectstorage" + objectstoresnapshotv1 "github.com/agent-substrate/substrate/pkg/proto/objectstoresnapshotpb/v1" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/health" + healthpb "google.golang.org/grpc/health/grpc_health_v1" + "google.golang.org/grpc/status" +) + +const ( + testBucket = "bucket" + testPrefix = "root/atespaces/team-a/actors/uid1/snapshots/snap1" + testURI = "gs://" + testBucket + "/" + testPrefix +) + +// memObjects is an in-memory backend implementing both objectstorage.ObjectStorage +// and objectstore.Store, keyed by "bucket/object". +type memObjects struct { + mu sync.Mutex + m map[string][]byte +} + +func newMemObjects() *memObjects { return &memObjects{m: map[string][]byte{}} } + +func (s *memObjects) PutObject(_ context.Context, bucket, object string, r io.Reader) error { + b, err := io.ReadAll(r) + if err != nil { + return err + } + s.mu.Lock() + defer s.mu.Unlock() + s.m[bucket+"/"+object] = b + return nil +} + +func (s *memObjects) GetObject(_ context.Context, bucket, object string) (io.ReadCloser, error) { + s.mu.Lock() + defer s.mu.Unlock() + b, ok := s.m[bucket+"/"+object] + if !ok { + return nil, fmt.Errorf("%w: %s/%s", objectstorage.ErrObjectNotFound, bucket, object) + } + return io.NopCloser(bytes.NewReader(b)), nil +} + +func (s *memObjects) List(_ context.Context, bucket, prefix string) ([]string, error) { + s.mu.Lock() + defer s.mu.Unlock() + var out []string + for k := range s.m { + if name, ok := strings.CutPrefix(k, bucket+"/"); ok && strings.HasPrefix(name, prefix) { + out = append(out, name) + } + } + return out, nil +} + +func (s *memObjects) Delete(_ context.Context, bucket, object string) error { + s.mu.Lock() + defer s.mu.Unlock() + delete(s.m, bucket+"/"+object) + return nil +} + +func (s *memObjects) Copy(_ context.Context, srcBucket, srcObject, dstBucket, dstObject string) error { + s.mu.Lock() + defer s.mu.Unlock() + s.m[dstBucket+"/"+dstObject] = s.m[srcBucket+"/"+srcObject] + return nil +} + +func (s *memObjects) keys() []string { + s.mu.Lock() + defer s.mu.Unlock() + var out []string + for k := range s.m { + out = append(out, k) + } + sort.Strings(out) + return out +} + +// serve starts the plugins on a real Unix socket, as the sidecar does, and +// returns a connection to it. +func serve(t *testing.T, backend *memObjects, root string) *grpc.ClientConn { + t.Helper() + // Unix socket paths are length-limited; t.TempDir can exceed that on macOS. + sockDir, err := os.MkdirTemp("", "snapplug") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { os.RemoveAll(sockDir) }) + lis, err := Listen(filepath.Join(sockDir, "plugin.sock")) + if err != nil { + t.Fatal(err) + } + node, err := NewNodePlugin(backend, root) + if err != nil { + t.Fatal(err) + } + srv := grpc.NewServer() + objectstoresnapshotv1.RegisterNodeProviderServer(srv, node) + objectstoresnapshotv1.RegisterControlProviderServer(srv, NewControlPlugin(backend)) + healthpb.RegisterHealthServer(srv, health.NewServer()) + go srv.Serve(lis) + t.Cleanup(srv.Stop) + + conn, err := Dial(filepath.Join(sockDir, "plugin.sock")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { conn.Close() }) + return conn +} + +func TestWaitReady(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if err := WaitReady(ctx, serve(t, newMemObjects(), t.TempDir())); err != nil { + t.Errorf("WaitReady against a serving plugin = %v", err) + } + + // A socket nobody listens on must fail when ctx ends, not hang: Dial + // makes every call wait for the plugin. + conn, err := Dial(filepath.Join(t.TempDir(), "missing.sock")) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + short, cancelShort := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancelShort() + err = WaitReady(short, conn) + if status.Code(err) != codes.DeadlineExceeded { + t.Errorf("WaitReady against a missing socket = %v, want %s", err, codes.DeadlineExceeded) + } +} + +func TestUploadFetchRoundTrip(t *testing.T) { + ctx := context.Background() + backend := newMemObjects() + root := t.TempDir() + client := objectstoresnapshotv1.NewNodeProviderClient(serve(t, backend, root)) + + src := filepath.Join(root, "src") + dst := filepath.Join(root, "dst") + for _, d := range []string{src, dst} { + if err := os.Mkdir(d, 0o755); err != nil { + t.Fatal(err) + } + } + files := map[string][]byte{ + "memory.img": bytes.Repeat([]byte("abcdefgh"), 1<<16), + "state.bin": []byte("vm state"), + manifestFile: []byte(`{"snapshotFiles":["memory.img","state.bin"]}`), + } + for name, content := range files { + if err := os.WriteFile(filepath.Join(src, name), content, 0o644); err != nil { + t.Fatal(err) + } + } + + if _, err := client.UploadSnapshot(ctx, &objectstoresnapshotv1.UploadSnapshotRequest{ + SnapshotUri: testURI, LocalPath: src, Files: []string{"memory.img", "state.bin"}, + }); err != nil { + t.Fatalf("UploadSnapshot(data) = %v", err) + } + if _, err := client.UploadSnapshot(ctx, &objectstoresnapshotv1.UploadSnapshotRequest{ + SnapshotUri: testURI, LocalPath: src, Files: []string{manifestFile}, + }); err != nil { + t.Fatalf("UploadSnapshot(manifest) = %v", err) + } + + // The stored layout is the one atelet has always written. + want := []string{ + testBucket + "/" + testPrefix + "/manifest.json", + testBucket + "/" + testPrefix + "/memory.img.zstd", + testBucket + "/" + testPrefix + "/state.bin.zstd", + } + if got := backend.keys(); !equal(got, want) { + t.Fatalf("stored objects = %v, want %v", got, want) + } + if got := backend.m[want[0]]; !bytes.Equal(got, files[manifestFile]) { + t.Errorf("stored manifest = %q, want it uncompressed", got) + } + + if _, err := client.FetchSnapshot(ctx, &objectstoresnapshotv1.FetchSnapshotRequest{ + SnapshotUri: testURI, WritePath: dst, Files: []string{manifestFile, "memory.img", "state.bin"}, + }); err != nil { + t.Fatalf("FetchSnapshot = %v", err) + } + for name, content := range files { + got, err := os.ReadFile(filepath.Join(dst, name)) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, content) { + t.Errorf("fetched %s differs from the uploaded file", name) + } + } +} + +func TestFetchMissingIsNotFound(t *testing.T) { + root := t.TempDir() + client := objectstoresnapshotv1.NewNodeProviderClient(serve(t, newMemObjects(), root)) + for _, file := range []string{manifestFile, "memory.img"} { + _, err := client.FetchSnapshot(context.Background(), &objectstoresnapshotv1.FetchSnapshotRequest{ + SnapshotUri: testURI, WritePath: root, Files: []string{file}, + }) + if status.Code(err) != codes.NotFound { + t.Errorf("FetchSnapshot(%s) = %v, want NotFound", file, err) + } + } +} + +func TestNodeRequestValidation(t *testing.T) { + root := t.TempDir() + client := objectstoresnapshotv1.NewNodeProviderClient(serve(t, newMemObjects(), root)) + for _, tc := range []struct { + name string + uri string + dir string + files []string + }{ + {"bad uri", "gs://bucket/not-a-snapshot", root, []string{"a"}}, + {"relative dir", testURI, "relative/dir", []string{"a"}}, + {"dir outside root", testURI, filepath.Dir(root), []string{"a"}}, + {"dir escapes root", testURI, root + "/../elsewhere", []string{"a"}}, + {"sibling sharing root's prefix", testURI, root + "-other", []string{"a"}}, + {"no files", testURI, root, nil}, + {"empty file", testURI, root, []string{""}}, + {"nested file", testURI, root, []string{"sub/a"}}, + {"parent file", testURI, root, []string{".."}}, + {"duplicate file", testURI, root, []string{"a", "b", "a"}}, + } { + t.Run(tc.name, func(t *testing.T) { + _, fetchErr := client.FetchSnapshot(context.Background(), &objectstoresnapshotv1.FetchSnapshotRequest{ + SnapshotUri: tc.uri, WritePath: tc.dir, Files: tc.files, + }) + _, uploadErr := client.UploadSnapshot(context.Background(), &objectstoresnapshotv1.UploadSnapshotRequest{ + SnapshotUri: tc.uri, LocalPath: tc.dir, Files: tc.files, + }) + for _, err := range []error{fetchErr, uploadErr} { + if status.Code(err) != codes.InvalidArgument { + t.Errorf("got %v, want InvalidArgument", err) + } + } + }) + } +} + +func TestUploadRejectsSymlinkOutsideDir(t *testing.T) { + for _, file := range []string{manifestFile, "checkpoint.img"} { + t.Run(file, func(t *testing.T) { + parent := t.TempDir() + backend := newMemObjects() + client := objectstoresnapshotv1.NewNodeProviderClient(serve(t, backend, parent)) + checkpointDir := filepath.Join(parent, "checkpoint-state") + if err := os.Mkdir(checkpointDir, 0o700); err != nil { + t.Fatal(err) + } + outside := filepath.Join(parent, "outside") + if err := os.WriteFile(outside, []byte("secret"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outside, filepath.Join(checkpointDir, file)); err != nil { + t.Fatal(err) + } + + _, err := client.UploadSnapshot(context.Background(), &objectstoresnapshotv1.UploadSnapshotRequest{ + SnapshotUri: testURI, LocalPath: checkpointDir, Files: []string{file}, + }) + if err == nil { + t.Fatal("UploadSnapshot followed a symlink outside the snapshot directory") + } + if got := backend.keys(); len(got) != 0 { + t.Fatalf("uploaded objects = %v, want none", got) + } + }) + } +} + +func TestFetchRejectsSymlinkOutsideDir(t *testing.T) { + for _, file := range []string{manifestFile, "checkpoint.img"} { + t.Run(file, func(t *testing.T) { + ctx := context.Background() + parent := t.TempDir() + backend := newMemObjects() + client := objectstoresnapshotv1.NewNodeProviderClient(serve(t, backend, parent)) + src := filepath.Join(parent, "src") + restoreDir := filepath.Join(parent, "restore-state") + for _, d := range []string{src, restoreDir} { + if err := os.Mkdir(d, 0o700); err != nil { + t.Fatal(err) + } + } + if err := os.WriteFile(filepath.Join(src, file), []byte("replacement"), 0o600); err != nil { + t.Fatal(err) + } + if _, err := client.UploadSnapshot(ctx, &objectstoresnapshotv1.UploadSnapshotRequest{ + SnapshotUri: testURI, LocalPath: src, Files: []string{file}, + }); err != nil { + t.Fatal(err) + } + outside := filepath.Join(parent, "outside") + if err := os.WriteFile(outside, []byte("keep me"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outside, filepath.Join(restoreDir, file)); err != nil { + t.Fatal(err) + } + + _, err := client.FetchSnapshot(ctx, &objectstoresnapshotv1.FetchSnapshotRequest{ + SnapshotUri: testURI, WritePath: restoreDir, Files: []string{file}, + }) + if err == nil { + t.Fatal("FetchSnapshot followed a symlink outside the restore directory") + } + if got, err := os.ReadFile(outside); err != nil || string(got) != "keep me" { + t.Fatalf("outside file = %q, %v; want unchanged", got, err) + } + }) + } +} + +// A directory below --root that is a symlink to elsewhere passes the lexical +// check, so only opening it through os.Root keeps the transfer inside --root. +func TestRejectsDirSymlinkOutsideRoot(t *testing.T) { + ctx := context.Background() + parent := t.TempDir() + root := filepath.Join(parent, "root") + outside := filepath.Join(parent, "outside") + for _, d := range []string{root, outside} { + if err := os.Mkdir(d, 0o700); err != nil { + t.Fatal(err) + } + } + if err := os.WriteFile(filepath.Join(outside, "checkpoint.img"), []byte("secret"), 0o600); err != nil { + t.Fatal(err) + } + link := filepath.Join(root, "link") + if err := os.Symlink(outside, link); err != nil { + t.Fatal(err) + } + backend := newMemObjects() + client := objectstoresnapshotv1.NewNodeProviderClient(serve(t, backend, root)) + + if _, err := client.UploadSnapshot(ctx, &objectstoresnapshotv1.UploadSnapshotRequest{ + SnapshotUri: testURI, LocalPath: link, Files: []string{"checkpoint.img"}, + }); err == nil { + t.Error("UploadSnapshot read through a directory symlink outside --root") + } + if got := backend.keys(); len(got) != 0 { + t.Errorf("uploaded objects = %v, want none", got) + } + + backend.m[testBucket+"/"+testPrefix+"/"+manifestFile] = []byte("m") + if _, err := client.FetchSnapshot(ctx, &objectstoresnapshotv1.FetchSnapshotRequest{ + SnapshotUri: testURI, WritePath: link, Files: []string{manifestFile}, + }); err == nil { + t.Error("FetchSnapshot succeeded through a directory symlink outside --root") + } + if _, err := os.Stat(filepath.Join(outside, manifestFile)); !os.IsNotExist(err) { + t.Errorf("FetchSnapshot created a file outside --root: %v", err) + } +} + +func TestCleanupSnapshot(t *testing.T) { + ctx := context.Background() + backend := newMemObjects() + client := objectstoresnapshotv1.NewControlProviderClient(serve(t, backend, t.TempDir())) + // uid10 shares a string prefix with uid1, so it survives only if cleanup + // stops at a path-segment boundary. + keep := testBucket + "/root/atespaces/team-a/actors/uid10/snapshots/snap1/manifest.json" + backend.m[testBucket+"/"+testPrefix+"/manifest.json"] = []byte("m") + backend.m[testBucket+"/"+testPrefix+"/memory.img.zstd"] = []byte("d") + backend.m[keep] = []byte("other actor") + + // An owner prefix collects every snapshot below it, and nothing beside it. + owner := "gs://" + testBucket + "/root/atespaces/team-a/actors/uid1" + if _, err := client.CleanupSnapshot(ctx, &objectstoresnapshotv1.CleanupSnapshotRequest{SnapshotUri: owner}); err != nil { + t.Fatalf("CleanupSnapshot = %v", err) + } + if got := backend.keys(); !equal(got, []string{keep}) { + t.Errorf("objects left = %v, want only %s", got, keep) + } + // Cleaning up again is a no-op. + if _, err := client.CleanupSnapshot(ctx, &objectstoresnapshotv1.CleanupSnapshotRequest{SnapshotUri: owner}); err != nil { + t.Errorf("repeated CleanupSnapshot = %v", err) + } + // A bare bucket is never a valid prefix. + for _, uri := range []string{"gs://" + testBucket, "gs://" + testBucket + "/", "not a uri?x=1"} { + _, err := client.CleanupSnapshot(ctx, &objectstoresnapshotv1.CleanupSnapshotRequest{SnapshotUri: uri}) + if status.Code(err) != codes.InvalidArgument { + t.Errorf("CleanupSnapshot(%q) = %v, want InvalidArgument", uri, err) + } + } +} + +func TestCopySnapshot(t *testing.T) { + ctx := context.Background() + backend := newMemObjects() + client := objectstoresnapshotv1.NewControlProviderClient(serve(t, backend, t.TempDir())) + backend.m[testBucket+"/"+testPrefix+"/manifest.json"] = []byte("m") + tag := "gs://" + testBucket + "/root/atespaces/team-a/tags/tag1" + + if _, err := client.CopySnapshot(ctx, &objectstoresnapshotv1.CopySnapshotRequest{SrcUri: testURI, DstUri: tag}); err != nil { + t.Fatalf("CopySnapshot = %v", err) + } + if got := backend.m[testBucket+"/root/atespaces/team-a/tags/tag1/manifest.json"]; string(got) != "m" { + t.Errorf("copied manifest = %q, want %q", got, "m") + } + + empty := "gs://" + testBucket + "/root/atespaces/team-a/actors/uid9/snapshots/none" + _, err := client.CopySnapshot(ctx, &objectstoresnapshotv1.CopySnapshotRequest{SrcUri: empty, DstUri: tag}) + if status.Code(err) != codes.FailedPrecondition { + t.Errorf("CopySnapshot(empty source) = %v, want FailedPrecondition", err) + } +} + +func equal(a, b []string) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} diff --git a/internal/objectstoreplugin/socket.go b/internal/objectstoreplugin/socket.go new file mode 100644 index 0000000000..14138dc8de --- /dev/null +++ b/internal/objectstoreplugin/socket.go @@ -0,0 +1,83 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package objectstoreplugin + +import ( + "context" + "errors" + "fmt" + "io/fs" + "net" + "os" + "path/filepath" + + "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + healthpb "google.golang.org/grpc/health/grpc_health_v1" +) + +// Listen removes any stale socket at path and listens on a fresh one that +// only the socket's owner can connect to. +func Listen(path string) (net.Listener, error) { + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return nil, fmt.Errorf("while creating socket directory: %w", err) + } + if err := os.Remove(path); err != nil && !errors.Is(err, fs.ErrNotExist) { + return nil, fmt.Errorf("while removing stale socket %s: %w", path, err) + } + lis, err := net.Listen("unix", path) + if err != nil { + return nil, fmt.Errorf("while listening on %s: %w", path, err) + } + if err := os.Chmod(path, 0o600); err != nil { + lis.Close() + return nil, fmt.Errorf("while restricting socket %s: %w", path, err) + } + return lis, nil +} + +// Dial returns a long-lived client connection to a plugin socket. Calls wait +// for the plugin to come up rather than failing while it starts, so a caller +// that must not hang on a plugin that never comes up checks WaitReady first. +// +// The connection is unauthenticated: the socket is reachable only from +// inside the pod, and only by its owner. +func Dial(path string) (*grpc.ClientConn, error) { + conn, err := grpc.NewClient("unix://"+path, + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithStatsHandler(otelgrpc.NewClientHandler()), + grpc.WithDefaultCallOptions(grpc.WaitForReady(true)), + ) + if err != nil { + return nil, fmt.Errorf("while dialing snapshot plugin at %s: %w", path, err) + } + return conn, nil +} + +// WaitReady blocks until the plugin behind conn reports that it is serving, +// or ctx ends. Bounding ctx is what turns a plugin that never comes up (a +// wrong socket path, a crash loop) into an error at startup rather than a +// transfer that hangs. +func WaitReady(ctx context.Context, conn *grpc.ClientConn) error { + resp, err := healthpb.NewHealthClient(conn).Check(ctx, &healthpb.HealthCheckRequest{}) + if err != nil { + return fmt.Errorf("while waiting for the snapshot plugin at %s: %w", conn.Target(), err) + } + if resp.GetStatus() != healthpb.HealthCheckResponse_SERVING { + return fmt.Errorf("snapshot plugin at %s is %s", conn.Target(), resp.GetStatus()) + } + return nil +} diff --git a/internal/objectstoreplugin/status.go b/internal/objectstoreplugin/status.go new file mode 100644 index 0000000000..aa9008eb3b --- /dev/null +++ b/internal/objectstoreplugin/status.go @@ -0,0 +1,41 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package objectstoreplugin + +import ( + "context" + "errors" + + "github.com/agent-substrate/substrate/internal/objectstore" + "github.com/agent-substrate/substrate/pkg/objectstorage" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// toStatus maps a plugin error to a gRPC status, keeping NotFound +// distinguishable so callers can probe for a snapshot. +func toStatus(err error) error { + switch { + case errors.Is(err, objectstorage.ErrObjectNotFound): + return status.Error(codes.NotFound, err.Error()) + case errors.Is(err, objectstore.ErrEmptySource): + return status.Error(codes.FailedPrecondition, err.Error()) + case errors.Is(err, context.Canceled): + return status.Error(codes.Canceled, err.Error()) + case errors.Is(err, context.DeadlineExceeded): + return status.Error(codes.DeadlineExceeded, err.Error()) + } + return status.Error(codes.Internal, err.Error()) +} diff --git a/internal/resources/snapshot.go b/internal/resources/snapshot.go index ab4f932faf..42c01e68a1 100644 --- a/internal/resources/snapshot.go +++ b/internal/resources/snapshot.go @@ -52,6 +52,23 @@ func (p StoragePrefix) IsZero() bool { return p == StoragePrefix{} } func (p StoragePrefix) String() string { return p.uri } +// ParseStoragePrefix parses a storage prefix received as a string, such as a +// snapshot URI or an owner's prefix. It rejects a bare bucket, so a prefix +// can never name every object in a bucket. +func ParseStoragePrefix(uri string) (StoragePrefix, error) { + if err := ValidateSnapshotLocation(uri); err != nil { + return StoragePrefix{}, err + } + u, err := url.Parse(uri) + if err != nil { + return StoragePrefix{}, fmt.Errorf("invalid storage prefix %q: %v", uri, err) + } + if strings.Trim(u.Path, "/") == "" { + return StoragePrefix{}, fmt.Errorf("invalid storage prefix %q: missing path", uri) + } + return StoragePrefix{uri: uri}, nil +} + // SnapshotOwner is the resource whose objects a snapshot's prefix holds: the // Actor that took it, or the Tag that copied it. Deletion is expressed as // "collect everything under my own prefix", so an owner cannot name another's diff --git a/internal/resources/snapshot_test.go b/internal/resources/snapshot_test.go index 195ef8da90..e461025dd0 100644 --- a/internal/resources/snapshot_test.go +++ b/internal/resources/snapshot_test.go @@ -595,3 +595,27 @@ func TestSnapshotURIZeroValue(t *testing.T) { t.Errorf("%q reports IsZero", uri) } } + +func TestParseStoragePrefix(t *testing.T) { + for _, tc := range []struct { + uri string + wantErr bool + }{ + {uri: "gs://bucket/root/atespaces/team-a/actors/uid1"}, + {uri: "s3://bucket/root/atespaces/team-a/actors/uid1/snapshots/snap1"}, + {uri: "gs://bucket", wantErr: true}, + {uri: "gs://bucket/", wantErr: true}, + {uri: "gs:///root", wantErr: true}, + {uri: "gs://bucket/root?x=1", wantErr: true}, + {uri: "gs://user@bucket/root", wantErr: true}, + } { + got, err := ParseStoragePrefix(tc.uri) + if (err != nil) != tc.wantErr { + t.Errorf("ParseStoragePrefix(%q) error = %v, wantErr %v", tc.uri, err, tc.wantErr) + continue + } + if err == nil && got.String() != tc.uri { + t.Errorf("ParseStoragePrefix(%q) = %q", tc.uri, got) + } + } +} diff --git a/internal/resources/snapshotfiles.go b/internal/resources/snapshotfiles.go new file mode 100644 index 0000000000..4487344999 --- /dev/null +++ b/internal/resources/snapshotfiles.go @@ -0,0 +1,37 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package resources + +import ( + "fmt" + "path/filepath" +) + +// ValidateSnapshotFileNames requires each name to be a distinct plain file name +// in the checkpoint directory. Actual file access must still use os.Root so +// symlinks cannot escape that directory. +func ValidateSnapshotFileNames(files []string) error { + seen := make(map[string]bool, len(files)) + for i, name := range files { + switch { + case name != filepath.Base(name) || !filepath.IsLocal(name) || name == ".": + return fmt.Errorf("snapshotFiles[%d] %q is not a file name in the checkpoint directory", i, name) + case seen[name]: + return fmt.Errorf("snapshotFiles[%d] %q is duplicated", i, name) + } + seen[name] = true + } + return nil +} diff --git a/internal/resources/snapshotfiles_test.go b/internal/resources/snapshotfiles_test.go new file mode 100644 index 0000000000..cba3bb3d5c --- /dev/null +++ b/internal/resources/snapshotfiles_test.go @@ -0,0 +1,42 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package resources + +import "testing" + +func TestValidateSnapshotFileNames(t *testing.T) { + for _, tc := range []struct { + name string + files []string + wantErr bool + }{ + {name: "empty list", files: nil}, + {name: "plain names", files: []string{"manifest.json", "memory.img", "durable-dir.tar"}}, + {name: "empty name", files: []string{""}, wantErr: true}, + {name: "dot", files: []string{"."}, wantErr: true}, + {name: "dot dot", files: []string{".."}, wantErr: true}, + {name: "absolute", files: []string{"/etc/passwd"}, wantErr: true}, + {name: "nested", files: []string{"a/b"}, wantErr: true}, + {name: "escaping", files: []string{"../x"}, wantErr: true}, + {name: "duplicate", files: []string{"a", "b", "a"}, wantErr: true}, + } { + t.Run(tc.name, func(t *testing.T) { + err := ValidateSnapshotFileNames(tc.files) + if gotErr := err != nil; gotErr != tc.wantErr { + t.Errorf("ValidateSnapshotFileNames(%q) = %v, want error: %v", tc.files, err, tc.wantErr) + } + }) + } +} diff --git a/manifests/ate-install/ate-api-server.yaml b/manifests/ate-install/ate-api-server.yaml index 90724822e1..5718480cec 100644 --- a/manifests/ate-install/ate-api-server.yaml +++ b/manifests/ate-install/ate-api-server.yaml @@ -83,6 +83,47 @@ spec: # the readinessProbe below reaching NotReady, plus propagation) + # --drain-timeout. terminationGracePeriodSeconds: 40 + # The control snapshot plugin deletes and copies external snapshots for + # ate-api-server, over a Unix socket in a pod-local emptyDir. A native + # sidecar starts before ate-api-server and stops after it. ate-api-server + # starts only once the startup probe sees the plugin serving. + initContainers: + - name: snapshot-plugin + image: ko://github.com/agent-substrate/substrate/cmd/snapshot-plugin + restartPolicy: Always + args: + - control + - --socket=/run/snapshot-plugin/control.sock + # The plugin listens on a Unix socket, which a gRPC probe cannot + # reach, and the image has no shell, so the binary checks itself. + startupProbe: + exec: + command: + - /ko-app/snapshot-plugin + - healthcheck + - --socket=/run/snapshot-plugin/control.sock + periodSeconds: 2 + failureThreshold: 30 + securityContext: + capabilities: + drop: + - ALL + env: + # The sidecar, not ate-api-server, holds the object storage settings: + # it copies external snapshots for tags and deletes the ones nothing + # refers to any more, so it needs the same backend atelet's sidecar + # reads and writes them through. Unset or "gcs" selects GCS with + # ambient credentials; "s3" with the AWS_* variables selects S3 (see + # the kind overlay). ate-api-server exits if ATE_STORAGE_BACKEND is + # set on it. + - name: ATE_STORAGE_BACKEND + value: "gcs" + envFrom: + - configMapRef: + name: ate-otel-config + volumeMounts: + - name: snapshot-plugin + mountPath: /run/snapshot-plugin containers: - name: ate-api-server image: ko://github.com/agent-substrate/substrate/cmd/ateapi @@ -107,6 +148,7 @@ spec: # Graceful shutdown knobs. The sum must fit within terminationGracePeriodSeconds. - --drain-delay=13s - --drain-timeout=15s + - --snapshot-plugin-socket=/run/snapshot-plugin/control.sock env: - name: POD_NAME valueFrom: @@ -122,11 +164,6 @@ spec: fieldPath: metadata.uid - name: OTEL_RESOURCE_ATTRIBUTES value: k8s.namespace.name=$(POD_NAMESPACE),k8s.pod.name=$(POD_NAME),k8s.pod.uid=$(POD_UID),service.instance.id=$(POD_UID) - # ate-api copies external snapshots for tags and deletes the ones - # nothing refers to any more, so it needs the same backend atelet reads - # and writes them through. - - name: ATE_STORAGE_BACKEND - value: "gcs" # OTLP exporter settings are shared by every control plane component; # see manifests/ate-install/ate-otel-config.yaml. Listed first so the # per-developer ConfigMap below can still override a key. @@ -147,6 +184,8 @@ spec: name: ate-api-server-secret-envvars optional: true volumeMounts: + - name: snapshot-plugin + mountPath: /run/snapshot-plugin # servicedns: the apiserver's own gRPC serving cert (DNS SAN api.ate-system.svc). - name: "servicedns" mountPath: "/run/servicedns.podcert.ate.dev" @@ -212,6 +251,8 @@ spec: initialDelaySeconds: 10 periodSeconds: 10 volumes: + - name: snapshot-plugin + emptyDir: {} - name: "servicedns" projected: sources: diff --git a/manifests/ate-install/atelet.yaml b/manifests/ate-install/atelet.yaml index 1d6556f636..7abddfc91f 100644 --- a/manifests/ate-install/atelet.yaml +++ b/manifests/ate-install/atelet.yaml @@ -147,6 +147,58 @@ spec: - key: ate.dev/sandboxClass operator: Exists effect: NoSchedule + # The node snapshot plugin moves external snapshots between this node and + # object storage for atelet, over a Unix socket in a pod-local emptyDir. + # A native sidecar (restartPolicy: Always) starts before atelet and stops + # after it, so uploads still in flight during atelet's drain finish. + # atelet starts only once the startup probe sees the plugin serving. + initContainers: + - name: snapshot-plugin + image: ko://github.com/agent-substrate/substrate/cmd/snapshot-plugin + restartPolicy: Always + args: + - node + - --socket=/run/snapshot-plugin/node.sock + - --root=/var/lib/ate + # The plugin listens on a Unix socket, which a gRPC probe cannot + # reach, and the image has no shell, so the binary checks itself. + startupProbe: + exec: + command: + - /ko-app/snapshot-plugin + - healthcheck + - --socket=/run/snapshot-plugin/node.sock + periodSeconds: 2 + failureThreshold: 30 + # Reads and writes snapshot files under the /var/lib/ate hostPath as + # root, like atelet; needs no capabilities. + securityContext: + runAsUser: 0 + runAsGroup: 0 + capabilities: + drop: + - ALL + # Same reservation as atelet and no limits: the plugin now does the + # zstd work atelet used to, and must not get less CPU for it. + resources: + requests: + cpu: 50m + memory: 128Mi + env: + # Object storage settings for snapshot transfers. Unset or "gcs" + # selects GCS with ambient credentials; "s3" with the AWS_* variables + # selects S3 (see the kind overlay). atelet keeps its own copy below + # for sandbox asset downloads, which do not go through the plugin. + - name: ATE_STORAGE_BACKEND + value: "gcs" + envFrom: + - configMapRef: + name: ate-otel-config + volumeMounts: + - name: run-ateom + mountPath: /var/lib/ate + - name: snapshot-plugin + mountPath: /run/snapshot-plugin containers: - name: atelet image: ko://github.com/agent-substrate/substrate/cmd/atelet @@ -164,6 +216,7 @@ spec: # terminationGracePeriodSeconds above. - --drain-delay=0s - --drain-timeout=5m + - --snapshot-plugin-socket=/run/snapshot-plugin/node.sock # atelet does no mounts, netlink, device, or namespace operations (those # live in the ateom worker pod) — it only reads/writes the # /var/lib/ate hostPath as root. CAP_DAC_OVERRIDE is for one thing: @@ -238,6 +291,8 @@ spec: volumeMounts: - name: run-ateom mountPath: /var/lib/ate + - name: snapshot-plugin + mountPath: /run/snapshot-plugin - name: podidentity mountPath: /run/podidentity.podcert.ate.dev - name: servicedns-ca @@ -257,6 +312,8 @@ spec: mountPath: /run/image-credential-provider/config.yaml readOnly: true volumes: + - name: snapshot-plugin + emptyDir: {} - name: run-ateom hostPath: path: /var/lib/ate diff --git a/manifests/ate-install/kind/atelet/kustomization.yaml b/manifests/ate-install/kind/atelet/kustomization.yaml index 023b922ee3..0963cc3ebc 100644 --- a/manifests/ate-install/kind/atelet/kustomization.yaml +++ b/manifests/ate-install/kind/atelet/kustomization.yaml @@ -70,3 +70,22 @@ patches: $patch: delete - name: image-credential-provider-config $patch: delete + # The snapshot plugin transfers atelet's external snapshots, so it + # needs the same rustfs settings. atelet keeps its own copy for + # sandbox assets. + initContainers: + - name: snapshot-plugin + env: + - name: ATE_STORAGE_BACKEND + value: s3 + - name: AWS_REGION + value: us-east-1 + - name: AWS_ENDPOINT_URL + value: http://rustfs.ate-system.svc:9000 + - name: AWS_S3_USE_PATH_STYLE + value: "true" + # TODO: use a secret / identity management + - name: AWS_ACCESS_KEY_ID + value: rustfsadmin + - name: AWS_SECRET_ACCESS_KEY + value: rustfsadmin diff --git a/manifests/ate-install/kind/kustomization.yaml b/manifests/ate-install/kind/kustomization.yaml index b4feace682..f124e8ff9b 100644 --- a/manifests/ate-install/kind/kustomization.yaml +++ b/manifests/ate-install/kind/kustomization.yaml @@ -43,9 +43,9 @@ resources: # shared by every component via envFrom -- putting always_on there would pin the # router and the control plane to 100% too, undoing their per-component ratios. # -# The storage settings point ate-api at the same rustfs the ./atelet overlay -# points atelet at: both ends of an external snapshot's life have to agree on -# where it lives. +# The storage settings point ate-api-server's snapshot plugin at the same +# rustfs the ./atelet overlay points atelet's at: both ends of an external +# snapshot's life have to agree on where it lives. patches: - patch: |- apiVersion: apps/v1 @@ -61,6 +61,9 @@ patches: env: - name: OTEL_TRACES_SAMPLER value: parentbased_always_on + initContainers: + - name: snapshot-plugin + env: - name: ATE_STORAGE_BACKEND value: s3 - name: AWS_REGION diff --git a/pkg/objectstorage/env.go b/pkg/objectstorage/env.go new file mode 100644 index 0000000000..537c8d3169 --- /dev/null +++ b/pkg/objectstorage/env.go @@ -0,0 +1,65 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package objectstorage + +import ( + "context" + "fmt" + "log/slog" + "os" + + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/s3" +) + +// UsesS3 reports whether ATE_STORAGE_BACKEND selects S3. Any other value +// selects GCS, the default. +func UsesS3() bool { + return os.Getenv("ATE_STORAGE_BACKEND") == "s3" +} + +// NewS3ClientFromEnv builds an S3 client from the standard AWS environment +// variables. AWS_S3_USE_PATH_STYLE=true selects path-style addressing. +func NewS3ClientFromEnv(ctx context.Context) (*s3.Client, error) { + slog.InfoContext(ctx, "Using S3 storage backend") + // Depends on the standard AWS environment variables, which have to be set + // on every pod that uses this client. + cfg, err := config.LoadDefaultConfig(ctx) + if err != nil { + return nil, fmt.Errorf("loading S3 config: %w", err) + } + return s3.NewFromConfig(cfg, func(o *s3.Options) { + if usePathStyle := os.Getenv("AWS_S3_USE_PATH_STYLE"); usePathStyle == "true" { + o.UsePathStyle = true + } + }), nil +} + +// NewFromEnv builds the ObjectStorage for the backend UsesS3 selects. +func NewFromEnv(ctx context.Context) (ObjectStorage, error) { + if UsesS3() { + client, err := NewS3ClientFromEnv(ctx) + if err != nil { + return nil, err + } + return NewS3Client(client), nil + } + // GCS is currently the default, TODO: we assume workload identity / ADC + client, err := NewGCSClient(ctx) + if err != nil { + return nil, fmt.Errorf("creating GCS client: %w", err) + } + return client, nil +} diff --git a/pkg/objectstorage/env_test.go b/pkg/objectstorage/env_test.go new file mode 100644 index 0000000000..5b9b9e7e4e --- /dev/null +++ b/pkg/objectstorage/env_test.go @@ -0,0 +1,69 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package objectstorage + +import ( + "context" + "path/filepath" + "testing" +) + +func TestNewFromEnv(t *testing.T) { + for _, tc := range []struct { + name string + backend string + pathStyle string + wantS3 bool + wantPathStyle bool + }{ + {name: "unset selects GCS", backend: ""}, + {name: "unknown value selects GCS", backend: "gcs"}, + {name: "s3", backend: "s3", wantS3: true}, + {name: "s3 with path style", backend: "s3", pathStyle: "true", wantS3: true, wantPathStyle: true}, + {name: "s3 path style needs exactly true", backend: "s3", pathStyle: "1", wantS3: true}, + } { + t.Run(tc.name, func(t *testing.T) { + // GCS: the emulator host makes storage.NewClient skip credential lookup. + t.Setenv("STORAGE_EMULATOR_HOST", "127.0.0.1:1") + // S3: keep the developer's AWS config out of the test. + missing := filepath.Join(t.TempDir(), "missing") + t.Setenv("AWS_CONFIG_FILE", missing) + t.Setenv("AWS_SHARED_CREDENTIALS_FILE", missing) + t.Setenv("AWS_REGION", "us-east-1") + t.Setenv("ATE_STORAGE_BACKEND", tc.backend) + t.Setenv("AWS_S3_USE_PATH_STYLE", tc.pathStyle) + + got, err := NewFromEnv(context.Background()) + if err != nil { + t.Fatalf("NewFromEnv: %v", err) + } + switch c := got.(type) { + case *s3Client: + if !tc.wantS3 { + t.Fatalf("NewFromEnv returned an S3 client, want GCS") + } + if got := c.client.Options().UsePathStyle; got != tc.wantPathStyle { + t.Errorf("UsePathStyle = %v, want %v", got, tc.wantPathStyle) + } + case *gcsClient: + if tc.wantS3 { + t.Fatalf("NewFromEnv returned a GCS client, want S3") + } + default: + t.Fatalf("NewFromEnv returned %T", got) + } + }) + } +} From 8ccefc038a9e6108fe2a18f9f17261f06c2469f0 Mon Sep 17 00:00:00 2001 From: Siyan Shen Date: Thu, 8 Oct 2026 22:11:03 +0000 Subject: [PATCH 15/44] atespace isolation tests (#1966) Add an e2e suite for the actor isolation boundary Tests added (internal/e2e/suites/security): - TestCrossTenantIsolation: the victim tenant writes a marker to its durable-dir volume, reads it back, and is suspended; the attacker tenant then resumes on the same worker. Controls confirm the probe reports missing files as errors and can read its own durable data. - VictimDurableMarkerNotVisibleThroughAttackerMount: the victim's marker is not readable through the attacker's own /data mount. - VictimHostStateNotReachableFromAttackerSandbox: the victim's durable directory, projected actor identity, and sandbox asset record are not readable at their host paths (taken from internal/ateompath), and the attacker cannot write into the victim's durable directory. - TagScopedToAtespaceCannotSeedAnotherTenant: an ATESPACE-scoped Tag created from the attacker's snapshot cannot seed an Actor in the other atespace. The test asserts the tag-scope rejection message, not just FailedPrecondition, since other preconditions on that path return the same code. A same-atespace clone from the tag restores the source's durable data, proving the tag was usable. Fixtures (internal/e2e/fixtures/security): a pool-bearing tenant, a namespace-only second tenant that must schedule onto the first's pool, and a shared probe-based ActorTemplate with a system-info volume and a durable-dir volume. The pool-bearing half is registered in sandbox_test.go's fixture rendering checks. Fixes # > It's a good idea to open an issue first for discussion. - [ ] Tests pass - [ ] Appropriate changes to documentation are included in the PR --- .../security/atespace-a-pool.yaml.tmpl | 50 ++ .../security/atespace-b-namespace.yaml.tmpl | 29 ++ .../security/atespace-template.yaml.tmpl | 76 +++ internal/e2e/sandbox_test.go | 4 + internal/e2e/suites/security/README.md | 119 +++++ .../e2e/suites/security/crossatespace_test.go | 453 ++++++++++++++++++ internal/e2e/suites/security/testmain_test.go | 24 + 7 files changed, 755 insertions(+) create mode 100644 internal/e2e/fixtures/security/atespace-a-pool.yaml.tmpl create mode 100644 internal/e2e/fixtures/security/atespace-b-namespace.yaml.tmpl create mode 100644 internal/e2e/fixtures/security/atespace-template.yaml.tmpl create mode 100644 internal/e2e/suites/security/README.md create mode 100644 internal/e2e/suites/security/crossatespace_test.go create mode 100644 internal/e2e/suites/security/testmain_test.go diff --git a/internal/e2e/fixtures/security/atespace-a-pool.yaml.tmpl b/internal/e2e/fixtures/security/atespace-a-pool.yaml.tmpl new file mode 100644 index 0000000000..32fd9bc1b1 --- /dev/null +++ b/internal/e2e/fixtures/security/atespace-a-pool.yaml.tmpl @@ -0,0 +1,50 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Atespace A of the cross-atespace security fixture: a namespace plus the only +# WorkerPool. Atespace B deploys atespace-b-namespace.yaml.tmpl, a namespace +# with no pool, so its actors run on this one too. +# +# replicas is 1 on purpose. Pools are not scoped to an atespace: the scheduler +# matches on sandbox class, labels and capacity, never on the actor's +# atespace. The test runs the victim, suspends it, then starts the attacker. +# With one worker the attacker has to land on the worker the victim just +# left, which is where the victim's per-actor host state is. With more +# workers the attacker could boot on a fresh one and the host-state checks +# would prove nothing. +# +# The workload label is a fixed string rather than ${FIXTURE_SUFFIX}-suffixed +# because both atespaces' templates must select this one pool and they render +# with different suffixes. Sharing the label across the gVisor and micro-VM +# lanes is safe: they run one after the other, and sandbox class is itself a +# scheduling constraint. + +apiVersion: v1 +kind: Namespace +metadata: + name: ate-e2e${FIXTURE_SUFFIX} + +--- + +apiVersion: ate.dev/v1alpha1 +kind: WorkerPool +metadata: + name: shared + namespace: ate-e2e${FIXTURE_SUFFIX} + labels: + workload: ate-e2e-security +spec: + replicas: 1 + workerImage: ${ATEOM_IMAGE} +${WORKERPOOL_RUNTIME} diff --git a/internal/e2e/fixtures/security/atespace-b-namespace.yaml.tmpl b/internal/e2e/fixtures/security/atespace-b-namespace.yaml.tmpl new file mode 100644 index 0000000000..6dfd9038ce --- /dev/null +++ b/internal/e2e/fixtures/security/atespace-b-namespace.yaml.tmpl @@ -0,0 +1,29 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Atespace B of the cross-atespace security fixture: a namespace only. +# +# Atespace B deliberately has no WorkerPool. Pools are not scoped to an +# atespace, so B's actors are placed on the single-worker pool that +# atespace-a-pool.yaml.tmpl deploys in atespace A's namespace. That forces +# A's and B's actors to take turns on one worker, which is the reuse boundary +# the suite tests. +# +# The namespace is still needed: Substrate reconciles an atespace's +# namespaced objects (network policies, for example) into it. + +apiVersion: v1 +kind: Namespace +metadata: + name: ate-e2e${FIXTURE_SUFFIX} diff --git a/internal/e2e/fixtures/security/atespace-template.yaml.tmpl b/internal/e2e/fixtures/security/atespace-template.yaml.tmpl new file mode 100644 index 0000000000..a54d995349 --- /dev/null +++ b/internal/e2e/fixtures/security/atespace-template.yaml.tmpl @@ -0,0 +1,76 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The ActorTemplate of the cross-atespace security fixture: a +# protojson-shaped ateapipb.ActorTemplate created through the ate API by +# e2e.DeploySubstrateFixture. +# +# BOTH atespaces deploy this same file. They differ only in the ${FIXTURE_SUFFIX} +# they render with, which is what gives each its own atespace and its own +# snapshot prefix, so the suite compares two actors that are identical in +# every respect except the boundary under test. +# +# The container is the shared e2e probe image. Its /readfile and /writefile +# endpoints let the test try reads and writes from inside the running actor, +# which is the only place the isolation claim can be checked: what the actor +# actually sees at /data, not what atelet's mount config says it should see. +# +# The workerSelector names the single shared pool (see atespace-a-pool.yaml.tmpl), +# not a per-atespace one. + +metadata: + atespace: ate-e2e${FIXTURE_SUFFIX} + name: probe +workerSelector: + matchLabels: + workload: ate-e2e-security +containers: +- name: probe + image: ko://github.com/agent-substrate/substrate/internal/e2e/fixtures/probe + command: ["/ko-app/probe"] + volumeMounts: + # The probe expects its identity files under /run/ate. The mount also makes + # atelet write a per-actor identity file on the worker host, which is one + # of the host paths the attacker is checked against. + - name: system-info + mountPath: /run/ate + # The per-actor durable directory whose isolation is under test. + - name: data + mountPath: /data + # Gate the golden snapshot on the probe actually serving, so a read or write + # right after a resume cannot race a sandbox checkpointed before the + # listener came up. + wakeupProbe: + httpGet: + path: /healthz + port: 80 + timeoutSeconds: 60 +volumes: +- name: system-info + systemInfo: + dataSources: + - actorMetadata: + items: + - field: ACTOR_METADATA_FIELD_NAME + path: actor-id + - field: ACTOR_METADATA_FIELD_ATESPACE + path: atespace + - field: ACTOR_METADATA_FIELD_UID + path: actor-uid +- name: data + durableDir: {} +${TEMPLATE_RESOURCES} +${TEMPLATE_SANDBOX_CONFIG} +snapshotConfig: + storageLocation: gs://${BUCKET_NAME}/ate-e2e${FIXTURE_SUFFIX}/ diff --git a/internal/e2e/sandbox_test.go b/internal/e2e/sandbox_test.go index f0510b9309..ac443fd8ec 100644 --- a/internal/e2e/sandbox_test.go +++ b/internal/e2e/sandbox_test.go @@ -40,6 +40,10 @@ var substrateFixtures = []struct { Pool: "internal/e2e/fixtures/probe/probe-sized.yaml.tmpl", Template: "internal/e2e/fixtures/probe/probe-sized-template.yaml.tmpl", }, 1}, + {SubstrateFixtureManifests{ + Pool: "internal/e2e/fixtures/security/atespace-a-pool.yaml.tmpl", + Template: "internal/e2e/fixtures/security/atespace-template.yaml.tmpl", + }, 1}, {SubstrateFixtureManifests{ Pool: "internal/e2e/fixtures/capabilities/capabilities.yaml.tmpl", Template: "internal/e2e/fixtures/capabilities/capabilities-templates.yaml.tmpl", diff --git a/internal/e2e/suites/security/README.md b/internal/e2e/suites/security/README.md new file mode 100644 index 0000000000..382f9e2bc0 --- /dev/null +++ b/internal/e2e/suites/security/README.md @@ -0,0 +1,119 @@ +# Security e2e suite + +End-to-end tests for Substrate's isolation boundaries. This is the runbook +for running them on a local kind cluster and for watching what they do while +they run. + +Prereqs: Go, kubectl, a running Docker. kind is auto-managed. + +## Steps to run the security tests + +1. Create the cluster (in the repo root): + +``` +hack/create-kind-cluster.sh +``` + +2. Install the control plane (slow step — pulls ~570 MB): + +``` +hack/install-ate-kind.sh --deploy-ate-system +``` + +Stop here — the NFS/CSI and demo steps CI runs are for other suites; this suite needs none of them. + +3. Run the cross-atespace isolation test: + +``` +hack/run-e2e-kind.sh ./internal/e2e/suites/security -v -run TestCrossAtespaceIsolation +``` + +The wrapper sets KO_DOCKER_REPO, BUCKET_NAME, the kube context, and builds the probe image via ko. To run only the durable-mount and host-state subtests and skip the tag-scope one: + +``` +hack/run-e2e-kind.sh ./internal/e2e/suites/security -v \ + -run 'TestCrossAtespaceIsolation/(VictimDurableMarkerNotVisibleThroughAttackerMount|VictimHostStateNotReachableFromAttackerSandbox)' +``` + +Note the setup (victim run/write/suspend, attacker reuse) runs regardless — the subtest filter only selects which assertions execute. + +4. On failure, a failed suite keeps its namespaces so you can read the worker logs: + + +kubectl get ns -l ate.dev/e2e +kubectl -n logs -c ateom +KIND_CLUSTER_NAME=kind hack/cleanup-e2e.sh # reclaim when done + + + +## Run the GitHub checks locally + +The PR workflow (`.github/workflows/pr-workflow.yaml`) runs three checks: +`run-tests`, `E2E ()`, and `e2e-test`, which only aggregates the +E2E matrix. Every one of them compiles this suite, so a build error here +fails all three. + +Fast, no cluster needed. This is what CI's `run-tests` job does, minus the +JUnit wrapping, and it also type-checks every E2E suite: + +``` +go vet ./... && make verify && (cd tools/apitool && go test -race ./...) +``` + +`make verify` runs `go test -race ./...` and then `hack/verify-all.sh` +(gofmt, boilerplate, licenses, go.mod, metrics registry, generated code). +Commit or stash first: the generated-code check refuses a dirty tree. The +metrics check needs Docker or a local `weaver` at the version pinned in +`hack/verify/metrics.sh`. The root-gated tests CI runs under sudo need Linux: + +``` +hack/run-root-tests.sh -race -v +``` + +Full E2E, same as CI's gVisor lane. Do steps 1 and 2 above, then run every +suite with CI's flags instead of the `-run` filter from step 3: + +``` +hack/run-e2e-kind.sh -v -args --no-color +``` + +CI also deploys the NFS CSI driver and the demos before this, and repeats the +run with `E2E_SANDBOX_CLASS=microvm`. Neither matters for this suite. + +## Commands to run while the test is running to observe +Watch the lifecycle (start these first, before/at test launch) + +# The two atespace namespaces appear and vanish — this is the run's heartbeat +kubectl get ns -l ate.dev/e2e -w + +# The one shared worker pod: watch the victim's pod get reused, not replaced +kubectl get pods -n ate-e2e-sec-a -w +The reuse itself (the point of the test) +Actors aren't in k8s, so use the plugin. This is where you see victim B run, suspend, then attacker A bind to the same worker: + + +# Actor state + worker assignment, atespace B (victim) then A (attacker) +watch -n1 'kubectl ate get actors -a ate-e2e-sec-b; echo ---; kubectl ate get actors -a ate-e2e-sec-a' + +# Which worker each actor is on — the name should match across the two atespaces +kubectl ate get workers +kubectl ate top workers # capacity: Actors used/total (you'll see 1/1 then 0/1 then 1/1) +The worker name that B's victim shows, then A's attacker shows after the suspend, is the same string requireReusedWorker asserts on. Seeing them match live is the manual version of that check. + +The shared worker pod on the host + +# The pod hosting both actors in turn +kubectl get pods -n ate-e2e-sec-a -o wide + +# ateom logs — the sandbox lifecycle, checkpoints, resumes +kubectl logs -n ate-e2e-sec-a -l app=ateom -c ateom -f + +# The on-host per-actor layout the host-state probes assert against. +# Two UID-keyed durable dirs (victim's + attacker's) live side by side here: +POD=$(kubectl get pods -n ate-e2e-sec-a -l app=ateom -o name | head -1) +kubectl exec -n ate-e2e-sec-a "$POD" -c ateom -- ls -la /var/lib/ateom-gvisor/actors +Control plane / snapshots (for the tag-scope subtest) + +kubectl ate get tags -a ate-e2e-sec-a # atespace-a-tag appears during the last subtest +kubectl ate get atespaces # ate-e2e-sec-a / -b listed while live +kubectl logs -n ate-system -l app=ateapi -f # Create \ No newline at end of file diff --git a/internal/e2e/suites/security/crossatespace_test.go b/internal/e2e/suites/security/crossatespace_test.go new file mode 100644 index 0000000000..f40fa865fd --- /dev/null +++ b/internal/e2e/suites/security/crossatespace_test.go @@ -0,0 +1,453 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package security holds the end-to-end tests for Substrate's security +// boundaries. Every assertion here is negative: something must not be +// possible. Each is paired with a positive control showing the probe could +// have seen the violation, because a negative test that stops exercising its +// boundary keeps passing. +// +// This file covers the boundary between two atespaces whose actors are +// identical except for the atespace they run in. The victim actor runs, writes +// state, and vacates its worker; the attacker actor, in the other atespace, +// then reuses that worker and must find none of it. A single-worker pool +// (fixtures/security/atespace-a-pool.yaml.tmpl) forces the reuse, and the test +// checks that it happened. +package security + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/url" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/agent-substrate/substrate/internal/e2e" + "github.com/agent-substrate/substrate/internal/nodepath" + "github.com/agent-substrate/substrate/internal/resources" + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +const ( + // templateName is the ActorTemplate name both atespaces deploy. + templateName = "probe" + + // dataMount is where atespace-template.yaml.tmpl mounts the durable-dir + // volume, and durableVolume is that volume's name, which is also the last + // segment of its mount point on the worker. Keep both in sync with the + // manifest. + dataMount = "/data" + durableVolume = "data" + + // writtenByProbe is the fixed body the probe's /writefile endpoint writes. + // Actors are told apart by the path they write, not by the content. + writtenByProbe = "written by probe" + + // markerA and markerB are named for the atespace that owns them, so a file + // seen across the boundary is attributable. + markerA = dataMount + "/atespace-a-marker" + markerB = dataMount + "/atespace-b-marker" + + // neverWritten is a path no actor creates. Reading it proves /readfile + // reports a missing file as an error, which every negative assertion in + // this file relies on. + neverWritten = dataMount + "/never-written" +) + +// fileResponse mirrors the probe's /readfile and /writefile responses. +type fileResponse struct { + Path string `json:"path"` + Content string `json:"content"` + OK string `json:"ok"` + Error string `json:"error"` +} + +// atespaces is the pair of atespaces under test. +type atespaces struct { + a, b string +} + +// TestCrossAtespaceIsolation checks that the atespace boundary holds across +// worker reuse for an actor's stored data and its snapshots. +// +// The victim, in atespace B, runs, writes its durable marker, and is +// suspended. The attacker, in atespace A, then starts on the worker the victim +// left and must find none of its state. +// +// The fixture is deployed once for the whole test. Deploying waits for each +// atespace's golden snapshot, which on the micro-VM lane is a cold boot plus a +// checkpoint, so per-subtest fixtures would dominate the runtime. +func TestCrossAtespaceIsolation(t *testing.T) { + env, err := e2e.CheckEnv("BUCKET_NAME", "KO_DOCKER_REPO") + if err != nil { + t.Fatalf("CheckEnv failed: %v", err) + } + ctx := context.Background() + clients := e2e.GetClients() + + ns := deployAtespaces(t, ctx, clients, env["BUCKET_NAME"]) + + rc, err := e2e.NewRouterClient(ctx) + if err != nil { + t.Fatalf("NewRouterClient: %v", err) + } + defer rc.Close() + + // Victim: the actor in atespace B runs first. Its UID and worker are recorded while it is + // RUNNING: the attacker probes host paths keyed by that UID and must land + // on that worker. + const victim = "victim" + createAndResumeActor(t, ctx, clients, ns.b, victim) + victimActor := resources.ActorRef{Atespace: ns.b, Name: victim} + victimUID := actorUID(t, ctx, clients, ns.b, victim) + victimWorker := assignedWorker(t, ctx, clients, ns.b, victim) + + writeFile(t, ctx, rc, victimActor, markerB) + // Control: the victim reads back its own marker, so the attacker's failure + // to read it below means the boundary held, not that the marker was never + // written. + if got := readFile(t, ctx, rc, victimActor, markerB); got.Content != writtenByProbe { + t.Fatalf("control: victim %s reading its OWN %s = %q (error %q), want %q — the probe cannot see its own durable data, so nothing below is meaningful", + victimActor, markerB, got.Content, got.Error, writtenByProbe) + } + + // Suspend the victim to free the worker. The durable marker stays on the + // worker; the snapshot goes to object storage. + if _, err := clients.SubstrateAPI.SuspendActor(ctx, &ateapipb.SuspendActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: ns.b, Name: victim}, + }); err != nil { + t.Fatalf("SuspendActor %s/%s: %v", ns.b, victim, err) + } + waitForActorState(t, ctx, clients, ns.b, victim, ateapipb.ActorState_ACTOR_STATE_SUSPENDED) + + // Attacker: the actor in atespace A takes the freed worker. + const attacker = "attacker" + createAndResumeActor(t, ctx, clients, ns.a, attacker) + attackerActor := resources.ActorRef{Atespace: ns.a, Name: attacker} + requireReusedWorker(t, ctx, clients, ns.a, attacker, victimWorker) + + // Controls for every negative assertion below. A failure is fatal: the + // probe cannot tell a denial from a missing file, so the subtests would + // pass for the wrong reason. + if got := readFile(t, ctx, rc, attackerActor, neverWritten); got.Error == "" { + t.Fatalf("control: attacker %s read %s, which nothing ever created, and reported no error (content %q) — /readfile does not report misses as errors, so every negative assertion here would pass vacuously", + attackerActor, neverWritten, got.Content) + } + writeFile(t, ctx, rc, attackerActor, markerA) + if got := readFile(t, ctx, rc, attackerActor, markerA); got.Content != writtenByProbe { + t.Fatalf("control: attacker %s reading its OWN %s = %q (error %q), want %q — the probe cannot see its own durable data", + attackerActor, markerA, got.Content, got.Error, writtenByProbe) + } + + // Durable mount: the attacker's /data must be its own. The victim's + // marker is known to exist, so reading it through the attacker's mount + // means durable directories are shared or mis-keyed across reuse. + t.Run("VictimDurableMarkerNotVisibleThroughAttackerMount", func(t *testing.T) { + if got := readFile(t, ctx, rc, attackerActor, markerB); got.Error == "" { + t.Errorf("attacker %s read %s, the victim's durable marker in atespace %s, through its own mount: content %q — durable-dir volumes are shared across the worker-reuse boundary", + attackerActor, markerB, ns.b, got.Content) + } + }) + + // Host state: the mount is confined (above); this checks the sandbox + // does not expose the host directories that mount is built from. The paths + // come from internal/nodepath so they move with the worker layout; see + // actorHostPath. + t.Run("VictimHostStateNotReachableFromAttackerSandbox", func(t *testing.T) { + for _, probe := range []struct { + what string + path string + }{ + {"the victim's durable marker on the host", actorHostPath(victimUID, "durable-dir", durableVolume, filepath.Base(markerB))}, + {"the victim's projected actor identity", actorHostPath(victimUID, "system-info", "system-info", "actor-id")}, + {"the victim's sandbox asset record", actorHostPath(victimUID, "sandbox-assets.json")}, + } { + if got := readFile(t, ctx, rc, attackerActor, probe.path); got.Error == "" { + t.Errorf("attacker %s read %s (%s): content %q — the sandbox exposes the worker's per-actor host state across the reuse boundary", + attackerActor, probe.path, probe.what, got.Content) + } + } + + // A write is the more serious direction of the same boundary: reaching the + // victim's durable directory means an actor can corrupt another atespace's data. + injected := actorHostPath(victimUID, "durable-dir", durableVolume, "injected") + if got := writeFileAllowingFailure(t, ctx, rc, attackerActor, injected); got.Error == "" { + t.Errorf("attacker %s wrote %s, inside the victim's durable directory (atespace %s) on the worker — an actor can corrupt another atespace's stored data", + attackerActor, injected, ns.b) + } + }) + + // Tag scope. Runs last: it suspends the attacker to snapshot it. + t.Run("TagScopedToAtespaceCannotSeedAnotherAtespace", func(t *testing.T) { + testTagScopeBoundary(t, ctx, clients, rc, ns, attacker) + }) +} + +// actorHostPath is nodepath.ActorsDir// on the worker host. The +// leaves (durable-dir, system-info, sandbox-assets.json) mirror +// cmd/atelet/internal/ateletpath, which this suite cannot import. +func actorHostPath(actorUID string, elem ...string) string { + return filepath.Join(append([]string{nodepath.ActorsDir, actorUID}, elem...)...) +} + +// testTagScopeBoundary checks that an Actor cannot be seeded from a snapshot +// taken in another atespace. A snapshot carries the source's memory, root filesystem +// and durable data, so loading one across the boundary hands over everything. +// +// Substrate enforces this through the Tag's scope in ateapi's CreateActor. The +// negative case matches the error message, not just the code. +// +// source is a running actor in atespace A holding markerA; it is suspended here +// to produce the snapshot the tag captures. +func testTagScopeBoundary(t *testing.T, ctx context.Context, clients *e2e.Clients, rc *e2e.RouterClient, ns atespaces, source string) { + t.Helper() + + suspended, err := clients.SubstrateAPI.SuspendActor(ctx, &ateapipb.SuspendActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: ns.a, Name: source}, + }) + if err != nil { + t.Fatalf("SuspendActor %s/%s: %v", ns.a, source, err) + } + if suspended.GetActor().GetStatus().GetExternalSnapshot().GetSnapshotUri() == "" { + t.Fatalf("suspended actor %s/%s has no external snapshot, so there is nothing for a tag to capture", ns.a, source) + } + + tagRef := &ateapipb.ObjectRef{Atespace: ns.a, Name: "atespace-a-tag"} + t.Cleanup(func() { + _, _ = clients.SubstrateAPI.DeleteTag(context.Background(), &ateapipb.DeleteTagRequest{Tag: tagRef}) + }) + // Tags outlive the fixture namespace, so an interrupted run can leave one + // behind and wedge reruns on AlreadyExists. Clear it first, best-effort. + _, _ = clients.SubstrateAPI.DeleteTag(ctx, &ateapipb.DeleteTagRequest{Tag: tagRef}) + tag, err := clients.SubstrateAPI.CreateTag(ctx, &ateapipb.CreateTagRequest{ + Tag: &ateapipb.Tag{ + Metadata: &ateapipb.ResourceMetadata{Atespace: ns.a, Name: tagRef.GetName()}, + Scope: ateapipb.TagScope_TAG_SCOPE_ATESPACE, + SourceActor: &ateapipb.ObjectRef{Atespace: ns.a, Name: source}, + }, + }) + if err != nil { + t.Fatalf("CreateTag %s/%s: %v", ns.a, tagRef.GetName(), err) + } + if tag.GetStatus().GetSnapshot().GetSnapshotUri() == "" { + t.Fatalf("tag %s/%s has no snapshot uri, so it could seed an Actor in NO atespace and the negative case below would be vacuous", + ns.a, tagRef.GetName()) + } + + // The boundary: the other atespace may not name this tag. + crossRef := &ateapipb.ObjectRef{Atespace: ns.b, Name: "cross-atespace-clone"} + t.Cleanup(func() { + _, _ = clients.SubstrateAPI.DeleteActor(context.Background(), &ateapipb.DeleteActorRequest{Actor: crossRef}) + }) + _, err = clients.SubstrateAPI.CreateActor(ctx, &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: ns.b, Name: crossRef.GetName()}, + ActorTemplate: &ateapipb.ObjectRef{Atespace: ns.b, Name: templateName}, + SourceTag: tagRef, + }}) + switch { + case err == nil: + t.Errorf("CreateActor %s/%s seeded from tag %s/%s (scope ATESPACE) succeeded — an actor can be seeded from another atespace's snapshot", + ns.b, crossRef.GetName(), ns.a, tagRef.GetName()) + case status.Code(err) != codes.FailedPrecondition: + t.Errorf("CreateActor %s/%s seeded from tag %s/%s: got %v, want FailedPrecondition", + ns.b, crossRef.GetName(), ns.a, tagRef.GetName(), err) + case !strings.Contains(status.Convert(err).Message(), "not published outside its Atespace"): + // Rejected by some other precondition, not the scope check. + t.Errorf("CreateActor %s/%s seeded from tag %s/%s was rejected by a different precondition: %q; want the tag-scope rejection", + ns.b, crossRef.GetName(), ns.a, tagRef.GetName(), status.Convert(err).Message()) + } + + // Positive control: the same tag seeds an Actor in its own atespace and the + // clone carries the source's durable data. Otherwise an unusable tag would + // make the rejection above look like enforcement. + const clone = "same-atespace-clone" + cloneRef := &ateapipb.ObjectRef{Atespace: ns.a, Name: clone} + t.Cleanup(func() { + cleanupCtx := context.Background() + _, _ = clients.SubstrateAPI.SuspendActor(cleanupCtx, &ateapipb.SuspendActorRequest{Actor: cloneRef}) + _, _ = clients.SubstrateAPI.DeleteActor(cleanupCtx, &ateapipb.DeleteActorRequest{Actor: cloneRef}) + }) + if _, err := clients.SubstrateAPI.CreateActor(ctx, &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: ns.a, Name: clone}, + ActorTemplate: &ateapipb.ObjectRef{Atespace: ns.a, Name: templateName}, + SourceTag: tagRef, + }}); err != nil { + t.Fatalf("control: CreateActor %s/%s from tag %s in its own atespace: %v — the tag is unusable, so the cross-atespace rejection above proves nothing", + ns.a, clone, tagRef.GetName(), err) + } + if _, err := e2e.ResumeActorAwaitCapacity(t, ctx, clients, &ateapipb.ResumeActorRequest{Actor: cloneRef}); err != nil { + t.Fatalf("control: ResumeActor %s/%s: %v", ns.a, clone, err) + } + waitForActorState(t, ctx, clients, ns.a, clone, ateapipb.ActorState_ACTOR_STATE_RUNNING) + if got := readFile(t, ctx, rc, resources.ActorRef{Atespace: ns.a, Name: clone}, markerA); got.Content != writtenByProbe { + t.Fatalf("control: actor %s/%s restored from tag %s reads %s = %q (error %q), want %q — the tag carries no durable data, so the cross-atespace rejection above proves nothing", + ns.a, clone, tagRef.GetName(), markerA, got.Content, got.Error, writtenByProbe) + } +} + +// deployAtespaces installs both atespaces and returns their names. A goes +// first: it brings the only WorkerPool, and B's golden snapshot needs a worker +// to boot on. +func deployAtespaces(t *testing.T, ctx context.Context, clients *e2e.Clients, bucket string) atespaces { + t.Helper() + a, _ := e2e.DeploySubstrateFixture(t, ctx, clients, e2e.SubstrateFixtureManifests{ + Pool: "internal/e2e/fixtures/security/atespace-a-pool.yaml.tmpl", + Template: "internal/e2e/fixtures/security/atespace-template.yaml.tmpl", + }, bucket, "sec-a", false) + b, _ := e2e.DeploySubstrateFixture(t, ctx, clients, e2e.SubstrateFixtureManifests{ + Pool: "internal/e2e/fixtures/security/atespace-b-namespace.yaml.tmpl", + Template: "internal/e2e/fixtures/security/atespace-template.yaml.tmpl", + }, bucket, "sec-b", false) + if a == b { + t.Fatalf("both fixtures rendered to atespace %q — the fixture no longer isolates anything", a) + } + return atespaces{a: a, b: b} +} + +// requireReusedWorker fails unless the actor landed on priorWorker. The fixture +// runs one worker for both atespaces, so any other worker means the attacker never +// touched the host the victim left its state on and the test proves nothing. +func requireReusedWorker(t *testing.T, ctx context.Context, clients *e2e.Clients, atespace, name, priorWorker string) { + t.Helper() + got := assignedWorker(t, ctx, clients, atespace, name) + if got != priorWorker { + t.Fatalf("actor %s/%s landed on worker %q, not the worker %q the victim vacated; the fixture deploys one worker for both atespaces, so the reuse boundary under test is not being exercised", + atespace, name, got, priorWorker) + } + t.Logf("actor %s/%s reused worker %s", atespace, name, priorWorker) +} + +func assignedWorker(t *testing.T, ctx context.Context, clients *e2e.Clients, atespace, name string) string { + t.Helper() + actor, err := clients.SubstrateAPI.GetActor(ctx, &ateapipb.GetActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: atespace, Name: name}, + }) + if err != nil { + t.Fatalf("GetActor %s/%s: %v", atespace, name, err) + } + worker := actor.GetStatus().GetWorkerAssignment().GetWorker().GetName() + if worker == "" { + t.Fatalf("actor %s/%s has no worker assignment while RUNNING", atespace, name) + } + return worker +} + +func actorUID(t *testing.T, ctx context.Context, clients *e2e.Clients, atespace, name string) string { + t.Helper() + actor, err := clients.SubstrateAPI.GetActor(ctx, &ateapipb.GetActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: atespace, Name: name}, + }) + if err != nil { + t.Fatalf("GetActor %s/%s: %v", atespace, name, err) + } + uid := actor.GetMetadata().GetUid() + if uid == "" { + t.Fatalf("actor %s/%s has no uid", atespace, name) + } + return uid +} + +// createAndResumeActor creates an actor from its atespace's template and resumes +// it from the golden snapshot, removing it when the test ends. +func createAndResumeActor(t *testing.T, ctx context.Context, clients *e2e.Clients, atespace, name string) { + t.Helper() + ref := &ateapipb.ObjectRef{Atespace: atespace, Name: name} + // Actor records outlive the fixture namespace, so an interrupted run can + // leave one behind and wedge reruns on AlreadyExists. DeleteActor needs + // SUSPENDED or CRASHED, hence the suspend; both are best-effort. + _, _ = clients.SubstrateAPI.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: ref}) + _, _ = clients.SubstrateAPI.DeleteActor(ctx, &ateapipb.DeleteActorRequest{Actor: ref}) + if _, err := clients.SubstrateAPI.CreateActor(ctx, &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: atespace, Name: name}, + ActorTemplate: &ateapipb.ObjectRef{Atespace: atespace, Name: templateName}, + }}); err != nil { + t.Fatalf("CreateActor %s/%s: %v", atespace, name, err) + } + t.Cleanup(func() { + cleanupCtx := context.Background() + _, _ = clients.SubstrateAPI.SuspendActor(cleanupCtx, &ateapipb.SuspendActorRequest{Actor: ref}) + if _, err := clients.SubstrateAPI.DeleteActor(cleanupCtx, &ateapipb.DeleteActorRequest{Actor: ref}); err != nil { + t.Logf("cleanup: DeleteActor %s/%s failed, actor leaked (remove with: kubectl ate delete actor %s -a %s): %v", atespace, name, name, atespace, err) + } + }) + + if _, err := e2e.ResumeActorAwaitCapacity(t, ctx, clients, &ateapipb.ResumeActorRequest{Actor: ref}); err != nil { + t.Fatalf("ResumeActor %s/%s: %v", atespace, name, err) + } + waitForActorState(t, ctx, clients, atespace, name, ateapipb.ActorState_ACTOR_STATE_RUNNING) +} + +func waitForActorState(t *testing.T, ctx context.Context, clients *e2e.Clients, atespace, name string, want ateapipb.ActorState) { + t.Helper() + deadline := time.Now().Add(60 * time.Second) + for time.Now().Before(deadline) { + resp, err := clients.SubstrateAPI.GetActor(ctx, &ateapipb.GetActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: atespace, Name: name}, + }) + if err == nil && resp.GetStatus().GetState() == want { + return + } + time.Sleep(time.Second) + } + t.Fatalf("timed out waiting for actor %s/%s to reach state %v", atespace, name, want) +} + +// writeFile writes the probe's fixed body at path and fails the test if the +// write did not land, so a later negative assertion cannot pass on missing +// state. +func writeFile(t *testing.T, ctx context.Context, rc *e2e.RouterClient, actor resources.ActorRef, path string) { + t.Helper() + if got := writeFileAllowingFailure(t, ctx, rc, actor, path); got.OK != "true" { + t.Fatalf("actor %s writing %s: %q", actor, path, got.Error) + } +} + +// writeFileAllowingFailure is writeFile for assertions that expect the write +// to be refused; it returns the probe's report instead of failing. +func writeFileAllowingFailure(t *testing.T, ctx context.Context, rc *e2e.RouterClient, actor resources.ActorRef, path string) fileResponse { + t.Helper() + return probeFileOp(t, ctx, rc, actor, "/writefile", path) +} + +func readFile(t *testing.T, ctx context.Context, rc *e2e.RouterClient, actor resources.ActorRef, path string) fileResponse { + t.Helper() + return probeFileOp(t, ctx, rc, actor, "/readfile", path) +} + +// probeFileOp calls one of the probe's file endpoints. A transport or HTTP +// failure is fatal rather than returned: the probe never ran the operation, +// which is not the same as the actor being denied. +func probeFileOp(t *testing.T, ctx context.Context, rc *e2e.RouterClient, actor resources.ActorRef, endpoint, path string) fileResponse { + t.Helper() + resp, err := rc.Get(ctx, actor, endpoint+"?path="+url.QueryEscape(path)) + if err != nil { + t.Fatalf("GET %s for %s: %v", endpoint, actor, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + t.Fatalf("GET %s for %s: status %d, body %q", endpoint, actor, resp.StatusCode, body) + } + var out fileResponse + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + t.Fatalf("decoding %s for %s: %v", endpoint, actor, err) + } + return out +} diff --git a/internal/e2e/suites/security/testmain_test.go b/internal/e2e/suites/security/testmain_test.go new file mode 100644 index 0000000000..3f30d513f4 --- /dev/null +++ b/internal/e2e/suites/security/testmain_test.go @@ -0,0 +1,24 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package security + +import ( + "os" + "testing" + + "github.com/agent-substrate/substrate/internal/e2e" +) + +func TestMain(m *testing.M) { os.Exit(e2e.RunTestMain(m)) } From 9b76a6988a098a5069cfe2d192e269802b774c67 Mon Sep 17 00:00:00 2001 From: Zoe Zhao <35669350+zoez7@users.noreply.github.com> Date: Thu, 8 Oct 2026 22:17:01 +0000 Subject: [PATCH 16/44] SandboxConfig now holds multiple named versions with a default (#2333) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A SandboxConfig can now carry several sandbox versions, with `spec.defaultVersion` selecting the one actors boot with: ```yaml kind: SandboxConfig metadata: name: gvisor-default annotations: sandboxconfig.ate.dev/is-class-default: "true" spec: sandboxClass: gvisor defaultVersion: v1 versions: - name: v1 state: Enabled pauseImage: "registry.k8s.io/pause:3.10.2@sha256:..." assets: {...} - name: v0 state: Disabled pauseImage: "registry.k8s.io/pause:3.10.2@sha256:..." assets: {...} ``` - `spec.pauseImage` and `spec.assets` move into `spec.versions[]`. Each entry has `name` (a DNS label, unique in the list), `state` (`Enabled` by default, or `Disabled`), `pauseImage` and `assets`. A config holds 1–16 versions. - CEL requires `defaultVersion` to name an existing version that is not `Disabled`, and applies the per-class pauseImage rule to every version. - The ValidatingAdmissionPolicy checks the per-class required assets on every version. - The shipped `gvisor-default` and `microvm` configs now use `defaultVersion: v1` and carry the `sandboxconfig.ate.dev/is-class-default` annotation. More than one config of a class may carry it. Nothing reads it yet; a later change resolves it newest-wins. - ate-api and atelet prewarm read only the default version for now. #### Breaking change Every SandboxConfig must use the new shape. The apiVersion stays `v1alpha1` and there is no conversion, so once the CRD is updated, `spec.pauseImage` and `spec.assets` are pruned from existing objects, and ate-api rejects actors that name them (FailedPrecondition) until they are re-applied with `defaultVersion` and `versions`. The install re-applies `gvisor-default` and `microvm`; custom SandboxConfigs must be rewritten by hand. Roll out the CRD together with ate-api: an older ate-api reading a new-shape config gets an empty pause image. - [x] Tests pass - [x] Appropriate changes to documentation are included in the PR --- .golangci-kal.yaml | 4 +- .../controlapi/actor_template_test.go | 10 +- .../controlapi/functionaltest/common_test.go | 28 +-- .../internal/controlapi/sandbox_assets.go | 21 +- .../controlapi/sandbox_assets_test.go | 58 +++++- .../controlapi/workflow_resume_test.go | 19 +- cmd/atelet/sandbox_prewarm.go | 33 ++- cmd/atelet/sandbox_prewarm_test.go | 86 ++++++-- docs/api-guide.md | 41 ++-- .../generated/ate.dev_sandboxconfigs.yaml | 173 ++++++++++------ .../ate-install/sandboxconfig-gvisor.yaml | 34 +-- .../ate-install/sandboxconfig-validation.yaml | 22 +- .../microvm/sandboxconfig-microvm.yaml.tmpl | 64 +++--- pkg/api/v1alpha1/sandboxconfig_types.go | 103 ++++++++-- pkg/api/v1alpha1/sandboxconfig_types_test.go | 108 ++++++++++ .../v1alpha1/sandboxconfig_validation_test.go | 194 ++++++++++++++++-- pkg/api/v1alpha1/zz_generated.deepcopy.go | 33 ++- 17 files changed, 809 insertions(+), 222 deletions(-) create mode 100644 pkg/api/v1alpha1/sandboxconfig_types_test.go diff --git a/.golangci-kal.yaml b/.golangci-kal.yaml index 19efe7d4ac..fdfbdafb0b 100644 --- a/.golangci-kal.yaml +++ b/.golangci-kal.yaml @@ -96,7 +96,9 @@ linters: - path: 'pkg/api/v1alpha1/sandboxconfig_types\.go' text: '^requiredfields: .*\bAssetFile\.(SHA256|URL)\b' - path: 'pkg/api/v1alpha1/sandboxconfig_types\.go' - text: '^(nomaps|optionalfields|requiredfields): .*\bSandboxConfigSpec\.(Assets|PauseImage|SandboxClass)\b' + text: '^(optionalfields|requiredfields): .*\bSandboxConfigSpec\.SandboxClass\b' + - path: 'pkg/api/v1alpha1/sandboxconfig_types\.go' + text: '^(nomaps|optionalfields): .*\bSandboxVersionConfig\.(Assets|PauseImage)\b' - path: 'pkg/api/v1alpha1/sandboxconfig_types\.go' text: '^(nonpointerstructs|requiredfields): .*\bSandboxConfig\.Spec\b' - path: 'pkg/api/v1alpha1/csidriverconfig_types\.go' diff --git a/cmd/ateapi/internal/controlapi/actor_template_test.go b/cmd/ateapi/internal/controlapi/actor_template_test.go index 7fe9c23f70..06bc3e8cdf 100644 --- a/cmd/ateapi/internal/controlapi/actor_template_test.go +++ b/cmd/ateapi/internal/controlapi/actor_template_test.go @@ -62,9 +62,13 @@ func gvisorDefaultLister(t *testing.T) listersv1alpha1.SandboxConfigLister { return sandboxConfigListerFor(t, []*atev1alpha1.SandboxConfig{{ ObjectMeta: metav1.ObjectMeta{Name: "gvisor-default"}, Spec: atev1alpha1.SandboxConfigSpec{ - SandboxClass: atev1alpha1.SandboxClassGvisor, - PauseImage: "registry.k8s.io/pause@sha256:x", - Assets: testAssets(), + SandboxClass: atev1alpha1.SandboxClassGvisor, + DefaultVersion: "v1", + Versions: []atev1alpha1.SandboxVersionConfig{{ + Name: "v1", + PauseImage: "registry.k8s.io/pause@sha256:x", + Assets: testAssets(), + }}, }, }}) } diff --git a/cmd/ateapi/internal/controlapi/functionaltest/common_test.go b/cmd/ateapi/internal/controlapi/functionaltest/common_test.go index 7814bea8bb..5396522108 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/common_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/common_test.go @@ -505,18 +505,22 @@ func ensureGvisorSandboxConfig(t *testing.T, tc *testContext, name string) { sc := &atev1alpha1.SandboxConfig{ ObjectMeta: metav1.ObjectMeta{Name: name}, Spec: atev1alpha1.SandboxConfigSpec{ - SandboxClass: atev1alpha1.SandboxClassGvisor, - PauseImage: testPauseImage, - Assets: map[string]map[string]atev1alpha1.AssetFile{ - "amd64": {"runsc": { - URL: "gs://gvisor/releases/nightly/2026-05-19/x86_64/runsc", - SHA256: "a397be1abc2420d26bce6c70e6e2ff96c73aaaab929756c56f5e2089ea842b63", - }}, - "arm64": {"runsc": { - URL: "gs://gvisor/releases/nightly/2026-05-19/aarch64/runsc", - SHA256: "1ba2366ae2efceba166046f51a4104f9261c9cb72c6db8f5b3fe2dc57dea86b9", - }}, - }, + SandboxClass: atev1alpha1.SandboxClassGvisor, + DefaultVersion: "v1", + Versions: []atev1alpha1.SandboxVersionConfig{{ + Name: "v1", + PauseImage: testPauseImage, + Assets: map[string]map[string]atev1alpha1.AssetFile{ + "amd64": {"runsc": { + URL: "gs://gvisor/releases/nightly/2026-05-19/x86_64/runsc", + SHA256: "a397be1abc2420d26bce6c70e6e2ff96c73aaaab929756c56f5e2089ea842b63", + }}, + "arm64": {"runsc": { + URL: "gs://gvisor/releases/nightly/2026-05-19/aarch64/runsc", + SHA256: "1ba2366ae2efceba166046f51a4104f9261c9cb72c6db8f5b3fe2dc57dea86b9", + }}, + }, + }}, }, } if _, err := tc.substrateClient.ApiV1alpha1().SandboxConfigs().Create(context.Background(), sc, metav1.CreateOptions{}); err != nil && !apierrors.IsAlreadyExists(err) { diff --git a/cmd/ateapi/internal/controlapi/sandbox_assets.go b/cmd/ateapi/internal/controlapi/sandbox_assets.go index 51160fcc54..b7aa641cce 100644 --- a/cmd/ateapi/internal/controlapi/sandbox_assets.go +++ b/cmd/ateapi/internal/controlapi/sandbox_assets.go @@ -49,7 +49,7 @@ func resolveTemplateSandboxConfig( // resolveSandboxAssets determines the sandbox binaries and pause image an actor // should boot with and projects them onto the ateletpb.SandboxAssets atelet -// fetches: the SandboxConfig the ActorTemplate names via +// fetches: the default version of the SandboxConfig the ActorTemplate names via // sandbox_config.config_name (required; enforced by CreateActorTemplate), // checked against the template's sandbox_class. func resolveSandboxAssets( @@ -67,18 +67,23 @@ func resolveSandboxAssets( if err != nil { return nil, err } - return sandboxAssetsProto(sc), nil + ver, ok := sc.Spec.DefaultVersionConfig() + if !ok { + return nil, apierror.FailedPrecondition( + "SandboxConfig %q has no enabled version %q named by its defaultVersion", sc.Name, sc.Spec.DefaultVersion) + } + return sandboxAssetsProto(sc, ver), nil } -// sandboxAssetsProto converts a resolved SandboxConfig into the proto atelet -// consumes. -func sandboxAssetsProto(sc *atev1alpha1.SandboxConfig) *ateletpb.SandboxAssets { +// sandboxAssetsProto converts a version of a resolved SandboxConfig into the +// proto atelet consumes. +func sandboxAssetsProto(sc *atev1alpha1.SandboxConfig, ver *atev1alpha1.SandboxVersionConfig) *ateletpb.SandboxAssets { out := &ateletpb.SandboxAssets{ SandboxClass: string(sc.Spec.SandboxClass), - PauseImage: sc.Spec.PauseImage, - Assets: make(map[string]*ateletpb.ArchAssets, len(sc.Spec.Assets)), + PauseImage: ver.PauseImage, + Assets: make(map[string]*ateletpb.ArchAssets, len(ver.Assets)), } - for arch, files := range sc.Spec.Assets { + for arch, files := range ver.Assets { archAssets := &ateletpb.ArchAssets{Files: make(map[string]*ateletpb.AssetFile, len(files))} for name, f := range files { archAssets.Files[name] = &ateletpb.AssetFile{Url: f.URL, Sha256: f.SHA256} diff --git a/cmd/ateapi/internal/controlapi/sandbox_assets_test.go b/cmd/ateapi/internal/controlapi/sandbox_assets_test.go index b037714c23..a9a98e3a8b 100644 --- a/cmd/ateapi/internal/controlapi/sandbox_assets_test.go +++ b/cmd/ateapi/internal/controlapi/sandbox_assets_test.go @@ -23,6 +23,7 @@ import ( "github.com/agent-substrate/substrate/pkg/proto/ateapipb" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/cache" + "k8s.io/utils/ptr" ) // sandboxConfigListerFor builds a lister over the given SandboxConfigs, using @@ -45,17 +46,46 @@ func testAssets() map[string]map[string]atev1alpha1.AssetFile { } // TestResolveSandboxAssets pins the template-side resolution: the config the -// template names is resolved (with its class checked), an empty name or an -// unrecognized class is an error, and the pause image travels with the -// sandbox binaries. +// template names is resolved (with its class checked) to its default version, +// a missing default version, an empty name or an unrecognized class is an +// error, and the pause image travels with the sandbox binaries. func TestResolveSandboxAssets(t *testing.T) { const namedPause = "gcr.io/gke-release/pause@sha256:named" namedConfig := &atev1alpha1.SandboxConfig{ ObjectMeta: metav1.ObjectMeta{Name: "gvisor-custom"}, Spec: atev1alpha1.SandboxConfigSpec{ - SandboxClass: atev1alpha1.SandboxClassGvisor, - PauseImage: namedPause, - Assets: testAssets(), + SandboxClass: atev1alpha1.SandboxClassGvisor, + DefaultVersion: "v2", + Versions: []atev1alpha1.SandboxVersionConfig{{ + Name: "v1", + PauseImage: "gcr.io/gke-release/pause@sha256:old", + Assets: testAssets(), + }, { + Name: "v2", + PauseImage: namedPause, + Assets: testAssets(), + }}, + }, + } + // A config whose defaultVersion names no entry; the CRD rejects this, but + // an object stored before the versions schema reads back this way. + noDefaultConfig := &atev1alpha1.SandboxConfig{ + ObjectMeta: metav1.ObjectMeta{Name: "gvisor-no-default"}, + Spec: atev1alpha1.SandboxConfigSpec{SandboxClass: atev1alpha1.SandboxClassGvisor}, + } + // A config whose defaultVersion names a Disabled entry; the CRD rejects + // this too, and it must not resolve to the disabled version. + disabledDefaultConfig := &atev1alpha1.SandboxConfig{ + ObjectMeta: metav1.ObjectMeta{Name: "gvisor-disabled-default"}, + Spec: atev1alpha1.SandboxConfigSpec{ + SandboxClass: atev1alpha1.SandboxClassGvisor, + DefaultVersion: "v1", + Versions: []atev1alpha1.SandboxVersionConfig{{ + Name: "v1", + State: ptr.To(atev1alpha1.SandboxVersionStateDisabled), + PauseImage: namedPause, + Assets: testAssets(), + }}, }, } @@ -71,6 +101,20 @@ func TestResolveSandboxAssets(t *testing.T) { ConfigName: "gvisor-custom", }, wantPauseImage: namedPause, + }, { + name: "named config without its default version", + sandbox: &ateapipb.SandboxConfig{ + SandboxClass: ateapipb.SandboxClass_SANDBOX_CLASS_GVISOR, + ConfigName: "gvisor-no-default", + }, + wantErr: `SandboxConfig "gvisor-no-default" has no enabled version ""`, + }, { + name: "named config whose default version is disabled", + sandbox: &ateapipb.SandboxConfig{ + SandboxClass: ateapipb.SandboxClass_SANDBOX_CLASS_GVISOR, + ConfigName: "gvisor-disabled-default", + }, + wantErr: `SandboxConfig "gvisor-disabled-default" has no enabled version "v1"`, }, { name: "named config class mismatch", sandbox: &ateapipb.SandboxConfig{ @@ -99,7 +143,7 @@ func TestResolveSandboxAssets(t *testing.T) { }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - configLister := sandboxConfigListerFor(t, []*atev1alpha1.SandboxConfig{namedConfig}) + configLister := sandboxConfigListerFor(t, []*atev1alpha1.SandboxConfig{namedConfig, noDefaultConfig, disabledDefaultConfig}) got, err := resolveSandboxAssets(configLister, tt.sandbox) if tt.wantErr != "" { diff --git a/cmd/ateapi/internal/controlapi/workflow_resume_test.go b/cmd/ateapi/internal/controlapi/workflow_resume_test.go index 859a03862f..e519d182f2 100644 --- a/cmd/ateapi/internal/controlapi/workflow_resume_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_resume_test.go @@ -1110,9 +1110,13 @@ func newWireCaptureWorkflow(t *testing.T, persistence store.Interface) (*ActorWo lister := sandboxConfigListerFor(t, []*atev1alpha1.SandboxConfig{{ ObjectMeta: metav1.ObjectMeta{Name: "gvisor"}, Spec: atev1alpha1.SandboxConfigSpec{ - SandboxClass: atev1alpha1.SandboxClassGvisor, - PauseImage: "pause@sha256:abc", - Assets: testAssets(), + SandboxClass: atev1alpha1.SandboxClassGvisor, + DefaultVersion: "v1", + Versions: []atev1alpha1.SandboxVersionConfig{{ + Name: "v1", + PauseImage: "pause@sha256:abc", + Assets: testAssets(), + }}, }, }}) @@ -1385,11 +1389,10 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { // Every request atelet receives, Run or Restore, carries the sandbox the // template's SandboxConfig resolves to. - wantSandboxAssets := sandboxAssetsProto(&atev1alpha1.SandboxConfig{Spec: atev1alpha1.SandboxConfigSpec{ - SandboxClass: atev1alpha1.SandboxClassGvisor, - PauseImage: "pause@sha256:abc", - Assets: testAssets(), - }}) + wantSandboxAssets := sandboxAssetsProto( + &atev1alpha1.SandboxConfig{Spec: atev1alpha1.SandboxConfigSpec{SandboxClass: atev1alpha1.SandboxClassGvisor}}, + &atev1alpha1.SandboxVersionConfig{Name: "v1", PauseImage: "pause@sha256:abc", Assets: testAssets()}, + ) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/cmd/atelet/sandbox_prewarm.go b/cmd/atelet/sandbox_prewarm.go index deaa413057..1395643cea 100644 --- a/cmd/atelet/sandbox_prewarm.go +++ b/cmd/atelet/sandbox_prewarm.go @@ -257,18 +257,27 @@ func (p *sandboxPrewarmer) prewarm(ctx context.Context, cfg *v1alpha1.SandboxCon defer cancel() t := time.Now() + // TODO: prewarm every Enabled version, not just the default. + ver, ok := cfg.Spec.DefaultVersionConfig() + if !ok { + // The CRD rejects this shape, so this should not be reachable. + slog.WarnContext(ctx, "Skipping sandbox asset prewarm: config has no enabled default version", + slog.String("config", cfg.Name), slog.String("defaultVersion", cfg.Spec.DefaultVersion)) + return nil + } + var imageErr error var wg sync.WaitGroup // schedule prewarm pause image if provided - if cfg.Spec.PauseImage != "" { + if ver.PauseImage != "" { wg.Go(func() { - if _, err := p.images.EnsureImage(ctx, cfg.Spec.PauseImage); err != nil { - imageErr = fmt.Errorf("while prewarming pause image %q: %w", cfg.Spec.PauseImage, err) + if _, err := p.images.EnsureImage(ctx, ver.PauseImage); err != nil { + imageErr = fmt.Errorf("while prewarming pause image %q: %w", ver.PauseImage, err) } }) } // schedule prewarm sandbox assets if provided - rec, assetErr := recordFromSandboxConfig(cfg) + rec, assetErr := recordFromSandboxConfig(cfg, ver) switch { case errors.Is(assetErr, errNoAssetsForArch): // Permanent until the config changes, and a change re-enqueues: @@ -292,8 +301,9 @@ func (p *sandboxPrewarmer) prewarm(ctx context.Context, cfg *v1alpha1.SandboxCon } slog.InfoContext(ctx, "Sandbox assets prewarmed", slog.String("config", cfg.Name), + slog.String("version", ver.Name), slog.Int("assets", assets), - slog.String("pauseImage", cfg.Spec.PauseImage), + slog.String("pauseImage", ver.PauseImage), slog.Duration("duration", time.Since(t))) return nil } @@ -303,17 +313,18 @@ func (p *sandboxPrewarmer) prewarm(ctx context.Context, cfg *v1alpha1.SandboxCon // config changes, so prewarm treats it as nothing-to-do rather than retrying. var errNoAssetsForArch = errors.New("no sandbox assets for this architecture") -// recordFromSandboxConfig projects a SandboxConfig's per-architecture assets -// onto the local node's architecture, mirroring recordFromRequest. -func recordFromSandboxConfig(cfg *v1alpha1.SandboxConfig) (*sandboxAssetsRecord, error) { +// recordFromSandboxConfig projects the per-architecture assets of one version +// of a SandboxConfig onto the local node's architecture, mirroring +// recordFromRequest. +func recordFromSandboxConfig(cfg *v1alpha1.SandboxConfig, ver *v1alpha1.SandboxVersionConfig) (*sandboxAssetsRecord, error) { arch := runtime.GOARCH - files := cfg.Spec.Assets[arch] + files := ver.Assets[arch] if len(files) == 0 { - return nil, fmt.Errorf("sandbox config %q, architecture %q: %w", cfg.Name, arch, errNoAssetsForArch) + return nil, fmt.Errorf("sandbox config %q, version %q, architecture %q: %w", cfg.Name, ver.Name, arch, errNoAssetsForArch) } rec := &sandboxAssetsRecord{ SandboxClass: string(cfg.Spec.SandboxClass), - PauseImage: cfg.Spec.PauseImage, + PauseImage: ver.PauseImage, Assets: make(map[string]assetEntry, len(files)), } for name, f := range files { diff --git a/cmd/atelet/sandbox_prewarm_test.go b/cmd/atelet/sandbox_prewarm_test.go index e29d7cd83f..004fda2e00 100644 --- a/cmd/atelet/sandbox_prewarm_test.go +++ b/cmd/atelet/sandbox_prewarm_test.go @@ -30,6 +30,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/cache" + "k8s.io/utils/ptr" "github.com/agent-substrate/substrate/cmd/atelet/internal/ateletpath" "github.com/agent-substrate/substrate/internal/imagecache" @@ -72,13 +73,17 @@ func gvisorConfig(name, url, sha string) *v1alpha1.SandboxConfig { return &v1alpha1.SandboxConfig{ ObjectMeta: metav1.ObjectMeta{Name: name}, Spec: v1alpha1.SandboxConfigSpec{ - SandboxClass: v1alpha1.SandboxClassGvisor, - PauseImage: "registry.k8s.io/pause@sha256:abc", - Assets: map[string]map[string]v1alpha1.AssetFile{ - runtime.GOARCH: { - runscAssetName: {URL: url, SHA256: sha}, + SandboxClass: v1alpha1.SandboxClassGvisor, + DefaultVersion: "v1", + Versions: []v1alpha1.SandboxVersionConfig{{ + Name: "v1", + PauseImage: "registry.k8s.io/pause@sha256:abc", + Assets: map[string]map[string]v1alpha1.AssetFile{ + runtime.GOARCH: { + runscAssetName: {URL: url, SHA256: sha}, + }, }, - }, + }}, }, } } @@ -87,15 +92,15 @@ func TestRecordFromSandboxConfig(t *testing.T) { sha := fmt.Sprintf("%x", sha256.Sum256([]byte("runsc"))) cfg := gvisorConfig("gvisor-default", "gs://bucket/runsc", sha) - rec, err := recordFromSandboxConfig(cfg) + rec, err := recordFromSandboxConfig(cfg, &cfg.Spec.Versions[0]) if err != nil { t.Fatalf("recordFromSandboxConfig: %v", err) } if rec.SandboxClass != string(v1alpha1.SandboxClassGvisor) { t.Errorf("SandboxClass = %q, want %q", rec.SandboxClass, v1alpha1.SandboxClassGvisor) } - if rec.PauseImage != cfg.Spec.PauseImage { - t.Errorf("PauseImage = %q, want %q", rec.PauseImage, cfg.Spec.PauseImage) + if rec.PauseImage != cfg.Spec.Versions[0].PauseImage { + t.Errorf("PauseImage = %q, want %q", rec.PauseImage, cfg.Spec.Versions[0].PauseImage) } want := assetEntry{URL: "gs://bucket/runsc", SHA256: sha} if got := rec.Assets[runscAssetName]; got != want { @@ -105,10 +110,10 @@ func TestRecordFromSandboxConfig(t *testing.T) { // A config with no assets for this node's architecture cannot be // projected, and the error carries the sentinel that keeps prewarm from // retrying a condition only a config change can clear. - cfg.Spec.Assets = map[string]map[string]v1alpha1.AssetFile{ + cfg.Spec.Versions[0].Assets = map[string]map[string]v1alpha1.AssetFile{ "other-arch": {runscAssetName: {URL: "gs://bucket/runsc", SHA256: sha}}, } - if _, err := recordFromSandboxConfig(cfg); !errors.Is(err, errNoAssetsForArch) { + if _, err := recordFromSandboxConfig(cfg, &cfg.Spec.Versions[0]); !errors.Is(err, errNoAssetsForArch) { t.Errorf("recordFromSandboxConfig with no assets for the local architecture = %v, want errNoAssetsForArch", err) } } @@ -170,7 +175,7 @@ func TestPrewarmProcessRetries(t *testing.T) { ctx := context.Background() cfg := gvisorConfig("gvisor-default", "gs://bucket/runsc", fmt.Sprintf("%x", sha256.Sum256([]byte("runsc")))) // No pause image, so a failing prewarm exercises only the asset path. - cfg.Spec.PauseImage = "" + cfg.Spec.Versions[0].PauseImage = "" indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{}) if err := indexer.Add(cfg); err != nil { @@ -268,7 +273,7 @@ func TestPrewarmPauseImage(t *testing.T) { ctx := context.Background() content := []byte("runsc binary bytes") cfg := gvisorConfig("gvisor-default", "gs://bucket/runsc", fmt.Sprintf("%x", sha256.Sum256(content))) - cfg.Spec.PauseImage = pauseRef + cfg.Spec.Versions[0].PauseImage = pauseRef p := &sandboxPrewarmer{ assets: &AteomHerder{anonGCSClient: fakeObjectStorage{data: content}}, @@ -282,7 +287,7 @@ func TestPrewarmPauseImage(t *testing.T) { // failing object storage is actually consulted — and must not keep the // pause image from being pulled. failCfg := gvisorConfig("gvisor-default", "gs://bucket/runsc", fmt.Sprintf("%x", sha256.Sum256([]byte("other runsc")))) - failCfg.Spec.PauseImage = pauseRef + failCfg.Spec.Versions[0].PauseImage = pauseRef p = &sandboxPrewarmer{ assets: &AteomHerder{anonGCSClient: fakeObjectStorage{err: errors.New("bucket unavailable")}}, images: failStore, @@ -295,10 +300,10 @@ func TestPrewarmPauseImage(t *testing.T) { // pause image pulled, and prewarm succeeds: the missing assets are // permanent until the config changes, not a retryable failure. archCfg := gvisorConfig("gvisor-default", "gs://bucket/runsc", fmt.Sprintf("%x", sha256.Sum256(content))) - archCfg.Spec.Assets = map[string]map[string]v1alpha1.AssetFile{ - "other-arch": {runscAssetName: archCfg.Spec.Assets[runtime.GOARCH][runscAssetName]}, + archCfg.Spec.Versions[0].Assets = map[string]map[string]v1alpha1.AssetFile{ + "other-arch": {runscAssetName: archCfg.Spec.Versions[0].Assets[runtime.GOARCH][runscAssetName]}, } - archCfg.Spec.PauseImage = pauseRef + archCfg.Spec.Versions[0].PauseImage = pauseRef p = &sandboxPrewarmer{images: archStore} if err := p.prewarm(ctx, archCfg); err != nil { t.Errorf("prewarm with no assets for the local architecture: %v", err) @@ -335,7 +340,7 @@ func TestPrewarmTimeout(t *testing.T) { t.Cleanup(func() { nodepath.StaticFilesDir, prewarmTimeout = origDir, origTimeout }) cfg := gvisorConfig("gvisor-default", "gs://bucket/runsc", fmt.Sprintf("%x", sha256.Sum256([]byte("hung runsc")))) - cfg.Spec.PauseImage = "" + cfg.Spec.Versions[0].PauseImage = "" p := &sandboxPrewarmer{assets: &AteomHerder{anonGCSClient: hangingObjectStorage{}}} done := make(chan error, 1) @@ -350,6 +355,49 @@ func TestPrewarmTimeout(t *testing.T) { } } +// TestPrewarmDefaultVersionOnly verifies prewarm fetches only the version +// defaultVersion names, and that a config without its default version, or +// whose default version is Disabled, is nothing to do rather than a retryable +// failure. +func TestPrewarmDefaultVersionOnly(t *testing.T) { + origDir := nodepath.StaticFilesDir + nodepath.StaticFilesDir = t.TempDir() + t.Cleanup(func() { nodepath.StaticFilesDir = origDir }) + + ctx := context.Background() + content := []byte("runsc binary bytes") + cfg := gvisorConfig("gvisor-default", "gs://bucket/runsc", fmt.Sprintf("%x", sha256.Sum256(content))) + cfg.Spec.Versions[0].PauseImage = "" + // A non-default version whose asset the fake bucket cannot serve: fetching + // it would fail the hash check. + otherSHA := fmt.Sprintf("%x", sha256.Sum256([]byte("other runsc"))) + cfg.Spec.Versions = append([]v1alpha1.SandboxVersionConfig{{ + Name: "v0", + Assets: map[string]map[string]v1alpha1.AssetFile{runtime.GOARCH: {runscAssetName: {URL: "gs://bucket/other-runsc", SHA256: otherSHA}}}, + }}, cfg.Spec.Versions...) + + p := &sandboxPrewarmer{assets: &AteomHerder{anonGCSClient: fakeObjectStorage{data: content}}} + if err := p.prewarm(ctx, cfg); err != nil { + t.Fatalf("prewarm: %v", err) + } + if _, err := os.Stat(ateletpath.RunSCBinaryPath(otherSHA)); !errors.Is(err, os.ErrNotExist) { + t.Errorf("non-default version asset was prewarmed (stat err = %v)", err) + } + + // Any fetch would fail, so a nil error means nothing was attempted. + cfg.Spec.DefaultVersion = "missing" + p = &sandboxPrewarmer{assets: &AteomHerder{anonGCSClient: fakeObjectStorage{err: errors.New("bucket unavailable")}}} + if err := p.prewarm(ctx, cfg); err != nil { + t.Errorf("prewarm of a config without its default version: %v", err) + } + + cfg.Spec.DefaultVersion = "v0" + cfg.Spec.Versions[0].State = ptr.To(v1alpha1.SandboxVersionStateDisabled) + if err := p.prewarm(ctx, cfg); err != nil { + t.Errorf("prewarm of a config whose default version is Disabled: %v", err) + } +} + // TestSandboxAssetPrewarmDownloads runs the whole path: a SandboxConfig in a // fake clientset flows through the informer into the prewarm worker, which // lands the asset in the static-files cache without any Run/Restore request. @@ -366,7 +414,7 @@ func TestSandboxAssetPrewarmDownloads(t *testing.T) { content := []byte("runsc binary bytes") sha := fmt.Sprintf("%x", sha256.Sum256(content)) cfg := gvisorConfig("gvisor-default", "gs://bucket/runsc", sha) - cfg.Spec.PauseImage = pauseRef + cfg.Spec.Versions[0].PauseImage = pauseRef ctx := t.Context() client := fake.NewSimpleClientset(cfg) diff --git a/docs/api-guide.md b/docs/api-guide.md index 75c13b2e86..4d4a00787c 100644 --- a/docs/api-guide.md +++ b/docs/api-guide.md @@ -396,7 +396,7 @@ One consequence worth planning for: **a published snapshot is read from the ates ## 3. SandboxConfig: The Sandbox Itself -`SandboxConfig` is a **cluster-scoped** resource that decouples the sandbox — its binaries (the gVisor `runsc` binary, or a micro-VM kernel/firmware/config) and the `pauseImage` that holds the sandbox's namespaces — from the workload definition in the `ActorTemplate`. An actor's cold boot resolves the sandbox binaries from the config its `ActorTemplate` names via `sandboxConfig.configName`. +`SandboxConfig` is a **cluster-scoped** resource that decouples the sandbox — its binaries (the gVisor `runsc` binary, or a micro-VM kernel/firmware/config) and the `pauseImage` that holds the sandbox's namespaces — from the workload definition in the `ActorTemplate`. An actor's cold boot resolves the sandbox binaries from the default version of the config its `ActorTemplate` names via `sandboxConfig.configName`. This means a single, cluster-managed config pins the sandbox runtime version for many templates: snapshots stay restorable because the version is recorded in each snapshot's manifest, and operators upgrade the runtime in one place. @@ -405,8 +405,17 @@ This means a single, cluster-managed config pins the sandbox runtime version for | Field | Type | Description | | :--- | :--- | :--- | | `sandboxClass` | `string` | **Required.** Runtime family this config applies to: `gvisor` (default) or `microvm`. An `ActorTemplate` only uses `SandboxConfig`s whose `sandboxClass` matches its own. | +| `defaultVersion` | `string` | **Required.** Name of the entry in `versions` that actors boot with. It must name an existing version that is not `Disabled`. | +| `versions` | `[]SandboxVersionConfig` | **Required.** 1–16 sandbox versions, unique by `name`. | + +Each entry of `versions` (`SandboxVersionConfig`): + +| Field | Type | Description | +| :--- | :--- | :--- | +| `name` | `string` | **Required.** Identifies the version within the config. A DNS label (lower-case alphanumerics and `-`, at most 63 characters). | +| `state` | `string` | Optional. `Enabled` (default) or `Disabled`. The version named by `defaultVersion` cannot be `Disabled`. | | `pauseImage` | `string` | **Required for `gvisor`; not allowed for `microvm`**, which runs no pause container. The image for the sandbox's root container (e.g. `registry.k8s.io/pause`, or `gcr.io/gke-release/pause` on GKE). Must include a digest (`...@sha256:...`) — it is recorded in each snapshot's manifest so a restore rebuilds the sandbox from the same image. | -| `assets` | `map[arch]map[name]AssetFile` | Optional. Content-addressed files atelet fetches, keyed by architecture (`amd64`, `arm64`) then asset name. gVisor expects a `gvisor` asset (the release's `gvisor.tar.zstd`), which atelet auto-extracts. A micro-VM backend expects several. Each `AssetFile` is a `{ url, sha256 }` pair. | +| `assets` | `map[arch]map[name]AssetFile` | Content-addressed files atelet fetches, keyed by architecture (`amd64`, `arm64`) then asset name. gVisor expects a `gvisor` asset (the release's `gvisor.tar.zstd`), which atelet auto-extracts. A micro-VM backend expects several. Each `AssetFile` is a `{ url, sha256 }` pair. A `ValidatingAdmissionPolicy` enforces each class's required assets on every version. | A cluster-wide gVisor `SandboxConfig` (`gvisor-default`) is installed with the platform, so gVisor templates can name it via `sandboxConfig.configName` without any extra setup. @@ -417,23 +426,29 @@ apiVersion: ate.dev/v1alpha1 kind: SandboxConfig metadata: name: gvisor-default + annotations: + sandboxconfig.ate.dev/is-class-default: "true" spec: sandboxClass: gvisor - pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" - assets: - amd64: - gvisor: - url: "gs://gvisor/releases/nightly/2026-09-02/x86_64/gvisor.tar.zstd" - sha256: "d547d81401461fd1c679c5c4fa0a6c2b8ef7dc3c22ce23c9e25dcc4c69cfd06f" - arm64: - gvisor: - url: "gs://gvisor/releases/nightly/2026-09-02/aarch64/gvisor.tar.zstd" - sha256: "a64916f9813ce7e4841a30480a599337f7dda07b421c6bf0123db2212aa7d1df" + defaultVersion: v1 + versions: + - name: v1 + state: Enabled + pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" + assets: + amd64: + gvisor: + url: "gs://gvisor/releases/nightly/2026-09-02/x86_64/gvisor.tar.zstd" + sha256: "d547d81401461fd1c679c5c4fa0a6c2b8ef7dc3c22ce23c9e25dcc4c69cfd06f" + arm64: + gvisor: + url: "gs://gvisor/releases/nightly/2026-09-02/aarch64/gvisor.tar.zstd" + sha256: "a64916f9813ce7e4841a30480a599337f7dda07b421c6bf0123db2212aa7d1df" ``` ### Micro-VM SandboxConfig -A `microvm` `SandboxConfig` supplies the [Kata Containers](https://katacontainers.io/) + [Cloud Hypervisor](https://www.cloudhypervisor.org/) toolchain instead of `runsc`. Each architecture must define the full asset set — `cloud-hypervisor`, `virtiofsd`, `kata-kernel`, and `kata-image` — which a `ValidatingAdmissionPolicy` enforces at apply time. Worker pods for a micro-VM pool require `/dev/kvm` and nested-virtualization-capable nodes. The controller requests those devices on the pod automatically, and atelet advertises them only where they exist, so placement follows the hardware rather than a node label. Clusters that reserve nested-virt nodes with an `ate.dev/sandboxClass=microvm` taint are still tolerated: advertising a device attracts these pods to capable nodes but repels nothing else from them. The same convention applies to every class: worker pods of a pool tolerate `ate.dev/sandboxClass=:NoSchedule`, so a cluster can reserve a node pool per sandbox class with that taint, and the atelet DaemonSet tolerates the key for any value. +A `microvm` `SandboxConfig` supplies the [Kata Containers](https://katacontainers.io/) + [Cloud Hypervisor](https://www.cloudhypervisor.org/) toolchain instead of `runsc`. Each architecture of each version must define the full asset set — `cloud-hypervisor`, `virtiofsd`, `kata-kernel`, and `kata-image` — which a `ValidatingAdmissionPolicy` enforces at apply time. Worker pods for a micro-VM pool require `/dev/kvm` and nested-virtualization-capable nodes. The controller requests those devices on the pod automatically, and atelet advertises them only where they exist, so placement follows the hardware rather than a node label. Clusters that reserve nested-virt nodes with an `ate.dev/sandboxClass=microvm` taint are still tolerated: advertising a device attracts these pods to capable nodes but repels nothing else from them. The same convention applies to every class: worker pods of a pool tolerate `ate.dev/sandboxClass=:NoSchedule`, so a cluster can reserve a node pool per sandbox class with that taint, and the atelet DaemonSet tolerates the key for any value. See [`hack/microvm-assets/`](../hack/microvm-assets/) for scripts that assemble and stage these assets, plus a worked counter demo (`demos/counter/counter-microvm.yaml.tmpl`) that suspends and resumes an in-RAM counter across worker pods. diff --git a/manifests/ate-install/generated/ate.dev_sandboxconfigs.yaml b/manifests/ate-install/generated/ate.dev_sandboxconfigs.yaml index a41ada36a9..7d3e919371 100644 --- a/manifests/ate-install/generated/ate.dev_sandboxconfigs.yaml +++ b/manifests/ate-install/generated/ate.dev_sandboxconfigs.yaml @@ -34,6 +34,9 @@ spec: - jsonPath: .spec.sandboxClass name: Class type: string + - jsonPath: .spec.defaultVersion + name: Default + type: string - jsonPath: .metadata.creationTimestamp name: Age type: date @@ -41,10 +44,10 @@ spec: schema: openAPIV3Schema: description: |- - SandboxConfig is cluster-scoped configuration describing the sandbox binaries - for a sandbox runtime family. It is referenced by an ActorTemplate's - sandbox_config.config_name (required) and decouples - sandbox binary selection from the workload definition. + SandboxConfig is cluster-scoped configuration describing the versions of the + sandbox binaries for a sandbox runtime family. It is referenced by an + ActorTemplate's sandbox_config.config_name (required) and decouples sandbox + binary selection from the workload definition. properties: apiVersion: description: |- @@ -66,60 +69,13 @@ spec: spec: description: spec defines the desired state of SandboxConfig properties: - assets: - additionalProperties: - additionalProperties: - description: |- - AssetFile is one content-addressed file that atelet fetches for a sandbox - runtime (e.g. the gVisor runsc binary, or a micro-VM kernel/firmware/config). - properties: - sha256: - description: |- - SHA256 is the lower-case hex SHA256 of the asset. It both names the cached - file (preventing collisions) and verifies the download's integrity. - pattern: ^[a-f0-9]{64}$ - type: string - url: - description: |- - URL is where to download the asset from (e.g. a gs:// URL). It may be - fetched anonymously or with credentials depending on atelet's - configuration. - minLength: 1 - type: string - required: - - sha256 - - url - type: object - type: object + defaultVersion: description: |- - Assets is the set of files atelet fetches for this runtime, keyed first by - architecture (GOARCH, e.g. "amd64", "arm64") and then by asset name. The - asset names are interpreted by the sandbox backend: gVisor expects a - "gvisor" asset (the release's gvisor.tar.zstd, which atelet extracts so - the gvisor-bin/ helpers sit next to runsc; a legacy bare-binary "runsc" - asset is still accepted); a micro-VM backend expects several (e.g. - "cloud-hypervisor", "kata-kernel", "kata-image"). The schema is - intentionally generic; per-class requirements are enforced by a - ValidatingAdmissionPolicy. - type: object - pauseImage: - description: |- - PauseImage is the container image used as the root sandbox container. - It holds the sandbox's namespaces and runs no workload code, so it is an - implementation detail of the sandbox rather than something actor authors - choose. Required for gvisor; not allowed for microvm, which runs no pause - container. It is captured in the snapshot manifest alongside the sandbox - binaries, so a restore always re-creates the sandbox from the same image - the snapshot was taken with. - - Typically, set it to [1] for on-gcp, and [2] for off-gcp - - - [1] gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da - - [2] registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4 + defaultVersion names the entry in versions that new sandboxes boot with. + It must name an existing, non-Disabled version. + maxLength: 63 + minLength: 1 type: string - x-kubernetes-validations: - - message: All images must include a digest - rule: self.contains('@') sandboxClass: default: gvisor description: |- @@ -130,13 +86,112 @@ spec: - gvisor - microvm type: string + versions: + description: versions are the sandbox versions this config offers, + keyed by name. + items: + description: |- + SandboxVersionConfig is one version of a SandboxConfig: the exact pause image + and asset set a sandbox boots with. + properties: + assets: + additionalProperties: + additionalProperties: + description: |- + AssetFile is one content-addressed file that atelet fetches for a sandbox + runtime (e.g. the gVisor runsc binary, or a micro-VM kernel/firmware/config). + properties: + sha256: + description: |- + SHA256 is the lower-case hex SHA256 of the asset. It both names the cached + file (preventing collisions) and verifies the download's integrity. + pattern: ^[a-f0-9]{64}$ + type: string + url: + description: |- + URL is where to download the asset from (e.g. a gs:// URL). It may be + fetched anonymously or with credentials depending on atelet's + configuration. + minLength: 1 + type: string + required: + - sha256 + - url + type: object + type: object + description: |- + Assets is the set of files atelet fetches for this runtime, keyed first by + architecture (GOARCH, e.g. "amd64", "arm64") and then by asset name. The + asset names are interpreted by the sandbox backend: gVisor expects a + "gvisor" asset (the release's gvisor.tar.zstd, which atelet extracts so + the gvisor-bin/ helpers sit next to runsc; a legacy bare-binary "runsc" + asset is still accepted); a micro-VM backend expects several (e.g. + "cloud-hypervisor", "kata-kernel", "kata-image"). The schema is + intentionally generic; per-class requirements are enforced by a + ValidatingAdmissionPolicy. + type: object + name: + description: |- + name identifies this version within its SandboxConfig. It is a DNS label + (lower-case alphanumerics and '-'). + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + pauseImage: + description: |- + PauseImage is the container image used as the root sandbox container. + It holds the sandbox's namespaces and runs no workload code, so it is an + implementation detail of the sandbox rather than something actor authors + choose. Required for gvisor; not allowed for microvm, which runs no pause + container. It is captured in the snapshot manifest alongside the sandbox + binaries, so a restore always re-creates the sandbox from the same image + the snapshot was taken with. + + Typically, set it to [1] for on-gcp, and [2] for off-gcp + + - [1] gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da + - [2] registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4 + maxLength: 512 + minLength: 1 + type: string + x-kubernetes-validations: + - message: All images must include a digest + rule: self.contains('@') + state: + default: Enabled + description: |- + state is whether this version may be used: Enabled (default) or + Disabled. + enum: + - Enabled + - Disabled + type: string + required: + - name + type: object + maxItems: 16 + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map required: + - defaultVersion - sandboxClass + - versions type: object x-kubernetes-validations: - - message: pauseImage is required for gvisor and not allowed for other - sandbox classes - rule: 'self.sandboxClass == ''gvisor'' ? has(self.pauseImage) : !has(self.pauseImage)' + - message: defaultVersion must name an entry in versions + rule: self.versions.exists(v, v.name == self.defaultVersion) + - message: the version named by defaultVersion must not be Disabled + rule: '!self.versions.exists(v, v.name == self.defaultVersion && has(v.state) + && v.state == ''Disabled'')' + - message: pauseImage is required on every version for gvisor and not + allowed for other sandbox classes + rule: 'self.sandboxClass == ''gvisor'' ? self.versions.all(v, has(v.pauseImage) + && size(v.pauseImage) > 0) : self.versions.all(v, !has(v.pauseImage) + || size(v.pauseImage) == 0)' required: - spec type: object diff --git a/manifests/ate-install/sandboxconfig-gvisor.yaml b/manifests/ate-install/sandboxconfig-gvisor.yaml index 5ab2d1e251..77f3654bf9 100644 --- a/manifests/ate-install/sandboxconfig-gvisor.yaml +++ b/manifests/ate-install/sandboxconfig-gvisor.yaml @@ -17,23 +17,29 @@ # sandboxConfig.configName. atelet fetches the gVisor release tarball # (gvisor.tar.zstd: runsc plus the gvisor-bin/ helpers runsc requires next to # it) matching the worker node's architecture and extracts it locally. To pin a -# different release, edit the assets below or create another SandboxConfig and -# name it from the ActorTemplate. +# different release, add a version below and point defaultVersion at it, or +# create another SandboxConfig and name it from the ActorTemplate. apiVersion: ate.dev/v1alpha1 kind: SandboxConfig metadata: name: gvisor-default + annotations: + sandboxconfig.ate.dev/is-class-default: "true" spec: sandboxClass: gvisor - # The root sandbox container's image. On GCP, prefer the in-project mirror - # gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da. - pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" - assets: - amd64: - gvisor: - url: "gs://gvisor/releases/nightly/2026-09-02/x86_64/gvisor.tar.zstd" - sha256: "d547d81401461fd1c679c5c4fa0a6c2b8ef7dc3c22ce23c9e25dcc4c69cfd06f" - arm64: - gvisor: - url: "gs://gvisor/releases/nightly/2026-09-02/aarch64/gvisor.tar.zstd" - sha256: "a64916f9813ce7e4841a30480a599337f7dda07b421c6bf0123db2212aa7d1df" + defaultVersion: v1 + versions: + - name: v1 + state: Enabled + # The root sandbox container's image. On GCP, prefer the in-project mirror + # gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da. + pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" + assets: + amd64: + gvisor: + url: "gs://gvisor/releases/nightly/2026-09-02/x86_64/gvisor.tar.zstd" + sha256: "d547d81401461fd1c679c5c4fa0a6c2b8ef7dc3c22ce23c9e25dcc4c69cfd06f" + arm64: + gvisor: + url: "gs://gvisor/releases/nightly/2026-09-02/aarch64/gvisor.tar.zstd" + sha256: "a64916f9813ce7e4841a30480a599337f7dda07b421c6bf0123db2212aa7d1df" diff --git a/manifests/ate-install/sandboxconfig-validation.yaml b/manifests/ate-install/sandboxconfig-validation.yaml index 354acfecec..1e77e6af02 100644 --- a/manifests/ate-install/sandboxconfig-validation.yaml +++ b/manifests/ate-install/sandboxconfig-validation.yaml @@ -14,7 +14,8 @@ # Per-sandbox-class asset requirements for SandboxConfig. The CRD schema is # generic (any arch -> any asset name -> {url, sha256}); this policy enforces the -# requirements a given sandbox class actually needs, fail-closed at apply time. +# requirements a given sandbox class actually needs on every version under +# spec.versions, fail-closed at apply time. # (url/sha256 being required and well-formed is enforced by the CRD schema.) apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingAdmissionPolicy @@ -35,19 +36,20 @@ spec: # tarball. - expression: >- object.spec.sandboxClass != 'gvisor' || - (has(object.spec.assets) && size(object.spec.assets) > 0 && - object.spec.assets.all(arch, - 'gvisor' in object.spec.assets[arch] || 'runsc' in object.spec.assets[arch])) - message: "a gvisor SandboxConfig must define a 'gvisor' (release tarball) or legacy 'runsc' asset for every architecture under spec.assets" + object.spec.versions.all(v, + has(v.assets) && size(v.assets) > 0 && + v.assets.all(arch, 'gvisor' in v.assets[arch] || 'runsc' in v.assets[arch])) + message: "a gvisor SandboxConfig must define a 'gvisor' (release tarball) or legacy 'runsc' asset for every architecture of every version under spec.versions[].assets" # The micro-VM (cloud-hypervisor) runtime needs its asset set for every # architecture it advertises. - expression: >- object.spec.sandboxClass != 'microvm' || - (has(object.spec.assets) && size(object.spec.assets) > 0 && - object.spec.assets.all(arch, - ['cloud-hypervisor', 'virtiofsd', 'kata-kernel', 'kata-image'] - .all(name, name in object.spec.assets[arch]))) - message: "a microvm SandboxConfig must define cloud-hypervisor, virtiofsd, kata-kernel, and kata-image assets for every architecture under spec.assets" + object.spec.versions.all(v, + has(v.assets) && size(v.assets) > 0 && + v.assets.all(arch, + ['cloud-hypervisor', 'virtiofsd', 'kata-kernel', 'kata-image'] + .all(name, name in v.assets[arch]))) + message: "a microvm SandboxConfig must define cloud-hypervisor, virtiofsd, kata-kernel, and kata-image assets for every architecture of every version under spec.versions[].assets" --- apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingAdmissionPolicyBinding diff --git a/manifests/microvm/sandboxconfig-microvm.yaml.tmpl b/manifests/microvm/sandboxconfig-microvm.yaml.tmpl index b4975e79b0..9747a7c9c0 100644 --- a/manifests/microvm/sandboxconfig-microvm.yaml.tmpl +++ b/manifests/microvm/sandboxconfig-microvm.yaml.tmpl @@ -35,34 +35,40 @@ apiVersion: ate.dev/v1alpha1 kind: SandboxConfig metadata: name: microvm + annotations: + sandboxconfig.ate.dev/is-class-default: "true" spec: sandboxClass: microvm - assets: - arm64: - cloud-hypervisor: - url: "gs://${BUCKET_NAME}/kata-assets/cloud-hypervisor" - sha256: "f192b510eea1c710cbc439d716bb0573c223fc463dbe3e6523788a2b7ef62850" - # virtiofsd serves the overlay RO lower (virtio-fs). - virtiofsd: - url: "gs://${BUCKET_NAME}/kata-assets/virtiofsd" - sha256: "3aa9240045d2261930ef04138fe05cf288f94c867e63e5d0845758bac5e3d458" - kata-kernel: - url: "gs://${BUCKET_NAME}/kata-assets/vmlinux" - sha256: "4960f6cf781274eaa445fc9f88bb8c5ce951649278bc9caa13afb1e99cfe80e6" - kata-image: - url: "gs://${BUCKET_NAME}/kata-assets/rootfs.img" - sha256: "7e3bdd500eb92de17c377bd84f8ed57e50d6bb68c7ec08c88f5e1916cd49f30d" - # amd64 assets are kata 4.1.0 (assemble.sh ARCH=amd64). - amd64: - cloud-hypervisor: - url: "gs://${BUCKET_NAME}/kata-assets/cloud-hypervisor" - sha256: "448af3d4e59b22c2987f7df94c213ad40fb53a10d437e42b5ee6c4fce7c29ecc" - virtiofsd: - url: "gs://${BUCKET_NAME}/kata-assets/virtiofsd" - sha256: "15b2e72a78cc08a9bd8a6943e89fb69c88cb3cbeb63069efceade835342ac7d4" - kata-kernel: - url: "gs://${BUCKET_NAME}/kata-assets/vmlinux" - sha256: "8e9dbc3a6e4c26adb089d23d31201edebce905c7f3e31aa67b32bdd41df1b86f" - kata-image: - url: "gs://${BUCKET_NAME}/kata-assets/rootfs.img" - sha256: "96497f64da1de9c7473fef46c3d29ddd0805d334731cf9d903a21a5b2c33cefb" + defaultVersion: v1 + versions: + - name: v1 + state: Enabled + assets: + arm64: + cloud-hypervisor: + url: "gs://${BUCKET_NAME}/kata-assets/cloud-hypervisor" + sha256: "f192b510eea1c710cbc439d716bb0573c223fc463dbe3e6523788a2b7ef62850" + # virtiofsd serves the overlay RO lower (virtio-fs). + virtiofsd: + url: "gs://${BUCKET_NAME}/kata-assets/virtiofsd" + sha256: "3aa9240045d2261930ef04138fe05cf288f94c867e63e5d0845758bac5e3d458" + kata-kernel: + url: "gs://${BUCKET_NAME}/kata-assets/vmlinux" + sha256: "4960f6cf781274eaa445fc9f88bb8c5ce951649278bc9caa13afb1e99cfe80e6" + kata-image: + url: "gs://${BUCKET_NAME}/kata-assets/rootfs.img" + sha256: "7e3bdd500eb92de17c377bd84f8ed57e50d6bb68c7ec08c88f5e1916cd49f30d" + # amd64 assets are kata 4.1.0 (assemble.sh ARCH=amd64). + amd64: + cloud-hypervisor: + url: "gs://${BUCKET_NAME}/kata-assets/cloud-hypervisor" + sha256: "448af3d4e59b22c2987f7df94c213ad40fb53a10d437e42b5ee6c4fce7c29ecc" + virtiofsd: + url: "gs://${BUCKET_NAME}/kata-assets/virtiofsd" + sha256: "15b2e72a78cc08a9bd8a6943e89fb69c88cb3cbeb63069efceade835342ac7d4" + kata-kernel: + url: "gs://${BUCKET_NAME}/kata-assets/vmlinux" + sha256: "8e9dbc3a6e4c26adb089d23d31201edebce905c7f3e31aa67b32bdd41df1b86f" + kata-image: + url: "gs://${BUCKET_NAME}/kata-assets/rootfs.img" + sha256: "96497f64da1de9c7473fef46c3d29ddd0805d334731cf9d903a21a5b2c33cefb" diff --git a/pkg/api/v1alpha1/sandboxconfig_types.go b/pkg/api/v1alpha1/sandboxconfig_types.go index 3667cc034e..f88c4b7b3e 100644 --- a/pkg/api/v1alpha1/sandboxconfig_types.go +++ b/pkg/api/v1alpha1/sandboxconfig_types.go @@ -49,18 +49,42 @@ type AssetFile struct { SHA256 string `json:"sha256"` } -// SandboxConfigSpec is the desired state of a SandboxConfig. +// SandboxConfigClassDefaultAnnotation marks a SandboxConfig as a default for its +// sandboxClass when set to "true". More than one SandboxConfig of the same +// class may carry it; the newest one wins. +const SandboxConfigClassDefaultAnnotation = "sandboxconfig.ate.dev/is-class-default" + +// SandboxVersionState is whether a SandboxConfig version may be used. // -// +kubebuilder:validation:XValidation:rule="self.sandboxClass == 'gvisor' ? has(self.pauseImage) : !has(self.pauseImage)",message="pauseImage is required for gvisor and not allowed for other sandbox classes" -type SandboxConfigSpec struct { - // SandboxClass is the sandbox runtime family this config applies to. An - // ActorTemplate only uses SandboxConfigs whose SandboxClass matches its - // sandbox_config.sandbox_class. +// +kubebuilder:validation:Enum=Enabled;Disabled +type SandboxVersionState string + +const ( + // SandboxVersionStateEnabled marks a version as usable. Default. + SandboxVersionStateEnabled SandboxVersionState = "Enabled" + // SandboxVersionStateDisabled marks a version as unusable. The version + // named by defaultVersion cannot be disabled. + SandboxVersionStateDisabled SandboxVersionState = "Disabled" +) + +// SandboxVersionConfig is one version of a SandboxConfig: the exact pause image +// and asset set a sandbox boots with. +type SandboxVersionConfig struct { + // name identifies this version within its SandboxConfig. It is a DNS label + // (lower-case alphanumerics and '-'). // // +required - // +kubebuilder:validation:Enum=gvisor;microvm - // +kubebuilder:default=gvisor - SandboxClass SandboxClass `json:"sandboxClass"` + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=63 + // +kubebuilder:validation:Pattern=`^[a-z0-9]([-a-z0-9]*[a-z0-9])?$` + Name string `json:"name,omitempty"` + + // state is whether this version may be used: Enabled (default) or + // Disabled. + // + // +optional + // +kubebuilder:default=Enabled + State *SandboxVersionState `json:"state,omitempty"` // TODO: drop PauseImage once gVisor can run without a pause container: // https://github.com/google/gvisor/pull/13981 @@ -79,6 +103,8 @@ type SandboxConfigSpec struct { // - [2] registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4 // // +optional + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=512 // +kubebuilder:validation:XValidation:rule="self.contains('@')",message="All images must include a digest" PauseImage string `json:"pauseImage,omitempty"` @@ -96,10 +122,60 @@ type SandboxConfigSpec struct { Assets map[string]map[string]AssetFile `json:"assets,omitempty"` } -// SandboxConfig is cluster-scoped configuration describing the sandbox binaries -// for a sandbox runtime family. It is referenced by an ActorTemplate's -// sandbox_config.config_name (required) and decouples -// sandbox binary selection from the workload definition. +// SandboxConfigSpec is the desired state of a SandboxConfig. +// +// +kubebuilder:validation:XValidation:rule="self.versions.exists(v, v.name == self.defaultVersion)",message="defaultVersion must name an entry in versions" +// +kubebuilder:validation:XValidation:rule="!self.versions.exists(v, v.name == self.defaultVersion && has(v.state) && v.state == 'Disabled')",message="the version named by defaultVersion must not be Disabled" +// +kubebuilder:validation:XValidation:rule="self.sandboxClass == 'gvisor' ? self.versions.all(v, has(v.pauseImage) && size(v.pauseImage) > 0) : self.versions.all(v, !has(v.pauseImage) || size(v.pauseImage) == 0)",message="pauseImage is required on every version for gvisor and not allowed for other sandbox classes" +type SandboxConfigSpec struct { + // SandboxClass is the sandbox runtime family this config applies to. An + // ActorTemplate only uses SandboxConfigs whose SandboxClass matches its + // sandbox_config.sandbox_class. + // + // +required + // +kubebuilder:validation:Enum=gvisor;microvm + // +kubebuilder:default=gvisor + SandboxClass SandboxClass `json:"sandboxClass"` + + // defaultVersion names the entry in versions that new sandboxes boot with. + // It must name an existing, non-Disabled version. + // + // +required + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=63 + DefaultVersion string `json:"defaultVersion,omitempty"` + + // versions are the sandbox versions this config offers, keyed by name. + // + // +required + // +listType=map + // +listMapKey=name + // +kubebuilder:validation:MinItems=1 + // +kubebuilder:validation:MaxItems=16 + Versions []SandboxVersionConfig `json:"versions,omitempty"` +} + +// DefaultVersionConfig returns the entry of Versions named by DefaultVersion, +// and whether one exists. A Disabled entry is never returned: the CRD rejects +// a Disabled default, so a spec that reads back that way is treated as having +// no default version. +func (s *SandboxConfigSpec) DefaultVersionConfig() (*SandboxVersionConfig, bool) { + for i := range s.Versions { + if s.Versions[i].Name != s.DefaultVersion { + continue + } + if st := s.Versions[i].State; st != nil && *st == SandboxVersionStateDisabled { + return nil, false + } + return &s.Versions[i], true + } + return nil, false +} + +// SandboxConfig is cluster-scoped configuration describing the versions of the +// sandbox binaries for a sandbox runtime family. It is referenced by an +// ActorTemplate's sandbox_config.config_name (required) and decouples sandbox +// binary selection from the workload definition. // // +genclient // +genclient:nonNamespaced @@ -107,6 +183,7 @@ type SandboxConfigSpec struct { // +kubebuilder:object:root=true // +kubebuilder:resource:scope=Cluster,shortName=sandboxconfig // +kubebuilder:printcolumn:name="Class",type=string,JSONPath=`.spec.sandboxClass` +// +kubebuilder:printcolumn:name="Default",type=string,JSONPath=`.spec.defaultVersion` // +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` type SandboxConfig struct { metav1.TypeMeta `json:",inline"` diff --git a/pkg/api/v1alpha1/sandboxconfig_types_test.go b/pkg/api/v1alpha1/sandboxconfig_types_test.go new file mode 100644 index 0000000000..f5869ec696 --- /dev/null +++ b/pkg/api/v1alpha1/sandboxconfig_types_test.go @@ -0,0 +1,108 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1alpha1 + +import ( + "testing" + + "k8s.io/utils/ptr" +) + +func TestDefaultVersionConfig(t *testing.T) { + version := func(name string, state *SandboxVersionState) SandboxVersionConfig { + return SandboxVersionConfig{Name: name, State: state} + } + + tests := []struct { + name string + spec SandboxConfigSpec + wantName string // "" means no default is expected + }{{ + name: "default with nil state is returned", + spec: SandboxConfigSpec{ + DefaultVersion: "v1", + Versions: []SandboxVersionConfig{version("v1", nil)}, + }, + wantName: "v1", + }, { + name: "default explicitly Enabled is returned", + spec: SandboxConfigSpec{ + DefaultVersion: "v1", + Versions: []SandboxVersionConfig{version("v1", ptr.To(SandboxVersionStateEnabled))}, + }, + wantName: "v1", + }, { + name: "default picked out of several versions", + spec: SandboxConfigSpec{ + DefaultVersion: "v2", + Versions: []SandboxVersionConfig{ + version("v1", ptr.To(SandboxVersionStateDisabled)), + version("v2", nil), + version("v3", nil), + }, + }, + wantName: "v2", + }, { + name: "Disabled default is treated as missing", + spec: SandboxConfigSpec{ + DefaultVersion: "v1", + Versions: []SandboxVersionConfig{ + version("v1", ptr.To(SandboxVersionStateDisabled)), + version("v2", nil), + }, + }, + }, { + name: "defaultVersion names no entry", + spec: SandboxConfigSpec{ + DefaultVersion: "v9", + Versions: []SandboxVersionConfig{version("v1", nil)}, + }, + }, { + name: "no versions", + spec: SandboxConfigSpec{DefaultVersion: "v1"}, + }} + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, ok := tt.spec.DefaultVersionConfig() + if tt.wantName == "" { + if ok || got != nil { + t.Fatalf("DefaultVersionConfig() = %+v, %v; want nil, false", got, ok) + } + return + } + if !ok || got == nil { + t.Fatalf("DefaultVersionConfig() = %+v, %v; want %q, true", got, ok, tt.wantName) + } + if got.Name != tt.wantName { + t.Fatalf("DefaultVersionConfig().Name = %q; want %q", got.Name, tt.wantName) + } + // The pointer must alias the spec so callers see the real entry, + // not a copy. + if got != &tt.spec.Versions[indexOf(tt.spec.Versions, tt.wantName)] { + t.Fatalf("DefaultVersionConfig() returned a copy, want pointer into spec.Versions") + } + }) + } +} + +func indexOf(versions []SandboxVersionConfig, name string) int { + for i := range versions { + if versions[i].Name == name { + return i + } + } + return -1 +} diff --git a/pkg/api/v1alpha1/sandboxconfig_validation_test.go b/pkg/api/v1alpha1/sandboxconfig_validation_test.go index a1b4b4123c..5e819755c8 100644 --- a/pkg/api/v1alpha1/sandboxconfig_validation_test.go +++ b/pkg/api/v1alpha1/sandboxconfig_validation_test.go @@ -25,6 +25,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/yaml" ) @@ -37,27 +38,51 @@ const vapManifestPath = "../../../manifests/ate-install/sandboxconfig-validation const validPauseImage = "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" -// sandboxConfig returns a config with a pause image iff the class needs one. +// sandboxVersion returns a version with a pause image iff the class needs one. +func sandboxVersion(name string, class SandboxClass, assets map[string]map[string]AssetFile) SandboxVersionConfig { + v := SandboxVersionConfig{Name: name, Assets: assets} + if class == SandboxClassGvisor { + v.PauseImage = validPauseImage + } + return v +} + +// sandboxConfig returns a config with a single default version "v1". func sandboxConfig(name string, class SandboxClass, assets map[string]map[string]AssetFile) *SandboxConfig { - sc := &SandboxConfig{ + return &SandboxConfig{ ObjectMeta: metav1.ObjectMeta{Name: name}, Spec: SandboxConfigSpec{ - SandboxClass: class, - Assets: assets, + SandboxClass: class, + DefaultVersion: "v1", + Versions: []SandboxVersionConfig{sandboxVersion("v1", class, assets)}, }, } - if class == SandboxClassGvisor { - sc.Spec.PauseImage = validPauseImage - } - return sc } -// withPauseImage overrides the pause image on an otherwise-valid config. +// withPauseImage overrides the pause image of the first version on an +// otherwise-valid config. func withPauseImage(sc *SandboxConfig, image string) *SandboxConfig { - sc.Spec.PauseImage = image + sc.Spec.Versions[0].PauseImage = image + return sc +} + +// withVersions appends versions to a config. +func withVersions(sc *SandboxConfig, versions ...SandboxVersionConfig) *SandboxConfig { + sc.Spec.Versions = append(sc.Spec.Versions, versions...) + return sc +} + +// withDefaultVersion overrides the default version of a config. +func withDefaultVersion(sc *SandboxConfig, name string) *SandboxConfig { + sc.Spec.DefaultVersion = name return sc } +// gvisorAssets returns a valid gVisor asset set for amd64. +func gvisorAssets() map[string]map[string]AssetFile { + return map[string]map[string]AssetFile{"amd64": {"gvisor": gvisorAsset()}} +} + func runscAsset() AssetFile { return AssetFile{URL: "gs://bucket/runsc", SHA256: validSHA256} } func gvisorAsset() AssetFile { @@ -200,17 +225,106 @@ func TestSandboxConfigValidation(t *testing.T) { name: "gvisor missing pauseImage", sc: withPauseImage(sandboxConfig("bad-no-pause", SandboxClassGvisor, map[string]map[string]AssetFile{"amd64": {"gvisor": gvisorAsset()}}), ""), wantErr: true, - errMsg: "pauseImage is required for gvisor and not allowed", + errMsg: "pauseImage is required on every version for gvisor", }, { name: "microvm with pauseImage", sc: withPauseImage(sandboxConfig("bad-microvm-pause", SandboxClassMicroVM, map[string]map[string]AssetFile{"amd64": microVMAssets()}), validPauseImage), wantErr: true, - errMsg: "pauseImage is required for gvisor and not allowed", + errMsg: "pauseImage is required on every version for gvisor", }, { name: "unpinned pauseImage", sc: withPauseImage(sandboxConfig("bad-unpinned-pause", SandboxClassGvisor, map[string]map[string]AssetFile{"amd64": {"gvisor": gvisorAsset()}}), "registry.k8s.io/pause:3.10.2"), wantErr: true, errMsg: "All images must include a digest", + }, { + name: "valid gvisor with two versions", + sc: withVersions(sandboxConfig("ok-two-versions", SandboxClassGvisor, gvisorAssets()), sandboxVersion("v2", SandboxClassGvisor, gvisorAssets())), + wantErr: false, + }, { + name: "valid non-default disabled version", + sc: withVersions(sandboxConfig("ok-disabled-old", SandboxClassGvisor, gvisorAssets()), func() SandboxVersionConfig { + v := sandboxVersion("v0", SandboxClassGvisor, gvisorAssets()) + v.State = ptr.To(SandboxVersionStateDisabled) + return v + }()), + wantErr: false, + }, { + name: "defaultVersion not in versions", + sc: withDefaultVersion(sandboxConfig("bad-default-missing", SandboxClassGvisor, gvisorAssets()), "v9"), + wantErr: true, + errMsg: "defaultVersion must name an entry in versions", + }, { + name: "defaultVersion disabled", + sc: func() *SandboxConfig { + sc := sandboxConfig("bad-default-disabled", SandboxClassGvisor, gvisorAssets()) + sc.Spec.Versions[0].State = ptr.To(SandboxVersionStateDisabled) + return sc + }(), + wantErr: true, + errMsg: "must not be Disabled", + }, { + name: "no versions", + sc: &SandboxConfig{ObjectMeta: metav1.ObjectMeta{Name: "bad-no-versions"}, Spec: SandboxConfigSpec{SandboxClass: SandboxClassGvisor, DefaultVersion: "v1", Versions: []SandboxVersionConfig{}}}, + wantErr: true, + errMsg: "spec.versions", + }, { + name: "duplicate version names", + sc: withVersions(sandboxConfig("bad-dup-versions", SandboxClassGvisor, gvisorAssets()), sandboxVersion("v1", SandboxClassGvisor, gvisorAssets())), + wantErr: true, + errMsg: "Duplicate value", + }, { + name: "too many versions", + sc: func() *SandboxConfig { + sc := sandboxConfig("bad-many-versions", SandboxClassGvisor, gvisorAssets()) + for i := range 16 { + sc.Spec.Versions = append(sc.Spec.Versions, sandboxVersion(fmt.Sprintf("extra-%d", i), SandboxClassGvisor, gvisorAssets())) + } + return sc + }(), + wantErr: true, + errMsg: "spec.versions", + }, { + name: "version name not a DNS label", + sc: func() *SandboxConfig { + sc := sandboxConfig("bad-version-name", SandboxClassGvisor, gvisorAssets()) + sc.Spec.Versions[0].Name = "V_1" + sc.Spec.DefaultVersion = "V_1" + return sc + }(), + wantErr: true, + errMsg: "spec.versions[0].name", + }, { + name: "gvisor second version missing pauseImage", + sc: withVersions(sandboxConfig("bad-v2-no-pause", SandboxClassGvisor, gvisorAssets()), func() SandboxVersionConfig { + v := sandboxVersion("v2", SandboxClassGvisor, gvisorAssets()) + v.PauseImage = "" + return v + }()), + wantErr: true, + errMsg: "pauseImage is required on every version for gvisor", + }, { + name: "microvm second version with pauseImage", + sc: withVersions(sandboxConfig("bad-microvm-v2-pause", SandboxClassMicroVM, map[string]map[string]AssetFile{"amd64": microVMAssets()}), func() SandboxVersionConfig { + v := sandboxVersion("v2", SandboxClassMicroVM, map[string]map[string]AssetFile{"amd64": microVMAssets()}) + v.PauseImage = validPauseImage + return v + }()), + wantErr: true, + errMsg: "pauseImage is required on every version for gvisor", + }, { + name: "gvisor second version missing gvisor and runsc", + sc: withVersions(sandboxConfig("bad-v2-no-runsc", SandboxClassGvisor, gvisorAssets()), sandboxVersion("v2", SandboxClassGvisor, map[string]map[string]AssetFile{"amd64": {"notrunsc": runscAsset()}})), + wantErr: true, + errMsg: "runsc", + }, { + name: "microvm second version missing an asset", + sc: withVersions(sandboxConfig("bad-microvm-v2", SandboxClassMicroVM, map[string]map[string]AssetFile{"amd64": microVMAssets()}), sandboxVersion("v2", SandboxClassMicroVM, map[string]map[string]AssetFile{"amd64": func() map[string]AssetFile { + m := microVMAssets() + delete(m, "kata-kernel") + return m + }()})), + wantErr: true, + errMsg: "microvm SandboxConfig must define", }} for _, tt := range tests { @@ -233,3 +347,59 @@ func TestSandboxConfigValidation(t *testing.T) { }) } } + +func TestSandboxConfigUpdateRemovingDefaultVersion(t *testing.T) { + ctx := t.Context() + sc := withVersions(sandboxConfig("update-remove-default", SandboxClassGvisor, gvisorAssets()), sandboxVersion("v2", SandboxClassGvisor, gvisorAssets())) + if err := k8sClient.Create(ctx, sc); err != nil { + t.Fatalf("Create() unexpected error: %v", err) + } + t.Cleanup(func() { _ = k8sClient.Delete(ctx, sc) }) + + // Dropping the version defaultVersion names must be denied. + sc.Spec.Versions = sc.Spec.Versions[1:] + err := k8sClient.Update(ctx, sc) + if err == nil { + t.Fatalf("Update() removing the default version succeeded, want denied") + } + if !strings.Contains(err.Error(), "defaultVersion must name an entry in versions") { + t.Errorf("Update() error = %q, want it to mention defaultVersion", err.Error()) + } + + // Repointing defaultVersion in the same update is allowed. + sc.Spec.DefaultVersion = "v2" + if err := k8sClient.Update(ctx, sc); err != nil { + t.Fatalf("Update() repointing defaultVersion: %v", err) + } +} + +// TestShippedSandboxConfigManifests guards the SandboxConfigs the install ships +// against the CRD schema and the shipped ValidatingAdmissionPolicy. +func TestShippedSandboxConfigManifests(t *testing.T) { + ctx := t.Context() + applyVAP(t, ctx) + + for _, path := range []string{ + "../../../manifests/ate-install/sandboxconfig-gvisor.yaml", + "../../../manifests/microvm/sandboxconfig-microvm.yaml.tmpl", + } { + t.Run(path, func(t *testing.T) { + raw, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read manifest: %v", err) + } + raw = []byte(strings.ReplaceAll(string(raw), "${BUCKET_NAME}", "test-bucket")) + sc := &SandboxConfig{} + if err := yaml.UnmarshalStrict(raw, sc); err != nil { + t.Fatalf("decode manifest: %v", err) + } + if sc.Annotations[SandboxConfigClassDefaultAnnotation] != "true" { + t.Errorf("manifest is not annotated %s=true", SandboxConfigClassDefaultAnnotation) + } + if err := k8sClient.Create(ctx, sc); err != nil { + t.Fatalf("Create() unexpected error: %v", err) + } + t.Cleanup(func() { _ = k8sClient.Delete(ctx, sc) }) + }) + } +} diff --git a/pkg/api/v1alpha1/zz_generated.deepcopy.go b/pkg/api/v1alpha1/zz_generated.deepcopy.go index 98051918d8..e9e19cc475 100644 --- a/pkg/api/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/api/v1alpha1/zz_generated.deepcopy.go @@ -192,6 +192,33 @@ func (in *SandboxConfigList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SandboxConfigSpec) DeepCopyInto(out *SandboxConfigSpec) { *out = *in + if in.Versions != nil { + in, out := &in.Versions, &out.Versions + *out = make([]SandboxVersionConfig, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SandboxConfigSpec. +func (in *SandboxConfigSpec) DeepCopy() *SandboxConfigSpec { + if in == nil { + return nil + } + out := new(SandboxConfigSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SandboxVersionConfig) DeepCopyInto(out *SandboxVersionConfig) { + *out = *in + if in.State != nil { + in, out := &in.State, &out.State + *out = new(SandboxVersionState) + **out = **in + } if in.Assets != nil { in, out := &in.Assets, &out.Assets *out = make(map[string]map[string]AssetFile, len(*in)) @@ -212,12 +239,12 @@ func (in *SandboxConfigSpec) DeepCopyInto(out *SandboxConfigSpec) { } } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SandboxConfigSpec. -func (in *SandboxConfigSpec) DeepCopy() *SandboxConfigSpec { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SandboxVersionConfig. +func (in *SandboxVersionConfig) DeepCopy() *SandboxVersionConfig { if in == nil { return nil } - out := new(SandboxConfigSpec) + out := new(SandboxVersionConfig) in.DeepCopyInto(out) return out } From 66f8a88807ffa60bd67df7bdabcf4377068bb256 Mon Sep 17 00:00:00 2001 From: Max Thompson Date: Thu, 8 Oct 2026 22:24:26 +0000 Subject: [PATCH 17/44] Egress gateway can mint and cache actor JWTs (#2135) Part of #1660. - Adds the `egress/actorjwt` package, whose `Minter` mints actor JWTs through `Control.MintActorJWT`. Tokens are cached by actor, sorted audiences, and lifetime, keyed by the deterministic protobuf encoding of the mint request, and each is reused until a third of its lifetime remains. - Concurrent misses share one mint. It keeps running if the request that started it goes away and has no timeout of its own, so a slow mint still fills the cache. - The gateway knows the actor only by name, so it sends no UID (#2207). An actor recreated under the same name can get its predecessor's token until the refresh point. Nothing calls the minter yet, so `actor_jwt` entries still get a 501. The next PR in the series wires it into `applyEffects`. Testing: unit tests call a fake Control server over bufconn and cover reuse, keying, errors, and concurrent and canceled callers. They pass under `-race`. E2E hasn't run. - [x] Tests pass - [x] Appropriate changes to documentation are included in the PR (no user-facing change yet; docs come later in the #1660 series) --- .../router/egress/actorjwt/actorjwt.go | 110 ++++++++++ .../router/egress/actorjwt/actorjwt_test.go | 204 ++++++++++++++++++ cmd/atenet/internal/router/egress/egress.go | 4 + 3 files changed, 318 insertions(+) create mode 100644 cmd/atenet/internal/router/egress/actorjwt/actorjwt.go create mode 100644 cmd/atenet/internal/router/egress/actorjwt/actorjwt_test.go diff --git a/cmd/atenet/internal/router/egress/actorjwt/actorjwt.go b/cmd/atenet/internal/router/egress/actorjwt/actorjwt.go new file mode 100644 index 0000000000..1115b80520 --- /dev/null +++ b/cmd/atenet/internal/router/egress/actorjwt/actorjwt.go @@ -0,0 +1,110 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package actorjwt mints the actor JWTs that egress policies inject. +package actorjwt + +import ( + "context" + "fmt" + "slices" + "time" + + "golang.org/x/sync/singleflight" + "google.golang.org/protobuf/proto" + "k8s.io/apimachinery/pkg/util/cache" + + "github.com/agent-substrate/substrate/internal/resources" + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" +) + +// Minter mints actor JWTs and reuses each token until a third of its lifetime +// remains. Tokens are cached by actor name, so an actor recreated under the +// same name can get its predecessor's token until then. +type Minter struct { + client ateapipb.ControlClient + // tokens maps a tokenKey to a JWT until the token's refresh point. + tokens *cache.Expiring + // flight collapses concurrent mints of one token into a single call. + flight singleflight.Group +} + +// New returns a Minter that mints through client. +func New(client ateapipb.ControlClient) *Minter { + return &Minter{client: client, tokens: cache.NewExpiring()} +} + +// Token returns a JWT for the actor ref names, bound to src's audiences and +// lifetime. MintActorJWT errors are returned wrapped. +func (m *Minter) Token(ctx context.Context, ref resources.ActorRef, src *ateapipb.ActorJWTSource) (string, error) { + req := &ateapipb.MintActorJWTRequest{ + Actor: ref.ToObjectRef(), + Audience: slices.Sorted(slices.Values(src.GetAudiences())), + ExpirationSeconds: src.GetExpirationSeconds(), + } + key, err := tokenKey(req) + if err != nil { + return "", err + } + if jwt, ok := m.cached(key); ok { + return jwt, nil + } + + // The mint outlives the caller: the leader of a flight going away must + // not fail the callers that joined it. It has no timeout of its own, so a + // slow mint still lands in the cache for the next request. + ch := m.flight.DoChan(key, func() (any, error) { + return m.mint(context.WithoutCancel(ctx), key, ref, req) + }) + select { + case <-ctx.Done(): + return "", ctx.Err() + case res := <-ch: + if res.Err != nil { + return "", res.Err + } + return res.Val.(string), nil + } +} + +// mint sends req and caches the token under key, unless a flight that just +// finished has already stored one. +func (m *Minter) mint(ctx context.Context, key string, ref resources.ActorRef, req *ateapipb.MintActorJWTRequest) (string, error) { + if jwt, ok := m.cached(key); ok { + return jwt, nil + } + resp, err := m.client.MintActorJWT(ctx, req) + if err != nil { + return "", fmt.Errorf("minting an actor JWT for %s: %w", ref, err) + } + reuseFor := time.Until(resp.GetExpiresAt().AsTime()) - time.Duration(req.GetExpirationSeconds())*time.Second/3 + m.tokens.Set(key, resp.GetActorJwt(), reuseFor) + return resp.GetActorJwt(), nil +} + +func (m *Minter) cached(key string) (string, bool) { + v, ok := m.tokens.Get(key) + s, _ := v.(string) + return s, ok +} + +// tokenKey identifies the token req mints by req's own encoding. Deterministic +// encoding is stable within one binary, which is all an in-process cache needs. +func tokenKey(req *ateapipb.MintActorJWTRequest) (string, error) { + b, err := proto.MarshalOptions{Deterministic: true}.Marshal(req) + if err != nil { + return "", fmt.Errorf("encoding the token key: %w", err) + } + return string(b), nil +} diff --git a/cmd/atenet/internal/router/egress/actorjwt/actorjwt_test.go b/cmd/atenet/internal/router/egress/actorjwt/actorjwt_test.go new file mode 100644 index 0000000000..7fa35d5fe1 --- /dev/null +++ b/cmd/atenet/internal/router/egress/actorjwt/actorjwt_test.go @@ -0,0 +1,204 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package actorjwt + +import ( + "context" + "errors" + "fmt" + "net" + "sync" + "sync/atomic" + "testing" + "testing/synctest" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/status" + "google.golang.org/grpc/test/bufconn" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/timestamppb" + + "github.com/agent-substrate/substrate/internal/resources" + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" +) + +var testActor = resources.ActorRef{Atespace: "default", Name: "my-actor"} + +// fakeControl's nth MintActorJWT call returns "jwt-", expiring +// expiration_seconds from now. err, when set, is returned instead, and gate, +// when non-nil, blocks each call until it is closed. +type fakeControl struct { + ateapipb.UnimplementedControlServer + calls atomic.Int32 + last atomic.Pointer[ateapipb.MintActorJWTRequest] + err error + gate chan struct{} +} + +func (f *fakeControl) MintActorJWT(ctx context.Context, req *ateapipb.MintActorJWTRequest) (*ateapipb.MintActorJWTResponse, error) { + n := f.calls.Add(1) + f.last.Store(req) + if f.gate != nil { + select { + case <-f.gate: + case <-ctx.Done(): + return nil, status.FromContextError(ctx.Err()).Err() + } + } + if f.err != nil { + return nil, f.err + } + return &ateapipb.MintActorJWTResponse{ + ActorJwt: fmt.Sprintf("jwt-%d", n), + ExpiresAt: timestamppb.New(time.Now().Add(time.Duration(req.GetExpirationSeconds()) * time.Second)), + }, nil +} + +// newMinter returns a Minter whose client reaches ctl over an in-memory gRPC +// connection. +func newMinter(t *testing.T, ctl *fakeControl) *Minter { + t.Helper() + lis := bufconn.Listen(1 << 20) + srv := grpc.NewServer() + ateapipb.RegisterControlServer(srv, ctl) + go func() { _ = srv.Serve(lis) }() + t.Cleanup(srv.Stop) + + conn, err := grpc.NewClient("passthrough:///bufconn", + grpc.WithContextDialer(func(ctx context.Context, _ string) (net.Conn, error) { return lis.DialContext(ctx) }), + grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.NewClient: %v", err) + } + t.Cleanup(func() { _ = conn.Close() }) + return New(ateapipb.NewControlClient(conn)) +} + +func jwtSource(lifetime int64, audiences ...string) *ateapipb.ActorJWTSource { + return &ateapipb.ActorJWTSource{Audiences: audiences, ExpirationSeconds: lifetime} +} + +// wantToken calls m.Token and fails the test unless it returns want. +func wantToken(t *testing.T, m *Minter, ref resources.ActorRef, src *ateapipb.ActorJWTSource, want string) { + t.Helper() + got, err := m.Token(context.Background(), ref, src) + if err != nil { + t.Fatalf("Token: %v", err) + } + if got != want { + t.Fatalf("Token = %q, want %q", got, want) + } +} + +func TestReuseTokenUntilAThirdOfItsLifetimeRemains(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + ctl := &fakeControl{} + m := newMinter(t, ctl) + src := jwtSource(900, "https://b.example", "https://a.example") + + wantToken(t, m, testActor, src, "jwt-1") + want := &ateapipb.MintActorJWTRequest{ + Actor: &ateapipb.ObjectRef{Atespace: "default", Name: "my-actor"}, + Audience: []string{"https://a.example", "https://b.example"}, + ExpirationSeconds: 900, + } + if got := ctl.last.Load(); !proto.Equal(got, want) { + t.Errorf("MintActorJWT request = %v, want %v", got, want) + } + + time.Sleep(600*time.Second - time.Nanosecond) + wantToken(t, m, testActor, src, "jwt-1") + time.Sleep(time.Nanosecond) + wantToken(t, m, testActor, src, "jwt-2") + }) +} + +func TestKeyOnActorAudiencesAndLifetime(t *testing.T) { + m := newMinter(t, &fakeControl{}) + other := resources.ActorRef{Atespace: "default", Name: "other-actor"} + + wantToken(t, m, testActor, jwtSource(900, "a", "b"), "jwt-1") + wantToken(t, m, testActor, jwtSource(900, "b", "a"), "jwt-1") + wantToken(t, m, testActor, jwtSource(900, "a"), "jwt-2") + wantToken(t, m, testActor, jwtSource(600, "a", "b"), "jwt-3") + wantToken(t, m, other, jwtSource(900, "a", "b"), "jwt-4") + wantToken(t, m, testActor, jwtSource(900, "a b"), "jwt-5") +} + +func TestDoNotCacheErrors(t *testing.T) { + ctl := &fakeControl{err: status.Error(codes.Unavailable, "ateapi is down")} + m := newMinter(t, ctl) + src := jwtSource(900, "a") + + if _, err := m.Token(context.Background(), testActor, src); status.Code(err) != codes.Unavailable { + t.Fatalf("Token = %v, want the Unavailable MintActorJWT error", err) + } + ctl.err = nil + wantToken(t, m, testActor, src, "jwt-2") +} + +func TestCollapseConcurrentMints(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + ctl := &fakeControl{gate: make(chan struct{})} + m := newMinter(t, ctl) + + const callers = 8 + var wg sync.WaitGroup + for range callers { + wg.Go(func() { + if _, err := m.Token(context.Background(), testActor, jwtSource(900, "a")); err != nil { + t.Errorf("Token: %v", err) + } + }) + } + synctest.Wait() + close(ctl.gate) + wg.Wait() + if calls := ctl.calls.Load(); calls != 1 { + t.Errorf("MintActorJWT calls = %d, want 1 for %d concurrent callers", calls, callers) + } + }) +} + +// The leader's cancellation must not fail the mint it started, and the token +// still lands in the cache. +func TestMintOutlivesCanceledCaller(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + ctl := &fakeControl{gate: make(chan struct{})} + m := newMinter(t, ctl) + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { + _, err := m.Token(ctx, testActor, jwtSource(900, "a")) + done <- err + }() + synctest.Wait() + cancel() + if err := <-done; !errors.Is(err, context.Canceled) { + t.Fatalf("canceled caller got %v, want context.Canceled", err) + } + + close(ctl.gate) + synctest.Wait() + wantToken(t, m, testActor, jwtSource(900, "a"), "jwt-1") + if calls := ctl.calls.Load(); calls != 1 { + t.Errorf("MintActorJWT calls = %d, want 1", calls) + } + }) +} diff --git a/cmd/atenet/internal/router/egress/egress.go b/cmd/atenet/internal/router/egress/egress.go index 5731b01c48..38c678003a 100644 --- a/cmd/atenet/internal/router/egress/egress.go +++ b/cmd/atenet/internal/router/egress/egress.go @@ -41,6 +41,7 @@ import ( "google.golang.org/grpc/status" "google.golang.org/protobuf/types/known/structpb" + "github.com/agent-substrate/substrate/cmd/atenet/internal/router/egress/actorjwt" "github.com/agent-substrate/substrate/cmd/atenet/internal/router/extproc" "github.com/agent-substrate/substrate/internal/egresspolicy" "github.com/agent-substrate/substrate/internal/resources" @@ -83,6 +84,8 @@ type Handler struct { actorIdentityRoots *x509.CertPool // policies is the per-actor EgressPolicy cache every leg reads through. policies *policyCache + // actorJWTs mints the actor JWTs that credential injections ask for. + actorJWTs *actorjwt.Minter // provider resolves an egress policy's credential injections. Nil means // credential injection is not configured, and a request that needs one is // denied. @@ -107,6 +110,7 @@ func New(apiClient ateapipb.ControlClient, actorIdentityRoots *x509.CertPool, po apiClient: apiClient, actorIdentityRoots: actorIdentityRoots, policies: newPolicyCache(apiClient, policyCacheTTL), + actorJWTs: actorjwt.New(apiClient), provider: provider, providerName: providerName, } From b2f6cb8cf49138451bb0bdff3a2d884ac33567f3 Mon Sep 17 00:00:00 2001 From: Haven Xia Date: Thu, 8 Oct 2026 23:28:37 +0000 Subject: [PATCH 18/44] Build the OCI spec in ateom instead of atelet (#2293) Today atelet wrote each container's `config.json`, and each ateom loaded it and reshaped it for its runtime. To better decouple atelet and ateom, the ateom Container message now carries each container's `argv`, `environment`, `capabilities` and resource limits next to its volume mounts. atelet still resolves argv and env, because it pulled the image and has the image config. It also still resolves the capability set from the template. `ocispec.Build` now takes the ateom Container and the actor's directories. - ateom-gvisor builds and saves the spec before runsc create or restore, including the pause container's. - ateom-microvm builds it in memory. For the bundle, atelet only pulls images and writes the overlay spec. The pause container also starts with an empty environment instead of the pause image's ENV and a default PATH. - [x] Tests pass - [x] Appropriate changes to documentation are included in the PR --- cmd/atelet/main.go | 100 +++--- cmd/atelet/main_test.go | 73 ++++- cmd/atelet/oci.go | 38 +-- cmd/atelet/volumes_test.go | 2 +- cmd/ateom-gvisor/main.go | 2 + cmd/ateom-gvisor/runsc.go | 38 ++- cmd/ateom-gvisor/runsc_test.go | 49 +++ cmd/ateom-microvm/restore.go | 2 +- cmd/ateom-microvm/run.go | 14 +- internal/imagecache/README.md | 2 +- internal/imagecache/imagecache.go | 2 +- internal/imagecache/spec.go | 12 +- internal/ocispec/microvm_test.go | 10 +- internal/ocispec/ocispec.go | 107 +++---- internal/ocispec/ocispec_test.go | 96 +++--- internal/ocispec/parity_test.go | 40 +-- internal/proto/ateompb/ateom.pb.go | 376 ++++++++++++++++------- internal/proto/ateompb/ateom.proto | 29 ++ internal/protoredact/protoredact_test.go | 1 + 19 files changed, 623 insertions(+), 370 deletions(-) diff --git a/cmd/atelet/main.go b/cmd/atelet/main.go index 99bc760a51..e9945698e9 100644 --- a/cmd/atelet/main.go +++ b/cmd/atelet/main.go @@ -548,9 +548,10 @@ func (s *AteomHerder) Run(ctx context.Context, req *ateletpb.RunRequest) (resp * if registration, err = s.systemInfoVolumes.Register(actorUID, actorRef, systemInfoVolumesFor(actorUID, req.GetSpec())); err != nil { return nil, err } - if err := s.prepareOCIBundles(ctx, actorUID, actorRef, + containerSpecs, err := s.prepareOCIBundles(ctx, actorUID, actorRef, req.GetSpec(), sandboxRec.PauseImage, req.GetTargetAteomUid(), - ); err != nil { + ) + if err != nil { return nil, err } @@ -559,7 +560,7 @@ func (s *AteomHerder) Run(ctx context.Context, req *ateletpb.RunRequest) (resp * return nil, err } - spec, err := buildAteomWorkloadSpec(req.GetSpec()) + spec, err := buildAteomWorkloadSpec(req.GetSpec(), containerSpecs) if err != nil { return nil, apierror.InvalidArgument("invalid workload spec: %v", err) } @@ -692,7 +693,7 @@ func (s *AteomHerder) Checkpoint(ctx context.Context, req *ateletpb.CheckpointRe // Tell ateom to take the checkpoint and delete containers. ateom reports the // exact files it wrote so we ship precisely that set (gVisor's image files, // cloud-hypervisor's snapshot set, ...) rather than a hardcoded list. - spec, err := buildAteomWorkloadSpec(req.GetSpec()) + spec, err := buildAteomWorkloadSpec(req.GetSpec(), nil) if err != nil { return nil, apierror.InvalidArgument("invalid workload spec: %v", err) } @@ -1160,6 +1161,7 @@ func (s *AteomHerder) Restore(ctx context.Context, req *ateletpb.RestoreRequest) // fetch + image unpack hides whichever leg is shorter, and on a cold node // (uncached assets + image, ~2.5s unpack) that overlap is large. var assetPaths map[string]string + var containerSpecs []*ateompb.ContainerSpec // One per leg: a single field written from both goroutines would race. var downloadErr, prepErr error var prepFailedPhase string @@ -1199,7 +1201,7 @@ func (s *AteomHerder) Restore(ctx context.Context, req *ateletpb.RestoreRequest) return err } t := time.Now() - err = s.prepareOCIBundles(gctx, actorUID, actorRef, req.GetSpec(), runtimeRec.PauseImage, req.GetTargetAteomUid()) + containerSpecs, err = s.prepareOCIBundles(gctx, actorUID, actorRef, req.GetSpec(), runtimeRec.PauseImage, req.GetTargetAteomUid()) dBundles = time.Since(t) if err != nil { prepFailedPhase = ateattr.SnapshotPhaseOCIUnpack @@ -1224,7 +1226,7 @@ func (s *AteomHerder) Restore(ctx context.Context, req *ateletpb.RestoreRequest) // Tell ateom to do runsc create + runsc restore for pause container and // all application containers. - spec, err := buildAteomWorkloadSpec(req.GetSpec()) + spec, err := buildAteomWorkloadSpec(req.GetSpec(), containerSpecs) if err != nil { return nil, apierror.InvalidArgument("invalid workload spec: %v", err) } @@ -1295,7 +1297,7 @@ func (s *AteomHerder) Terminate(ctx context.Context, req *ateletpb.TerminateRequ return nil, fmt.Errorf("failed to dial ateom for terminate (actor: %s, actorUID: %s): %w", actorRef, actorUID, err) } - spec, err := buildAteomWorkloadSpec(req.GetSpec()) + spec, err := buildAteomWorkloadSpec(req.GetSpec(), nil) if err != nil { return nil, apierror.InvalidArgument("invalid workload spec: %v", err) } @@ -1454,7 +1456,8 @@ func (s *AteomHerder) downloadExternalCheckpoint(ctx context.Context, snapshotUR // comes from the sandbox record, not the workload spec: it is sandbox // configuration, and on a restore it must be the image the snapshot was taken // with. It is empty for sandboxes without a pause container, which get no -// pause bundle. +// pause bundle. It returns each application container's spec, indexed like +// spec.GetContainers(). func (s *AteomHerder) prepareOCIBundles( ctx context.Context, actorUID string, @@ -1462,14 +1465,14 @@ func (s *AteomHerder) prepareOCIBundles( spec *ateletpb.WorkloadSpec, pauseImage string, targetAteomUid string, -) error { +) ([]*ateompb.ContainerSpec, error) { // Prepare host folders for volume types that need them. for _, vol := range spec.GetVolumes() { switch vol.GetSource().(type) { case *ateletpb.Volume_DurableDir: volPath := ateletpath.DurableDirVolumeMountPoint(actorUID, vol.GetName()) if err := os.MkdirAll(volPath, 0o700); err != nil { - return fmt.Errorf("while creating %q: %w", volPath, err) + return nil, fmt.Errorf("while creating %q: %w", volPath, err) } } } @@ -1478,21 +1481,8 @@ func (s *AteomHerder) prepareOCIBundles( if pauseImage != "" { g.Go(func() error { - if err := prepareOCIDirectory( - gCtx, - s.imageCache, - actorUID, - ocispec.PauseContainer, - pauseImage, - []string{"/pause"}, - nil, - nil, - nodepath.ActorNetNSPath(actorUID), - nil, // pause is sandbox infra; it mounts no volumes. - nil, - nil, // pause only reaps; it needs no capabilities. - nil, // pause carries no user-declared limits. - ); err != nil { + // pause is sandbox infra; it mounts no volumes. + if _, err := prepareOCIDirectory(gCtx, s.imageCache, actorUID, ocispec.PauseContainer, pauseImage, nil, nil); err != nil { return wrapFileSystemErr("while creating pause OCI bundle", err) } return nil @@ -1500,35 +1490,34 @@ func (s *AteomHerder) prepareOCIBundles( } // Application containers. - for _, ctr := range spec.GetContainers() { - ctr := ctr + containerSpecs := make([]*ateompb.ContainerSpec, len(spec.GetContainers())) + for i, ctr := range spec.GetContainers() { var envs []string for _, env := range ctr.GetEnv() { envs = append(envs, fmt.Sprintf("%s=%s", env.GetName(), env.GetValue())) } g.Go(func() error { - if err := prepareOCIDirectory( - gCtx, - s.imageCache, - actorUID, - ctr.GetName(), - ctr.GetImage(), - ctr.GetCommand(), - ctr.GetArgs(), - envs, - nodepath.ActorNetNSPath(actorUID), - spec.GetVolumes(), - ctr.GetVolumeMounts(), - resolveCapabilities(ctr.GetSecurityContext().GetCapabilities()), - ctr.GetResources(), - ); err != nil { + imageConfig, err := prepareOCIDirectory(gCtx, s.imageCache, actorUID, ctr.GetName(), ctr.GetImage(), spec.GetVolumes(), ctr.GetVolumeMounts()) + if err != nil { return wrapFileSystemErr(fmt.Sprintf("while creating %q OCI bundle", ctr.GetName()), err) } + args, err := resolveProcessArgs(imageConfig, ctr.GetCommand(), ctr.GetArgs()) + if err != nil { + return fmt.Errorf("while resolving process args for container %q: %w", ctr.GetName(), err) + } + containerSpecs[i] = &ateompb.ContainerSpec{ + Args: args, + Env: resolveActorEnv(imageConfig, envs), + Capabilities: resolveCapabilities(ctr.GetSecurityContext().GetCapabilities()), + Resources: toAteomResourceLimits(ctr.GetResources()), + } return nil }) } - - return g.Wait() + if err := g.Wait(); err != nil { + return nil, err + } + return containerSpecs, nil } // dialAteom opens (or reuses) the gRPC connection to the target ateom @@ -1542,8 +1531,10 @@ func (s *AteomHerder) dialAteom(ctx context.Context, targetAteomUid string) (ate } // buildAteomWorkloadSpec projects the atelet-facing workload spec onto -// the ateom-facing one. -func buildAteomWorkloadSpec(spec *ateletpb.WorkloadSpec) (*ateompb.WorkloadSpec, error) { +// the ateom-facing one. containerSpecs holds each container's spec from +// prepareOCIBundles, indexed like spec.GetContainers(); it is nil for RPCs that +// start no process. +func buildAteomWorkloadSpec(spec *ateletpb.WorkloadSpec, containerSpecs []*ateompb.ContainerSpec) (*ateompb.WorkloadSpec, error) { volumes := make(map[string]*ateletpb.Volume) for _, vol := range spec.GetVolumes() { name := vol.GetName() @@ -1554,7 +1545,7 @@ func buildAteomWorkloadSpec(spec *ateletpb.WorkloadSpec) (*ateompb.WorkloadSpec, } out := &ateompb.WorkloadSpec{} - for _, ctr := range spec.GetContainers() { + for i, ctr := range spec.GetContainers() { var ddMounts []*ateompb.DurableDirVolumeMount var csiMounts []*ateompb.VolumeMount var siMounts []*ateompb.SystemInfoVolumeMount @@ -1591,18 +1582,29 @@ func buildAteomWorkloadSpec(spec *ateletpb.WorkloadSpec) (*ateompb.WorkloadSpec, return nil, fmt.Errorf("container %q mounts volume %q with unsupported source %T", ctr.GetName(), volName, vol.GetSource()) } } - out.Containers = append(out.Containers, &ateompb.Container{ + container := &ateompb.Container{ Name: ctr.GetName(), DurableDirVolumeMounts: ddMounts, CsiVolumeMounts: csiMounts, SystemInfoVolumeMounts: siMounts, ImageVolumeMounts: imgMounts, WakeupProbe: toAteomWakeupProbe(ctr.GetWakeupProbe()), - }) + } + if containerSpecs != nil { + container.ContainerSpec = containerSpecs[i] + } + out.Containers = append(out.Containers, container) } return out, nil } +func toAteomResourceLimits(limits *ateletpb.ResourceLimits) *ateompb.ResourceLimits { + if limits == nil { + return nil + } + return &ateompb.ResourceLimits{MemoryBytes: limits.GetMemoryBytes(), CpuMillis: limits.GetCpuMillis()} +} + func toAteomEgressGateway(gateway *ateletpb.EgressGateway) *ateompb.EgressGateway { if gateway == nil { return nil diff --git a/cmd/atelet/main_test.go b/cmd/atelet/main_test.go index f9a09dea3c..3bfda17eb4 100644 --- a/cmd/atelet/main_test.go +++ b/cmd/atelet/main_test.go @@ -37,6 +37,7 @@ import ( "github.com/agent-substrate/substrate/internal/apierror" "github.com/agent-substrate/substrate/internal/ateattr" "github.com/agent-substrate/substrate/internal/atelet" + "github.com/agent-substrate/substrate/internal/imagecache" "github.com/agent-substrate/substrate/internal/nodepath" "github.com/agent-substrate/substrate/internal/ocispec" "github.com/agent-substrate/substrate/internal/proto/ateletpb" @@ -258,10 +259,10 @@ func TestPrepareOCIBundlesPause(t *testing.T) { useTempNodeDirs(t) const actorUID = "actor-uid-1" s := &AteomHerder{imageCache: newImageVolumeStore(t)} - if err := s.prepareOCIBundles(t.Context(), actorUID, resources.ActorRef{}, spec, tc.pauseImage, "ateom-uid-1"); err != nil { + if _, err := s.prepareOCIBundles(t.Context(), actorUID, resources.ActorRef{}, spec, tc.pauseImage, "ateom-uid-1"); err != nil { t.Fatalf("prepareOCIBundles: %v", err) } - if _, err := os.Stat(filepath.Join(ateletpath.OCIBundlePath(actorUID, "app"), "config.json")); err != nil { + if _, err := os.Stat(filepath.Join(ateletpath.OCIBundlePath(actorUID, "app"), imagecache.OverlaySpecFileName)); err != nil { t.Errorf("app bundle: %v", err) } _, err := os.Stat(ateletpath.OCIBundlePath(actorUID, ocispec.PauseContainer)) @@ -272,6 +273,43 @@ func TestPrepareOCIBundlesPause(t *testing.T) { } } +// TestPrepareOCIBundlesFillsContainerSpec pins that each container's spec +// carries its resolved args, env, capabilities and resource limits. +func TestPrepareOCIBundlesFillsContainerSpec(t *testing.T) { + useTempNodeDirs(t) + host := imageVolumeTestRegistry(t) + image := host + "/actor:v1" + pushTestImage(t, image, singleFileLayer(t, "bin/app", "app")) + spec := &ateletpb.WorkloadSpec{ + Containers: []*ateletpb.Container{{ + Name: "app", + Image: image, + Command: []string{"/bin/app"}, + Args: []string{"serve"}, + Env: []*ateletpb.EnvEntry{{Name: "GREETING", Value: "hello"}}, + SecurityContext: &ateletpb.SecurityContext{ + Capabilities: &ateletpb.Capabilities{Add: []string{"NET_ADMIN"}, Drop: []string{"KILL"}}, + }, + Resources: &ateletpb.ResourceLimits{MemoryBytes: 1 << 30, CpuMillis: 500}, + }}, + } + + s := &AteomHerder{imageCache: newImageVolumeStore(t)} + got, err := s.prepareOCIBundles(t.Context(), "actor-uid-1", resources.ActorRef{}, spec, "", "ateom-uid-1") + if err != nil { + t.Fatalf("prepareOCIBundles: %v", err) + } + want := []*ateompb.ContainerSpec{{ + Args: []string{"/bin/app", "serve"}, + Env: []string{"GREETING=hello", "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"}, + Capabilities: []string{"CAP_AUDIT_WRITE", "CAP_NET_ADMIN", "CAP_NET_BIND_SERVICE"}, + Resources: &ateompb.ResourceLimits{MemoryBytes: 1 << 30, CpuMillis: 500}, + }} + if diff := cmp.Diff(want, got, protocmp.Transform()); diff != "" { + t.Errorf("prepareOCIBundles container specs mismatch (-want +got):\n%s", diff) + } +} + func TestSnapshotManifestRejectsNonLocalFile(t *testing.T) { manifest, err := json.Marshal(sandboxAssetsRecord{ SandboxClass: "gvisor", @@ -935,7 +973,30 @@ func TestBuildAteomWorkloadSpecForwardsWakeupProbe(t *testing.T) { {Name: "without-probe"}, }, } - got, err := buildAteomWorkloadSpec(in) + got, err := buildAteomWorkloadSpec(in, nil) + if err != nil { + t.Fatalf("buildAteomWorkloadSpec failed: %v", err) + } + if diff := cmp.Diff(want, got, protocmp.Transform()); diff != "" { + t.Errorf("buildAteomWorkloadSpec mismatch (-want +got):\n%s", diff) + } +} + +func TestBuildAteomWorkloadSpecForwardsContainerSpecs(t *testing.T) { + in := &ateletpb.WorkloadSpec{ + Containers: []*ateletpb.Container{{Name: "main"}, {Name: "sidecar"}}, + } + containerSpecs := []*ateompb.ContainerSpec{ + {Args: []string{"/app", "serve"}}, + {Args: []string{"/sidecar"}}, + } + want := &ateompb.WorkloadSpec{ + Containers: []*ateompb.Container{ + {Name: "main", ContainerSpec: &ateompb.ContainerSpec{Args: []string{"/app", "serve"}}}, + {Name: "sidecar", ContainerSpec: &ateompb.ContainerSpec{Args: []string{"/sidecar"}}}, + }, + } + got, err := buildAteomWorkloadSpec(in, containerSpecs) if err != nil { t.Fatalf("buildAteomWorkloadSpec failed: %v", err) } @@ -997,7 +1058,7 @@ func TestBuildAteomWorkloadSpecForwardsDurableDirMounts(t *testing.T) { {Name: "no-volumes"}, }, } - got, err := buildAteomWorkloadSpec(in) + got, err := buildAteomWorkloadSpec(in, nil) if err != nil { t.Fatalf("buildAteomWorkloadSpec failed: %v", err) } @@ -1068,7 +1129,7 @@ func TestBuildAteomWorkloadSpecValidation(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - _, err := buildAteomWorkloadSpec(tc.in) + _, err := buildAteomWorkloadSpec(tc.in, nil) if err == nil { t.Fatal("expected error, got nil") } @@ -1348,7 +1409,7 @@ func TestBuildAteomWorkloadSpec_ImageVolumeMounts(t *testing.T) { }}, } - got, err := buildAteomWorkloadSpec(spec) + got, err := buildAteomWorkloadSpec(spec, nil) if err != nil { t.Fatalf("buildAteomWorkloadSpec: %v", err) } diff --git a/cmd/atelet/oci.go b/cmd/atelet/oci.go index f82c279d7c..21bbb0040f 100644 --- a/cmd/atelet/oci.go +++ b/cmd/atelet/oci.go @@ -24,7 +24,6 @@ import ( "github.com/agent-substrate/substrate/cmd/atelet/internal/ateletpath" "github.com/agent-substrate/substrate/internal/imagecache" - "github.com/agent-substrate/substrate/internal/ocispec" "github.com/agent-substrate/substrate/internal/proto/ateletpb" v1 "github.com/google/go-containerregistry/pkg/v1" "go.opentelemetry.io/otel" @@ -75,7 +74,7 @@ func resolveCapabilities(caps *ateletpb.Capabilities) []string { return out } -func prepareOCIDirectory(ctx context.Context, imageCache *imagecache.Store, actorUID, containerName, ref string, command, args []string, env []string, netns string, volumes []*ateletpb.Volume, volumeMounts []*ateletpb.VolumeMount, capabilities []string, resources *ateletpb.ResourceLimits) error { +func prepareOCIDirectory(ctx context.Context, imageCache *imagecache.Store, actorUID, containerName, ref string, volumes []*ateletpb.Volume, volumeMounts []*ateletpb.VolumeMount) (*v1.Config, error) { tracer := otel.Tracer("prepareOCIDirectory") ctx, span := tracer.Start(ctx, "prepareOCIDirectory") @@ -87,7 +86,7 @@ func prepareOCIDirectory(ctx context.Context, imageCache *imagecache.Store, acto // Clear any previous bundle contents (belt and suspenders: resetActorDirs // already wiped the bundle dir on the Run/Restore path). if err := imagecache.RemoveAllWritable(bundlePath); err != nil { - return fmt.Errorf("while clearing bundle %q: %w", bundlePath, err) + return nil, fmt.Errorf("while clearing bundle %q: %w", bundlePath, err) } // The bundle's rootfs is composed by ateom as an overlay mount just before @@ -98,7 +97,7 @@ func prepareOCIDirectory(ctx context.Context, imageCache *imagecache.Store, acto // it deliberately runs with no capabilities, so it cannot mount. for _, d := range []string{"rootfs", "upper", "work"} { if err := os.MkdirAll(path.Join(bundlePath, d), 0o700); err != nil { - return fmt.Errorf("in os.MkdirAll for container bundle dir: %w", err) + return nil, fmt.Errorf("in os.MkdirAll for container bundle dir: %w", err) } } @@ -120,16 +119,8 @@ func prepareOCIDirectory(ctx context.Context, imageCache *imagecache.Store, acto return err }) if err := g.Wait(); err != nil { - return err - } - - // Argv and env need only the image config; resolve them before writing - // any spec so an invalid container config fails fast. - resolvedArgs, err := resolveProcessArgs(&img.Config, command, args) - if err != nil { - return fmt.Errorf("while resolving process args for container %q: %w", containerName, err) + return nil, err } - resolvedEnv := resolveActorEnv(&img.Config, env) // Every bind target must exist in the rootfs for the mount to attach; // ateom creates them through the mounted overlay (they land in the @@ -144,27 +135,10 @@ func prepareOCIDirectory(ctx context.Context, imageCache *imagecache.Store, acto ExtraDirs: extraDirs, ImageVolumes: imageVolumes, }); err != nil { - return fmt.Errorf("while writing overlay spec: %w", err) - } - - // Write the runtime-neutral OCI spec to config.json. - if err := ocispec.Save(bundlePath, ocispec.Build(ocispec.Options{ - Args: resolvedArgs, - Env: resolvedEnv, - NetNSPath: netns, - Volumes: volumes, - VolumeMounts: volumeMounts, - Capabilities: capabilities, - Resources: resources, - DurableDirVolumeMountsDir: ateletpath.DurableDirVolumeMountsDir(actorUID), - VolumesDir: ateletpath.VolumesDir(actorUID), - SystemInfoVolumeRootsDir: ateletpath.SystemInfoVolumeRootsDir(actorUID), - BundlePath: bundlePath, - })); err != nil { - return fmt.Errorf("while writing OCI spec: %w", err) + return nil, fmt.Errorf("while writing overlay spec: %w", err) } - return nil + return &img.Config, nil } // resolveImageVolumes pulls the image behind every image-typed volume this diff --git a/cmd/atelet/volumes_test.go b/cmd/atelet/volumes_test.go index 375153e020..e3b13ac9b3 100644 --- a/cmd/atelet/volumes_test.go +++ b/cmd/atelet/volumes_test.go @@ -76,7 +76,7 @@ var _ volume.VolumePluginWorkerPlane = (*fakeWorkerPlugin)(nil) // withTempActorsDir redirects nodepath.ActorsDir at a temp dir for the // duration of the test. Every path derived from ActorsDir moves with it, -// including the ones resetActorDirs and the OCI spec builder compute +// including the ones resetActorDirs and the bundle preparation compute // independently of the volume mount code. func withTempActorsDir(t *testing.T) { t.Helper() diff --git a/cmd/ateom-gvisor/main.go b/cmd/ateom-gvisor/main.go index a74f3cc02f..c72c027f5a 100644 --- a/cmd/ateom-gvisor/main.go +++ b/cmd/ateom-gvisor/main.go @@ -594,6 +594,7 @@ func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkload actorDirs: req.GetActorDirs(), size: sizing.FromLimits(req.GetCpuMilli(), req.GetMemoryBytes()), durableVolumes: durableVolumeNames(req.GetSpec()), + containers: req.GetSpec().GetContainers(), } var containersToDelete []string defer func() { @@ -951,6 +952,7 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore actorDirs: req.GetActorDirs(), size: sizing.FromLimits(req.GetCpuMilli(), req.GetMemoryBytes()), durableVolumes: durableVolumeNames(req.GetSpec()), + containers: req.GetSpec().GetContainers(), } var containersToDelete []string defer func() { diff --git a/cmd/ateom-gvisor/runsc.go b/cmd/ateom-gvisor/runsc.go index 6c55706465..1745603333 100644 --- a/cmd/ateom-gvisor/runsc.go +++ b/cmd/ateom-gvisor/runsc.go @@ -43,6 +43,8 @@ type runsc struct { size sizing.SandboxSize // durableVolumes are the durable-dir volume names declared to the sandbox. durableVolumes []string + // containers are the workload's containers, from the request. + containers []*ateompb.Container } // durableVolumeNames returns the sorted, deduplicated durable-dir volume names @@ -58,13 +60,27 @@ func durableVolumeNames(spec *ateompb.WorkloadSpec) []string { return slices.Compact(names) } -// shapeSpec loads, shapes for gVisor, and saves the container's OCI spec. -func (r *runsc) shapeSpec(containerName string) error { - bundle := ociBundlePath(r.actorDirs, containerName) - spec, err := ocispec.Load(bundle) - if err != nil { - return err +// pauseContainer is the sandbox root container. It only reaps, so it gets no +// capabilities and mounts nothing. +func pauseContainer() *ateompb.Container { + return &ateompb.Container{ + Name: ocispec.PauseContainer, + ContainerSpec: &ateompb.ContainerSpec{Args: []string{"/pause"}}, + } +} + +// writeSpec builds the container's OCI spec, shapes it for gVisor, and saves it +// into the container's bundle. +func (r *runsc) writeSpec(containerName string) error { + container := pauseContainer() + if containerName != ocispec.PauseContainer { + i := slices.IndexFunc(r.containers, func(c *ateompb.Container) bool { return c.GetName() == containerName }) + if i < 0 { + return fmt.Errorf("container %q is not in the workload spec", containerName) + } + container = r.containers[i] } + spec := ocispec.Build(ocispec.Options{ActorUID: r.actorUID, ActorDirs: r.actorDirs, Container: container}) ocispec.ShapeGVisor(spec, ocispec.GVisorOptions{ ActorUID: r.actorUID, ContainerName: containerName, @@ -72,14 +88,14 @@ func (r *runsc) shapeSpec(containerName string) error { Size: r.size, ResolvConf: resolvConfPath(r.actorDirs), }) - return ocispec.Save(bundle, spec) + return ocispec.Save(ociBundlePath(r.actorDirs, containerName), spec) } func (r *runsc) cmdCreate(ctx context.Context, out io.Writer, containerName string, additionalArgs []string) error { slog.InfoContext(ctx, "About to run runsc create", slog.String("container", containerName)) - if err := r.shapeSpec(containerName); err != nil { - return fmt.Errorf("while shaping the OCI spec for %q: %w", containerName, err) + if err := r.writeSpec(containerName); err != nil { + return fmt.Errorf("while writing the OCI spec for %q: %w", containerName, err) } args := []string{ @@ -290,8 +306,8 @@ func (r *runsc) restoreArgs(containerName, checkpointPath string) []string { func (r *runsc) cmdRestore(ctx context.Context, out io.Writer, containerName, checkpointPath string) error { slog.InfoContext(ctx, "About to run runsc restore", slog.String("container", containerName)) - if err := r.shapeSpec(containerName); err != nil { - return fmt.Errorf("while shaping the OCI spec for %q: %w", containerName, err) + if err := r.writeSpec(containerName); err != nil { + return fmt.Errorf("while writing the OCI spec for %q: %w", containerName, err) } cmd := exec.CommandContext(ctx, r.path, r.restoreArgs(containerName, checkpointPath)...) diff --git a/cmd/ateom-gvisor/runsc_test.go b/cmd/ateom-gvisor/runsc_test.go index 58be691329..f9a8bcc709 100644 --- a/cmd/ateom-gvisor/runsc_test.go +++ b/cmd/ateom-gvisor/runsc_test.go @@ -18,9 +18,14 @@ package main import ( "context" + "encoding/json" + "os" + "path/filepath" "reflect" + "slices" "testing" + "github.com/opencontainers/runtime-spec/specs-go" "google.golang.org/grpc/codes" "github.com/agent-substrate/substrate/internal/actorlock" @@ -34,6 +39,50 @@ var testActorDirs = &ateompb.ActorDirs{ OciBundleDir: "/node/actors/test-actor-123/bundle", } +// writeSpec builds a workload container's spec from the request and the pause +// container's from ateom's own description. +func TestWriteSpec(t *testing.T) { + root := t.TempDir() + actorDirs := &ateompb.ActorDirs{RootDir: root, OciBundleDir: filepath.Join(root, "bundles")} + r := &runsc{ + actorUID: "uid-a", + actorDirs: actorDirs, + containers: []*ateompb.Container{{Name: "app", ContainerSpec: &ateompb.ContainerSpec{Args: []string{"/app", "serve"}}}}, + } + for name, wantArgs := range map[string][]string{ + "app": {"/app", "serve"}, + ocispec.PauseContainer: {"/pause"}, + } { + bundle := ociBundlePath(actorDirs, name) + if err := os.MkdirAll(bundle, 0o700); err != nil { + t.Fatal(err) + } + if err := r.writeSpec(name); err != nil { + t.Fatalf("writeSpec(%q) = %v", name, err) + } + b, err := os.ReadFile(filepath.Join(bundle, "config.json")) + if err != nil { + t.Fatal(err) + } + var spec specs.Spec + if err := json.Unmarshal(b, &spec); err != nil { + t.Fatal(err) + } + if !slices.Equal(spec.Process.Args, wantArgs) { + t.Errorf("%s args = %v, want %v", name, spec.Process.Args, wantArgs) + } + if got := spec.Annotations["io.kubernetes.cri.container-name"]; got != name { + t.Errorf("%s container-name annotation = %q, want the spec shaped for gVisor", name, got) + } + if name == ocispec.PauseContainer && len(spec.Process.Capabilities.Bounding) != 0 { + t.Errorf("pause capabilities = %v, want none", spec.Process.Capabilities.Bounding) + } + } + if err := r.writeSpec("missing"); err == nil { + t.Error("writeSpec() = nil for a container outside the workload spec") + } +} + func TestKillArgs(t *testing.T) { r := &runsc{ path: "/usr/bin/runsc", diff --git a/cmd/ateom-microvm/restore.go b/cmd/ateom-microvm/restore.go index 561710101f..fa0578a10c 100644 --- a/cmd/ateom-microvm/restore.go +++ b/cmd/ateom-microvm/restore.go @@ -270,7 +270,7 @@ func (s *AteomService) restoreFullScope(ctx context.Context, p actorBootParams, if len(containers) > maxActorContainers { return apierror.Unimplemented("ateom-microvm supports at most %d containers, got %d", maxActorContainers, len(containers)) } - ctrs, err := s.buildActorContainers(p.actorDirs, containers) + ctrs, err := s.buildActorContainers(p.actorUID, p.actorDirs, containers) if err != nil { return err } diff --git a/cmd/ateom-microvm/run.go b/cmd/ateom-microvm/run.go index 50c8356104..b7b64bdd06 100644 --- a/cmd/ateom-microvm/run.go +++ b/cmd/ateom-microvm/run.go @@ -229,7 +229,8 @@ func (s *AteomService) resolveRuntime(paths map[string]string) resolvedRuntime { // Contract with atelet: // - The runtime assets (guest kernel, guest OS image, cloud-hypervisor, virtiofsd) // are on disk and passed as runtime asset paths. -// - The OCI bundle (config.json + populated rootfs/) is prepared per container. +// - Each container's bundle holds the overlay spec its rootfs/ is composed +// from. ateom builds the OCI spec itself. func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkloadRequest) (resp *ateompb.RunWorkloadResponse, retErr error) { if err := validateActorDirs(req.GetActorDirs()); err != nil { return nil, err @@ -437,7 +438,7 @@ func (s *AteomService) coldBootActor(ctx context.Context, p actorBootParams) (re // Prepare each container's OCI spec + record its bundle rootfs (the overlay // lower the host merges under the container's writable upper). - ctrs, err := s.buildActorContainers(p.actorDirs, containers) + ctrs, err := s.buildActorContainers(p.actorUID, p.actorDirs, containers) if err != nil { return err } @@ -621,19 +622,16 @@ func (s *AteomService) coldBootActor(ctx context.Context, p actorBootParams) (re } // buildActorContainers prepares each of the actor's containers for the shared -// micro-VM: it loads the OCI spec from the per-container bundle, injects guest DNS, +// micro-VM: it builds the OCI spec from the request, injects guest DNS, // and records the bundle rootfs that backs the overlay's RO lower. No host disk is // mounted here — the merged overlays are assembled in stageMergedRootfs after the // sandbox state is clean. Both RunWorkload and RestoreWorkload go through here. -func (s *AteomService) buildActorContainers(actorDirs *ateompb.ActorDirs, containers []*ateompb.Container) ([]actorContainer, error) { +func (s *AteomService) buildActorContainers(actorUID string, actorDirs *ateompb.ActorDirs, containers []*ateompb.Container) ([]actorContainer, error) { ctrs := make([]actorContainer, len(containers)) for i, c := range containers { cn := c.GetName() bundle := ociBundlePath(actorDirs, cn) - spec, err := ocispec.Load(bundle) - if err != nil { - return nil, fmt.Errorf("while reading the OCI spec for %q: %w", cn, err) - } + spec := ocispec.Build(ocispec.Options{ActorUID: actorUID, ActorDirs: actorDirs, Container: c}) if err := ocispec.ShapeMicroVM(spec, ocispec.MicroVMOptions{ActorDirs: actorDirs, ContainerID: cn}); err != nil { return nil, fmt.Errorf("while shaping the OCI spec for %q: %w", cn, err) } diff --git a/internal/imagecache/README.md b/internal/imagecache/README.md index 63492fabc5..1b81f4c272 100644 --- a/internal/imagecache/README.md +++ b/internal/imagecache/README.md @@ -107,7 +107,7 @@ layer diffIDs in order — layers shared by N images exist once. requested digest (and the per-platform child digest for multi-arch refs). `prepareOCIDirectory` in atelet then writes `rootfs-overlay.json` -(`OverlaySpec`) into the bundle next to `config.json`, listing the layer +(`OverlaySpec`) into the bundle, listing the layer directories bottom-first plus any `ExtraDirs` (in-rootfs bind-mount targets, e.g. the actor identity mount at `/run/ate`), and creates the empty bundle-local `rootfs/`, `upper/`, and `work/` directories. diff --git a/internal/imagecache/imagecache.go b/internal/imagecache/imagecache.go index ec01d77780..8ab0da44f6 100644 --- a/internal/imagecache/imagecache.go +++ b/internal/imagecache/imagecache.go @@ -21,7 +21,7 @@ // // - atelet (plain root, all capabilities dropped) pulls layers and unpacks // them into the pool (Store.EnsureImage), and writes a rootfs-overlay.json -// next to each bundle's config.json (WriteSpec). Whiteout entries are +// into each bundle (WriteSpec). Whiteout entries are // recorded in per-layer metadata rather than materialized, because // overlayfs whiteouts are char devices (CAP_MKNOD) with trusted.* xattrs // for opaque dirs (CAP_SYS_ADMIN). diff --git a/internal/imagecache/spec.go b/internal/imagecache/spec.go index 47c2f5db13..e1e0873440 100644 --- a/internal/imagecache/spec.go +++ b/internal/imagecache/spec.go @@ -25,16 +25,16 @@ import ( ) // ImageVolumeMountPath is where one image volume is composed inside a -// bundle. atelet writes it into config.json as the bind source; the ateom -// mounts the volume there (see setupImageVolumes). +// bundle. The ateom mounts the volume there (see setupImageVolumes) and binds +// it into the container. func ImageVolumeMountPath(bundlePath, volumeName string) string { return filepath.Join(bundlePath, "volumes", volumeName) } -// OverlaySpecFileName is the file atelet writes into each container bundle, -// next to config.json, describing how to compose the bundle's rootfs from -// cached layers. Its absence means the bundle's rootfs is a plain directory -// (e.g. one prepared by a pre-imagecache atelet) and needs no mount. +// OverlaySpecFileName is the file atelet writes into each container bundle +// describing how to compose the bundle's rootfs from cached layers. Its +// absence means the bundle's rootfs is a plain directory (e.g. one prepared by +// a pre-imagecache atelet) and needs no mount. const OverlaySpecFileName = "rootfs-overlay.json" // OverlaySpec is the contract between atelet (which cannot mount) and the diff --git a/internal/ocispec/microvm_test.go b/internal/ocispec/microvm_test.go index fb3f808fa5..bfde5f6595 100644 --- a/internal/ocispec/microvm_test.go +++ b/internal/ocispec/microvm_test.go @@ -17,7 +17,7 @@ package ocispec import ( "testing" - "github.com/agent-substrate/substrate/internal/proto/ateletpb" + "github.com/agent-substrate/substrate/internal/proto/ateompb" "github.com/opencontainers/runtime-spec/specs-go" ) @@ -121,10 +121,10 @@ func eqInt64Ptr(a, b *int64) bool { // unbound the container. func TestShapeMicroVM_KeepsDeclaredContainerLimits(t *testing.T) { const declared = 64 * 1024 * 1024 - spec := Build(Options{ + spec := Build(Options{Container: &ateompb.Container{ContainerSpec: &ateompb.ContainerSpec{ Args: []string{"/app"}, - Resources: &ateletpb.ResourceLimits{MemoryBytes: declared}, - }) + Resources: &ateompb.ResourceLimits{MemoryBytes: declared}, + }}}) if err := ShapeMicroVM(spec, MicroVMOptions{ActorDirs: parityActorDirs, ContainerID: "app"}); err != nil { t.Fatalf("ShapeMicroVM() = %v", err) } @@ -140,7 +140,7 @@ func TestShapeMicroVM_KeepsDeclaredContainerLimits(t *testing.T) { // A container that declares nothing must stay unbounded inside the guest: guest // RAM is the real ceiling, and a cap equal to the whole guest can never bind. func TestShapeMicroVM_LeavesUndeclaredContainerUnlimited(t *testing.T) { - spec := Build(Options{Args: []string{"/app"}}) + spec := Build(Options{Container: &ateompb.Container{ContainerSpec: &ateompb.ContainerSpec{Args: []string{"/app"}}}}) if err := ShapeMicroVM(spec, MicroVMOptions{ActorDirs: parityActorDirs, ContainerID: "app"}); err != nil { t.Fatalf("ShapeMicroVM() = %v", err) } diff --git a/internal/ocispec/ocispec.go b/internal/ocispec/ocispec.go index 7b672f2453..dbaec1cf85 100644 --- a/internal/ocispec/ocispec.go +++ b/internal/ocispec/ocispec.go @@ -24,7 +24,8 @@ import ( "path/filepath" "github.com/agent-substrate/substrate/internal/imagecache" - "github.com/agent-substrate/substrate/internal/proto/ateletpb" + "github.com/agent-substrate/substrate/internal/nodepath" + "github.com/agent-substrate/substrate/internal/proto/ateompb" "github.com/opencontainers/runtime-spec/specs-go" ) @@ -34,27 +35,15 @@ const specFile = "config.json" // hostname is the UTS hostname for actor containers. const hostname = "actor" -// Options describes one actor container. Args, Env and Capabilities arrive -// already resolved. +// Options describes one actor container. type Options struct { - Args []string - Env []string - // NetNSPath is the network namespace the ateom runs the actor in. - NetNSPath string - Volumes []*ateletpb.Volume - VolumeMounts []*ateletpb.VolumeMount - Capabilities []string - // Resources are the container's own declared limits, or nil for none. - Resources *ateletpb.ResourceLimits - - // The actor's directories the bind mount sources are joined from, one - // subdirectory per volume name. - DurableDirVolumeMountsDir string - VolumesDir string - SystemInfoVolumeRootsDir string - // BundlePath is this container's bundle, where its image volumes are - // composed. - BundlePath string + // ActorUID names the actor's network namespace. + ActorUID string + // ActorDirs are the actor's directories the bind mount sources are joined + // from, one subdirectory per volume name. + ActorDirs *ateompb.ActorDirs + // Container carries the process and its volume mounts. + Container *ateompb.Container } const ( @@ -71,7 +60,7 @@ const ( // ociResources maps the resolved limits onto the OCI spec's linux.resources. // Returns nil when nothing is set, so the spec carries no linux.resources at // all for a container that declares no limits. -func ociResources(r *ateletpb.ResourceLimits) *specs.LinuxResources { +func ociResources(r *ateompb.ResourceLimits) *specs.LinuxResources { if r == nil || (r.GetMemoryBytes() <= 0 && r.GetCpuMillis() <= 0) { return nil } @@ -89,19 +78,21 @@ func ociResources(r *ateletpb.ResourceLimits) *specs.LinuxResources { // Build returns a runtime-neutral OCI spec for an actor container. func Build(o Options) *specs.Spec { + container := o.Container + containerSpec := container.GetContainerSpec() spec := &specs.Spec{ Process: &specs.Process{ User: specs.User{ UID: 0, GID: 0, }, - Args: o.Args, - Env: o.Env, + Args: containerSpec.GetArgs(), + Env: containerSpec.GetEnv(), Cwd: "/", Capabilities: &specs.LinuxCapabilities{ - Bounding: o.Capabilities, - Effective: o.Capabilities, - Permitted: o.Capabilities, + Bounding: containerSpec.GetCapabilities(), + Effective: containerSpec.GetCapabilities(), + Permitted: containerSpec.GetCapabilities(), // Inheritable and Ambient stay empty; capabilities inherit via // Bounding. // @@ -152,7 +143,7 @@ func Build(o Options) *specs.Spec { }, { Type: "network", - Path: o.NetNSPath, // Will be created by ateom + Path: nodepath.ActorNetNSPath(o.ActorUID), }, { Type: "ipc", @@ -164,54 +155,36 @@ func Build(o Options) *specs.Spec { Type: "mount", }, }, - Resources: ociResources(o.Resources), + Resources: ociResources(containerSpec.GetResources()), }, } - volumesByName := make(map[string]*ateletpb.Volume, len(o.Volumes)) - for _, vol := range o.Volumes { - volumesByName[vol.GetName()] = vol - } - for _, vm := range o.VolumeMounts { - var srcPath string - options := []string{"bind", "rw"} - switch volumesByName[vm.GetName()].GetSource().(type) { - case *ateletpb.Volume_DurableDir: - srcPath = filepath.Join(o.DurableDirVolumeMountsDir, vm.GetName()) - case *ateletpb.Volume_External: - srcPath = filepath.Join(o.VolumesDir, vm.GetName()) - case *ateletpb.Volume_SystemInfo: - srcPath = filepath.Join(o.SystemInfoVolumeRootsDir, vm.GetName()) - options = []string{"bind", "ro"} - case *ateletpb.Volume_Image: - srcPath = imagecache.ImageVolumeMountPath(o.BundlePath, vm.GetName()) - options = []string{"bind", "ro"} - default: - continue - } + // Mount order across volume kinds does not matter: the API rejects mounts + // that nest under one another. + bind := func(dest, src, mode string) { spec.Mounts = append(spec.Mounts, specs.Mount{ - Destination: vm.GetMountPath(), + Destination: dest, Type: "bind", - Source: srcPath, - Options: options, + Source: src, + Options: []string{"bind", mode}, }) } - - return spec -} - -// Load reads the OCI spec of the bundle at bundlePath. -func Load(bundlePath string) (*specs.Spec, error) { - specPath := path.Join(bundlePath, specFile) - b, err := os.ReadFile(specPath) - if err != nil { - return nil, fmt.Errorf("reading %q: %w", specPath, err) + actorDirs := o.ActorDirs + for _, m := range container.GetDurableDirVolumeMounts() { + bind(m.GetMountPath(), filepath.Join(actorDirs.GetDurableDirVolumeMountsDir(), m.GetVolumeName()), "rw") + } + for _, m := range container.GetCsiVolumeMounts() { + bind(m.GetMountPath(), filepath.Join(actorDirs.GetVolumesDir(), m.GetVolumeName()), "rw") } - var spec specs.Spec - if err := json.Unmarshal(b, &spec); err != nil { - return nil, fmt.Errorf("parsing %q: %w", specPath, err) + for _, m := range container.GetSystemInfoVolumeMounts() { + bind(m.GetMountPath(), filepath.Join(actorDirs.GetSystemInfoVolumeRootsDir(), m.GetVolumeName()), "ro") } - return &spec, nil + bundlePath := filepath.Join(actorDirs.GetOciBundleDir(), container.GetName()) + for _, m := range container.GetImageVolumeMounts() { + bind(m.GetMountPath(), imagecache.ImageVolumeMountPath(bundlePath, m.GetVolumeName()), "ro") + } + + return spec } // Save writes spec as the OCI spec of the bundle at bundlePath. diff --git a/internal/ocispec/ocispec_test.go b/internal/ocispec/ocispec_test.go index d19d4ae80a..2d06ed5c3d 100644 --- a/internal/ocispec/ocispec_test.go +++ b/internal/ocispec/ocispec_test.go @@ -15,11 +15,15 @@ package ocispec import ( + "encoding/json" + "os" + "path/filepath" "slices" "testing" "github.com/agent-substrate/substrate/internal/imagecache" - "github.com/agent-substrate/substrate/internal/proto/ateletpb" + "github.com/agent-substrate/substrate/internal/nodepath" + "github.com/agent-substrate/substrate/internal/proto/ateompb" "github.com/opencontainers/runtime-spec/specs-go" ) @@ -35,18 +39,8 @@ func mountFor(t *testing.T, spec *specs.Spec, dest string) specs.Mount { return specs.Mount{} } -func durableVolume(name string) *ateletpb.Volume { - return &ateletpb.Volume{Name: name, Source: &ateletpb.Volume_DurableDir{DurableDir: &ateletpb.DurableDirVolume{}}} -} - // Each volume mount becomes a bind of its host directory, rw or ro by kind. func TestBuild_VolumeMounts(t *testing.T) { - volumes := []*ateletpb.Volume{ - durableVolume("data"), - {Name: "sysinfo", Source: &ateletpb.Volume_SystemInfo{SystemInfo: &ateletpb.SystemInfoVolume{}}}, - {Name: "csi", Source: &ateletpb.Volume_External{External: &ateletpb.ExternalVolumeSource{}}}, - {Name: "agent", Source: &ateletpb.Volume_Image{Image: &ateletpb.ImageVolumeSource{}}}, - } const ( durableDir = "/node/actors/a/durable-dir" volumesDir = "/node/actors/a/volumes" @@ -54,19 +48,23 @@ func TestBuild_VolumeMounts(t *testing.T) { bundle = "/node/actors/a/bundles/app" ) spec := Build(Options{ - Args: []string{"/app"}, - Volumes: volumes, - VolumeMounts: []*ateletpb.VolumeMount{ - {Name: "data", MountPath: "/var/data"}, - {Name: "data", MountPath: "/home/counter"}, - {Name: "sysinfo", MountPath: "/run/ate"}, - {Name: "csi", MountPath: "/mnt/csi"}, - {Name: "agent", MountPath: "/ate"}, + ActorDirs: &ateompb.ActorDirs{ + OciBundleDir: "/node/actors/a/bundles", + DurableDirVolumeMountsDir: durableDir, + VolumesDir: volumesDir, + SystemInfoVolumeRootsDir: sysInfoDir, + }, + Container: &ateompb.Container{ + Name: "app", + ContainerSpec: &ateompb.ContainerSpec{Args: []string{"/app"}}, + DurableDirVolumeMounts: []*ateompb.DurableDirVolumeMount{ + {VolumeName: "data", MountPath: "/var/data"}, + {VolumeName: "data", MountPath: "/home/counter"}, + }, + SystemInfoVolumeMounts: []*ateompb.SystemInfoVolumeMount{{VolumeName: "sysinfo", MountPath: "/run/ate"}}, + CsiVolumeMounts: []*ateompb.VolumeMount{{VolumeName: "csi", MountPath: "/mnt/csi"}}, + ImageVolumeMounts: []*ateompb.ImageVolumeMount{{VolumeName: "agent", MountPath: "/ate"}}, }, - DurableDirVolumeMountsDir: durableDir, - VolumesDir: volumesDir, - SystemInfoVolumeRootsDir: sysInfoDir, - BundlePath: bundle, }) for _, tc := range []struct { @@ -93,22 +91,22 @@ func TestBuild_VolumeMounts(t *testing.T) { } } -// A mount naming an undeclared volume is skipped. -func TestBuild_UnknownVolumeMountSkipped(t *testing.T) { - spec := Build(Options{ - VolumeMounts: []*ateletpb.VolumeMount{{Name: "missing", MountPath: "/mnt/missing"}}, - }) - for _, m := range spec.Mounts { - if m.Destination == "/mnt/missing" { - t.Fatalf("mount for an undeclared volume was emitted: %v", m) - } +// The container joins the network namespace ateom creates for the actor. +func TestBuild_JoinsTheActorNetNS(t *testing.T) { + spec := Build(Options{ActorUID: "uid-a", Container: &ateompb.Container{ContainerSpec: &ateompb.ContainerSpec{Args: []string{"/app"}}}}) + i := slices.IndexFunc(spec.Linux.Namespaces, func(ns specs.LinuxNamespace) bool { return ns.Type == specs.NetworkNamespace }) + if i < 0 { + t.Fatal("spec has no network namespace") + } + if got, want := spec.Linux.Namespaces[i].Path, nodepath.ActorNetNSPath("uid-a"); got != want { + t.Errorf("network namespace path = %q, want %q", got, want) } } // The resolved set lands in bounding, effective and permitted only. func TestBuild_Capabilities(t *testing.T) { want := []string{"CAP_CHOWN", "CAP_KILL"} - spec := Build(Options{Args: []string{"/app"}, Capabilities: want}) + spec := Build(Options{Container: &ateompb.Container{ContainerSpec: &ateompb.ContainerSpec{Args: []string{"/app"}, Capabilities: want}}}) caps := spec.Process.Capabilities if caps == nil { @@ -141,7 +139,7 @@ func TestBuild_Capabilities(t *testing.T) { // The pause container gets no capabilities. func TestBuild_NoCapabilitiesForPause(t *testing.T) { - spec := Build(Options{Args: []string{"/pause"}}) + spec := Build(Options{Container: &ateompb.Container{ContainerSpec: &ateompb.ContainerSpec{Args: []string{"/pause"}}}}) caps := spec.Process.Capabilities if caps == nil { @@ -163,15 +161,19 @@ func TestBuild_NoCapabilitiesForPause(t *testing.T) { } } -func TestSaveLoadRoundTrip(t *testing.T) { +func TestSave(t *testing.T) { bundle := t.TempDir() - want := Build(Options{Args: []string{"/app"}, NetNSPath: "/run/netns/x"}) + want := Build(Options{ActorUID: "uid-a", Container: &ateompb.Container{ContainerSpec: &ateompb.ContainerSpec{Args: []string{"/app"}}}}) if err := Save(bundle, want); err != nil { t.Fatalf("Save() = %v", err) } - got, err := Load(bundle) + b, err := os.ReadFile(filepath.Join(bundle, "config.json")) if err != nil { - t.Fatalf("Load() = %v", err) + t.Fatal(err) + } + var got specs.Spec + if err := json.Unmarshal(b, &got); err != nil { + t.Fatal(err) } if got.Hostname != want.Hostname || !slices.Equal(got.Process.Args, want.Process.Args) || len(got.Mounts) != len(want.Mounts) { t.Errorf("round trip changed the spec:\n got %+v\nwant %+v", got, want) @@ -182,11 +184,11 @@ func TestOCIResources(t *testing.T) { if got := ociResources(nil); got != nil { t.Errorf("ociResources(nil) = %v, want nil", got) } - if got := ociResources(&ateletpb.ResourceLimits{}); got != nil { + if got := ociResources(&ateompb.ResourceLimits{}); got != nil { t.Errorf("ociResources(zero) = %v, want nil so the spec is unchanged", got) } - got := ociResources(&ateletpb.ResourceLimits{MemoryBytes: 268435456, CpuMillis: 200}) + got := ociResources(&ateompb.ResourceLimits{MemoryBytes: 268435456, CpuMillis: 200}) if got == nil { t.Fatal("ociResources() = nil, want limits") } @@ -213,7 +215,7 @@ func TestOCIResources(t *testing.T) { // EINVAL at container create. func TestOCIResources_ClampsQuotaToKernelMinimum(t *testing.T) { for _, millis := range []int64{1, 5, 9} { - got := ociResources(&ateletpb.ResourceLimits{CpuMillis: millis}) + got := ociResources(&ateompb.ResourceLimits{CpuMillis: millis}) if got == nil || got.CPU == nil || got.CPU.Quota == nil { t.Fatalf("cpu=%dm: ociResources() = %v, want a quota", millis, got) } @@ -223,7 +225,7 @@ func TestOCIResources_ClampsQuotaToKernelMinimum(t *testing.T) { } } // At the floor the quota is exact, not clamped. - got := ociResources(&ateletpb.ResourceLimits{CpuMillis: 10}) + got := ociResources(&ateompb.ResourceLimits{CpuMillis: 10}) if *got.CPU.Quota != cpuQuotaMinUS { t.Errorf("cpu=10m: quota = %d, want exactly %d", *got.CPU.Quota, cpuQuotaMinUS) } @@ -232,24 +234,24 @@ func TestOCIResources_ClampsQuotaToKernelMinimum(t *testing.T) { // A negative limit must not produce a non-nil but empty LinuxResources, which // would put a bare "resources": {} into the spec. func TestOCIResources_NegativeIsUnset(t *testing.T) { - if got := ociResources(&ateletpb.ResourceLimits{MemoryBytes: -1, CpuMillis: -1}); got != nil { + if got := ociResources(&ateompb.ResourceLimits{MemoryBytes: -1, CpuMillis: -1}); got != nil { t.Errorf("ociResources(negative) = %+v, want nil", got) } } // A container without limits carries no linux.resources at all. func TestBuild_NoResourcesLeavesLinuxUntouched(t *testing.T) { - spec := Build(Options{Args: []string{"/pause"}}) + spec := Build(Options{Container: &ateompb.Container{ContainerSpec: &ateompb.ContainerSpec{Args: []string{"/app"}}}}) if spec.Linux.Resources != nil { t.Errorf("Linux.Resources = %v, want nil when no limits are declared", spec.Linux.Resources) } } func TestBuild_ResourcesApplied(t *testing.T) { - spec := Build(Options{ + spec := Build(Options{Container: &ateompb.Container{ContainerSpec: &ateompb.ContainerSpec{ Args: []string{"/app"}, - Resources: &ateletpb.ResourceLimits{MemoryBytes: 67108864}, - }) + Resources: &ateompb.ResourceLimits{MemoryBytes: 67108864}, + }}}) if spec.Linux.Resources == nil || spec.Linux.Resources.Memory == nil { t.Fatalf("Linux.Resources = %v, want a memory limit", spec.Linux.Resources) } diff --git a/internal/ocispec/parity_test.go b/internal/ocispec/parity_test.go index 4fbd80fe31..468c511721 100644 --- a/internal/ocispec/parity_test.go +++ b/internal/ocispec/parity_test.go @@ -16,12 +16,10 @@ package ocispec import ( "encoding/json" - "path/filepath" "slices" "strings" "testing" - "github.com/agent-substrate/substrate/internal/proto/ateletpb" "github.com/agent-substrate/substrate/internal/proto/ateompb" "github.com/agent-substrate/substrate/internal/sizing" "github.com/opencontainers/runtime-spec/specs-go" @@ -39,26 +37,21 @@ var parityActorDirs = &ateompb.ActorDirs{ // parityOptions mounts one volume of every kind. var parityOptions = Options{ - Args: []string{"/app"}, - DurableDirVolumeMountsDir: parityActorDirs.GetDurableDirVolumeMountsDir(), - VolumesDir: parityActorDirs.GetVolumesDir(), - SystemInfoVolumeRootsDir: parityActorDirs.GetSystemInfoVolumeRootsDir(), - BundlePath: filepath.Join(parityActorDirs.GetOciBundleDir(), "app"), - Volumes: []*ateletpb.Volume{ - durableVolume("data"), - {Name: "sysinfo", Source: &ateletpb.Volume_SystemInfo{SystemInfo: &ateletpb.SystemInfoVolume{}}}, - {Name: "csi", Source: &ateletpb.Volume_External{External: &ateletpb.ExternalVolumeSource{}}}, - {Name: "agent", Source: &ateletpb.Volume_Image{Image: &ateletpb.ImageVolumeSource{}}}, - }, - VolumeMounts: []*ateletpb.VolumeMount{ - {Name: "data", MountPath: "/var/data"}, - // System-info volume mount. - {Name: "sysinfo", MountPath: "/run/ate"}, - {Name: "csi", MountPath: "/mnt/csi"}, - {Name: "agent", MountPath: "/ate"}, + ActorUID: testActorUID, + ActorDirs: parityActorDirs, + Container: &ateompb.Container{ + Name: "app", + ContainerSpec: &ateompb.ContainerSpec{Args: []string{"/app"}}, + DurableDirVolumeMounts: []*ateompb.DurableDirVolumeMount{{VolumeName: "data", MountPath: "/var/data"}}, + SystemInfoVolumeMounts: []*ateompb.SystemInfoVolumeMount{{VolumeName: "sysinfo", MountPath: "/run/ate"}}, + CsiVolumeMounts: []*ateompb.VolumeMount{{VolumeName: "csi", MountPath: "/mnt/csi"}}, + ImageVolumeMounts: []*ateompb.ImageVolumeMount{{VolumeName: "agent", MountPath: "/ate"}}, }, } +// parityMountPaths are where parityOptions mounts its volumes. +var parityMountPaths = []string{"/var/data", "/run/ate", "/mnt/csi", "/ate"} + var paritySize = sizing.SandboxSize{MilliCPU: 500, MemoryBytes: 512 << 20} // Build's output contains no runtime-specific literals. @@ -96,10 +89,9 @@ func TestShapers_PreserveEveryVolumeMount(t *testing.T) { if err := tc.shape(spec); err != nil { t.Fatalf("shaping the spec: %v", err) } - for _, vm := range parityOptions.VolumeMounts { - m := mountFor(t, spec, vm.GetMountPath()) - if m.Source == "" { - t.Errorf("mount %q has no source after shaping", vm.GetMountPath()) + for _, dest := range parityMountPaths { + if m := mountFor(t, spec, dest); m.Source == "" { + t.Errorf("mount %q has no source after shaping", dest) } } // Neither shaper overrides the hostname. @@ -139,7 +131,7 @@ func TestShapeMicroVM_TranslatesSourcesIntoTheShare(t *testing.T) { // ShapeMicroVM errors on a bind that is not staged into the share. func TestShapeMicroVM_UnstagedSourceIsAnError(t *testing.T) { - spec := Build(Options{Args: []string{"/app"}}) + spec := Build(Options{Container: &ateompb.Container{ContainerSpec: &ateompb.ContainerSpec{Args: []string{"/app"}}}}) spec.Mounts = append(spec.Mounts, specs.Mount{Destination: "/mnt/new", Type: "bind", Source: "/var/lib/ate/new-kind/x"}) if err := ShapeMicroVM(spec, MicroVMOptions{ActorDirs: parityActorDirs, ContainerID: "app"}); err == nil { t.Fatal("ShapeMicroVM() = nil, want an error for a bind that is not staged into the share") diff --git a/internal/proto/ateompb/ateom.pb.go b/internal/proto/ateompb/ateom.pb.go index 5ed1349db9..ea2b111232 100644 --- a/internal/proto/ateompb/ateom.pb.go +++ b/internal/proto/ateompb/ateom.pb.go @@ -691,8 +691,11 @@ type Container struct { SystemInfoVolumeMounts []*SystemInfoVolumeMount `protobuf:"bytes,5,rep,name=system_info_volume_mounts,json=systemInfoVolumeMounts,proto3" json:"system_info_volume_mounts,omitempty"` // image_volume_mounts are the image volumes this container mounts, if any. ImageVolumeMounts []*ImageVolumeMount `protobuf:"bytes,6,rep,name=image_volume_mounts,json=imageVolumeMounts,proto3" json:"image_volume_mounts,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // container_spec is what ateom builds the container's OCI spec from, + // besides its volume mounts. Unset on RPCs that start no process. + ContainerSpec *ContainerSpec `protobuf:"bytes,7,opt,name=container_spec,json=containerSpec,proto3" json:"container_spec,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *Container) Reset() { @@ -767,6 +770,143 @@ func (x *Container) GetImageVolumeMounts() []*ImageVolumeMount { return nil } +func (x *Container) GetContainerSpec() *ContainerSpec { + if x != nil { + return x.ContainerSpec + } + return nil +} + +// ContainerSpec is a container's process and limits, resolved by atelet. +type ContainerSpec struct { + state protoimpl.MessageState `protogen:"open.v1"` + // args is the process argv, already resolved against the image's + // ENTRYPOINT and CMD. + Args []string `protobuf:"bytes,1,rep,name=args,proto3" json:"args,omitempty"` + // env is the process environment as KEY=VALUE, already merged with the + // image's ENV. + Env []string `protobuf:"bytes,2,rep,name=env,proto3" json:"env,omitempty"` + // capabilities is the container's capability set, CAP_-prefixed. + Capabilities []string `protobuf:"bytes,3,rep,name=capabilities,proto3" json:"capabilities,omitempty"` + // resources are the container's own declared limits, if any. + Resources *ResourceLimits `protobuf:"bytes,4,opt,name=resources,proto3" json:"resources,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ContainerSpec) Reset() { + *x = ContainerSpec{} + mi := &file_ateom_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ContainerSpec) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ContainerSpec) ProtoMessage() {} + +func (x *ContainerSpec) ProtoReflect() protoreflect.Message { + mi := &file_ateom_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ContainerSpec.ProtoReflect.Descriptor instead. +func (*ContainerSpec) Descriptor() ([]byte, []int) { + return file_ateom_proto_rawDescGZIP(), []int{7} +} + +func (x *ContainerSpec) GetArgs() []string { + if x != nil { + return x.Args + } + return nil +} + +func (x *ContainerSpec) GetEnv() []string { + if x != nil { + return x.Env + } + return nil +} + +func (x *ContainerSpec) GetCapabilities() []string { + if x != nil { + return x.Capabilities + } + return nil +} + +func (x *ContainerSpec) GetResources() *ResourceLimits { + if x != nil { + return x.Resources + } + return nil +} + +// ResourceLimits are a container's declared limits. +type ResourceLimits struct { + state protoimpl.MessageState `protogen:"open.v1"` + // memory_bytes is the memory limit in bytes. 0 means unset. + MemoryBytes int64 `protobuf:"varint,1,opt,name=memory_bytes,json=memoryBytes,proto3" json:"memory_bytes,omitempty"` + // cpu_millis is the CPU limit in milli-cores (1000 = one core). 0 means unset. + CpuMillis int64 `protobuf:"varint,2,opt,name=cpu_millis,json=cpuMillis,proto3" json:"cpu_millis,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ResourceLimits) Reset() { + *x = ResourceLimits{} + mi := &file_ateom_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ResourceLimits) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ResourceLimits) ProtoMessage() {} + +func (x *ResourceLimits) ProtoReflect() protoreflect.Message { + mi := &file_ateom_proto_msgTypes[8] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ResourceLimits.ProtoReflect.Descriptor instead. +func (*ResourceLimits) Descriptor() ([]byte, []int) { + return file_ateom_proto_rawDescGZIP(), []int{8} +} + +func (x *ResourceLimits) GetMemoryBytes() int64 { + if x != nil { + return x.MemoryBytes + } + return 0 +} + +func (x *ResourceLimits) GetCpuMillis() int64 { + if x != nil { + return x.CpuMillis + } + return 0 +} + // VolumeMount is one volume mounted into a container. type VolumeMount struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -778,7 +918,7 @@ type VolumeMount struct { func (x *VolumeMount) Reset() { *x = VolumeMount{} - mi := &file_ateom_proto_msgTypes[7] + mi := &file_ateom_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -790,7 +930,7 @@ func (x *VolumeMount) String() string { func (*VolumeMount) ProtoMessage() {} func (x *VolumeMount) ProtoReflect() protoreflect.Message { - mi := &file_ateom_proto_msgTypes[7] + mi := &file_ateom_proto_msgTypes[9] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -803,7 +943,7 @@ func (x *VolumeMount) ProtoReflect() protoreflect.Message { // Deprecated: Use VolumeMount.ProtoReflect.Descriptor instead. func (*VolumeMount) Descriptor() ([]byte, []int) { - return file_ateom_proto_rawDescGZIP(), []int{7} + return file_ateom_proto_rawDescGZIP(), []int{9} } func (x *VolumeMount) GetVolumeName() string { @@ -834,7 +974,7 @@ type DurableDirVolumeMount struct { func (x *DurableDirVolumeMount) Reset() { *x = DurableDirVolumeMount{} - mi := &file_ateom_proto_msgTypes[8] + mi := &file_ateom_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -846,7 +986,7 @@ func (x *DurableDirVolumeMount) String() string { func (*DurableDirVolumeMount) ProtoMessage() {} func (x *DurableDirVolumeMount) ProtoReflect() protoreflect.Message { - mi := &file_ateom_proto_msgTypes[8] + mi := &file_ateom_proto_msgTypes[10] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -859,7 +999,7 @@ func (x *DurableDirVolumeMount) ProtoReflect() protoreflect.Message { // Deprecated: Use DurableDirVolumeMount.ProtoReflect.Descriptor instead. func (*DurableDirVolumeMount) Descriptor() ([]byte, []int) { - return file_ateom_proto_rawDescGZIP(), []int{8} + return file_ateom_proto_rawDescGZIP(), []int{10} } func (x *DurableDirVolumeMount) GetVolumeName() string { @@ -892,7 +1032,7 @@ type SystemInfoVolumeMount struct { func (x *SystemInfoVolumeMount) Reset() { *x = SystemInfoVolumeMount{} - mi := &file_ateom_proto_msgTypes[9] + mi := &file_ateom_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -904,7 +1044,7 @@ func (x *SystemInfoVolumeMount) String() string { func (*SystemInfoVolumeMount) ProtoMessage() {} func (x *SystemInfoVolumeMount) ProtoReflect() protoreflect.Message { - mi := &file_ateom_proto_msgTypes[9] + mi := &file_ateom_proto_msgTypes[11] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -917,7 +1057,7 @@ func (x *SystemInfoVolumeMount) ProtoReflect() protoreflect.Message { // Deprecated: Use SystemInfoVolumeMount.ProtoReflect.Descriptor instead. func (*SystemInfoVolumeMount) Descriptor() ([]byte, []int) { - return file_ateom_proto_rawDescGZIP(), []int{9} + return file_ateom_proto_rawDescGZIP(), []int{11} } func (x *SystemInfoVolumeMount) GetVolumeName() string { @@ -949,7 +1089,7 @@ type ImageVolumeMount struct { func (x *ImageVolumeMount) Reset() { *x = ImageVolumeMount{} - mi := &file_ateom_proto_msgTypes[10] + mi := &file_ateom_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -961,7 +1101,7 @@ func (x *ImageVolumeMount) String() string { func (*ImageVolumeMount) ProtoMessage() {} func (x *ImageVolumeMount) ProtoReflect() protoreflect.Message { - mi := &file_ateom_proto_msgTypes[10] + mi := &file_ateom_proto_msgTypes[12] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -974,7 +1114,7 @@ func (x *ImageVolumeMount) ProtoReflect() protoreflect.Message { // Deprecated: Use ImageVolumeMount.ProtoReflect.Descriptor instead. func (*ImageVolumeMount) Descriptor() ([]byte, []int) { - return file_ateom_proto_rawDescGZIP(), []int{10} + return file_ateom_proto_rawDescGZIP(), []int{12} } func (x *ImageVolumeMount) GetVolumeName() string { @@ -1004,7 +1144,7 @@ type WakeupProbe struct { func (x *WakeupProbe) Reset() { *x = WakeupProbe{} - mi := &file_ateom_proto_msgTypes[11] + mi := &file_ateom_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1016,7 +1156,7 @@ func (x *WakeupProbe) String() string { func (*WakeupProbe) ProtoMessage() {} func (x *WakeupProbe) ProtoReflect() protoreflect.Message { - mi := &file_ateom_proto_msgTypes[11] + mi := &file_ateom_proto_msgTypes[13] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1029,7 +1169,7 @@ func (x *WakeupProbe) ProtoReflect() protoreflect.Message { // Deprecated: Use WakeupProbe.ProtoReflect.Descriptor instead. func (*WakeupProbe) Descriptor() ([]byte, []int) { - return file_ateom_proto_rawDescGZIP(), []int{11} + return file_ateom_proto_rawDescGZIP(), []int{13} } func (x *WakeupProbe) GetHttpGet() *HTTPGetAction { @@ -1059,7 +1199,7 @@ type HTTPGetAction struct { func (x *HTTPGetAction) Reset() { *x = HTTPGetAction{} - mi := &file_ateom_proto_msgTypes[12] + mi := &file_ateom_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1071,7 +1211,7 @@ func (x *HTTPGetAction) String() string { func (*HTTPGetAction) ProtoMessage() {} func (x *HTTPGetAction) ProtoReflect() protoreflect.Message { - mi := &file_ateom_proto_msgTypes[12] + mi := &file_ateom_proto_msgTypes[14] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1084,7 +1224,7 @@ func (x *HTTPGetAction) ProtoReflect() protoreflect.Message { // Deprecated: Use HTTPGetAction.ProtoReflect.Descriptor instead. func (*HTTPGetAction) Descriptor() ([]byte, []int) { - return file_ateom_proto_rawDescGZIP(), []int{12} + return file_ateom_proto_rawDescGZIP(), []int{14} } func (x *HTTPGetAction) GetPath() string { @@ -1109,7 +1249,7 @@ type RunWorkloadResponse struct { func (x *RunWorkloadResponse) Reset() { *x = RunWorkloadResponse{} - mi := &file_ateom_proto_msgTypes[13] + mi := &file_ateom_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1121,7 +1261,7 @@ func (x *RunWorkloadResponse) String() string { func (*RunWorkloadResponse) ProtoMessage() {} func (x *RunWorkloadResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateom_proto_msgTypes[13] + mi := &file_ateom_proto_msgTypes[15] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1134,7 +1274,7 @@ func (x *RunWorkloadResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RunWorkloadResponse.ProtoReflect.Descriptor instead. func (*RunWorkloadResponse) Descriptor() ([]byte, []int) { - return file_ateom_proto_rawDescGZIP(), []int{13} + return file_ateom_proto_rawDescGZIP(), []int{15} } type CheckpointWorkloadRequest struct { @@ -1169,7 +1309,7 @@ type CheckpointWorkloadRequest struct { func (x *CheckpointWorkloadRequest) Reset() { *x = CheckpointWorkloadRequest{} - mi := &file_ateom_proto_msgTypes[14] + mi := &file_ateom_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1181,7 +1321,7 @@ func (x *CheckpointWorkloadRequest) String() string { func (*CheckpointWorkloadRequest) ProtoMessage() {} func (x *CheckpointWorkloadRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateom_proto_msgTypes[14] + mi := &file_ateom_proto_msgTypes[16] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1194,7 +1334,7 @@ func (x *CheckpointWorkloadRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use CheckpointWorkloadRequest.ProtoReflect.Descriptor instead. func (*CheckpointWorkloadRequest) Descriptor() ([]byte, []int) { - return file_ateom_proto_rawDescGZIP(), []int{14} + return file_ateom_proto_rawDescGZIP(), []int{16} } func (x *CheckpointWorkloadRequest) GetAtespace() string { @@ -1289,7 +1429,7 @@ type CheckpointWorkloadResponse struct { func (x *CheckpointWorkloadResponse) Reset() { *x = CheckpointWorkloadResponse{} - mi := &file_ateom_proto_msgTypes[15] + mi := &file_ateom_proto_msgTypes[17] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1301,7 +1441,7 @@ func (x *CheckpointWorkloadResponse) String() string { func (*CheckpointWorkloadResponse) ProtoMessage() {} func (x *CheckpointWorkloadResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateom_proto_msgTypes[15] + mi := &file_ateom_proto_msgTypes[17] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1314,7 +1454,7 @@ func (x *CheckpointWorkloadResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use CheckpointWorkloadResponse.ProtoReflect.Descriptor instead. func (*CheckpointWorkloadResponse) Descriptor() ([]byte, []int) { - return file_ateom_proto_rawDescGZIP(), []int{15} + return file_ateom_proto_rawDescGZIP(), []int{17} } func (x *CheckpointWorkloadResponse) GetSnapshotFiles() []string { @@ -1366,7 +1506,7 @@ type RestoreWorkloadRequest struct { func (x *RestoreWorkloadRequest) Reset() { *x = RestoreWorkloadRequest{} - mi := &file_ateom_proto_msgTypes[16] + mi := &file_ateom_proto_msgTypes[18] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1378,7 +1518,7 @@ func (x *RestoreWorkloadRequest) String() string { func (*RestoreWorkloadRequest) ProtoMessage() {} func (x *RestoreWorkloadRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateom_proto_msgTypes[16] + mi := &file_ateom_proto_msgTypes[18] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1391,7 +1531,7 @@ func (x *RestoreWorkloadRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use RestoreWorkloadRequest.ProtoReflect.Descriptor instead. func (*RestoreWorkloadRequest) Descriptor() ([]byte, []int) { - return file_ateom_proto_rawDescGZIP(), []int{16} + return file_ateom_proto_rawDescGZIP(), []int{18} } func (x *RestoreWorkloadRequest) GetAtespace() string { @@ -1507,7 +1647,7 @@ type RestoreWorkloadResponse struct { func (x *RestoreWorkloadResponse) Reset() { *x = RestoreWorkloadResponse{} - mi := &file_ateom_proto_msgTypes[17] + mi := &file_ateom_proto_msgTypes[19] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1519,7 +1659,7 @@ func (x *RestoreWorkloadResponse) String() string { func (*RestoreWorkloadResponse) ProtoMessage() {} func (x *RestoreWorkloadResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateom_proto_msgTypes[17] + mi := &file_ateom_proto_msgTypes[19] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1532,7 +1672,7 @@ func (x *RestoreWorkloadResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RestoreWorkloadResponse.ProtoReflect.Descriptor instead. func (*RestoreWorkloadResponse) Descriptor() ([]byte, []int) { - return file_ateom_proto_rawDescGZIP(), []int{17} + return file_ateom_proto_rawDescGZIP(), []int{19} } type GetWorkloadStatsRequest struct { @@ -1548,7 +1688,7 @@ type GetWorkloadStatsRequest struct { func (x *GetWorkloadStatsRequest) Reset() { *x = GetWorkloadStatsRequest{} - mi := &file_ateom_proto_msgTypes[18] + mi := &file_ateom_proto_msgTypes[20] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1560,7 +1700,7 @@ func (x *GetWorkloadStatsRequest) String() string { func (*GetWorkloadStatsRequest) ProtoMessage() {} func (x *GetWorkloadStatsRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateom_proto_msgTypes[18] + mi := &file_ateom_proto_msgTypes[20] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1573,7 +1713,7 @@ func (x *GetWorkloadStatsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetWorkloadStatsRequest.ProtoReflect.Descriptor instead. func (*GetWorkloadStatsRequest) Descriptor() ([]byte, []int) { - return file_ateom_proto_rawDescGZIP(), []int{18} + return file_ateom_proto_rawDescGZIP(), []int{20} } func (x *GetWorkloadStatsRequest) GetActorUid() string { @@ -1637,7 +1777,7 @@ type WorkloadStatsSample struct { func (x *WorkloadStatsSample) Reset() { *x = WorkloadStatsSample{} - mi := &file_ateom_proto_msgTypes[19] + mi := &file_ateom_proto_msgTypes[21] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1649,7 +1789,7 @@ func (x *WorkloadStatsSample) String() string { func (*WorkloadStatsSample) ProtoMessage() {} func (x *WorkloadStatsSample) ProtoReflect() protoreflect.Message { - mi := &file_ateom_proto_msgTypes[19] + mi := &file_ateom_proto_msgTypes[21] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1662,7 +1802,7 @@ func (x *WorkloadStatsSample) ProtoReflect() protoreflect.Message { // Deprecated: Use WorkloadStatsSample.ProtoReflect.Descriptor instead. func (*WorkloadStatsSample) Descriptor() ([]byte, []int) { - return file_ateom_proto_rawDescGZIP(), []int{19} + return file_ateom_proto_rawDescGZIP(), []int{21} } func (x *WorkloadStatsSample) GetAtespace() string { @@ -1765,7 +1905,7 @@ type GetWorkloadStatsResponse struct { func (x *GetWorkloadStatsResponse) Reset() { *x = GetWorkloadStatsResponse{} - mi := &file_ateom_proto_msgTypes[20] + mi := &file_ateom_proto_msgTypes[22] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1777,7 +1917,7 @@ func (x *GetWorkloadStatsResponse) String() string { func (*GetWorkloadStatsResponse) ProtoMessage() {} func (x *GetWorkloadStatsResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateom_proto_msgTypes[20] + mi := &file_ateom_proto_msgTypes[22] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1790,7 +1930,7 @@ func (x *GetWorkloadStatsResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use GetWorkloadStatsResponse.ProtoReflect.Descriptor instead. func (*GetWorkloadStatsResponse) Descriptor() ([]byte, []int) { - return file_ateom_proto_rawDescGZIP(), []int{20} + return file_ateom_proto_rawDescGZIP(), []int{22} } func (x *GetWorkloadStatsResponse) GetSample() *WorkloadStatsSample { @@ -1808,7 +1948,7 @@ type GetActiveWorkloadStatsRequest struct { func (x *GetActiveWorkloadStatsRequest) Reset() { *x = GetActiveWorkloadStatsRequest{} - mi := &file_ateom_proto_msgTypes[21] + mi := &file_ateom_proto_msgTypes[23] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1820,7 +1960,7 @@ func (x *GetActiveWorkloadStatsRequest) String() string { func (*GetActiveWorkloadStatsRequest) ProtoMessage() {} func (x *GetActiveWorkloadStatsRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateom_proto_msgTypes[21] + mi := &file_ateom_proto_msgTypes[23] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1833,7 +1973,7 @@ func (x *GetActiveWorkloadStatsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetActiveWorkloadStatsRequest.ProtoReflect.Descriptor instead. func (*GetActiveWorkloadStatsRequest) Descriptor() ([]byte, []int) { - return file_ateom_proto_rawDescGZIP(), []int{21} + return file_ateom_proto_rawDescGZIP(), []int{23} } type GetActiveWorkloadStatsResponse struct { @@ -1853,7 +1993,7 @@ type GetActiveWorkloadStatsResponse struct { func (x *GetActiveWorkloadStatsResponse) Reset() { *x = GetActiveWorkloadStatsResponse{} - mi := &file_ateom_proto_msgTypes[22] + mi := &file_ateom_proto_msgTypes[24] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1865,7 +2005,7 @@ func (x *GetActiveWorkloadStatsResponse) String() string { func (*GetActiveWorkloadStatsResponse) ProtoMessage() {} func (x *GetActiveWorkloadStatsResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateom_proto_msgTypes[22] + mi := &file_ateom_proto_msgTypes[24] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1878,7 +2018,7 @@ func (x *GetActiveWorkloadStatsResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use GetActiveWorkloadStatsResponse.ProtoReflect.Descriptor instead. func (*GetActiveWorkloadStatsResponse) Descriptor() ([]byte, []int) { - return file_ateom_proto_rawDescGZIP(), []int{22} + return file_ateom_proto_rawDescGZIP(), []int{24} } func (x *GetActiveWorkloadStatsResponse) GetSamples() []*WorkloadStatsSample { @@ -1942,14 +2082,24 @@ const file_ateom_proto_rawDesc = "" + "\fWorkloadSpec\x120\n" + "\n" + "containers\x18\x01 \x03(\v2\x10.ateom.ContainerR\n" + - "containers\"\x91\x03\n" + + "containers\"\xce\x03\n" + "\tContainer\x12\x12\n" + "\x04name\x18\x01 \x01(\tR\x04name\x125\n" + "\fwakeup_probe\x18\x02 \x01(\v2\x12.ateom.WakeupProbeR\vwakeupProbe\x12W\n" + "\x19durable_dir_volume_mounts\x18\x03 \x03(\v2\x1c.ateom.DurableDirVolumeMountR\x16durableDirVolumeMounts\x12>\n" + "\x11csi_volume_mounts\x18\x04 \x03(\v2\x12.ateom.VolumeMountR\x0fcsiVolumeMounts\x12W\n" + "\x19system_info_volume_mounts\x18\x05 \x03(\v2\x1c.ateom.SystemInfoVolumeMountR\x16systemInfoVolumeMounts\x12G\n" + - "\x13image_volume_mounts\x18\x06 \x03(\v2\x17.ateom.ImageVolumeMountR\x11imageVolumeMounts\"M\n" + + "\x13image_volume_mounts\x18\x06 \x03(\v2\x17.ateom.ImageVolumeMountR\x11imageVolumeMounts\x12;\n" + + "\x0econtainer_spec\x18\a \x01(\v2\x14.ateom.ContainerSpecR\rcontainerSpec\"\x93\x01\n" + + "\rContainerSpec\x12\x12\n" + + "\x04args\x18\x01 \x03(\tR\x04args\x12\x15\n" + + "\x03env\x18\x02 \x03(\tB\x03\x80\x01\x01R\x03env\x12\"\n" + + "\fcapabilities\x18\x03 \x03(\tR\fcapabilities\x123\n" + + "\tresources\x18\x04 \x01(\v2\x15.ateom.ResourceLimitsR\tresources\"R\n" + + "\x0eResourceLimits\x12!\n" + + "\fmemory_bytes\x18\x01 \x01(\x03R\vmemoryBytes\x12\x1d\n" + + "\n" + + "cpu_millis\x18\x02 \x01(\x03R\tcpuMillis\"M\n" + "\vVolumeMount\x12\x1f\n" + "\vvolume_name\x18\x01 \x01(\tR\n" + "volumeName\x12\x1d\n" + @@ -2080,7 +2230,7 @@ func file_ateom_proto_rawDescGZIP() []byte { } var file_ateom_proto_enumTypes = make([]protoimpl.EnumInfo, 3) -var file_ateom_proto_msgTypes = make([]protoimpl.MessageInfo, 26) +var file_ateom_proto_msgTypes = make([]protoimpl.MessageInfo, 28) var file_ateom_proto_goTypes = []any{ (SnapshotScope)(0), // 0: ateom.SnapshotScope (SandboxClass)(0), // 1: ateom.SandboxClass @@ -2092,70 +2242,74 @@ var file_ateom_proto_goTypes = []any{ (*EgressGateway)(nil), // 7: ateom.EgressGateway (*WorkloadSpec)(nil), // 8: ateom.WorkloadSpec (*Container)(nil), // 9: ateom.Container - (*VolumeMount)(nil), // 10: ateom.VolumeMount - (*DurableDirVolumeMount)(nil), // 11: ateom.DurableDirVolumeMount - (*SystemInfoVolumeMount)(nil), // 12: ateom.SystemInfoVolumeMount - (*ImageVolumeMount)(nil), // 13: ateom.ImageVolumeMount - (*WakeupProbe)(nil), // 14: ateom.WakeupProbe - (*HTTPGetAction)(nil), // 15: ateom.HTTPGetAction - (*RunWorkloadResponse)(nil), // 16: ateom.RunWorkloadResponse - (*CheckpointWorkloadRequest)(nil), // 17: ateom.CheckpointWorkloadRequest - (*CheckpointWorkloadResponse)(nil), // 18: ateom.CheckpointWorkloadResponse - (*RestoreWorkloadRequest)(nil), // 19: ateom.RestoreWorkloadRequest - (*RestoreWorkloadResponse)(nil), // 20: ateom.RestoreWorkloadResponse - (*GetWorkloadStatsRequest)(nil), // 21: ateom.GetWorkloadStatsRequest - (*WorkloadStatsSample)(nil), // 22: ateom.WorkloadStatsSample - (*GetWorkloadStatsResponse)(nil), // 23: ateom.GetWorkloadStatsResponse - (*GetActiveWorkloadStatsRequest)(nil), // 24: ateom.GetActiveWorkloadStatsRequest - (*GetActiveWorkloadStatsResponse)(nil), // 25: ateom.GetActiveWorkloadStatsResponse - nil, // 26: ateom.RunWorkloadRequest.RuntimeAssetPathsEntry - nil, // 27: ateom.CheckpointWorkloadRequest.RuntimeAssetPathsEntry - nil, // 28: ateom.RestoreWorkloadRequest.RuntimeAssetPathsEntry + (*ContainerSpec)(nil), // 10: ateom.ContainerSpec + (*ResourceLimits)(nil), // 11: ateom.ResourceLimits + (*VolumeMount)(nil), // 12: ateom.VolumeMount + (*DurableDirVolumeMount)(nil), // 13: ateom.DurableDirVolumeMount + (*SystemInfoVolumeMount)(nil), // 14: ateom.SystemInfoVolumeMount + (*ImageVolumeMount)(nil), // 15: ateom.ImageVolumeMount + (*WakeupProbe)(nil), // 16: ateom.WakeupProbe + (*HTTPGetAction)(nil), // 17: ateom.HTTPGetAction + (*RunWorkloadResponse)(nil), // 18: ateom.RunWorkloadResponse + (*CheckpointWorkloadRequest)(nil), // 19: ateom.CheckpointWorkloadRequest + (*CheckpointWorkloadResponse)(nil), // 20: ateom.CheckpointWorkloadResponse + (*RestoreWorkloadRequest)(nil), // 21: ateom.RestoreWorkloadRequest + (*RestoreWorkloadResponse)(nil), // 22: ateom.RestoreWorkloadResponse + (*GetWorkloadStatsRequest)(nil), // 23: ateom.GetWorkloadStatsRequest + (*WorkloadStatsSample)(nil), // 24: ateom.WorkloadStatsSample + (*GetWorkloadStatsResponse)(nil), // 25: ateom.GetWorkloadStatsResponse + (*GetActiveWorkloadStatsRequest)(nil), // 26: ateom.GetActiveWorkloadStatsRequest + (*GetActiveWorkloadStatsResponse)(nil), // 27: ateom.GetActiveWorkloadStatsResponse + nil, // 28: ateom.RunWorkloadRequest.RuntimeAssetPathsEntry + nil, // 29: ateom.CheckpointWorkloadRequest.RuntimeAssetPathsEntry + nil, // 30: ateom.RestoreWorkloadRequest.RuntimeAssetPathsEntry } var file_ateom_proto_depIdxs = []int32{ 8, // 0: ateom.TerminateWorkloadRequest.spec:type_name -> ateom.WorkloadSpec 3, // 1: ateom.TerminateWorkloadRequest.actor_dirs:type_name -> ateom.ActorDirs 8, // 2: ateom.RunWorkloadRequest.spec:type_name -> ateom.WorkloadSpec - 26, // 3: ateom.RunWorkloadRequest.runtime_asset_paths:type_name -> ateom.RunWorkloadRequest.RuntimeAssetPathsEntry + 28, // 3: ateom.RunWorkloadRequest.runtime_asset_paths:type_name -> ateom.RunWorkloadRequest.RuntimeAssetPathsEntry 7, // 4: ateom.RunWorkloadRequest.egress_gateway:type_name -> ateom.EgressGateway 3, // 5: ateom.RunWorkloadRequest.actor_dirs:type_name -> ateom.ActorDirs 9, // 6: ateom.WorkloadSpec.containers:type_name -> ateom.Container - 14, // 7: ateom.Container.wakeup_probe:type_name -> ateom.WakeupProbe - 11, // 8: ateom.Container.durable_dir_volume_mounts:type_name -> ateom.DurableDirVolumeMount - 10, // 9: ateom.Container.csi_volume_mounts:type_name -> ateom.VolumeMount - 12, // 10: ateom.Container.system_info_volume_mounts:type_name -> ateom.SystemInfoVolumeMount - 13, // 11: ateom.Container.image_volume_mounts:type_name -> ateom.ImageVolumeMount - 15, // 12: ateom.WakeupProbe.http_get:type_name -> ateom.HTTPGetAction - 8, // 13: ateom.CheckpointWorkloadRequest.spec:type_name -> ateom.WorkloadSpec - 27, // 14: ateom.CheckpointWorkloadRequest.runtime_asset_paths:type_name -> ateom.CheckpointWorkloadRequest.RuntimeAssetPathsEntry - 0, // 15: ateom.CheckpointWorkloadRequest.scope:type_name -> ateom.SnapshotScope - 3, // 16: ateom.CheckpointWorkloadRequest.actor_dirs:type_name -> ateom.ActorDirs - 8, // 17: ateom.RestoreWorkloadRequest.spec:type_name -> ateom.WorkloadSpec - 28, // 18: ateom.RestoreWorkloadRequest.runtime_asset_paths:type_name -> ateom.RestoreWorkloadRequest.RuntimeAssetPathsEntry - 0, // 19: ateom.RestoreWorkloadRequest.scope:type_name -> ateom.SnapshotScope - 7, // 20: ateom.RestoreWorkloadRequest.egress_gateway:type_name -> ateom.EgressGateway - 3, // 21: ateom.RestoreWorkloadRequest.actor_dirs:type_name -> ateom.ActorDirs - 1, // 22: ateom.WorkloadStatsSample.sandbox_class:type_name -> ateom.SandboxClass - 2, // 23: ateom.WorkloadStatsSample.source:type_name -> ateom.StatsSource - 22, // 24: ateom.GetWorkloadStatsResponse.sample:type_name -> ateom.WorkloadStatsSample - 22, // 25: ateom.GetActiveWorkloadStatsResponse.samples:type_name -> ateom.WorkloadStatsSample - 6, // 26: ateom.Ateom.RunWorkload:input_type -> ateom.RunWorkloadRequest - 17, // 27: ateom.Ateom.CheckpointWorkload:input_type -> ateom.CheckpointWorkloadRequest - 19, // 28: ateom.Ateom.RestoreWorkload:input_type -> ateom.RestoreWorkloadRequest - 21, // 29: ateom.Ateom.GetWorkloadStats:input_type -> ateom.GetWorkloadStatsRequest - 24, // 30: ateom.Ateom.GetActiveWorkloadStats:input_type -> ateom.GetActiveWorkloadStatsRequest - 4, // 31: ateom.Ateom.TerminateWorkload:input_type -> ateom.TerminateWorkloadRequest - 16, // 32: ateom.Ateom.RunWorkload:output_type -> ateom.RunWorkloadResponse - 18, // 33: ateom.Ateom.CheckpointWorkload:output_type -> ateom.CheckpointWorkloadResponse - 20, // 34: ateom.Ateom.RestoreWorkload:output_type -> ateom.RestoreWorkloadResponse - 23, // 35: ateom.Ateom.GetWorkloadStats:output_type -> ateom.GetWorkloadStatsResponse - 25, // 36: ateom.Ateom.GetActiveWorkloadStats:output_type -> ateom.GetActiveWorkloadStatsResponse - 5, // 37: ateom.Ateom.TerminateWorkload:output_type -> ateom.TerminateWorkloadResponse - 32, // [32:38] is the sub-list for method output_type - 26, // [26:32] is the sub-list for method input_type - 26, // [26:26] is the sub-list for extension type_name - 26, // [26:26] is the sub-list for extension extendee - 0, // [0:26] is the sub-list for field type_name + 16, // 7: ateom.Container.wakeup_probe:type_name -> ateom.WakeupProbe + 13, // 8: ateom.Container.durable_dir_volume_mounts:type_name -> ateom.DurableDirVolumeMount + 12, // 9: ateom.Container.csi_volume_mounts:type_name -> ateom.VolumeMount + 14, // 10: ateom.Container.system_info_volume_mounts:type_name -> ateom.SystemInfoVolumeMount + 15, // 11: ateom.Container.image_volume_mounts:type_name -> ateom.ImageVolumeMount + 10, // 12: ateom.Container.container_spec:type_name -> ateom.ContainerSpec + 11, // 13: ateom.ContainerSpec.resources:type_name -> ateom.ResourceLimits + 17, // 14: ateom.WakeupProbe.http_get:type_name -> ateom.HTTPGetAction + 8, // 15: ateom.CheckpointWorkloadRequest.spec:type_name -> ateom.WorkloadSpec + 29, // 16: ateom.CheckpointWorkloadRequest.runtime_asset_paths:type_name -> ateom.CheckpointWorkloadRequest.RuntimeAssetPathsEntry + 0, // 17: ateom.CheckpointWorkloadRequest.scope:type_name -> ateom.SnapshotScope + 3, // 18: ateom.CheckpointWorkloadRequest.actor_dirs:type_name -> ateom.ActorDirs + 8, // 19: ateom.RestoreWorkloadRequest.spec:type_name -> ateom.WorkloadSpec + 30, // 20: ateom.RestoreWorkloadRequest.runtime_asset_paths:type_name -> ateom.RestoreWorkloadRequest.RuntimeAssetPathsEntry + 0, // 21: ateom.RestoreWorkloadRequest.scope:type_name -> ateom.SnapshotScope + 7, // 22: ateom.RestoreWorkloadRequest.egress_gateway:type_name -> ateom.EgressGateway + 3, // 23: ateom.RestoreWorkloadRequest.actor_dirs:type_name -> ateom.ActorDirs + 1, // 24: ateom.WorkloadStatsSample.sandbox_class:type_name -> ateom.SandboxClass + 2, // 25: ateom.WorkloadStatsSample.source:type_name -> ateom.StatsSource + 24, // 26: ateom.GetWorkloadStatsResponse.sample:type_name -> ateom.WorkloadStatsSample + 24, // 27: ateom.GetActiveWorkloadStatsResponse.samples:type_name -> ateom.WorkloadStatsSample + 6, // 28: ateom.Ateom.RunWorkload:input_type -> ateom.RunWorkloadRequest + 19, // 29: ateom.Ateom.CheckpointWorkload:input_type -> ateom.CheckpointWorkloadRequest + 21, // 30: ateom.Ateom.RestoreWorkload:input_type -> ateom.RestoreWorkloadRequest + 23, // 31: ateom.Ateom.GetWorkloadStats:input_type -> ateom.GetWorkloadStatsRequest + 26, // 32: ateom.Ateom.GetActiveWorkloadStats:input_type -> ateom.GetActiveWorkloadStatsRequest + 4, // 33: ateom.Ateom.TerminateWorkload:input_type -> ateom.TerminateWorkloadRequest + 18, // 34: ateom.Ateom.RunWorkload:output_type -> ateom.RunWorkloadResponse + 20, // 35: ateom.Ateom.CheckpointWorkload:output_type -> ateom.CheckpointWorkloadResponse + 22, // 36: ateom.Ateom.RestoreWorkload:output_type -> ateom.RestoreWorkloadResponse + 25, // 37: ateom.Ateom.GetWorkloadStats:output_type -> ateom.GetWorkloadStatsResponse + 27, // 38: ateom.Ateom.GetActiveWorkloadStats:output_type -> ateom.GetActiveWorkloadStatsResponse + 5, // 39: ateom.Ateom.TerminateWorkload:output_type -> ateom.TerminateWorkloadResponse + 34, // [34:40] is the sub-list for method output_type + 28, // [28:34] is the sub-list for method input_type + 28, // [28:28] is the sub-list for extension type_name + 28, // [28:28] is the sub-list for extension extendee + 0, // [0:28] is the sub-list for field type_name } func init() { file_ateom_proto_init() } @@ -2164,14 +2318,14 @@ func file_ateom_proto_init() { return } file_ateom_proto_msgTypes[3].OneofWrappers = []any{} - file_ateom_proto_msgTypes[16].OneofWrappers = []any{} + file_ateom_proto_msgTypes[18].OneofWrappers = []any{} type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_ateom_proto_rawDesc), len(file_ateom_proto_rawDesc)), NumEnums: 3, - NumMessages: 26, + NumMessages: 28, NumExtensions: 0, NumServices: 1, }, diff --git a/internal/proto/ateompb/ateom.proto b/internal/proto/ateompb/ateom.proto index 7f234c0be5..1f5c49e1df 100644 --- a/internal/proto/ateompb/ateom.proto +++ b/internal/proto/ateompb/ateom.proto @@ -200,6 +200,35 @@ message Container { // image_volume_mounts are the image volumes this container mounts, if any. repeated ImageVolumeMount image_volume_mounts = 6; + + // container_spec is what ateom builds the container's OCI spec from, + // besides its volume mounts. Unset on RPCs that start no process. + ContainerSpec container_spec = 7; +} + +// ContainerSpec is a container's process and limits, resolved by atelet. +message ContainerSpec { + // args is the process argv, already resolved against the image's + // ENTRYPOINT and CMD. + repeated string args = 1; + + // env is the process environment as KEY=VALUE, already merged with the + // image's ENV. + repeated string env = 2 [ debug_redact = true ]; + + // capabilities is the container's capability set, CAP_-prefixed. + repeated string capabilities = 3; + + // resources are the container's own declared limits, if any. + ResourceLimits resources = 4; +} + +// ResourceLimits are a container's declared limits. +message ResourceLimits { + // memory_bytes is the memory limit in bytes. 0 means unset. + int64 memory_bytes = 1; + // cpu_millis is the CPU limit in milli-cores (1000 = one core). 0 means unset. + int64 cpu_millis = 2; } // VolumeMount is one volume mounted into a container. diff --git a/internal/protoredact/protoredact_test.go b/internal/protoredact/protoredact_test.go index d76b0df99c..ee482aba6b 100644 --- a/internal/protoredact/protoredact_test.go +++ b/internal/protoredact/protoredact_test.go @@ -412,6 +412,7 @@ func TestDebugRedactFieldsArePinned(t *testing.T) { "ateapi.EnvVar.value": true, "ateapi.MintActorJWTResponse.actor_jwt": true, "atelet.EnvEntry.value": true, + "ateom.ContainerSpec.env": true, "credprovider.FetchSecretResponse.opaque_bytes": true, "objectstoresnapshot.v1.FetchSnapshotRequest.actor_jwt": true, "objectstoresnapshot.v1.UploadSnapshotRequest.actor_jwt": true, From d91379a31895a1513b824ba5e60cb7ca2b26626d Mon Sep 17 00:00:00 2001 From: Benjamin Elder Date: Thu, 8 Oct 2026 23:42:08 +0000 Subject: [PATCH 19/44] security: prevent path traversal in snapshot storage locations (#2372) see: #2077 prevents path traversal. we append to the base path here with a random actor UID so it should be hard to exploit this but let's prevent path traversals anyhow. still discussing if we should make other changes, but this is cheap to do now. --- internal/resources/validate.go | 17 +++++++++++++---- internal/resources/validate_test.go | 10 ++++++++++ 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/internal/resources/validate.go b/internal/resources/validate.go index dba7ee96e3..a5f7d3803b 100644 --- a/internal/resources/validate.go +++ b/internal/resources/validate.go @@ -199,10 +199,7 @@ func ValidateRunscHash(sha256Hash string) error { // is a well-formed URI with a bucket, so a bad location fails fast instead of // deep inside an object-storage call. It deliberately does not restrict the // scheme: the storage layer only uses the host (bucket) and path, and which -// schemes are acceptable is a storage-backend policy, not a per-RPC one. The -// local paths used for snapshot upload/download are derived from the -// separately validated actor ref, not from this URI, so this is a sanity check -// rather than a path-traversal guard. +// schemes are acceptable is a storage-backend policy, not a per-RPC one. // // This validates the base that many snapshots share, not any one snapshot's // URI; SnapshotURI is the type for the latter, and it applies this check when @@ -223,6 +220,18 @@ func ValidateSnapshotLocation(location string) error { if u.Opaque != "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" { return fmt.Errorf("invalid snapshot location %q: must contain only a scheme, bucket, and path", location) } + // url.JoinPath cleans dot segments, and some backends trim spaces from + // segments (or windows, trailing dots) and treat '\' as a separator, so + // these could move snapshots outside of the location, e.g. into another + // atespace's prefix + if strings.Contains(u.Path, `\`) { + return fmt.Errorf(`invalid snapshot location %q: must not contain '\'`, location) + } + for segment := range strings.SplitSeq(u.Path, "/") { + if segment != "" && strings.Trim(segment, ". ") == "" { + return fmt.Errorf("invalid snapshot location %q: must not contain path segments of only '.' and ' '", location) + } + } return nil } diff --git a/internal/resources/validate_test.go b/internal/resources/validate_test.go index 8091868009..290ded5eb7 100644 --- a/internal/resources/validate_test.go +++ b/internal/resources/validate_test.go @@ -195,6 +195,16 @@ func TestValidateSnapshotLocation(t *testing.T) { // Opaque form (no //) parses with an empty host, so it is rejected // on either the bucket or the opaque check. {"opaque", "gs:bucket/path", true}, + {"dot dot", "gs://bucket/team-a/../team-b", true}, + {"trailing dot dot", "gs://bucket/path/..", true}, + {"escaped dot dot", "gs://bucket/%2e%2e/path", true}, + {"dot", "gs://bucket/./path", true}, + {"triple dot", "gs://bucket/.../path", true}, + {"space padded dot dot", "gs://bucket/a/%20..%20/b", true}, + {"trailing space dot dot", "gs://bucket/a/..%20/b", true}, + {"backslash", `gs://bucket/team-a\..\team-b`, true}, + {"escaped backslash", "gs://bucket/a%5Cb", true}, + {"dots in a name", "gs://bucket/a..b/.c/d.", false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { From eeec4eedbd0831fd65b60e03108aa818d8477fe8 Mon Sep 17 00:00:00 2001 From: Dmitry Berkovich <54346214+dberkov@users.noreply.github.com> Date: Fri, 9 Oct 2026 01:25:21 +0000 Subject: [PATCH 20/44] Replace SnapshotContentScope with the SnapshotFidelity ladder and preferred_fidelity (#2364) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **This is a rename, not a behavior change.** `SnapshotConfig.onCommit` becomes `preferredFidelity`, and the two-valued `SnapshotContentScope` enum becomes the ordered `SnapshotFidelity` enum, with the same vocabulary carried through atelet, ateom, and the metric label. `MEMORY` is exactly what `FULL` was and `VOLUMES` is exactly what `DATA` was: what gets captured on pause and suspend, what gets restored, the golden-actor override, and the paused-upload narrowing are all unchanged. The point is to put the lifecycle-v2 names in place before step III adds `required_fidelity` and `resumed_fidelity` on top of them. The one addition is `ROOTFS`, defined in the enum so it is complete but rejected everywhere, since no runtime can produce it yet. Part of #2102, step II as tracked in #2351. Step I (#2309) removed `on_pause`. Two commits, meant to be read in order. ## 1. ateapi: `SnapshotFidelity` and `preferred_fidelity` - New ordered enum `SnapshotFidelity { VOLUMES = 1, ROOTFS = 2, MEMORY = 3 }`, each level including the ones below it. `SnapshotContentScope` is deleted. MEMORY is the old FULL and VOLUMES the old DATA, so behavior is unchanged. - `SnapshotConfig.on_commit` becomes `preferred_fidelity`, documented as a best-effort ceiling: the system never packages more than this level and may package less when a layer cannot be captured or is too expensive at the time. A process resumed in place keeps its memory regardless. Defaults to MEMORY. Field numbers rearranged to `storage_location = 1`, `preferred_fidelity = 2`. - `ExternalSnapshot` and `LocalSnapshot` record `fidelity` instead of `content_scope`. - ROOTFS is in the enum so it is complete, but no sandbox runtime captures rootfs without memory yet, so a new field-level validator rejects it at template admission. Actors only take their fidelity from a template, so none can carry it. - kubectl-ate tag output, demos, benchmarking manifests, glossary, API guide, and architecture doc use the new vocabulary. The obsolete apitool exemption is removed. ## 2. atelet, ateom: the same enum on the wire - Both internal protos replace `SnapshotScope` with a `SnapshotFidelity` mirroring the public one; request fields become `fidelity` and `desired_fidelity`. Every hop maps level by level, ROOTFS included, and nothing is promoted: an unexpected value goes out as UNSPECIFIED for the next hop to reject. - Defense in depth for ROOTFS: atelet rejects it in checkpoint, restore, and paused-upload validation, and both ateom runtimes reject it on `CheckpointWorkload` and `RestoreWorkload` through a shared `resources.ValidateSnapshotFidelity`. - A golden snapshot must record MEMORY (UNSPECIFIED is no longer tolerated). - The metric attribute `ate.snapshot.scope` (`full`/`data`/`unknown`) becomes `ate.snapshot.fidelity` (`volumes`/`rootfs`/`memory`/`unknown`) in the OTel registry, `ateattr`, the ateom phase logs, and the snapshot manifest atelet writes and reads back when narrowing a paused upload. Dashboards keyed on the old label need updating. ## Behavior changes - **Breaking API change.** Templates that set `onCommit` or use `SNAPSHOT_CONTENT_SCOPE_*` values are rejected. Per the pre-v1.0 policy there is no compatibility shim, and previously stored templates and snapshots are not expected to exist. - Snapshot manifests written with the old `scope` key read back with no fidelity, which the upload path already refuses with a clear message. - Unchanged: what MEMORY and VOLUMES capture and restore, the golden-actor MEMORY override, and the paused-upload narrowing from MEMORY to VOLUMES. ## Relationship to #2289 #2289 restructures the snapshot status into `Snapshot` and `SnapshotStorage`. If it lands first, the `fidelity` field in this PR moves onto `SnapshotStorage` (per copy), which is where it belongs long-term; the enum, the template field, and the data-plane change are the same either way. ## Follow-ups (step III) `required_fidelity` with `required <= preferred` validation and the CRASHED rule, `ResumeActorResponse.resumed_fidelity`, and ROOTFS support in the micro-VM runtime. ## Verification - `go test` for ateapi (unit and functional), kubectl-ate, atelet, ateom-microvm, ateom-gvisor, ateomphaselog, ateattr, resources, and apitool pass; new validator at 100% coverage. - golangci-lint, kube-api-linter, python-protos, boilerplate verifiers pass; codegen, proto-fmt, licenses, go.mod show no drift. - Not run locally: the Weaver metrics-registry check and shellcheck, which need Docker; relying on CI for those. e2e suites compile but were not run against a cluster. - [x] Breaking change - [x] Not backward compatible 🤖 Generated with [Claude Code](https://claude.com/claude-code) --- .../glutton-durdir-data-template.yaml.tmpl | 2 +- .../glutton-durdir-full-template.yaml.tmpl | 2 +- .../manifests/glutton-template.yaml.tmpl | 2 +- .../manifests/kernelmem-template.yaml.tmpl | 2 +- .../manifests/sleep-template.yaml.tmpl | 2 +- .../swebench-astropy-7336-template.yaml.tmpl | 2 +- .../manifests/usermem-template.yaml.tmpl | 2 +- .../internal/apivalidation/actor_template.go | 9 + .../apivalidation/actor_template_test.go | 40 +++-- .../internal/apivalidation/actor_test.go | 20 +-- .../internal/apivalidation/validation_test.go | 42 ++--- .../apivalidation/zz_generated.validation.go | 56 +++--- cmd/ateapi/internal/controlapi/actor.go | 4 +- .../controlapi/actor_template_test.go | 6 +- cmd/ateapi/internal/controlapi/actor_test.go | 4 +- cmd/ateapi/internal/controlapi/converter.go | 17 +- .../internal/controlapi/converter_test.go | 42 +++-- .../functionaltest/actor_template_test.go | 4 +- .../controlapi/functionaltest/actor_test.go | 64 +++---- .../controlapi/functionaltest/common_test.go | 2 +- .../controlapi/functionaltest/tag_test.go | 4 +- cmd/ateapi/internal/controlapi/metrics.go | 4 +- .../internal/controlapi/metrics_test.go | 10 +- cmd/ateapi/internal/controlapi/tag_test.go | 8 +- .../controlapi/template_reconciler_test.go | 2 +- .../internal/controlapi/workflow_pause.go | 22 +-- .../controlapi/workflow_pause_test.go | 20 +-- .../internal/controlapi/workflow_resume.go | 48 +++-- .../controlapi/workflow_resume_test.go | 95 +++++----- .../internal/controlapi/workflow_suspend.go | 46 ++--- .../controlapi/workflow_suspend_test.go | 30 ++-- .../internal/controlapi/workflow_tag.go | 4 +- .../internal/controlapi/workflow_tag_test.go | 4 +- cmd/ateapi/internal/defaults/defaults.go | 4 +- cmd/ateapi/internal/defaults/defaults_test.go | 10 +- .../store/atepg/actor_template_test.go | 2 +- .../internal/store/storecontract/contract.go | 10 +- cmd/atelet/internal/ateletpath/ateletpath.go | 4 +- cmd/atelet/lifecycle_test.go | 16 +- cmd/atelet/main.go | 97 +++++----- cmd/atelet/main_test.go | 100 ++++++----- cmd/atelet/metrics.go | 4 +- cmd/atelet/metrics_test.go | 10 +- cmd/atelet/sandbox_assets.go | 12 +- cmd/atelet/snapshotlog_test.go | 10 +- cmd/ateom-gvisor/main.go | 53 ++++-- cmd/ateom-gvisor/phaselog_test.go | 12 +- cmd/ateom-microvm/checkpoint.go | 33 ++-- cmd/ateom-microvm/durable.go | 2 +- cmd/ateom-microvm/main.go | 9 + cmd/ateom-microvm/restore.go | 44 ++--- cmd/ateom-microvm/rootfsupper.go | 4 +- cmd/ateom-microvm/run.go | 6 +- .../internal/cmd/actortemplate_test.go | 6 +- cmd/kubectl-ate/internal/printer/printer.go | 8 +- .../internal/printer/printer_test.go | 12 +- .../autoscaled-workerpool-template.yaml.tmpl | 2 +- .../agent-luna-template.yaml.tmpl | 2 +- .../agent-mars-template.yaml.tmpl | 2 +- .../agent-orion-template.yaml.tmpl | 2 +- demos/counter/README.md | 4 +- .../counter-microvm-csi-test-template.yaml | 2 +- .../counter-microvm-template.yaml.tmpl | 4 +- demos/counter/counter-template.yaml.tmpl | 6 +- .../egress/egress-microvm-template.yaml.tmpl | 2 +- demos/egress/egress-template.yaml.tmpl | 2 +- demos/jupyter/jupyter-template.yaml.tmpl | 2 +- .../multi-template/counter-template.yaml.tmpl | 2 +- .../fspersist-template.yaml.tmpl | 2 +- demos/parking/parking-template.yaml.tmpl | 2 +- demos/sandbox/manual-test-multi-template.yaml | 2 +- demos/sandbox/sandbox-template.yaml.tmpl | 2 +- docs/api-guide.md | 6 +- docs/architecture.md | 2 +- docs/dev/best-practices/metrics.md | 2 +- docs/glossary.md | 40 +++-- docs/metrics/registry/metrics.yaml | 28 +-- docs/observability.md | 6 +- internal/ateattr/ateattr.go | 35 ++-- internal/ateattr/ateattr_test.go | 28 +-- internal/ateomphaselog/phaselog.go | 36 ++-- internal/ateomphaselog/phaselog_test.go | 33 ++-- internal/e2e/suites/demo/demo_test.go | 102 +++++------ internal/e2e/suites/demo/termination_test.go | 4 +- .../e2e/suites/demo/updatetemplate_test.go | 28 +-- internal/e2e/suites/metrics/metrics_test.go | 6 +- internal/proto/ateletpb/atelet.pb.go | 136 +++++++------- internal/proto/ateletpb/atelet.proto | 39 ++-- internal/proto/ateompb/ateom.pb.go | 116 ++++++------ internal/proto/ateompb/ateom.proto | 31 ++-- internal/resources/validate.go | 18 ++ internal/resources/validate_test.go | 26 +++ pkg/proto/ateapipb/ateapi.pb.go | 167 ++++++++++-------- pkg/proto/ateapipb/ateapi.proto | 68 ++++--- tools/apitool/exemptions.json | 5 - 95 files changed, 1146 insertions(+), 945 deletions(-) diff --git a/benchmarking/workloads/manifests/glutton-durdir-data-template.yaml.tmpl b/benchmarking/workloads/manifests/glutton-durdir-data-template.yaml.tmpl index 4e2d4b053e..5d984e4daf 100644 --- a/benchmarking/workloads/manifests/glutton-durdir-data-template.yaml.tmpl +++ b/benchmarking/workloads/manifests/glutton-durdir-data-template.yaml.tmpl @@ -44,7 +44,7 @@ volumes: - name: data durableDir: {} snapshotConfig: - onCommit: SNAPSHOT_CONTENT_SCOPE_DATA + preferredFidelity: SNAPSHOT_FIDELITY_VOLUMES storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/glutton-durdir-data/ sandboxConfig: sandboxClass: ${SANDBOX_CLASS_ENUM} diff --git a/benchmarking/workloads/manifests/glutton-durdir-full-template.yaml.tmpl b/benchmarking/workloads/manifests/glutton-durdir-full-template.yaml.tmpl index 4d174b896a..10c7272666 100644 --- a/benchmarking/workloads/manifests/glutton-durdir-full-template.yaml.tmpl +++ b/benchmarking/workloads/manifests/glutton-durdir-full-template.yaml.tmpl @@ -41,7 +41,7 @@ volumes: - name: data durableDir: {} snapshotConfig: - onCommit: SNAPSHOT_CONTENT_SCOPE_FULL + preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/glutton-durdir-full/ sandboxConfig: sandboxClass: ${SANDBOX_CLASS_ENUM} diff --git a/benchmarking/workloads/manifests/glutton-template.yaml.tmpl b/benchmarking/workloads/manifests/glutton-template.yaml.tmpl index e57c1dd838..bfa9469172 100644 --- a/benchmarking/workloads/manifests/glutton-template.yaml.tmpl +++ b/benchmarking/workloads/manifests/glutton-template.yaml.tmpl @@ -61,7 +61,7 @@ resources: - name: memory quantity: ${ACTOR_MEMORY} snapshotConfig: - onCommit: SNAPSHOT_CONTENT_SCOPE_FULL + preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/glutton/ sandboxConfig: sandboxClass: ${SANDBOX_CLASS_ENUM} diff --git a/benchmarking/workloads/manifests/kernelmem-template.yaml.tmpl b/benchmarking/workloads/manifests/kernelmem-template.yaml.tmpl index 38a5905700..7ffa43a637 100644 --- a/benchmarking/workloads/manifests/kernelmem-template.yaml.tmpl +++ b/benchmarking/workloads/manifests/kernelmem-template.yaml.tmpl @@ -31,7 +31,7 @@ containers: image: containerstack/alpine-stress:latest command: ["sh", "-c", "find /; for i in $(seq 50000); do sleep 10000 & done; sleep 10000"] snapshotConfig: - onCommit: SNAPSHOT_CONTENT_SCOPE_FULL + preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/kernelmem/ sandboxConfig: sandboxClass: ${SANDBOX_CLASS_ENUM} diff --git a/benchmarking/workloads/manifests/sleep-template.yaml.tmpl b/benchmarking/workloads/manifests/sleep-template.yaml.tmpl index 787fff639f..9549ca8672 100644 --- a/benchmarking/workloads/manifests/sleep-template.yaml.tmpl +++ b/benchmarking/workloads/manifests/sleep-template.yaml.tmpl @@ -35,7 +35,7 @@ resources: - name: memory quantity: ${ACTOR_MEMORY} snapshotConfig: - onCommit: SNAPSHOT_CONTENT_SCOPE_FULL + preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/sleep/ sandboxConfig: sandboxClass: ${SANDBOX_CLASS_ENUM} diff --git a/benchmarking/workloads/manifests/swebench-astropy-7336-template.yaml.tmpl b/benchmarking/workloads/manifests/swebench-astropy-7336-template.yaml.tmpl index f092ea551b..8d8fd737c9 100644 --- a/benchmarking/workloads/manifests/swebench-astropy-7336-template.yaml.tmpl +++ b/benchmarking/workloads/manifests/swebench-astropy-7336-template.yaml.tmpl @@ -50,5 +50,5 @@ sandboxConfig: sandboxClass: ${SANDBOX_CLASS_ENUM} configName: ${SANDBOX_CONFIG_NAME} snapshotConfig: - onCommit: SNAPSHOT_CONTENT_SCOPE_FULL + preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/swebench-astropy-7336/ diff --git a/benchmarking/workloads/manifests/usermem-template.yaml.tmpl b/benchmarking/workloads/manifests/usermem-template.yaml.tmpl index db8df41b3e..21eab89cb3 100644 --- a/benchmarking/workloads/manifests/usermem-template.yaml.tmpl +++ b/benchmarking/workloads/manifests/usermem-template.yaml.tmpl @@ -30,7 +30,7 @@ containers: image: containerstack/alpine-stress:latest command: ["stress", "--vm", "1", "--vm-bytes", "1G", "--vm-keep", "--vm-hang", "0"] snapshotConfig: - onCommit: SNAPSHOT_CONTENT_SCOPE_FULL + preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/usermem/ sandboxConfig: sandboxClass: ${SANDBOX_CLASS_ENUM} diff --git a/cmd/ateapi/internal/apivalidation/actor_template.go b/cmd/ateapi/internal/apivalidation/actor_template.go index f25107ed96..3366dd272f 100644 --- a/cmd/ateapi/internal/apivalidation/actor_template.go +++ b/cmd/ateapi/internal/apivalidation/actor_template.go @@ -237,6 +237,15 @@ func ValidateCustom_SnapshotConfig_StorageLocation(_ context.Context, _ operatio return nil } +// ValidateCustom_SnapshotConfig_PreferredFidelity rejects ROOTFS until a +// sandbox runtime can capture root filesystem changes without memory. +func ValidateCustom_SnapshotConfig_PreferredFidelity(_ context.Context, _ operation.Operation, fldPath *field.Path, value, _ *ateapipb.SnapshotFidelity) field.ErrorList { + if *value == ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS { + return field.ErrorList{field.Invalid(fldPath, value.String(), "ROOTFS fidelity is not supported yet")} + } + return nil +} + // envVarNameRE constrains env var names to any printable ASCII character // except '='. var envVarNameRE = regexp.MustCompile(`^[ -<>-~]+$`) diff --git a/cmd/ateapi/internal/apivalidation/actor_template_test.go b/cmd/ateapi/internal/apivalidation/actor_template_test.go index 5a12b817d3..d04127cbaa 100644 --- a/cmd/ateapi/internal/apivalidation/actor_template_test.go +++ b/cmd/ateapi/internal/apivalidation/actor_template_test.go @@ -63,11 +63,19 @@ func TestValidateCreateActorTemplateRequest(t *testing.T) { })}, field.ErrorList{field.Invalid(field.NewPath("actor_template", "metadata", "name"), "Tmpl_A", "").WithOrigin("format=k8s-short-name")}, }, { - "valid data-scoped snapshots", + "valid volumes fidelity", &ateapipb.CreateActorTemplateRequest{ActorTemplate: validActorTemplate(func(tmpl *ateapipb.ActorTemplate) { - tmpl.SnapshotConfig.OnCommit = ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA + tmpl.SnapshotConfig.PreferredFidelity = ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES })}, nil, + }, { + // ROOTFS is in the enum for the API's sake but no runtime captures it + // yet, so templates may not ask for it. + "rootfs fidelity not supported", + &ateapipb.CreateActorTemplateRequest{ActorTemplate: validActorTemplate(func(tmpl *ateapipb.ActorTemplate) { + tmpl.SnapshotConfig.PreferredFidelity = ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS + })}, + field.ErrorList{field.Invalid(field.NewPath("actor_template", "snapshot_config", "preferred_fidelity"), "SNAPSHOT_FIDELITY_ROOTFS", "")}, }, { "invalid worker_selector label key", &ateapipb.CreateActorTemplateRequest{ActorTemplate: validActorTemplate(func(tmpl *ateapipb.ActorTemplate) { @@ -136,14 +144,14 @@ func TestValidateCreateActorTemplateRequest(t *testing.T) { })}, field.ErrorList{field.Invalid(field.NewPath("actor_template", "snapshot_config", "storage_location"), "gs://my-bucket/snapshots?versions=true", "")}, }, { - // on_commit has no default of its own at this layer, so leaving it + // preferred_fidelity has no default of its own at this layer, so leaving it // unset is a required violation. - "on_commit unset", + "preferred_fidelity unset", &ateapipb.CreateActorTemplateRequest{ActorTemplate: validActorTemplate(func(tmpl *ateapipb.ActorTemplate) { - tmpl.SnapshotConfig.OnCommit = ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_UNSPECIFIED + tmpl.SnapshotConfig.PreferredFidelity = ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED })}, field.ErrorList{ - field.Required(field.NewPath("actor_template", "snapshot_config", "on_commit"), ""), + field.Required(field.NewPath("actor_template", "snapshot_config", "preferred_fidelity"), ""), }, }, { "missing sandbox_config", @@ -337,23 +345,23 @@ func TestValidateActorTemplate(t *testing.T) { }, { name: "unspecified snapshot scope", mutate: func(tmpl *ateapipb.ActorTemplate) { - tmpl.SnapshotConfig.OnCommit = ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_UNSPECIFIED + tmpl.SnapshotConfig.PreferredFidelity = ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED }, want: field.ErrorList{ - field.Required(field.NewPath("snapshot_config", "on_commit"), ""), + field.Required(field.NewPath("snapshot_config", "preferred_fidelity"), ""), }, }, { - name: "on_commit outside the enum", + name: "preferred_fidelity outside the enum", mutate: func(tmpl *ateapipb.ActorTemplate) { - tmpl.SnapshotConfig.OnCommit = ateapipb.SnapshotContentScope(99) + tmpl.SnapshotConfig.PreferredFidelity = ateapipb.SnapshotFidelity(99) }, - want: field.ErrorList{field.Invalid(field.NewPath("snapshot_config", "on_commit"), nil, "").WithOrigin("maximum")}, + want: field.ErrorList{field.Invalid(field.NewPath("snapshot_config", "preferred_fidelity"), nil, "").WithOrigin("maximum")}, }, { - name: "negative on_commit", + name: "negative preferred_fidelity", mutate: func(tmpl *ateapipb.ActorTemplate) { - tmpl.SnapshotConfig.OnCommit = ateapipb.SnapshotContentScope(-1) + tmpl.SnapshotConfig.PreferredFidelity = ateapipb.SnapshotFidelity(-1) }, - want: field.ErrorList{field.Invalid(field.NewPath("snapshot_config", "on_commit"), nil, "").WithOrigin("minimum")}, + want: field.ErrorList{field.Invalid(field.NewPath("snapshot_config", "preferred_fidelity"), nil, "").WithOrigin("minimum")}, }, { name: "no containers", mutate: func(tmpl *ateapipb.ActorTemplate) { tmpl.Containers = nil }, @@ -982,8 +990,8 @@ func validActorTemplate(mutations ...func(*ateapipb.ActorTemplate)) *ateapipb.Ac Metadata: &ateapipb.ResourceMetadata{Atespace: "ns1", Name: "tmpl-a"}, Containers: []*ateapipb.Container{{Name: "main", Image: "example.com/app:v1@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}}, SnapshotConfig: &ateapipb.SnapshotConfig{ - StorageLocation: "gs://my-bucket/snapshots", - OnCommit: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, + StorageLocation: "gs://my-bucket/snapshots", + PreferredFidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, }, SandboxConfig: &ateapipb.SandboxConfig{SandboxClass: ateapipb.SandboxClass_SANDBOX_CLASS_GVISOR, ConfigName: "gvisor-default"}, } diff --git a/cmd/ateapi/internal/apivalidation/actor_test.go b/cmd/ateapi/internal/apivalidation/actor_test.go index b383608746..4019e32999 100644 --- a/cmd/ateapi/internal/apivalidation/actor_test.go +++ b/cmd/ateapi/internal/apivalidation/actor_test.go @@ -453,8 +453,8 @@ func TestValidateActorUpdate(t *testing.T) { validInput(), validOutput(withStatus(func(s *ateapipb.ActorStatus) { s.ExternalSnapshot = &ateapipb.ExternalSnapshot{ - SnapshotUri: "gs://private/atespaces/as/actors/" + someActorUID + "/snapshots/snap-1", - ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, + SnapshotUri: "gs://private/atespaces/as/actors/" + someActorUID + "/snapshots/snap-1", + Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, } })), nil, @@ -497,26 +497,26 @@ func TestValidateActorUpdate(t *testing.T) { })), field.ErrorList{field.Invalid(field.NewPath("status", "local_snapshot", "snapshot_name"), nil, "").WithOrigin("format=k8s-short-name")}, }, { - "valid actor.status.local_snapshot.content_scope", + "valid actor.status.local_snapshot.fidelity", validInput(), validOutput(withStatus(func(s *ateapipb.ActorStatus) { - s.LocalSnapshot = &ateapipb.LocalSnapshot{ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA} + s.LocalSnapshot = &ateapipb.LocalSnapshot{Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES} })), nil, }, { - "negative actor.status.local_snapshot.content_scope", + "negative actor.status.local_snapshot.fidelity", validInput(), validOutput(withStatus(func(s *ateapipb.ActorStatus) { - s.LocalSnapshot = &ateapipb.LocalSnapshot{ContentScope: ateapipb.SnapshotContentScope(-1)} + s.LocalSnapshot = &ateapipb.LocalSnapshot{Fidelity: ateapipb.SnapshotFidelity(-1)} })), - field.ErrorList{field.Invalid(field.NewPath("status", "local_snapshot", "content_scope"), nil, "").WithOrigin("minimum")}, + field.ErrorList{field.Invalid(field.NewPath("status", "local_snapshot", "fidelity"), nil, "").WithOrigin("minimum")}, }, { - "invalid actor.status.local_snapshot.content_scope", + "invalid actor.status.local_snapshot.fidelity", validInput(), validOutput(withStatus(func(s *ateapipb.ActorStatus) { - s.LocalSnapshot = &ateapipb.LocalSnapshot{ContentScope: ateapipb.SnapshotContentScope(3)} + s.LocalSnapshot = &ateapipb.LocalSnapshot{Fidelity: ateapipb.SnapshotFidelity(4)} })), - field.ErrorList{field.Invalid(field.NewPath("status", "local_snapshot", "content_scope"), nil, "").WithOrigin("maximum")}, + field.ErrorList{field.Invalid(field.NewPath("status", "local_snapshot", "fidelity"), nil, "").WithOrigin("maximum")}, }, { "too many actor_volumes", validInput(), diff --git a/cmd/ateapi/internal/apivalidation/validation_test.go b/cmd/ateapi/internal/apivalidation/validation_test.go index 37d67bbee0..3b2e0e4461 100644 --- a/cmd/ateapi/internal/apivalidation/validation_test.go +++ b/cmd/ateapi/internal/apivalidation/validation_test.go @@ -1144,8 +1144,8 @@ func TestValidateDeleteOptions(t *testing.T) { func validExternalSnapshot(mutate ...func(*ateapipb.ExternalSnapshot)) *ateapipb.ExternalSnapshot { s := &ateapipb.ExternalSnapshot{ - SnapshotUri: "gs://private/atespaces/as/actors/" + someActorUID + "/snapshots/snap-1", - ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, + SnapshotUri: "gs://private/atespaces/as/actors/" + someActorUID + "/snapshots/snap-1", + Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, } for _, m := range mutate { m(s) @@ -1158,7 +1158,7 @@ func validExternalSnapshot(mutate ...func(*ateapipb.ExternalSnapshot)) *ateapipb func badExternalSnapshot(mutate ...func(*ateapipb.ExternalSnapshot)) *ateapipb.ExternalSnapshot { breakIt := func(s *ateapipb.ExternalSnapshot) { s.SnapshotUri = "" - s.ContentScope = ateapipb.SnapshotContentScope(3) + s.Fidelity = ateapipb.SnapshotFidelity(4) } return validExternalSnapshot(append([]func(*ateapipb.ExternalSnapshot){breakIt}, mutate...)...) } @@ -1166,7 +1166,7 @@ func badExternalSnapshot(mutate ...func(*ateapipb.ExternalSnapshot)) *ateapipb.E func TestValidateExternalSnapshot(t *testing.T) { valid := validExternalSnapshot uriPath := field.NewPath("snapshot_uri") - scopePath := field.NewPath("content_scope") + scopePath := field.NewPath("fidelity") tests := []struct { name string @@ -1178,17 +1178,17 @@ func TestValidateExternalSnapshot(t *testing.T) { obj: valid(), }, { - name: "valid content_scope: data", + name: "valid fidelity: data", obj: valid(func(s *ateapipb.ExternalSnapshot) { - s.ContentScope = ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA + s.Fidelity = ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES }), }, { // UNSPECIFIED reads as FULL, so optional lets the zero value skip // the bounds rather than failing the minimum. - name: "valid content_scope: unspecified", + name: "valid fidelity: unspecified", obj: valid(func(s *ateapipb.ExternalSnapshot) { - s.ContentScope = ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_UNSPECIFIED + s.Fidelity = ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED }), }, { @@ -1204,13 +1204,13 @@ func TestValidateExternalSnapshot(t *testing.T) { want: field.ErrorList{field.TooLong(uriPath, nil, 2048).WithOrigin("maxLength")}, }, { - name: "content_scope above the enum", - obj: valid(func(s *ateapipb.ExternalSnapshot) { s.ContentScope = ateapipb.SnapshotContentScope(3) }), + name: "fidelity above the enum", + obj: valid(func(s *ateapipb.ExternalSnapshot) { s.Fidelity = ateapipb.SnapshotFidelity(4) }), want: field.ErrorList{field.Invalid(scopePath, nil, "").WithOrigin("maximum")}, }, { - name: "negative content_scope", - obj: valid(func(s *ateapipb.ExternalSnapshot) { s.ContentScope = ateapipb.SnapshotContentScope(-1) }), + name: "negative fidelity", + obj: valid(func(s *ateapipb.ExternalSnapshot) { s.Fidelity = ateapipb.SnapshotFidelity(-1) }), want: field.ErrorList{field.Invalid(scopePath, nil, "").WithOrigin("minimum")}, }, { @@ -1233,7 +1233,7 @@ func TestValidateExternalSnapshot(t *testing.T) { func TestValidateExternalSnapshotUpdate(t *testing.T) { valid := validExternalSnapshot uriPath := field.NewPath("snapshot_uri") - scopePath := field.NewPath("content_scope") + scopePath := field.NewPath("fidelity") tests := []struct { name string @@ -1255,16 +1255,16 @@ func TestValidateExternalSnapshotUpdate(t *testing.T) { newObj: badExternalSnapshot(), }, { - name: "content_scope changed to a valid value", + name: "fidelity changed to a valid value", oldObj: valid(), newObj: valid(func(s *ateapipb.ExternalSnapshot) { - s.ContentScope = ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA + s.Fidelity = ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES }), }, { - name: "content_scope changed to a value outside the enum", + name: "fidelity changed to a value outside the enum", oldObj: valid(), - newObj: valid(func(s *ateapipb.ExternalSnapshot) { s.ContentScope = ateapipb.SnapshotContentScope(3) }), + newObj: valid(func(s *ateapipb.ExternalSnapshot) { s.Fidelity = ateapipb.SnapshotFidelity(4) }), want: field.ErrorList{field.Invalid(scopePath, nil, "").WithOrigin("maximum")}, }, { @@ -1276,10 +1276,10 @@ func TestValidateExternalSnapshotUpdate(t *testing.T) { { // The other side of the ratchet: a row that predates these rules // can still be repaired, one field at a time. - name: "content_scope repaired", + name: "fidelity repaired", oldObj: badExternalSnapshot(), newObj: badExternalSnapshot(func(s *ateapipb.ExternalSnapshot) { - s.ContentScope = ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL + s.Fidelity = ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY }), }, { @@ -1349,7 +1349,7 @@ func TestValidateNestedExternalSnapshot(t *testing.T) { t.Run(tt.name, func(t *testing.T) { want := field.ErrorList{ field.Required(tt.path.Child("snapshot_uri"), ""), - field.Invalid(tt.path.Child("content_scope"), nil, "").WithOrigin("maximum"), + field.Invalid(tt.path.Child("fidelity"), nil, "").WithOrigin("maximum"), } assertValidateErr(t, tt.validate(context.Background()), want) }) @@ -1557,7 +1557,7 @@ func TestValidateTagRequestPayloads(t *testing.T) { }, want: field.ErrorList{ field.Required(tagPath.Child("status", "snapshot", "snapshot_uri"), ""), - field.Invalid(tagPath.Child("status", "snapshot", "content_scope"), nil, "").WithOrigin("maximum"), + field.Invalid(tagPath.Child("status", "snapshot", "fidelity"), nil, "").WithOrigin("maximum"), }, }, { diff --git a/cmd/ateapi/internal/apivalidation/zz_generated.validation.go b/cmd/ateapi/internal/apivalidation/zz_generated.validation.go index a502173329..422aacb52a 100644 --- a/cmd/ateapi/internal/apivalidation/zz_generated.validation.go +++ b/cmd/ateapi/internal/apivalidation/zz_generated.validation.go @@ -3625,10 +3625,10 @@ func Validate_ExternalSnapshot( errs = append(errs, fn(fldPath.Child("snapshot_uri"), &obj.SnapshotUri, oldVal, oldObj != nil)...) } - { // field ateapipb.ExternalSnapshot.ContentScope + { // field ateapipb.ExternalSnapshot.Fidelity fn := func( fldPath *field.Path, - obj, oldObj *ateapipb.SnapshotContentScope, + obj, oldObj *ateapipb.SnapshotFidelity, oldValueCorrelated bool) (errs field.ErrorList) { // don't revalidate unchanged data if oldValueCorrelated && op.Type == operation.Update { @@ -3644,7 +3644,7 @@ func Validate_ExternalSnapshot( if earlyReturn { return // do not proceed } - if e := validate.Maximum(ctx, op, fldPath, obj, oldObj, 2); len(e) != 0 { + if e := validate.Maximum(ctx, op, fldPath, obj, oldObj, 3); len(e) != 0 { errs = append(errs, e...) } if e := validate.Minimum(ctx, op, fldPath, obj, oldObj, 1); len(e) != 0 { @@ -3653,10 +3653,10 @@ func Validate_ExternalSnapshot( return } oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.ExternalSnapshot) *ateapipb.SnapshotContentScope { - return &oldObj.ContentScope + func(oldObj *ateapipb.ExternalSnapshot) *ateapipb.SnapshotFidelity { + return &oldObj.Fidelity }) - errs = append(errs, fn(fldPath.Child("content_scope"), &obj.ContentScope, oldVal, oldObj != nil)...) + errs = append(errs, fn(fldPath.Child("fidelity"), &obj.Fidelity, oldVal, oldObj != nil)...) } { // field ateapipb.ExternalSnapshot.ActorTemplateUid @@ -5627,10 +5627,10 @@ func Validate_LocalSnapshot( errs = append(errs, fn(fldPath.Child("snapshot_name"), &obj.SnapshotName, oldVal, oldObj != nil)...) } - { // field ateapipb.LocalSnapshot.ContentScope + { // field ateapipb.LocalSnapshot.Fidelity fn := func( fldPath *field.Path, - obj, oldObj *ateapipb.SnapshotContentScope, + obj, oldObj *ateapipb.SnapshotFidelity, oldValueCorrelated bool) (errs field.ErrorList) { // don't revalidate unchanged data if oldValueCorrelated && op.Type == operation.Update { @@ -5646,7 +5646,7 @@ func Validate_LocalSnapshot( if earlyReturn { return // do not proceed } - if e := validate.Maximum(ctx, op, fldPath, obj, oldObj, 2); len(e) != 0 { + if e := validate.Maximum(ctx, op, fldPath, obj, oldObj, 3); len(e) != 0 { errs = append(errs, e...) } if e := validate.Minimum(ctx, op, fldPath, obj, oldObj, 1); len(e) != 0 { @@ -5655,10 +5655,10 @@ func Validate_LocalSnapshot( return } oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.LocalSnapshot) *ateapipb.SnapshotContentScope { - return &oldObj.ContentScope + func(oldObj *ateapipb.LocalSnapshot) *ateapipb.SnapshotFidelity { + return &oldObj.Fidelity }) - errs = append(errs, fn(fldPath.Child("content_scope"), &obj.ContentScope, oldVal, oldObj != nil)...) + errs = append(errs, fn(fldPath.Child("fidelity"), &obj.Fidelity, oldVal, oldObj != nil)...) } return errs @@ -7064,10 +7064,10 @@ func Validate_SnapshotConfig( ctx context.Context, op operation.Operation, fldPath *field.Path, obj, oldObj *ateapipb.SnapshotConfig) (errs field.ErrorList) { - { // field ateapipb.SnapshotConfig.OnCommit + { // field ateapipb.SnapshotConfig.StorageLocation fn := func( fldPath *field.Path, - obj, oldObj *ateapipb.SnapshotContentScope, + obj, oldObj *string, oldValueCorrelated bool) (errs field.ErrorList) { // don't revalidate unchanged data if oldValueCorrelated && op.Type == operation.Update { @@ -7084,25 +7084,26 @@ func Validate_SnapshotConfig( if earlyReturn { return // do not proceed } - if e := validate.Maximum(ctx, op, fldPath, obj, oldObj, 2); len(e) != 0 { + // custom validation + if e := ValidateCustom_SnapshotConfig_StorageLocation(ctx, op, fldPath, obj, oldObj); len(e) != 0 { errs = append(errs, e...) } - if e := validate.Minimum(ctx, op, fldPath, obj, oldObj, 1); len(e) != 0 { + if e := validate.MaxLength(ctx, op, fldPath, obj, oldObj, 1024); len(e) != 0 { errs = append(errs, e...) } return } oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.SnapshotConfig) *ateapipb.SnapshotContentScope { - return &oldObj.OnCommit + func(oldObj *ateapipb.SnapshotConfig) *string { + return &oldObj.StorageLocation }) - errs = append(errs, fn(fldPath.Child("on_commit"), &obj.OnCommit, oldVal, oldObj != nil)...) + errs = append(errs, fn(fldPath.Child("storage_location"), &obj.StorageLocation, oldVal, oldObj != nil)...) } - { // field ateapipb.SnapshotConfig.StorageLocation + { // field ateapipb.SnapshotConfig.PreferredFidelity fn := func( fldPath *field.Path, - obj, oldObj *string, + obj, oldObj *ateapipb.SnapshotFidelity, oldValueCorrelated bool) (errs field.ErrorList) { // don't revalidate unchanged data if oldValueCorrelated && op.Type == operation.Update { @@ -7120,19 +7121,22 @@ func Validate_SnapshotConfig( return // do not proceed } // custom validation - if e := ValidateCustom_SnapshotConfig_StorageLocation(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + if e := ValidateCustom_SnapshotConfig_PreferredFidelity(ctx, op, fldPath, obj, oldObj); len(e) != 0 { errs = append(errs, e...) } - if e := validate.MaxLength(ctx, op, fldPath, obj, oldObj, 1024); len(e) != 0 { + if e := validate.Maximum(ctx, op, fldPath, obj, oldObj, 3); len(e) != 0 { + errs = append(errs, e...) + } + if e := validate.Minimum(ctx, op, fldPath, obj, oldObj, 1); len(e) != 0 { errs = append(errs, e...) } return } oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.SnapshotConfig) *string { - return &oldObj.StorageLocation + func(oldObj *ateapipb.SnapshotConfig) *ateapipb.SnapshotFidelity { + return &oldObj.PreferredFidelity }) - errs = append(errs, fn(fldPath.Child("storage_location"), &obj.StorageLocation, oldVal, oldObj != nil)...) + errs = append(errs, fn(fldPath.Child("preferred_fidelity"), &obj.PreferredFidelity, oldVal, oldObj != nil)...) } return errs diff --git a/cmd/ateapi/internal/controlapi/actor.go b/cmd/ateapi/internal/controlapi/actor.go index 77bba00444..a3a75247ed 100644 --- a/cmd/ateapi/internal/controlapi/actor.go +++ b/cmd/ateapi/internal/controlapi/actor.go @@ -106,7 +106,7 @@ func (s *ServiceImpl) CreateActor(ctx context.Context, inActor *ateapipb.Actor) return nil, err } if inActor.GetSourceTag() == nil { - if err := validateGoldenSnapshotScope(sourceTag.GetStatus().GetSnapshot()); err != nil { + if err := validateGoldenSnapshotFidelity(sourceTag.GetStatus().GetSnapshot()); err != nil { return nil, err } } @@ -190,7 +190,7 @@ func (s *ServiceImpl) resolveTagSource(ctx context.Context, actorAtespace string if tag.GetStatus().GetSnapshot().GetSnapshotUri() == "" { return nil, apierror.FailedPrecondition("source Tag is still being created or failed creation") } - // TODO: Permit compatible DATA snapshots when runtimes can extract portable data. + // TODO: Permit compatible VOLUMES snapshots when runtimes can extract portable data. if tag.GetStatus().GetActorTemplateUid() != template.GetMetadata().GetUid() { return nil, apierror.FailedPrecondition("source Tag must be taken from an actor with ActorTemplate uid %q", tag.GetStatus().GetActorTemplateUid()) } diff --git a/cmd/ateapi/internal/controlapi/actor_template_test.go b/cmd/ateapi/internal/controlapi/actor_template_test.go index 06bc3e8cdf..8692040ed1 100644 --- a/cmd/ateapi/internal/controlapi/actor_template_test.go +++ b/cmd/ateapi/internal/controlapi/actor_template_test.go @@ -44,8 +44,8 @@ func validActorTemplate(mutations ...func(*ateapipb.ActorTemplate)) *ateapipb.Ac Metadata: &ateapipb.ResourceMetadata{Atespace: "ns1", Name: "tmpl-a"}, Containers: []*ateapipb.Container{{Name: "main", Image: "example.com/app:v1@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}}, SnapshotConfig: &ateapipb.SnapshotConfig{ - StorageLocation: "gs://my-bucket/snapshots", - OnCommit: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, + StorageLocation: "gs://my-bucket/snapshots", + PreferredFidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, }, SandboxConfig: &ateapipb.SandboxConfig{SandboxClass: ateapipb.SandboxClass_SANDBOX_CLASS_GVISOR, ConfigName: "gvisor-default"}, } @@ -232,7 +232,7 @@ func TestDeleteActorTemplate(t *testing.T) { actor = mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapipb.ActorStatus) { s.ExternalSnapshot = &ateapipb.ExternalSnapshot{ SnapshotUri: actorURI.String(), - ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, + Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, ActorTemplateUid: tmpl.GetMetadata().GetUid(), } }) diff --git a/cmd/ateapi/internal/controlapi/actor_test.go b/cmd/ateapi/internal/controlapi/actor_test.go index 12e91b3ea5..6a9eb5b627 100644 --- a/cmd/ateapi/internal/controlapi/actor_test.go +++ b/cmd/ateapi/internal/controlapi/actor_test.go @@ -814,7 +814,7 @@ func TestCreateActor_GoldenTagDefault(t *testing.T) { Scope: ateapipb.TagScope_TAG_SCOPE_PUBLISHED, Status: &ateapipb.TagStatus{ ActorTemplateUid: tmpl.GetMetadata().GetUid(), - Snapshot: &ateapipb.ExternalSnapshot{SnapshotUri: "gs://bucket/atespaces/ate-golden/tags/" + someActorUID, ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL}, + Snapshot: &ateapipb.ExternalSnapshot{SnapshotUri: "gs://bucket/atespaces/ate-golden/tags/" + someActorUID, Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY}, }, } wantCode := codes.OK @@ -828,7 +828,7 @@ func TestCreateActor_GoldenTagDefault(t *testing.T) { tag.Status.ActorTemplateUid = "other" wantCode = codes.FailedPrecondition case "data scope": - tag.Status.Snapshot.ContentScope = ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA + tag.Status.Snapshot.Fidelity = ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES wantCode = codes.FailedPrecondition } if scenario != "missing" { diff --git a/cmd/ateapi/internal/controlapi/converter.go b/cmd/ateapi/internal/controlapi/converter.go index 5d1e80080b..f34b9aaeaf 100644 --- a/cmd/ateapi/internal/controlapi/converter.go +++ b/cmd/ateapi/internal/controlapi/converter.go @@ -20,11 +20,20 @@ import ( "github.com/agent-substrate/substrate/pkg/proto/ateapipb" ) -func actorSnapshotContentScopeToAtelet(in ateapipb.SnapshotContentScope) ateletpb.SnapshotScope { - if in == ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA { - return ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA +// fidelityToAtelet maps the public fidelity ladder onto the identical atelet +// wire enum, level by level. Anything else, including UNSPECIFIED, goes out +// as UNSPECIFIED, which atelet rejects, rather than being silently promoted. +func fidelityToAtelet(in ateapipb.SnapshotFidelity) ateletpb.SnapshotFidelity { + switch in { + case ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES: + return ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES + case ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS: + return ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS + case ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY: + return ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY + default: + return ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED } - return ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL } // sandboxClassString renders the proto enum in the CRD's lower-case string diff --git a/cmd/ateapi/internal/controlapi/converter_test.go b/cmd/ateapi/internal/controlapi/converter_test.go index 9ebb8dc368..4a0824230b 100644 --- a/cmd/ateapi/internal/controlapi/converter_test.go +++ b/cmd/ateapi/internal/controlapi/converter_test.go @@ -21,36 +21,46 @@ import ( "github.com/agent-substrate/substrate/pkg/proto/ateapipb" ) -// TestSnapshotScopeToAtelet covers the wire scope derivation for template -// content scopes: an unset scope falls back to Full. -func TestSnapshotScopeToAtelet(t *testing.T) { +// TestFidelityToAtelet pins the level-by-level mapping onto the atelet enum: +// nothing is promoted, and an unset fidelity stays unset so atelet rejects it. +func TestFidelityToAtelet(t *testing.T) { tests := []struct { name string - in ateapipb.SnapshotContentScope - expected ateletpb.SnapshotScope + in ateapipb.SnapshotFidelity + expected ateletpb.SnapshotFidelity }{ { - name: "Full scope", - in: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, - expected: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL, + name: "memory", + in: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + expected: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, }, { - name: "Data scope", - in: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA, - expected: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA, + name: "rootfs", + in: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS, + expected: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS, }, { - name: "Default scope (unspecified)", - in: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_UNSPECIFIED, - expected: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL, + name: "volumes", + in: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, + expected: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, + }, + { + name: "unspecified stays unspecified", + in: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED, + expected: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED, + }, + { + name: "value outside the enum", + in: ateapipb.SnapshotFidelity(99), + expected: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - result := actorSnapshotContentScopeToAtelet(tt.in) + result := fidelityToAtelet(tt.in) if result != tt.expected { - t.Errorf("actorSnapshotContentScopeToAtelet(%v) = %v, want %v", tt.in, result, tt.expected) + t.Errorf("fidelityToAtelet(%v) = %v, want %v", tt.in, result, tt.expected) } }) } diff --git a/cmd/ateapi/internal/controlapi/functionaltest/actor_template_test.go b/cmd/ateapi/internal/controlapi/functionaltest/actor_template_test.go index 4f0041532b..f4c65b273c 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/actor_template_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/actor_template_test.go @@ -62,8 +62,8 @@ func TestActorTemplateCRUD(t *testing.T) { Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "tmpl-a", Version: 1}, Containers: []*ateapipb.Container{{Name: "main", Image: "example.com/app:v1@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}}, SnapshotConfig: &ateapipb.SnapshotConfig{ - StorageLocation: "gs://my-bucket/snapshots", - OnCommit: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, + StorageLocation: "gs://my-bucket/snapshots", + PreferredFidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, }, SandboxConfig: &ateapipb.SandboxConfig{ SandboxClass: ateapipb.SandboxClass_SANDBOX_CLASS_GVISOR, diff --git a/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go b/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go index 60656acb1e..6eba54a915 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go @@ -79,7 +79,7 @@ func TestCreateActor_Success(t *testing.T) { ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"}, Status: &ateapipb.ActorStatus{ State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, - ExternalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: goldenSnapshotURI(t), ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, ActorTemplateUid: tmpl.GetMetadata().GetUid()}, + ExternalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: goldenSnapshotURI(t), Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, ActorTemplateUid: tmpl.GetMetadata().GetUid()}, }, WorkerSelector: &ateapipb.Selector{MatchLabels: map[string]string{"tier": "free"}}, } @@ -294,7 +294,7 @@ func TestCreateActor_RejectsDifferentTemplateForDataSnapshot(t *testing.T) { createTemplateWithSelector(t, tc, "tmpl2", nil) seedTag(t, tc, "data-source", "data-snapshot", func(tag *ateapipb.Tag) { - tag.Status.Snapshot.ContentScope = ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA + tag.Status.Snapshot.Fidelity = ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES tag.Status.ActorTemplateUid = tmpl.GetMetadata().GetUid() }) @@ -390,8 +390,8 @@ func TestCreateActor_PendingTag(t *testing.T) { resources.TagRefFromTag(pending), store.PreconditionFrom(pending), func(toUpdate *ateapipb.Tag) error { toUpdate.Status.Snapshot = &ateapipb.ExternalSnapshot{ - SnapshotUri: snapshotURI.String(), - ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, + SnapshotUri: snapshotURI.String(), + Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, } return nil }); err != nil { @@ -700,7 +700,7 @@ func TestUpdateActor_Success(t *testing.T) { ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"}, Status: &ateapipb.ActorStatus{ State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, - ExternalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: goldenSnapshotURI(t), ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, ActorTemplateUid: tmpl.GetMetadata().GetUid()}, + ExternalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: goldenSnapshotURI(t), Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, ActorTemplateUid: tmpl.GetMetadata().GetUid()}, }, WorkerSelector: &ateapipb.Selector{ MatchLabels: map[string]string{"tier": "paid"}, @@ -846,7 +846,7 @@ func TestUpdateActor(t *testing.T) { ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"}, Status: &ateapipb.ActorStatus{ State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, - ExternalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: goldenSnapshotURI(t), ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, ActorTemplateUid: tmpl.GetMetadata().GetUid()}, + ExternalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: goldenSnapshotURI(t), Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, ActorTemplateUid: tmpl.GetMetadata().GetUid()}, }, WorkerSelector: &ateapipb.Selector{ MatchLabels: map[string]string{"tier": "paid"}, @@ -1120,7 +1120,7 @@ func TestDeleteActor_MissingSnapshotBucket(t *testing.T) { toUpdate.Status.ExternalSnapshot = nil if state == ateapipb.ActorState_ACTOR_STATE_SUSPENDED { toUpdate.Status.ExternalSnapshot = &ateapipb.ExternalSnapshot{ - SnapshotUri: uri.String(), ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, + SnapshotUri: uri.String(), Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, } } else { // A failed upload leaves only the in-progress URI, even @@ -2910,7 +2910,7 @@ func TestResumeActor(t *testing.T) { Status: &ateapipb.ActorStatus{ State: ateapipb.ActorState_ACTOR_STATE_RUNNING, AssignedNode: "node1", - ExternalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: goldenSnapshotURI(t), ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, ActorTemplateUid: tmpl.GetMetadata().GetUid()}, + ExternalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: goldenSnapshotURI(t), Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, ActorTemplateUid: tmpl.GetMetadata().GetUid()}, WorkerAssignment: &ateapipb.WorkerAssignment{ Worker: &ateapipb.ObjectRef{Name: podUID}, WorkerNamespace: ns, @@ -3022,10 +3022,10 @@ func TestResumeActorPassesLiteralEnv(t *testing.T) { } } -// createDataCommitTemplate creates "tmpl1" like createTemplate, but with -// onCommit DATA, so a resumed-after-suspend actor restores from a DATA +// createDataFidelityTemplate creates "tmpl1" like createTemplate, but with +// preferredFidelity DATA, so a resumed-after-suspend actor restores from a DATA // snapshot while the template also has a golden snapshot. -func createDataCommitTemplate(t *testing.T, tc *testContext, ns string) *ateapipb.ActorTemplate { +func createDataFidelityTemplate(t *testing.T, tc *testContext, ns string) *ateapipb.ActorTemplate { t.Helper() ensureDefaultGvisorSandboxConfig(t, tc) createWorkerPool(t, tc, ns, "pool1", map[string]string{poolLabelKey: ns}) @@ -3037,8 +3037,8 @@ func createDataCommitTemplate(t *testing.T, tc *testContext, ns string) *ateapip Name: "tmpl1", }, SnapshotConfig: &ateapipb.SnapshotConfig{ - StorageLocation: testStorageLocation, - OnCommit: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA, + StorageLocation: testStorageLocation, + PreferredFidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, }, SandboxConfig: &ateapipb.SandboxConfig{ SandboxClass: ateapipb.SandboxClass_SANDBOX_CLASS_GVISOR, @@ -3066,7 +3066,7 @@ func createDataCommitTemplate(t *testing.T, tc *testContext, ns string) *ateapip SourceActor: &ateapipb.ObjectRef{Atespace: resources.GoldenActorAtespace, Name: created.GetMetadata().GetUid()}, Scope: ateapipb.TagScope_TAG_SCOPE_PUBLISHED, Status: &ateapipb.TagStatus{ - Snapshot: &ateapipb.ExternalSnapshot{SnapshotUri: goldenSnapshotURI(t), ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL}, + Snapshot: &ateapipb.ExternalSnapshot{SnapshotUri: goldenSnapshotURI(t), Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY}, StorageLocation: testStorageLocation, ActorTemplateUid: created.GetMetadata().GetUid(), }, @@ -3098,7 +3098,7 @@ func TestResumeActor_DataSnapshotIgnoresGolden(t *testing.T) { tc := setupTest(t, ns) defer tc.cleanup() - createDataCommitTemplate(t, tc, ns) + createDataFidelityTemplate(t, tc, ns) workerName := createWorkerPod(t, tc, ns, "worker-1", "node1", "pool1") const name = "id1" @@ -3111,7 +3111,7 @@ func TestResumeActor_DataSnapshotIgnoresGolden(t *testing.T) { } // First resume runs fresh from the golden; the suspend then commits a - // DATA snapshot per onCommit. + // DATA snapshot per preferredFidelity. if _, err := tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{Actor: actorRef}); err != nil { t.Fatalf("ResumeActor (first) failed: %v", err) } @@ -3133,8 +3133,8 @@ func TestResumeActor_DataSnapshotIgnoresGolden(t *testing.T) { if restoreReq == nil { t.Fatal("second resume sent no Restore request to atelet") } - if got := restoreReq.GetScope(); got != ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA { - t.Fatalf("restore scope = %v, want SNAPSHOT_SCOPE_DATA", got) + if got := restoreReq.GetFidelity(); got != ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES { + t.Fatalf("restore scope = %v, want SNAPSHOT_FIDELITY_VOLUMES", got) } if got := restoreReq.GetExternalConfig().GetSnapshotUri(); got != actorSnapshotURI { t.Errorf("restore config snapshot uri = %q, want the actor's data snapshot %q", got, actorSnapshotURI) @@ -3644,7 +3644,7 @@ func TestSuspendActor(t *testing.T) { Scope: ateapipb.TagScope_TAG_SCOPE_ATESPACE, SourceActor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: name}, Status: &ateapipb.TagStatus{ - Snapshot: &ateapipb.ExternalSnapshot{SnapshotUri: tagSnapshotURI, ContentScope: sourceActor.GetStatus().GetExternalSnapshot().GetContentScope()}, + Snapshot: &ateapipb.ExternalSnapshot{SnapshotUri: tagSnapshotURI, Fidelity: sourceActor.GetStatus().GetExternalSnapshot().GetFidelity()}, ActorTemplateUid: tmpl.GetMetadata().GetUid(), StorageLocation: tmpl.GetSnapshotConfig().GetStorageLocation(), }, @@ -3750,7 +3750,7 @@ func TestSuspendActor(t *testing.T) { State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, ExternalSnapshot: &ateapipb.ExternalSnapshot{ SnapshotUri: snapshotURI, - ContentScope: sourceActor.GetStatus().GetExternalSnapshot().GetContentScope(), + Fidelity: sourceActor.GetStatus().GetExternalSnapshot().GetFidelity(), ActorTemplateUid: tmpl.GetMetadata().GetUid(), }, }, @@ -3802,19 +3802,19 @@ func TestResumeActor_RepointTemplateBeforeResume(t *testing.T) { // after it is created but before its first resume. moveActorToAnotherTemplate bool wantTemplate string - wantScope ateletpb.SnapshotScope + wantScope ateletpb.SnapshotFidelity }{ { name: "clone left on the tag's template", moveActorToAnotherTemplate: false, wantTemplate: "tmpl1", - wantScope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL, + wantScope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, }, { name: "clone repointed before its first resume", moveActorToAnotherTemplate: true, wantTemplate: "tmpl2", - wantScope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA, + wantScope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, }, } for _, tt := range tests { @@ -3897,7 +3897,7 @@ func TestResumeActor_RepointTemplateBeforeResume(t *testing.T) { if got := restoreReq.GetActorTemplateName(); got != tt.wantTemplate { t.Errorf("restore request to atelet had actor template = %q, want %q", got, tt.wantTemplate) } - if got := restoreReq.GetScope(); got != tt.wantScope { + if got := restoreReq.GetFidelity(); got != tt.wantScope { t.Errorf("restore request to atelet had scope = %v, want %v", got, tt.wantScope) } // Either way the restore reads the snapshot the clone borrowed @@ -3959,7 +3959,7 @@ func TestResumeActor_PausedAfterRepointUsesLocalProvenance(t *testing.T) { if _, err := tc.client.ResumeActor(ctx, &ateapipb.ResumeActorRequest{Actor: actorRef}); err != nil { t.Fatalf("ResumeActor(v2 from v1 snapshot) failed: %v", err) } - if got := tc.fakeAtelet.lastRestoreRequest().GetScope(); got != ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA { + if got := tc.fakeAtelet.lastRestoreRequest().GetFidelity(); got != ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES { t.Fatalf("first resume on v2 had scope = %v, want DATA", got) } if _, err := tc.client.PauseActor(ctx, &ateapipb.PauseActorRequest{Actor: actorRef}); err != nil { @@ -3976,7 +3976,7 @@ func TestResumeActor_PausedAfterRepointUsesLocalProvenance(t *testing.T) { if got := restoreReq.GetType(); got != ateletpb.CheckpointType_CHECKPOINT_TYPE_LOCAL { t.Errorf("restore request type = %v, want LOCAL", got) } - if got := restoreReq.GetScope(); got != ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL { + if got := restoreReq.GetFidelity(); got != ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY { t.Errorf("restore request scope = %v, want FULL (local checkpoint was captured on v2)", got) } } @@ -4042,9 +4042,9 @@ func TestPauseActor(t *testing.T) { State: ateapipb.ActorState_ACTOR_STATE_PAUSED, AssignedNode: "node1", LocalSnapshot: &ateapipb.LocalSnapshot{ - ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, + Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, }, - ExternalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: goldenSnapshotURI(t), ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, ActorTemplateUid: tmpl.GetMetadata().GetUid()}, + ExternalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: goldenSnapshotURI(t), Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, ActorTemplateUid: tmpl.GetMetadata().GetUid()}, }, } @@ -4737,8 +4737,8 @@ func TestSuspendActor_FromPaused(t *testing.T) { if got, want := upload.GetAtespace(), testAtespace; got != want { t.Errorf("upload atespace = %q, want %q", got, want) } - if got := upload.GetDesiredScope(); got != ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL { - t.Errorf("upload desired_scope = %v, want FULL (template default)", got) + if got := upload.GetDesiredFidelity(); got != ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY { + t.Errorf("upload desired_fidelity = %v, want MEMORY (template default)", got) } actor := suspended.GetActor() @@ -4751,8 +4751,8 @@ func TestSuspendActor_FromPaused(t *testing.T) { if got, want := actor.GetStatus().GetExternalSnapshot().GetSnapshotUri(), upload.GetDestinationSnapshotUri(); got != want { t.Errorf("snapshot URI = %q, want the upload destination %q", got, want) } - if got := actor.GetStatus().GetExternalSnapshot().GetContentScope(); got != ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL { - t.Errorf("snapshot ContentScope = %v, want FULL", got) + if got := actor.GetStatus().GetExternalSnapshot().GetFidelity(); got != ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY { + t.Errorf("snapshot Fidelity = %v, want FULL", got) } } diff --git a/cmd/ateapi/internal/controlapi/functionaltest/common_test.go b/cmd/ateapi/internal/controlapi/functionaltest/common_test.go index 5396522108..ad832bf39f 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/common_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/common_test.go @@ -459,7 +459,7 @@ func createTemplateWithContainersAndVolumes(t *testing.T, tc *testContext, ns st SourceActor: &ateapipb.ObjectRef{Atespace: resources.GoldenActorAtespace, Name: created.GetMetadata().GetUid()}, Scope: ateapipb.TagScope_TAG_SCOPE_PUBLISHED, Status: &ateapipb.TagStatus{ - Snapshot: &ateapipb.ExternalSnapshot{SnapshotUri: goldenSnapshotURI(t), ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL}, + Snapshot: &ateapipb.ExternalSnapshot{SnapshotUri: goldenSnapshotURI(t), Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY}, StorageLocation: testStorageLocation, ActorTemplateUid: created.GetMetadata().GetUid(), }, diff --git a/cmd/ateapi/internal/controlapi/functionaltest/tag_test.go b/cmd/ateapi/internal/controlapi/functionaltest/tag_test.go index 0562bc91a6..14ae066bf1 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/tag_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/tag_test.go @@ -47,7 +47,7 @@ func seedTag(t *testing.T, tc *testContext, actorName, tagName string, opts ...f t.Fatalf("NewActorSnapshotURI: %v", err) } actor, err = tc.persistence.UpdateActor(ctx, resources.ActorRefFromActor(actor), store.PreconditionFrom(actor), func(toUpdate *ateapipb.Actor) error { - toUpdate.Status.ExternalSnapshot = &ateapipb.ExternalSnapshot{SnapshotUri: actorSnapshotURI.String(), ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL} + toUpdate.Status.ExternalSnapshot = &ateapipb.ExternalSnapshot{SnapshotUri: actorSnapshotURI.String(), Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY} return nil }) if err != nil { @@ -58,7 +58,7 @@ func seedTag(t *testing.T, tc *testContext, actorName, tagName string, opts ...f Scope: ateapipb.TagScope_TAG_SCOPE_ATESPACE, SourceActor: resources.ActorRefFromActor(actor).ToObjectRef(), Status: &ateapipb.TagStatus{ - Snapshot: &ateapipb.ExternalSnapshot{ContentScope: actor.GetStatus().GetExternalSnapshot().GetContentScope()}, + Snapshot: &ateapipb.ExternalSnapshot{Fidelity: actor.GetStatus().GetExternalSnapshot().GetFidelity()}, StorageLocation: testStorageLocation, }, } diff --git a/cmd/ateapi/internal/controlapi/metrics.go b/cmd/ateapi/internal/controlapi/metrics.go index 291767fd74..a643e08ca7 100644 --- a/cmd/ateapi/internal/controlapi/metrics.go +++ b/cmd/ateapi/internal/controlapi/metrics.go @@ -202,7 +202,7 @@ func (i *Instruments) recordLifecycleOp(ctx context.Context, op string, start ti } // lifecycleOpAttrs builds the resume/suspend/pause dimensions from workflow -// state. Nil-safe, and omits the pool, snapshot-kind and snapshot-scope labels +// state. Nil-safe, and omits the pool, snapshot-kind and snapshot-fidelity labels // until they are known so a failure before the assign/restore steps never emits // an empty-string series. snapshotKind is empty for suspend/pause, which do not // restore; snapshotScope applies to all three and separates a full restore @@ -222,7 +222,7 @@ func lifecycleOpAttrs(actor *ateapipb.Actor, template *ateapipb.ActorTemplate, s attrs = append(attrs, ateattr.SnapshotKindKey.String(snapshotKind)) } if snapshotScope != "" { - attrs = append(attrs, ateattr.SnapshotScopeKey.String(snapshotScope)) + attrs = append(attrs, ateattr.SnapshotFidelityKey.String(snapshotScope)) } return attrs } diff --git a/cmd/ateapi/internal/controlapi/metrics_test.go b/cmd/ateapi/internal/controlapi/metrics_test.go index 8784b9c2ad..21e64e9862 100644 --- a/cmd/ateapi/internal/controlapi/metrics_test.go +++ b/cmd/ateapi/internal/controlapi/metrics_test.go @@ -249,7 +249,7 @@ func TestLifecycleOpDurationShape(t *testing.T) { SandboxConfig: &ateapipb.SandboxConfig{SandboxClass: ateapipb.SandboxClass_SANDBOX_CLASS_GVISOR}, } inst.recordLifecycleOp(context.Background(), ateattr.OperationResume, time.Now(), nil, - lifecycleOpAttrs(actor, template, ateattr.SnapshotKindLatest, ateattr.SnapshotScopeData)...) + lifecycleOpAttrs(actor, template, ateattr.SnapshotKindLatest, ateattr.SnapshotFidelityVolumes)...) dp := singleHistogramDP(t, reader, lifecycleOpDurationMetric) assertAttrKeys(t, dp, @@ -260,7 +260,7 @@ func TestLifecycleOpDurationShape(t *testing.T) { ateattr.WorkerPoolNameKey, ateattr.SandboxClassKey, ateattr.SnapshotKindKey, - ateattr.SnapshotScopeKey, + ateattr.SnapshotFidelityKey, ) if op, _ := attrString(dp, ateattr.ActorOperationNameKey); op != ateattr.OperationResume { t.Errorf("operation = %q, want %q", op, ateattr.OperationResume) @@ -272,8 +272,8 @@ func TestLifecycleOpDurationShape(t *testing.T) { } // Kind and scope are independent: a data restore of the actor's // own latest snapshot must stay distinguishable from one of a local snapshot. - if scope, _ := attrString(dp, ateattr.SnapshotScopeKey); scope != ateattr.SnapshotScopeData { - t.Errorf("snapshot scope = %q, want %q", scope, ateattr.SnapshotScopeData) + if scope, _ := attrString(dp, ateattr.SnapshotFidelityKey); scope != ateattr.SnapshotFidelityVolumes { + t.Errorf("snapshot fidelity = %q, want %q", scope, ateattr.SnapshotFidelityVolumes) } if kind, _ := attrString(dp, ateattr.SnapshotKindKey); kind != ateattr.SnapshotKindLatest { t.Errorf("snapshot kind = %q, want %q", kind, ateattr.SnapshotKindLatest) @@ -288,7 +288,7 @@ func TestLifecycleOpAttrsOmitsUnknownScope(t *testing.T) { actor := &ateapipb.Actor{ActorTemplate: &ateapipb.ObjectRef{Atespace: "ate-agents", Name: "support-agent"}} for _, kv := range lifecycleOpAttrs(actor, nil, "", "") { switch kv.Key { - case ateattr.SnapshotScopeKey, ateattr.SnapshotKindKey, ateattr.WorkerPoolNamespaceKey, ateattr.WorkerPoolNameKey: + case ateattr.SnapshotFidelityKey, ateattr.SnapshotKindKey, ateattr.WorkerPoolNamespaceKey, ateattr.WorkerPoolNameKey: t.Errorf("attribute %s must be omitted while unknown, got %q", kv.Key, kv.Value.AsString()) } } diff --git a/cmd/ateapi/internal/controlapi/tag_test.go b/cmd/ateapi/internal/controlapi/tag_test.go index 38002251c0..9c7b31d232 100644 --- a/cmd/ateapi/internal/controlapi/tag_test.go +++ b/cmd/ateapi/internal/controlapi/tag_test.go @@ -135,7 +135,7 @@ func TestUpdateTag(t *testing.T) { req: &ateapipb.Tag{ Scope: ateapipb.TagScope_TAG_SCOPE_PUBLISHED, Status: &ateapipb.TagStatus{ - Snapshot: &ateapipb.ExternalSnapshot{SnapshotUri: "gs://attacker/elsewhere", ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA}, + Snapshot: &ateapipb.ExternalSnapshot{SnapshotUri: "gs://attacker/elsewhere", Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES}, ActorTemplateUid: "other-template-uid", }, }, @@ -249,7 +249,7 @@ func newTestSuspendedActor(t *testing.T, ctx context.Context, st store.Interface t.Fatalf("NewActorSnapshotURI: %v", err) } return mustUpdateActorStatus(t, ctx, st, actor, func(status *ateapipb.ActorStatus) { - status.ExternalSnapshot = &ateapipb.ExternalSnapshot{SnapshotUri: uri.String(), ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL} + status.ExternalSnapshot = &ateapipb.ExternalSnapshot{SnapshotUri: uri.String(), Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY} }) } @@ -269,8 +269,8 @@ func newTestTag(t *testing.T, name string, actor *ateapipb.Actor) *ateapipb.Tag SourceActor: resources.ActorRefFromActor(actor).ToObjectRef(), Status: &ateapipb.TagStatus{ Snapshot: &ateapipb.ExternalSnapshot{ - SnapshotUri: uri.String(), - ContentScope: actor.GetStatus().GetExternalSnapshot().GetContentScope(), + SnapshotUri: uri.String(), + Fidelity: actor.GetStatus().GetExternalSnapshot().GetFidelity(), }, }, } diff --git a/cmd/ateapi/internal/controlapi/template_reconciler_test.go b/cmd/ateapi/internal/controlapi/template_reconciler_test.go index 70010aed7d..1a2d51bc35 100644 --- a/cmd/ateapi/internal/controlapi/template_reconciler_test.go +++ b/cmd/ateapi/internal/controlapi/template_reconciler_test.go @@ -606,7 +606,7 @@ func TestReconcileOne(t *testing.T) { // requests the happy path issues: the golden actor is named after the // template UID so recreated templates with the same name never collide, and // lives in the reserved ate-golden atespace so the suspend workflow commits -// it Full regardless of the template's onCommit scope. +// it Full regardless of the template's preferredFidelity scope. func TestReconcileOne_GoldenActorRequests(t *testing.T) { ctx := context.Background() st := newFakeTemplateStore(testTemplate()) diff --git a/cmd/ateapi/internal/controlapi/workflow_pause.go b/cmd/ateapi/internal/controlapi/workflow_pause.go index 522c573ff4..0c0f125e0f 100644 --- a/cmd/ateapi/internal/controlapi/workflow_pause.go +++ b/cmd/ateapi/internal/controlapi/workflow_pause.go @@ -37,7 +37,7 @@ func (w *ActorWorkflow) PauseActor(ctx context.Context, actorRef resources.Actor start := time.Now() var actor *ateapipb.Actor var actorTemplate *ateapipb.ActorTemplate - var wireSnapshotScope string + var wireFidelity string // Set just before finalize; nil until then, so earlier exits label // themselves from the record they hold. var finalAttrs []attribute.KeyValue @@ -45,7 +45,7 @@ func (w *ActorWorkflow) PauseActor(ctx context.Context, actorRef resources.Actor defer func() { attrs := finalAttrs if attrs == nil { - attrs = lifecycleOpAttrs(actor, actorTemplate, "", wireSnapshotScope) + attrs = lifecycleOpAttrs(actor, actorTemplate, "", wireFidelity) } w.instruments.recordLifecycleOp(ctx, ateattr.OperationPause, start, err, attrs...) }() @@ -72,7 +72,7 @@ func (w *ActorWorkflow) PauseActor(ctx context.Context, actorRef resources.Actor return nil, err } actor = marked - if wireSnapshotScope, err = w.ensureAteletPaused(leaseCtx, actorRef, actor, actorTemplate); err != nil { + if wireFidelity, err = w.ensureAteletPaused(leaseCtx, actorRef, actor, actorTemplate); err != nil { return nil, err } // TODO: There is no difference between suspend and pause for now, but we @@ -83,7 +83,7 @@ func (w *ActorWorkflow) PauseActor(ctx context.Context, actorRef resources.Actor } // FinalizePaused clears the WorkerAssignment the labels read, so snapshot // them here, as crash.go does for the crash counter. - finalAttrs = lifecycleOpAttrs(actor, actorTemplate, "", wireSnapshotScope) + finalAttrs = lifecycleOpAttrs(actor, actorTemplate, "", wireFidelity) var finalized *ateapipb.Actor if finalized, err = w.ensurePausedFinalized(leaseCtx, actorRef, actorTemplate); err != nil { return nil, err @@ -152,7 +152,7 @@ func (w *ActorWorkflow) ensureMarkedPausing(ctx context.Context, actorRef resour // the once-minted snapshot name, so a re-entered workflow re-sends the same // semantic request; once atelet's Checkpoint is idempotent on those keys this // step becomes fully reentrant with no changes here. -func (w *ActorWorkflow) ensureAteletPaused(ctx context.Context, actorRef resources.ActorRef, actor *ateapipb.Actor, actorTemplate *ateapipb.ActorTemplate) (wireSnapshotScope string, err error) { +func (w *ActorWorkflow) ensureAteletPaused(ctx context.Context, actorRef resources.ActorRef, actor *ateapipb.Actor, actorTemplate *ateapipb.ActorTemplate) (wireFidelity string, err error) { ctx, done := stepSpan(ctx, "CallAteletPause") defer func() { err = done(err) }() @@ -192,15 +192,15 @@ func (w *ActorWorkflow) ensureAteletPaused(ctx context.Context, actorRef resourc SnapshotName: actor.GetStatus().GetInProgressLocalSnapshotName(), }, }, - Scope: actorSnapshotContentScopeToAtelet(actorTemplate.GetSnapshotConfig().GetOnCommit()), + Fidelity: fidelityToAtelet(actorTemplate.GetSnapshotConfig().GetPreferredFidelity()), ActorUid: actor.GetMetadata().Uid, } - wireSnapshotScope = ateattr.SnapshotScopeValue(req.Scope) + wireFidelity = ateattr.SnapshotFidelityValue(req.Fidelity) if _, err = client.Checkpoint(ctx, req); err != nil { - return wireSnapshotScope, handleAteletError(ctx, w.store, actorRef, ateattr.OperationPause, "Checkpoint", false, err) + return wireFidelity, handleAteletError(ctx, w.store, actorRef, ateattr.OperationPause, "Checkpoint", false, err) } - return wireSnapshotScope, nil + return wireFidelity, nil } // ensurePausedFinalized releases the actor's worker (only when it is still @@ -257,7 +257,7 @@ func (w *ActorWorkflow) ensurePausedFinalized(ctx context.Context, actorRef reso newState = ateapipb.ActorState_ACTOR_STATE_CRASHED crashStatus = newActorCrash(ateattr.OperationPause, crashMessageLocalSnapshotNodeUnknown) } - contentScope := actorTemplate.GetSnapshotConfig().GetOnCommit() + fidelity := actorTemplate.GetSnapshotConfig().GetPreferredFidelity() sandboxClass := "" if worker != nil { sandboxClass = worker.GetSandboxClass() @@ -275,7 +275,7 @@ func (w *ActorWorkflow) ensurePausedFinalized(ctx context.Context, actorRef reso if toUpdate.GetStatus().GetInProgressLocalSnapshotName() != "" { localSnapshot := &ateapipb.LocalSnapshot{ SnapshotName: toUpdate.GetStatus().GetInProgressLocalSnapshotName(), - ContentScope: contentScope, + Fidelity: fidelity, } toUpdate.Status.LocalSnapshot = localSnapshot toUpdate.Status.InProgressLocalSnapshotName = "" diff --git a/cmd/ateapi/internal/controlapi/workflow_pause_test.go b/cmd/ateapi/internal/controlapi/workflow_pause_test.go index 9866baf230..9f8ee00448 100644 --- a/cmd/ateapi/internal/controlapi/workflow_pause_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_pause_test.go @@ -144,18 +144,18 @@ func TestEnsurePausedFinalized_AlreadyCrashed(t *testing.T) { } } -// TestEnsurePausedFinalized_RecordsContentScope verifies pause finalization -// records the scope the pause checkpoint captured (the template's onCommit) +// TestEnsurePausedFinalized_RecordsFidelity verifies pause finalization +// records the scope the pause checkpoint captured (the template's preferredFidelity) // in LocalSnapshot, so a later suspend or resume of the PAUSED actor knows // what the local snapshot contains. -func TestEnsurePausedFinalized_RecordsContentScope(t *testing.T) { +func TestEnsurePausedFinalized_RecordsFidelity(t *testing.T) { tests := []struct { name string - onCommit ateapipb.SnapshotContentScope - want ateapipb.SnapshotContentScope + fidelity ateapipb.SnapshotFidelity + want ateapipb.SnapshotFidelity }{ - {"data", ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA, ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA}, - {"full", ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL}, + {"data", ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES}, + {"full", ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { @@ -198,7 +198,7 @@ func TestEnsurePausedFinalized_RecordsContentScope(t *testing.T) { w := &ActorWorkflow{store: st} tmpl := &ateapipb.ActorTemplate{ - SnapshotConfig: &ateapipb.SnapshotConfig{OnCommit: tc.onCommit}, + SnapshotConfig: &ateapipb.SnapshotConfig{PreferredFidelity: tc.fidelity}, } got, err := w.ensurePausedFinalized(ctx, actorRef, tmpl) if err != nil { @@ -208,8 +208,8 @@ func TestEnsurePausedFinalized_RecordsContentScope(t *testing.T) { if got.GetStatus().GetState() != ateapipb.ActorState_ACTOR_STATE_PAUSED { t.Fatalf("state = %v, want PAUSED", got.GetStatus().GetState()) } - if scope := got.GetStatus().GetLocalSnapshot().GetContentScope(); scope != tc.want { - t.Errorf("LocalSnapshot.ContentScope = %v, want %v", scope, tc.want) + if scope := got.GetStatus().GetLocalSnapshot().GetFidelity(); scope != tc.want { + t.Errorf("LocalSnapshot.Fidelity = %v, want %v", scope, tc.want) } if got.GetStatus().GetAssignedNode() != "node1" { t.Errorf("AssignedNode = %q, want %q", got.GetStatus().GetAssignedNode(), "node1") diff --git a/cmd/ateapi/internal/controlapi/workflow_resume.go b/cmd/ateapi/internal/controlapi/workflow_resume.go index ee12bd5af1..85223ec7d1 100644 --- a/cmd/ateapi/internal/controlapi/workflow_resume.go +++ b/cmd/ateapi/internal/controlapi/workflow_resume.go @@ -43,19 +43,19 @@ type resumeSnapshotSource struct { // Zero means cold boot from the spec (unless the actor holds a local // snapshot, which takes precedence at restore). SnapshotURI resources.SnapshotURI - Scope ateapipb.SnapshotContentScope + Fidelity ateapipb.SnapshotFidelity // TemplateReplaced is true when the external snapshot's recorded template // UID differs from the actor's current template. TemplateReplaced bool } // restoreTelemetry labels the restore operation for the resume lifecycle -// metric. WireSnapshotScope describes the restore requested, not the stored -// snapshot's scope: a full snapshot restored under a replaced template goes +// metric. WireFidelity describes the restore requested, not the stored +// snapshot's scope: a MEMORY snapshot restored under a replaced template goes // out as data. type restoreTelemetry struct { - SnapshotKind string - WireSnapshotScope string + SnapshotKind string + WireFidelity string } // ResumeActor executes the workflow to resume a suspended actor. Idempotent: @@ -77,7 +77,7 @@ func (w *ActorWorkflow) ResumeActor(ctx context.Context, actorRef resources.Acto return } w.instruments.recordLifecycleOp(ctx, ateattr.OperationResume, start, err, - lifecycleOpAttrs(actor, actorTemplate, tele.SnapshotKind, tele.WireSnapshotScope)...) + lifecycleOpAttrs(actor, actorTemplate, tele.SnapshotKind, tele.WireFidelity)...) }() // Routed requests call ResumeActor even when the actor is already running. @@ -132,22 +132,18 @@ func (w *ActorWorkflow) ResumeActor(ctx context.Context, actorRef resources.Acto return actor, true, nil } -// validateGoldenSnapshotScope rejects a golden snapshot that does not carry +// validateGoldenSnapshotFidelity rejects a golden snapshot that does not carry // the guest state (memory + fs delta) a restore needs. Golden actors always -// commit Full (commitSnapshotScope), so this only trips on golden snapshots -// taken before that rule existed — surface a clear error instead of shipping -// a restore request atelet would reject (or that would boot an empty guest). -func validateGoldenSnapshotScope(snapshot *ateapipb.ExternalSnapshot) error { - scope := snapshot.GetContentScope() - switch scope { - case ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_UNSPECIFIED, - ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL: - return nil - default: +// commit MEMORY (preferredFidelity), so this only trips on a golden snapshot +// whose record drifted — surface a clear error instead of shipping a restore +// request atelet would reject (or that would boot an empty guest). +func validateGoldenSnapshotFidelity(snapshot *ateapipb.ExternalSnapshot) error { + if fidelity := snapshot.GetFidelity(); fidelity != ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY { return apierror.FailedPrecondition( - "ActorTemplate golden snapshot %q was taken with scope %s, not Full; regenerate the golden snapshot", - snapshot.GetSnapshotUri(), scope) + "ActorTemplate golden snapshot %q was taken with fidelity %s, not MEMORY; regenerate the golden snapshot", + snapshot.GetSnapshotUri(), fidelity) } + return nil } // loadActorForResume fetches the current actor record and its template, and @@ -180,7 +176,7 @@ func (w *ActorWorkflow) loadActorForResume(ctx context.Context, actorRef resourc if src.SnapshotURI, err = resources.ParseSnapshotURI(uri); err != nil { return nil, nil, src, apierror.DataLoss("Actor %s external snapshot: %v", actorRef, err) } - src.Scope = actor.GetStatus().GetExternalSnapshot().GetContentScope() + src.Fidelity = actor.GetStatus().GetExternalSnapshot().GetFidelity() capturedUnder := actor.GetStatus().GetExternalSnapshot().GetActorTemplateUid() src.TemplateReplaced = capturedUnder != "" && capturedUnder != actorTemplate.GetMetadata().GetUid() } @@ -719,8 +715,8 @@ func (w *ActorWorkflow) ensureAteletRestored(ctx context.Context, actorRef resou req.Config = &ateletpb.RestoreRequest_LocalConfig{ LocalConfig: &ateletpb.LocalCheckpointConfiguration{SnapshotName: local.GetSnapshotName()}, } - req.Scope = actorSnapshotContentScopeToAtelet(local.GetContentScope()) - tele.WireSnapshotScope = ateattr.SnapshotScopeValue(req.Scope) + req.Fidelity = fidelityToAtelet(local.GetFidelity()) + tele.WireFidelity = ateattr.SnapshotFidelityValue(req.Fidelity) if _, err = client.Restore(ctx, req); err != nil { return tele, handleAteletError(ctx, w.store, actorRef, ateattr.OperationResume, "Restore", false, err) @@ -729,11 +725,11 @@ func (w *ActorWorkflow) ensureAteletRestored(ctx context.Context, actorRef resou } else if !src.SnapshotURI.IsZero() { slog.InfoContext(ctx, "Actor has durable snapshot; Restoring from snapshot") tele.SnapshotKind = ateattr.SnapshotKindLatest - scope := actorSnapshotContentScopeToAtelet(src.Scope) + scope := fidelityToAtelet(src.Fidelity) if src.TemplateReplaced { - scope = ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA + scope = ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES } - tele.WireSnapshotScope = ateattr.SnapshotScopeValue(scope) + tele.WireFidelity = ateattr.SnapshotFidelityValue(scope) req := &ateletpb.RestoreRequest{ TargetAteomUid: assignment.GetWorkerPodUid(), Atespace: actor.GetMetadata().GetAtespace(), @@ -747,7 +743,7 @@ func (w *ActorWorkflow) ensureAteletRestored(ctx context.Context, actorRef resou SnapshotUri: src.SnapshotURI.String(), }, }, - Scope: scope, + Fidelity: scope, SandboxAssets: sandboxAssets, ActorUid: actor.GetMetadata().Uid, EgressGateway: egressGateway, diff --git a/cmd/ateapi/internal/controlapi/workflow_resume_test.go b/cmd/ateapi/internal/controlapi/workflow_resume_test.go index e519d182f2..4faa4d0752 100644 --- a/cmd/ateapi/internal/controlapi/workflow_resume_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_resume_test.go @@ -971,7 +971,7 @@ func TestLoadActorForResume_TemplateReplaced(t *testing.T) { seedOpts = append(seedOpts, func(a *ateapipb.Actor) { a.Status.ExternalSnapshot = &ateapipb.ExternalSnapshot{ SnapshotUri: someActorSnapshotURI(t, testStorageLocation, actorRef.Atespace, "snap-1"), - ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, + Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, ActorTemplateUid: resolve(tt.snapshotUID), } }) @@ -1126,7 +1126,7 @@ func newWireCaptureWorkflow(t *testing.T, persistence store.Interface) (*ActorWo // TestResumeActor_AteletWireRequest is the characteristic test for the // loadActorForResume + ensureAteletRestored seam: for every combination of // boot-source inputs it pins the exact request atelet receives — which RPC, -// req.Scope, and the snapshot the config names — and that a source-resolution +// req.Fidelity, and the snapshot the config names — and that a source-resolution // error never produces an atelet RPC. // // The rows are ordered strictly by input columns (local → external → tmplUID → @@ -1138,9 +1138,9 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { actorURI := someActorSnapshotURI(t, testStorageLocation, "team-a", "snap-1") goldenURI := someActorSnapshotURI(t, "gs://bucket/golden-root", "ate-golden", "golden-1") - fullScope := ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL - dataScope := ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA - unspecScope := ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_UNSPECIFIED + fullScope := ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY + dataScope := ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES + unspecScope := ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED // actorSeed is the actor status a row persists before resuming. type actorSeed struct { @@ -1176,7 +1176,7 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { // a row also pins that the other config is absent. snapshotName string snapshotURI string - scope ateletpb.SnapshotScope + scope ateletpb.SnapshotFidelity } tests := []struct { @@ -1191,50 +1191,50 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { }, { name: "02 inherited golden snapshot restores in Full", - actor: actorSeed{externalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: goldenURI, ContentScope: fullScope}, tmplUID: "current"}, - tmpl: templateSeed{golden: &ateapipb.ExternalSnapshot{SnapshotUri: goldenURI, ContentScope: fullScope}}, + actor: actorSeed{externalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: goldenURI, Fidelity: fullScope}, tmplUID: "current"}, + tmpl: templateSeed{golden: &ateapipb.ExternalSnapshot{SnapshotUri: goldenURI, Fidelity: fullScope}}, want: restoreWant{ checkpointType: ateletpb.CheckpointType_CHECKPOINT_TYPE_EXTERNAL, snapshotURI: goldenURI, - scope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL, + scope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, }, }, { name: "03 late non-Full golden does not change a cold boot", - tmpl: templateSeed{golden: &ateapipb.ExternalSnapshot{SnapshotUri: goldenURI, ContentScope: dataScope}}, + tmpl: templateSeed{golden: &ateapipb.ExternalSnapshot{SnapshotUri: goldenURI, Fidelity: dataScope}}, want: restoreWant{run: true}, }, { name: "04 inherited golden snapshot rejects a malformed URI", - actor: actorSeed{externalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: malformedURI, ContentScope: fullScope}, tmplUID: "current"}, - tmpl: templateSeed{golden: &ateapipb.ExternalSnapshot{SnapshotUri: malformedURI, ContentScope: fullScope}}, + actor: actorSeed{externalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: malformedURI, Fidelity: fullScope}, tmplUID: "current"}, + tmpl: templateSeed{golden: &ateapipb.ExternalSnapshot{SnapshotUri: malformedURI, Fidelity: fullScope}}, want: restoreWant{code: codes.DataLoss}, }, { name: "05 template repoint with a late golden still cold-boots", actor: actorSeed{tmplUID: "old-template-uid"}, - tmpl: templateSeed{golden: &ateapipb.ExternalSnapshot{SnapshotUri: goldenURI, ContentScope: fullScope}}, + tmpl: templateSeed{golden: &ateapipb.ExternalSnapshot{SnapshotUri: goldenURI, Fidelity: fullScope}}, want: restoreWant{run: true}, }, { name: "06 Full durable snapshot restores itself in Full", - actor: actorSeed{externalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: actorURI, ContentScope: fullScope}}, + actor: actorSeed{externalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: actorURI, Fidelity: fullScope}}, want: restoreWant{ checkpointType: ateletpb.CheckpointType_CHECKPOINT_TYPE_EXTERNAL, snapshotURI: actorURI, - scope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL, + scope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, }, }, { name: "07 durable snapshot built on the current template stays Full", actor: actorSeed{ - externalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: actorURI, ContentScope: fullScope}, + externalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: actorURI, Fidelity: fullScope}, tmplUID: "current", }, want: restoreWant{ checkpointType: ateletpb.CheckpointType_CHECKPOINT_TYPE_EXTERNAL, snapshotURI: actorURI, - scope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL, + scope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, }, }, { @@ -1243,61 +1243,64 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { // boots fresh and only the volume data carries over. name: "08 repointed actor's Full durable snapshot drops to Data", actor: actorSeed{ - externalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: actorURI, ContentScope: fullScope}, + externalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: actorURI, Fidelity: fullScope}, tmplUID: "mismatch", }, want: restoreWant{ checkpointType: ateletpb.CheckpointType_CHECKPOINT_TYPE_EXTERNAL, snapshotURI: actorURI, - scope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA, + scope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, }, }, { name: "09 Data durable snapshot restores as Data", - actor: actorSeed{externalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: actorURI, ContentScope: dataScope}}, + actor: actorSeed{externalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: actorURI, Fidelity: dataScope}}, want: restoreWant{ checkpointType: ateletpb.CheckpointType_CHECKPOINT_TYPE_EXTERNAL, snapshotURI: actorURI, - scope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA, + scope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, }, }, { // The template's golden snapshot never supplies guest state for a // Data snapshot: it restores as plain Data. name: "10 Data durable snapshot ignores the template's golden", - actor: actorSeed{externalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: actorURI, ContentScope: dataScope}}, - tmpl: templateSeed{golden: &ateapipb.ExternalSnapshot{SnapshotUri: goldenURI, ContentScope: fullScope}}, + actor: actorSeed{externalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: actorURI, Fidelity: dataScope}}, + tmpl: templateSeed{golden: &ateapipb.ExternalSnapshot{SnapshotUri: goldenURI, Fidelity: fullScope}}, want: restoreWant{ checkpointType: ateletpb.CheckpointType_CHECKPOINT_TYPE_EXTERNAL, snapshotURI: actorURI, - scope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA, + scope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, }, }, { - // Snapshots recorded before content_scope existed carry + // Snapshots recorded before fidelity existed carry // UNSPECIFIED; the conversion sends them out as Full. - name: "11 unspecified durable scope goes out as Full", - actor: actorSeed{externalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: actorURI, ContentScope: unspecScope}}, + // A stored snapshot always records its fidelity; one without is a + // drifted record, and ateapi forwards it unchanged for atelet's + // validation to reject rather than guessing MEMORY. + name: "11 unspecified durable fidelity goes out unspecified", + actor: actorSeed{externalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: actorURI, Fidelity: unspecScope}}, want: restoreWant{ checkpointType: ateletpb.CheckpointType_CHECKPOINT_TYPE_EXTERNAL, snapshotURI: actorURI, - scope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL, + scope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED, }, }, { name: "12 malformed durable snapshot URI fails with DataLoss", - actor: actorSeed{externalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: malformedURI, ContentScope: fullScope}}, + actor: actorSeed{externalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: malformedURI, Fidelity: fullScope}}, want: restoreWant{code: codes.DataLoss}, }, { name: "13 Full pause snapshot restores locally as Full", actor: actorSeed{ - localSnapshot: &ateapipb.LocalSnapshot{SnapshotName: localSnapshotName, ContentScope: fullScope}, + localSnapshot: &ateapipb.LocalSnapshot{SnapshotName: localSnapshotName, Fidelity: fullScope}, }, want: restoreWant{ checkpointType: ateletpb.CheckpointType_CHECKPOINT_TYPE_LOCAL, snapshotName: localSnapshotName, - scope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL, + scope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, }, }, { @@ -1306,38 +1309,38 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { // resume-source resolution is being reworked. name: "14 local snapshot built on the current template stays Full", actor: actorSeed{ - localSnapshot: &ateapipb.LocalSnapshot{SnapshotName: localSnapshotName, ContentScope: fullScope}, + localSnapshot: &ateapipb.LocalSnapshot{SnapshotName: localSnapshotName, Fidelity: fullScope}, tmplUID: "current", }, want: restoreWant{ checkpointType: ateletpb.CheckpointType_CHECKPOINT_TYPE_LOCAL, snapshotName: localSnapshotName, - scope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL, + scope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, }, }, { name: "15 Data pause snapshot restores locally as Data", actor: actorSeed{ - localSnapshot: &ateapipb.LocalSnapshot{SnapshotName: localSnapshotName, ContentScope: dataScope}, + localSnapshot: &ateapipb.LocalSnapshot{SnapshotName: localSnapshotName, Fidelity: dataScope}, }, want: restoreWant{ checkpointType: ateletpb.CheckpointType_CHECKPOINT_TYPE_LOCAL, snapshotName: localSnapshotName, - scope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA, + scope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, }, }, { name: "16 Data pause snapshot ignores the template's golden", actor: actorSeed{ - localSnapshot: &ateapipb.LocalSnapshot{SnapshotName: localSnapshotName, ContentScope: dataScope}, + localSnapshot: &ateapipb.LocalSnapshot{SnapshotName: localSnapshotName, Fidelity: dataScope}, }, tmpl: templateSeed{ - golden: &ateapipb.ExternalSnapshot{SnapshotUri: goldenURI, ContentScope: fullScope}, + golden: &ateapipb.ExternalSnapshot{SnapshotUri: goldenURI, Fidelity: fullScope}, }, want: restoreWant{ checkpointType: ateletpb.CheckpointType_CHECKPOINT_TYPE_LOCAL, snapshotName: localSnapshotName, - scope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA, + scope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, }, }, { @@ -1345,13 +1348,13 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { // scope wins over the durable snapshot's. name: "17 local snapshot wins over a Full durable snapshot", actor: actorSeed{ - localSnapshot: &ateapipb.LocalSnapshot{SnapshotName: localSnapshotName, ContentScope: dataScope}, - externalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: actorURI, ContentScope: fullScope}, + localSnapshot: &ateapipb.LocalSnapshot{SnapshotName: localSnapshotName, Fidelity: dataScope}, + externalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: actorURI, Fidelity: fullScope}, }, want: restoreWant{ checkpointType: ateletpb.CheckpointType_CHECKPOINT_TYPE_LOCAL, snapshotName: localSnapshotName, - scope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA, + scope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, }, }, { @@ -1361,21 +1364,21 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { // template. name: "18 local snapshot ignores an older external snapshot's template mismatch", actor: actorSeed{ - localSnapshot: &ateapipb.LocalSnapshot{SnapshotName: localSnapshotName, ContentScope: fullScope}, - externalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: actorURI, ContentScope: fullScope}, + localSnapshot: &ateapipb.LocalSnapshot{SnapshotName: localSnapshotName, Fidelity: fullScope}, + externalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: actorURI, Fidelity: fullScope}, tmplUID: "mismatch", }, want: restoreWant{ checkpointType: ateletpb.CheckpointType_CHECKPOINT_TYPE_LOCAL, snapshotName: localSnapshotName, - scope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL, + scope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, }, }, { // Restores take their sandbox from the template, like cold boots, // so an unresolvable SandboxConfig stops them before atelet. name: "19 durable snapshot restore with a missing SandboxConfig is rejected", - actor: actorSeed{externalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: actorURI, ContentScope: fullScope}}, + actor: actorSeed{externalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: actorURI, Fidelity: fullScope}}, tmpl: templateSeed{configName: "missing"}, want: restoreWant{code: codes.FailedPrecondition}, }, @@ -1504,7 +1507,7 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { if got := restore.GetExternalConfig().GetSnapshotUri(); got != tt.want.snapshotURI { t.Errorf("ExternalConfig.SnapshotUri = %q, want %q", got, tt.want.snapshotURI) } - if got := restore.GetScope(); got != tt.want.scope { + if got := restore.GetFidelity(); got != tt.want.scope { t.Errorf("restore scope = %v, want %v", got, tt.want.scope) } }) diff --git a/cmd/ateapi/internal/controlapi/workflow_suspend.go b/cmd/ateapi/internal/controlapi/workflow_suspend.go index a0892d40a1..1caa7d8270 100644 --- a/cmd/ateapi/internal/controlapi/workflow_suspend.go +++ b/cmd/ateapi/internal/controlapi/workflow_suspend.go @@ -40,7 +40,7 @@ func (w *ActorWorkflow) SuspendActor(ctx context.Context, actorRef resources.Act start := time.Now() var actor *ateapipb.Actor var actorTemplate *ateapipb.ActorTemplate - var wireSnapshotScope string + var wireFidelity string // Set just before finalize; nil until then, so earlier exits label // themselves from the record they hold. var finalAttrs []attribute.KeyValue @@ -48,7 +48,7 @@ func (w *ActorWorkflow) SuspendActor(ctx context.Context, actorRef resources.Act defer func() { attrs := finalAttrs if attrs == nil { - attrs = lifecycleOpAttrs(actor, actorTemplate, "", wireSnapshotScope) + attrs = lifecycleOpAttrs(actor, actorTemplate, "", wireFidelity) } w.instruments.recordLifecycleOp(ctx, ateattr.OperationSuspend, start, err, attrs...) }() @@ -79,9 +79,9 @@ func (w *ActorWorkflow) SuspendActor(ctx context.Context, actorRef resources.Act } actor = marked if fromPaused { - wireSnapshotScope, err = w.ensurePausedSnapshotUploaded(leaseCtx, actorRef, actor, actorTemplate) + wireFidelity, err = w.ensurePausedSnapshotUploaded(leaseCtx, actorRef, actor, actorTemplate) } else { - wireSnapshotScope, err = w.ensureAteletSuspended(leaseCtx, actorRef, actor, actorTemplate) + wireFidelity, err = w.ensureAteletSuspended(leaseCtx, actorRef, actor, actorTemplate) } if err != nil { return nil, err @@ -91,7 +91,7 @@ func (w *ActorWorkflow) SuspendActor(ctx context.Context, actorRef resources.Act } // FinalizeSuspended clears the WorkerAssignment the labels read, so snapshot // them here, as crash.go does for the crash counter. - finalAttrs = lifecycleOpAttrs(actor, actorTemplate, "", wireSnapshotScope) + finalAttrs = lifecycleOpAttrs(actor, actorTemplate, "", wireFidelity) var finalized *ateapipb.Actor if finalized, err = w.ensureSuspendedFinalized(leaseCtx, actorRef, actorTemplate); err != nil { return nil, err @@ -153,15 +153,15 @@ func (w *ActorWorkflow) ensureMarkedSuspending(ctx context.Context, actorRef res return storedActor, nil } -// commitSnapshotScope returns the scope a commit (suspend) snapshot is taken -// with. Golden actors always commit Full regardless of the template's -// onCommit: new actors borrow the golden snapshot and resume it Full, so it -// must carry the guest memory and filesystem. -func commitSnapshotScope(atespace string, tmpl *ateapipb.ActorTemplate) ateapipb.SnapshotContentScope { +// preferredFidelity returns the fidelity a suspend snapshot is taken with. +// Golden actors always commit MEMORY regardless of the template's +// preferredFidelity: new actors borrow the golden snapshot and resume it +// with memory, so it must carry the guest memory and filesystem. +func preferredFidelity(atespace string, tmpl *ateapipb.ActorTemplate) ateapipb.SnapshotFidelity { if atespace == resources.GoldenActorAtespace { - return ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL + return ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY } - return tmpl.GetSnapshotConfig().GetOnCommit() + return tmpl.GetSnapshotConfig().GetPreferredFidelity() } // isPausedOriginSuspend reports whether the suspend must upload a PAUSED @@ -183,7 +183,7 @@ func isPausedOriginSuspend(actor *ateapipb.Actor) bool { // once-minted snapshot location, so a re-entered workflow re-sends the same // semantic request; once atelet's Checkpoint is idempotent on those keys this // step becomes fully reentrant with no changes here. -func (w *ActorWorkflow) ensureAteletSuspended(ctx context.Context, actorRef resources.ActorRef, actor *ateapipb.Actor, actorTemplate *ateapipb.ActorTemplate) (wireSnapshotScope string, err error) { +func (w *ActorWorkflow) ensureAteletSuspended(ctx context.Context, actorRef resources.ActorRef, actor *ateapipb.Actor, actorTemplate *ateapipb.ActorTemplate) (wireFidelity string, err error) { ctx, done := stepSpan(ctx, "CallAteletSuspend") defer func() { err = done(err) }() @@ -223,15 +223,15 @@ func (w *ActorWorkflow) ensureAteletSuspended(ctx context.Context, actorRef reso SnapshotUri: actor.GetStatus().GetInProgressSnapshotUri(), }, }, - Scope: actorSnapshotContentScopeToAtelet(commitSnapshotScope(actor.GetMetadata().GetAtespace(), actorTemplate)), + Fidelity: fidelityToAtelet(preferredFidelity(actor.GetMetadata().GetAtespace(), actorTemplate)), ActorUid: actor.GetMetadata().Uid, } - wireSnapshotScope = ateattr.SnapshotScopeValue(req.Scope) + wireFidelity = ateattr.SnapshotFidelityValue(req.Fidelity) if _, err = client.Checkpoint(ctx, req); err != nil { - return wireSnapshotScope, handleAteletError(ctx, w.store, actorRef, ateattr.OperationSuspend, "Checkpoint", false, err) + return wireFidelity, handleAteletError(ctx, w.store, actorRef, ateattr.OperationSuspend, "Checkpoint", false, err) } - return wireSnapshotScope, nil + return wireFidelity, nil } // ensurePausedSnapshotUploaded suspends a PAUSED actor by telling the atelet @@ -240,7 +240,7 @@ func (w *ActorWorkflow) ensureAteletSuspended(ctx context.Context, actorRef reso // ateom to checkpoint. Retries re-send the same semantic request: the // destination is minted once and the upload overwrites deterministic object // names, with the remote manifest as the commit marker. -func (w *ActorWorkflow) ensurePausedSnapshotUploaded(ctx context.Context, actorRef resources.ActorRef, actor *ateapipb.Actor, actorTemplate *ateapipb.ActorTemplate) (wireSnapshotScope string, err error) { +func (w *ActorWorkflow) ensurePausedSnapshotUploaded(ctx context.Context, actorRef resources.ActorRef, actor *ateapipb.Actor, actorTemplate *ateapipb.ActorTemplate) (wireFidelity string, err error) { ctx, done := stepSpan(ctx, "UploadPausedCheckpoint") defer func() { err = done(err) }() @@ -274,14 +274,14 @@ func (w *ActorWorkflow) ensurePausedSnapshotUploaded(ctx context.Context, actorR DestinationSnapshotUri: actor.GetStatus().GetInProgressSnapshotUri(), // The commit scope, like a running-origin suspend; atelet converts // from the captured scope in the snapshot's manifest where possible. - DesiredScope: actorSnapshotContentScopeToAtelet(commitSnapshotScope(actor.GetMetadata().GetAtespace(), actorTemplate)), + DesiredFidelity: fidelityToAtelet(preferredFidelity(actor.GetMetadata().GetAtespace(), actorTemplate)), } - wireSnapshotScope = ateattr.SnapshotScopeValue(req.DesiredScope) + wireFidelity = ateattr.SnapshotFidelityValue(req.DesiredFidelity) if _, err = client.UploadPausedCheckpoint(ctx, req); err != nil { - return wireSnapshotScope, handleAteletError(ctx, w.store, actorRef, ateattr.OperationSuspend, "UploadPausedCheckpoint", false, err) + return wireFidelity, handleAteletError(ctx, w.store, actorRef, ateattr.OperationSuspend, "UploadPausedCheckpoint", false, err) } - return wireSnapshotScope, nil + return wireFidelity, nil } // newInProgressSnapshotURI is where the snapshot an actor is currently taking is @@ -368,7 +368,7 @@ func (w *ActorWorkflow) ensureSuspendedFinalized(ctx context.Context, actorRef r if inProgressSnapshotURI != "" { externalSnapshot = &ateapipb.ExternalSnapshot{ SnapshotUri: inProgressSnapshotURI, - ContentScope: commitSnapshotScope(actorRef.Atespace, actorTemplate), + Fidelity: preferredFidelity(actorRef.Atespace, actorTemplate), ActorTemplateUid: actorTemplate.GetMetadata().GetUid(), } } diff --git a/cmd/ateapi/internal/controlapi/workflow_suspend_test.go b/cmd/ateapi/internal/controlapi/workflow_suspend_test.go index 4ee20fdad1..2c651628c1 100644 --- a/cmd/ateapi/internal/controlapi/workflow_suspend_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_suspend_test.go @@ -593,32 +593,32 @@ func TestEnsureSuspendedFinalized_ReleasesOnlyOwnWorker(t *testing.T) { } } -// TestCommitSnapshotScope verifies golden actors always commit Full — new -// actors resume the golden snapshot Full, so the template's onCommit must not +// TestPreferredFidelity verifies golden actors always commit Full — new +// actors resume the golden snapshot Full, so the template's preferredFidelity must not // thin it down to a data-only capture. -func TestCommitSnapshotScope(t *testing.T) { - tmpl := func(onCommit ateapipb.SnapshotContentScope) *ateapipb.ActorTemplate { +func TestPreferredFidelity(t *testing.T) { + tmpl := func(fidelity ateapipb.SnapshotFidelity) *ateapipb.ActorTemplate { return &ateapipb.ActorTemplate{ - SnapshotConfig: &ateapipb.SnapshotConfig{OnCommit: onCommit}, + SnapshotConfig: &ateapipb.SnapshotConfig{PreferredFidelity: fidelity}, } } - fullScope := ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL - dataScope := ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA + fullScope := ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY + dataScope := ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES tests := []struct { name string atespace string - onCommit ateapipb.SnapshotContentScope - want ateapipb.SnapshotContentScope + fidelity ateapipb.SnapshotFidelity + want ateapipb.SnapshotFidelity }{ - {"golden actor ignores Data onCommit", resources.GoldenActorAtespace, dataScope, fullScope}, - {"golden actor keeps Full onCommit", resources.GoldenActorAtespace, fullScope, fullScope}, - {"regular actor uses Data onCommit", "team-a", dataScope, dataScope}, - {"regular actor uses Full onCommit", "team-a", fullScope, fullScope}, + {"golden actor ignores Data fidelity", resources.GoldenActorAtespace, dataScope, fullScope}, + {"golden actor keeps Full fidelity", resources.GoldenActorAtespace, fullScope, fullScope}, + {"regular actor uses Data fidelity", "team-a", dataScope, dataScope}, + {"regular actor uses Full fidelity", "team-a", fullScope, fullScope}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - if got := commitSnapshotScope(tc.atespace, tmpl(tc.onCommit)); got != tc.want { - t.Errorf("commitSnapshotScope(%q, onCommit=%s) = %s, want %s", tc.atespace, tc.onCommit, got, tc.want) + if got := preferredFidelity(tc.atespace, tmpl(tc.fidelity)); got != tc.want { + t.Errorf("preferredFidelity(%q, fidelity=%s) = %s, want %s", tc.atespace, tc.fidelity, got, tc.want) } }) } diff --git a/cmd/ateapi/internal/controlapi/workflow_tag.go b/cmd/ateapi/internal/controlapi/workflow_tag.go index 4f1d296d6f..c6a0ce6453 100644 --- a/cmd/ateapi/internal/controlapi/workflow_tag.go +++ b/cmd/ateapi/internal/controlapi/workflow_tag.go @@ -297,8 +297,8 @@ func (w *ActorWorkflow) ensureTagFinalized(ctx context.Context, tag *ateapipb.Ta tagRef := resources.TagRefFromTag(tag) // The copy is byte-identical to the source, so it carries the same content. finalSnapshot := &ateapipb.ExternalSnapshot{ - SnapshotUri: dst.String(), - ContentScope: snapshot.GetContentScope(), + SnapshotUri: dst.String(), + Fidelity: snapshot.GetFidelity(), } stored, err := w.store.UpdateTag(ctx, tagRef, store.PreconditionFrom(tag), func(toUpdate *ateapipb.Tag) error { toUpdate.Status.Snapshot = finalSnapshot diff --git a/cmd/ateapi/internal/controlapi/workflow_tag_test.go b/cmd/ateapi/internal/controlapi/workflow_tag_test.go index 6a8c6ca03d..e8536fb8fa 100644 --- a/cmd/ateapi/internal/controlapi/workflow_tag_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_tag_test.go @@ -46,7 +46,7 @@ func seedTagSource(t *testing.T, ctx context.Context, persistence store.Interfac actor = mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapipb.ActorStatus) { s.ExternalSnapshot = &ateapipb.ExternalSnapshot{ SnapshotUri: uri.String(), - ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, + Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, ActorTemplateUid: template.GetMetadata().GetUid(), } }) @@ -100,7 +100,7 @@ func TestTagActorSnapshot(t *testing.T) { if got, want := tag.GetStatus().GetActorTemplateUid(), template.GetMetadata().GetUid(); got != want { t.Errorf("actor template uid = %q, want %q", got, want) } - if got, want := tag.GetStatus().GetSnapshot().GetContentScope(), ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL; got != want { + if got, want := tag.GetStatus().GetSnapshot().GetFidelity(), ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY; got != want { t.Errorf("content scope = %v, want the source's %v", got, want) } diff --git a/cmd/ateapi/internal/defaults/defaults.go b/cmd/ateapi/internal/defaults/defaults.go index 3b4cfde90e..76c8bb2c7b 100644 --- a/cmd/ateapi/internal/defaults/defaults.go +++ b/cmd/ateapi/internal/defaults/defaults.go @@ -61,8 +61,8 @@ func applySnapshotConfigDefaults(sc *ateapipb.SnapshotConfig) { if sc == nil { return } - if sc.OnCommit == ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_UNSPECIFIED { - sc.OnCommit = ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL + if sc.PreferredFidelity == ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED { + sc.PreferredFidelity = ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY } } diff --git a/cmd/ateapi/internal/defaults/defaults_test.go b/cmd/ateapi/internal/defaults/defaults_test.go index 33ce8d1f9c..ca8082713b 100644 --- a/cmd/ateapi/internal/defaults/defaults_test.go +++ b/cmd/ateapi/internal/defaults/defaults_test.go @@ -25,8 +25,8 @@ import ( func TestApply(t *testing.T) { const ( - scopeFull = ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL - scopeData = ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA + scopeFull = ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY + scopeData = ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES ) tests := []struct { name string @@ -44,15 +44,15 @@ func TestApply(t *testing.T) { name: "empty snapshot_config gets every default", in: &ateapipb.ActorTemplate{SnapshotConfig: &ateapipb.SnapshotConfig{}}, want: &ateapipb.ActorTemplate{SnapshotConfig: &ateapipb.SnapshotConfig{ - OnCommit: scopeFull, + PreferredFidelity: scopeFull, }}, }, { name: "set scopes are kept", in: &ateapipb.ActorTemplate{SnapshotConfig: &ateapipb.SnapshotConfig{ - OnCommit: scopeData, + PreferredFidelity: scopeData, }}, want: &ateapipb.ActorTemplate{SnapshotConfig: &ateapipb.SnapshotConfig{ - OnCommit: scopeData, + PreferredFidelity: scopeData, }}, }, { name: "container without wakeup probe stays without one", diff --git a/cmd/ateapi/internal/store/atepg/actor_template_test.go b/cmd/ateapi/internal/store/atepg/actor_template_test.go index 0254d38bd7..0e04e5b6c3 100644 --- a/cmd/ateapi/internal/store/atepg/actor_template_test.go +++ b/cmd/ateapi/internal/store/atepg/actor_template_test.go @@ -97,7 +97,7 @@ func TestGetActorTemplate_BackfillsDefaults(t *testing.T) { t.Fatalf("GetActorTemplate failed: %v", err) } want := &ateapipb.SnapshotConfig{ - OnCommit: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, + PreferredFidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, } if diff := cmp.Diff(want, got.GetSnapshotConfig(), protocmp.Transform()); diff != "" { t.Errorf("GetActorTemplate did not backfill snapshot_config defaults (-want +got):\n%s", diff) diff --git a/cmd/ateapi/internal/store/storecontract/contract.go b/cmd/ateapi/internal/store/storecontract/contract.go index 5a82a3b411..c597100393 100644 --- a/cmd/ateapi/internal/store/storecontract/contract.go +++ b/cmd/ateapi/internal/store/storecontract/contract.go @@ -1146,8 +1146,8 @@ func newTestSuspendedActor(atespace, name string) *ateapipb.Actor { Status: &ateapipb.ActorStatus{ State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, ExternalSnapshot: &ateapipb.ExternalSnapshot{ - SnapshotUri: testActorSnapshotURI("gs://private", atespace, name), - ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, + SnapshotUri: testActorSnapshotURI("gs://private", atespace, name), + Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, }, }, } @@ -1338,7 +1338,7 @@ func runTagContractTests(t *testing.T, setup func(t *testing.T) store.Interface) { name: "snapshot content scope", mutate: func(toUpdate *ateapipb.Tag) { - toUpdate.Status.Snapshot.ContentScope = ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA + toUpdate.Status.Snapshot.Fidelity = ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES }, }, { @@ -1497,8 +1497,8 @@ func runTagContractTests(t *testing.T, setup func(t *testing.T) store.Interface) // transaction does: it names the copy that landed under the tag UID. func finalizeTag(toUpdate *ateapipb.Tag) error { toUpdate.Status.Snapshot = &ateapipb.ExternalSnapshot{ - SnapshotUri: testTagSnapshotURI(toUpdate.GetStatus().GetStorageLocation(), toUpdate.GetMetadata().GetAtespace(), toUpdate.GetMetadata().GetUid()), - ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, + SnapshotUri: testTagSnapshotURI(toUpdate.GetStatus().GetStorageLocation(), toUpdate.GetMetadata().GetAtespace(), toUpdate.GetMetadata().GetUid()), + Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, } return nil } diff --git a/cmd/atelet/internal/ateletpath/ateletpath.go b/cmd/atelet/internal/ateletpath/ateletpath.go index f7590e81d6..aa80821d7d 100644 --- a/cmd/atelet/internal/ateletpath/ateletpath.go +++ b/cmd/atelet/internal/ateletpath/ateletpath.go @@ -129,8 +129,8 @@ func DurableDirVolumeMountPoint(actorUID, volumeName string) string { // DurableDirVolumeMountsDir (see ateom-microvm's tarDurableVolumes), so // system-info roots are excluded by living in this separate directory. // - gVisor captures by declaration: durable mounts are registered with -// the sandbox (mount-hint annotations for FULL checkpoints, the -// enumerated durable mount paths for DATA fscheckpoints); system-info +// the sandbox (mount-hint annotations for MEMORY checkpoints, the +// enumerated durable mount paths for VOLUMES fscheckpoints); system-info // mounts are plain undeclared binds, never captured regardless of host // layout. // diff --git a/cmd/atelet/lifecycle_test.go b/cmd/atelet/lifecycle_test.go index b32db7e605..b80a54d6ae 100644 --- a/cmd/atelet/lifecycle_test.go +++ b/cmd/atelet/lifecycle_test.go @@ -207,7 +207,7 @@ func TestLocalSnapshotGC(t *testing.T) { ActorTemplateName: "counter", TargetAteomUid: ateomUID, Spec: spec, - Scope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL, + Fidelity: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, Type: ateletpb.CheckpointType_CHECKPOINT_TYPE_LOCAL, Config: &ateletpb.CheckpointRequest_LocalConfig{ LocalConfig: &ateletpb.LocalCheckpointConfiguration{SnapshotName: snapshotName}, @@ -230,7 +230,7 @@ func TestLocalSnapshotGC(t *testing.T) { TargetAteomUid: ateomUID, SandboxAssets: sandboxAssets, Spec: spec, - Scope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL, + Fidelity: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, Type: ateletpb.CheckpointType_CHECKPOINT_TYPE_LOCAL, Config: &ateletpb.RestoreRequest_LocalConfig{ LocalConfig: &ateletpb.LocalCheckpointConfiguration{SnapshotName: snapshotName}, @@ -366,7 +366,7 @@ func TestRestoreUsesRequestSandboxAssets(t *testing.T) { ActorTemplateName: "counter", TargetAteomUid: ateomUID, Spec: spec, - Scope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL, + Fidelity: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, Type: ateletpb.CheckpointType_CHECKPOINT_TYPE_LOCAL, Config: &ateletpb.CheckpointRequest_LocalConfig{ LocalConfig: &ateletpb.LocalCheckpointConfiguration{SnapshotName: snapshotName}, @@ -396,7 +396,7 @@ func TestRestoreUsesRequestSandboxAssets(t *testing.T) { TargetAteomUid: ateomUID, SandboxAssets: assetsWithPause(restorePause), Spec: spec, - Scope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL, + Fidelity: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, Type: ateletpb.CheckpointType_CHECKPOINT_TYPE_LOCAL, Config: &ateletpb.RestoreRequest_LocalConfig{ LocalConfig: &ateletpb.LocalCheckpointConfiguration{SnapshotName: snapshotName}, @@ -492,7 +492,7 @@ func TestActivationFailureBeforeRegistration(t *testing.T) { _, err = s.Restore(ctx, &ateletpb.RestoreRequest{ Atespace: "team-a", ActorName: "actor-1", ActorUid: actorUID, TargetAteomUid: "ateom-uid-1", SandboxAssets: assets, Spec: spec, - Scope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL, Type: ateletpb.CheckpointType_CHECKPOINT_TYPE_LOCAL, + Fidelity: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, Type: ateletpb.CheckpointType_CHECKPOINT_TYPE_LOCAL, Config: &ateletpb.RestoreRequest_LocalConfig{LocalConfig: &ateletpb.LocalCheckpointConfiguration{SnapshotName: snapshotName}}, }) } else { @@ -623,9 +623,9 @@ func TestRestoreFailureAfterRegistrationRemovesOwnRegistration(t *testing.T) { runscAssetName: {Url: "gs://test-bucket/runsc", Sha256: assetHash}, }}}, }, - Spec: spec, - Scope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL, - Type: ateletpb.CheckpointType_CHECKPOINT_TYPE_LOCAL, + Spec: spec, + Fidelity: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + Type: ateletpb.CheckpointType_CHECKPOINT_TYPE_LOCAL, Config: &ateletpb.RestoreRequest_LocalConfig{ LocalConfig: &ateletpb.LocalCheckpointConfiguration{SnapshotName: snapshotName}, }, diff --git a/cmd/atelet/main.go b/cmd/atelet/main.go index e9945698e9..461acd247f 100644 --- a/cmd/atelet/main.go +++ b/cmd/atelet/main.go @@ -643,7 +643,7 @@ func (s *AteomHerder) Checkpoint(ctx context.Context, req *ateletpb.CheckpointRe templateNamespace: req.GetActorTemplateAtespace(), templateName: req.GetActorTemplateName(), kind: checkpointSnapshotKind(req), - scope: ateattr.SnapshotScopeValue(req.GetScope()), + fidelity: ateattr.SnapshotFidelityValue(req.GetFidelity()), } attribution := resources.ActorAttribution{ Ref: actorRef, @@ -707,7 +707,7 @@ func (s *AteomHerder) Checkpoint(ctx context.Context, req *ateletpb.CheckpointRe RunscPath: runscPathFor(assetPaths), RuntimeAssetPaths: assetPaths, Spec: spec, - Scope: toAteomSnapshotScope(req.GetScope()), + Fidelity: toAteomFidelity(req.GetFidelity()), ActorUid: actorUID, ActorDirs: ateletpath.ActorDirs(actorUID), }) @@ -729,7 +729,7 @@ func (s *AteomHerder) Checkpoint(ctx context.Context, req *ateletpb.CheckpointRe sandboxRec.ActorUID = req.GetActorUid() sandboxRec.ActorTemplateAtespace = req.GetActorTemplateAtespace() sandboxRec.ActorTemplateName = req.GetActorTemplateName() - sandboxRec.Scope = ateattr.SnapshotScopeValue(req.GetScope()) + sandboxRec.Fidelity = ateattr.SnapshotFidelityValue(req.GetFidelity()) // No earlier pause snapshot can ever be restored again, so remove them // all: the actor's current state was just captured by CheckpointWorkload, @@ -795,13 +795,19 @@ func checkpointSnapshotFiles(resp *ateompb.CheckpointWorkloadResponse, required return files, dataFiles, nil } -func toAteomSnapshotScope(scope ateletpb.SnapshotScope) ateompb.SnapshotScope { - // assumption the request already been validated and scope is in the valid values set - switch scope { - case ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA: - return ateompb.SnapshotScope_SNAPSHOT_SCOPE_DATA +// toAteomFidelity maps the atelet enum onto the identical ateom one, level by +// level; the request was validated already, so anything else is a bug and +// goes out as UNSPECIFIED for ateom to reject. +func toAteomFidelity(fidelity ateletpb.SnapshotFidelity) ateompb.SnapshotFidelity { + switch fidelity { + case ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES: + return ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES + case ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS: + return ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS + case ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY: + return ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY default: - return ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL + return ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED } } @@ -857,7 +863,7 @@ func (s *AteomHerder) moveLocalCheckpoint(ctx context.Context, req *ateletpb.Che // shouldHaveSnapshots returns true if the checkpoint request is expected to produce snapshot files. func shouldHaveSnapshots(req *ateletpb.CheckpointRequest) bool { - if req.GetScope() != ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA { + if req.GetFidelity() != ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES { return true } @@ -927,8 +933,8 @@ func (s *AteomHerder) UploadPausedCheckpoint(ctx context.Context, req *ateletpb. templateName: req.GetActorTemplateName(), // Always the actor's durable latest: golden actors are never paused // (validation above rejects the golden atespace). - kind: ateattr.SnapshotKindLatest, - scope: ateattr.SnapshotScopeValue(req.GetDesiredScope()), + kind: ateattr.SnapshotKindLatest, + fidelity: ateattr.SnapshotFidelityValue(req.GetDesiredFidelity()), } defer func() { s.instruments.recordCheckpoint(ctx, op, @@ -991,20 +997,20 @@ func (s *AteomHerder) uploadLocalCheckpointDir(ctx context.Context, req *ateletp return "", err } - capturedScope := rec.Scope - if capturedScope == "" { - return rec.SandboxClass, apierror.FailedPrecondition("local snapshot %q has no scope recorded in its manifest (written by an older atelet); resume and pause the actor again before suspending it", req.GetLocalSnapshotName()) + capturedFidelity := rec.Fidelity + if capturedFidelity == "" { + return rec.SandboxClass, apierror.FailedPrecondition("local snapshot %q has no fidelity recorded in its manifest; resume and pause the actor again before suspending it", req.GetLocalSnapshotName()) } - desiredScope := ateattr.SnapshotScopeValue(req.GetDesiredScope()) + desiredFidelity := ateattr.SnapshotFidelityValue(req.GetDesiredFidelity()) switch { - case capturedScope == desiredScope: - case capturedScope == ateattr.SnapshotScopeData && desiredScope == ateattr.SnapshotScopeFull: + case capturedFidelity == desiredFidelity: + case capturedFidelity == ateattr.SnapshotFidelityVolumes && desiredFidelity == ateattr.SnapshotFidelityMemory: // The control plane rejects this before marking SUSPENDING; reaching // it here means the template changed mid-flight or store state drifted. - return rec.SandboxClass, apierror.FailedPrecondition("pause snapshot captured %s; cannot upload it as %s (memory was never captured)", capturedScope, desiredScope) - default: // captured FULL, DATA wanted - if err := narrowFullCaptureToData(rec); err != nil { + return rec.SandboxClass, apierror.FailedPrecondition("pause snapshot captured %s; cannot upload it as %s (memory was never captured)", capturedFidelity, desiredFidelity) + default: // captured MEMORY, VOLUMES wanted + if err := narrowMemoryCaptureToVolumes(rec); err != nil { return rec.SandboxClass, err } } @@ -1021,16 +1027,16 @@ func readSnapshotManifest(dir string) ([]byte, error) { return root.ReadFile(sandboxManifestName) } -// narrowFullCaptureToData rewrites rec so a FULL capture uploads as a DATA -// snapshot holding only the data-scope files ateom reported at checkpoint. -func narrowFullCaptureToData(rec *sandboxAssetsRecord) error { +// narrowMemoryCaptureToVolumes rewrites rec so a MEMORY capture uploads as a +// VOLUMES snapshot holding only the volume files ateom reported at checkpoint. +func narrowMemoryCaptureToVolumes(rec *sandboxAssetsRecord) error { if len(rec.DataSnapshotFiles) == 0 { // Either no durable-dir volumes were attached at pause, or the // manifest predates the list. Neither is retryable. - return apierror.FailedPrecondition("full capture lists no data-scope files; the actor has no durable data to upload as %s", ateattr.SnapshotScopeData) + return apierror.FailedPrecondition("memory capture lists no volume files; the actor has no durable data to upload as %s", ateattr.SnapshotFidelityVolumes) } rec.SnapshotFiles = rec.DataSnapshotFiles - rec.Scope = ateattr.SnapshotScopeData + rec.Fidelity = ateattr.SnapshotFidelityVolumes return nil } @@ -1061,7 +1067,7 @@ func (s *AteomHerder) Restore(ctx context.Context, req *ateletpb.RestoreRequest) op := snapshotOp{ templateNamespace: req.GetActorTemplateAtespace(), templateName: req.GetActorTemplateName(), - scope: ateattr.SnapshotScopeValue(req.GetScope()), + fidelity: ateattr.SnapshotFidelityValue(req.GetFidelity()), sandboxClass: req.GetSandboxAssets().GetSandboxClass(), } attribution := resources.ActorAttribution{ @@ -1245,7 +1251,7 @@ func (s *AteomHerder) Restore(ctx context.Context, req *ateletpb.RestoreRequest) RunscPath: runscPathFor(assetPaths), RuntimeAssetPaths: assetPaths, Spec: spec, - Scope: toAteomSnapshotScope(req.GetScope()), + Fidelity: toAteomFidelity(req.GetFidelity()), ActorUid: req.GetActorUid(), ActorDirs: actorDirs, PreserveRestoreDir: directLocal, @@ -1728,7 +1734,7 @@ func validateCheckpointRequest(req *ateletpb.CheckpointRequest) error { return err } - if err := validateSnapshotScope(req.GetScope()); err != nil { + if err := validateFidelity(req.GetFidelity()); err != nil { return err } @@ -1767,7 +1773,7 @@ func validateRestoreRequest(req *ateletpb.RestoreRequest) error { return err } - if err := validateSnapshotScope(req.GetScope()); err != nil { + if err := validateFidelity(req.GetFidelity()); err != nil { return err } @@ -1810,15 +1816,20 @@ func validateTerminateRequest(req *ateletpb.TerminateRequest) error { return resources.ValidateContainerNames(names) } -func validateSnapshotScope(scope ateletpb.SnapshotScope) error { - switch scope { - case ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL, - ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA: +// validateFidelity rejects a fidelity no runtime can serve. ROOTFS is in the +// enum but not captured by any sandbox runtime yet, so it is refused here as +// well as at template admission. +func validateFidelity(fidelity ateletpb.SnapshotFidelity) error { + switch fidelity { + case ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES: return nil - case ateletpb.SnapshotScope_SNAPSHOT_SCOPE_UNSPECIFIED: - return fmt.Errorf("snapshot scope must be non-zero") + case ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED: + return fmt.Errorf("snapshot fidelity must be non-zero") + case ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS: + return fmt.Errorf("ROOTFS fidelity is not supported yet") default: - return fmt.Errorf("invalid snapshot scope: %v", scope) + return fmt.Errorf("invalid snapshot fidelity: %v", fidelity) } } @@ -1836,12 +1847,14 @@ func validateUploadPausedCheckpointRequest(req *ateletpb.UploadPausedCheckpointR if _, err := resources.ParseSnapshotURI(req.GetDestinationSnapshotUri()); err != nil { errs = append(errs, field.Invalid(field.NewPath("destination_snapshot_uri"), req.GetDestinationSnapshotUri(), err.Error())) } - // Uploads only ever produce FULL or DATA snapshots. - switch req.GetDesiredScope() { - case ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL, ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA: + // Uploads only ever produce MEMORY or VOLUMES snapshots. + switch req.GetDesiredFidelity() { + case ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES: + case ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS: + errs = append(errs, field.Invalid(field.NewPath("desired_fidelity"), req.GetDesiredFidelity().String(), "ROOTFS fidelity is not supported yet")) default: - errs = append(errs, field.NotSupported(field.NewPath("desired_scope"), req.GetDesiredScope(), - []string{ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL.String(), ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA.String()})) + errs = append(errs, field.NotSupported(field.NewPath("desired_fidelity"), req.GetDesiredFidelity(), + []string{ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY.String(), ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES.String()})) } return errs.ToAggregate() } diff --git a/cmd/atelet/main_test.go b/cmd/atelet/main_test.go index 3bfda17eb4..14f6f39b26 100644 --- a/cmd/atelet/main_test.go +++ b/cmd/atelet/main_test.go @@ -174,38 +174,38 @@ func TestSnapshotManifestActorMetadata(t *testing.T) { ActorUID: "actor-uid", ActorTemplateAtespace: "templates", ActorTemplateName: "agent", - Scope: ateattr.SnapshotScopeFull, + Fidelity: ateattr.SnapshotFidelityMemory, } got, err := json.Marshal(rec) if err != nil { t.Fatal(err) } - for _, want := range []string{`"atespace":"team-a"`, `"actorName":"actor-1"`, `"actorUid":"actor-uid"`, `"actorTemplateAtespace":"templates"`, `"actorTemplateName":"agent"`, `"scope":"full"`} { + for _, want := range []string{`"atespace":"team-a"`, `"actorName":"actor-1"`, `"actorUid":"actor-uid"`, `"actorTemplateAtespace":"templates"`, `"actorTemplateName":"agent"`, `"fidelity":"memory"`} { if !bytes.Contains(got, []byte(want)) { t.Errorf("manifest %s missing %s", got, want) } } } -// TestSnapshotManifestScopeAbsent pins backward compatibility: manifests -// written before the scope field existed must still parse, reporting an empty -// scope, and a scope-less record must not serialize a scope key at all. -func TestSnapshotManifestScopeAbsent(t *testing.T) { +// TestSnapshotManifestFidelityAbsent pins that the on-node record written at +// Run/Restore, which has no fidelity, parses back with an empty fidelity and +// does not serialize a fidelity key at all. +func TestSnapshotManifestFidelityAbsent(t *testing.T) { legacy := []byte(`{"sandboxClass":"gvisor","pauseImage":"` + testPauseImage + `","snapshotFiles":["checkpoint.img"]}`) rec, err := unmarshalSandboxRecord(legacy) if err != nil { t.Fatalf("unmarshalSandboxRecord(legacy manifest): %v", err) } - if rec.Scope != "" { - t.Errorf("legacy manifest scope = %q, want empty", rec.Scope) + if rec.Fidelity != "" { + t.Errorf("manifest fidelity = %q, want empty", rec.Fidelity) } got, err := json.Marshal(sandboxAssetsRecord{SandboxClass: "gvisor"}) if err != nil { t.Fatal(err) } - if bytes.Contains(got, []byte(`"scope"`)) { - t.Errorf("scope-less record serialized a scope key: %s", got) + if bytes.Contains(got, []byte(`"fidelity"`)) { + t.Errorf("fidelity-less record serialized a fidelity key: %s", got) } } @@ -567,7 +567,7 @@ func validCheckpointRequest() *ateletpb.CheckpointRequest { SnapshotUri: testSnapshotURI, }, }, - Scope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL, + Fidelity: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, } } @@ -586,7 +586,7 @@ func validRestoreRequest() *ateletpb.RestoreRequest { SnapshotUri: testSnapshotURI, }, }, - Scope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL, + Fidelity: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, SandboxAssets: &ateletpb.SandboxAssets{ SandboxClass: "gvisor", PauseImage: testPauseImage, @@ -669,8 +669,13 @@ func TestValidateCheckpointRequest(t *testing.T) { r.Config = &ateletpb.CheckpointRequest_LocalConfig{LocalConfig: &ateletpb.LocalCheckpointConfiguration{SnapshotName: ".."}} }), true}, {"unspecified snapshot type", makeReq(func(r *ateletpb.CheckpointRequest) { r.Type = ateletpb.CheckpointType_CHECKPOINT_TYPE_UNSPECIFIED }), true}, - {"unspecified snapshot scope", makeReq(func(r *ateletpb.CheckpointRequest) { r.Scope = ateletpb.SnapshotScope_SNAPSHOT_SCOPE_UNSPECIFIED }), true}, - {"invalid snapshot scope", makeReq(func(r *ateletpb.CheckpointRequest) { r.Scope = ateletpb.SnapshotScope(23) }), true}, + {"unspecified snapshot fidelity", makeReq(func(r *ateletpb.CheckpointRequest) { + r.Fidelity = ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED + }), true}, + {"invalid snapshot fidelity", makeReq(func(r *ateletpb.CheckpointRequest) { r.Fidelity = ateletpb.SnapshotFidelity(23) }), true}, + {"rootfs fidelity not supported yet", makeReq(func(r *ateletpb.CheckpointRequest) { + r.Fidelity = ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS + }), true}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { @@ -727,8 +732,11 @@ func TestValidateRestoreRequest(t *testing.T) { r.Config = &ateletpb.RestoreRequest_LocalConfig{LocalConfig: &ateletpb.LocalCheckpointConfiguration{SnapshotName: ".."}} }), true}, {"unspecified snapshot type", makeReq(func(r *ateletpb.RestoreRequest) { r.Type = ateletpb.CheckpointType_CHECKPOINT_TYPE_UNSPECIFIED }), true}, - {"unspecified snapshot scope", makeReq(func(r *ateletpb.RestoreRequest) { r.Scope = ateletpb.SnapshotScope_SNAPSHOT_SCOPE_UNSPECIFIED }), true}, - {"invalid snapshot scope", makeReq(func(r *ateletpb.RestoreRequest) { r.Scope = ateletpb.SnapshotScope(23) }), true}, + {"unspecified snapshot fidelity", makeReq(func(r *ateletpb.RestoreRequest) { r.Fidelity = ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED }), true}, + {"invalid snapshot fidelity", makeReq(func(r *ateletpb.RestoreRequest) { r.Fidelity = ateletpb.SnapshotFidelity(23) }), true}, + {"rootfs fidelity not supported yet", makeReq(func(r *ateletpb.RestoreRequest) { + r.Fidelity = ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS + }), true}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { @@ -740,17 +748,20 @@ func TestValidateRestoreRequest(t *testing.T) { } // Every valid atelet scope must map to its ateom counterpart. -func TestToAteomSnapshotScope(t *testing.T) { +func TestToAteomFidelity(t *testing.T) { tests := []struct { - in ateletpb.SnapshotScope - want ateompb.SnapshotScope + in ateletpb.SnapshotFidelity + want ateompb.SnapshotFidelity }{ - {ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL, ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL}, - {ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA, ateompb.SnapshotScope_SNAPSHOT_SCOPE_DATA}, + {ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY}, + {ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES}, + {ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS, ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS}, + {ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED, ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED}, + {ateletpb.SnapshotFidelity(99), ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED}, } for _, tc := range tests { - if got := toAteomSnapshotScope(tc.in); got != tc.want { - t.Errorf("toAteomSnapshotScope(%v) = %v, want %v", tc.in, got, tc.want) + if got := toAteomFidelity(tc.in); got != tc.want { + t.Errorf("toAteomFidelity(%v) = %v, want %v", tc.in, got, tc.want) } } } @@ -1505,7 +1516,7 @@ func validUploadPausedCheckpointRequest() *ateletpb.UploadPausedCheckpointReques ActorTemplateName: "counter", LocalSnapshotName: "pause-snap-1", DestinationSnapshotUri: pausedSnapshotURI, - DesiredScope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL, + DesiredFidelity: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, } } @@ -1521,7 +1532,7 @@ func TestUploadLocalCheckpointDir(t *testing.T) { PauseImage: testPauseImage, SnapshotFiles: []string{"config.json", "memory-ranges", "data.tar"}, DataSnapshotFiles: []string{"data.tar"}, - Scope: ateattr.SnapshotScopeFull, + Fidelity: ateattr.SnapshotFidelityMemory, } } @@ -1558,8 +1569,8 @@ func TestUploadLocalCheckpointDir(t *testing.T) { if got := store.keys(); !slices.Equal(got, want) { t.Errorf("uploaded objects = %v, want %v", got, want) } - if rec := remoteManifest(t, store); rec.Scope != ateattr.SnapshotScopeFull { - t.Errorf("uploaded manifest scope = %q, want %q", rec.Scope, ateattr.SnapshotScopeFull) + if rec := remoteManifest(t, store); rec.Fidelity != ateattr.SnapshotFidelityMemory { + t.Errorf("uploaded manifest fidelity = %q, want %q", rec.Fidelity, ateattr.SnapshotFidelityMemory) } }) @@ -1572,7 +1583,7 @@ func TestUploadLocalCheckpointDir(t *testing.T) { }) req := validUploadPausedCheckpointRequest() - req.DesiredScope = ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA + req.DesiredFidelity = ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES if _, err := s.uploadLocalCheckpointDir(ctx, req, dir, uri); err != nil { t.Fatalf("uploadLocalCheckpointDir: %v", err) } @@ -1584,8 +1595,8 @@ func TestUploadLocalCheckpointDir(t *testing.T) { t.Errorf("uploaded objects = %v, want %v", got, want) } rec := remoteManifest(t, store) - if rec.Scope != ateattr.SnapshotScopeData { - t.Errorf("uploaded manifest scope = %q, want %q", rec.Scope, ateattr.SnapshotScopeData) + if rec.Fidelity != ateattr.SnapshotFidelityVolumes { + t.Errorf("uploaded manifest fidelity = %q, want %q", rec.Fidelity, ateattr.SnapshotFidelityVolumes) } if want := []string{"data.tar"}; !slices.Equal(rec.SnapshotFiles, want) { t.Errorf("uploaded manifest files = %v, want %v", rec.SnapshotFiles, want) @@ -1603,7 +1614,7 @@ func TestUploadLocalCheckpointDir(t *testing.T) { }) req := validUploadPausedCheckpointRequest() - req.DesiredScope = ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA + req.DesiredFidelity = ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES _, err := s.uploadLocalCheckpointDir(ctx, req, dir, uri) if got := apierror.Code(err); got != codes.FailedPrecondition { t.Fatalf("status.Code = %v (err %v), want FailedPrecondition", got, err) @@ -1620,7 +1631,7 @@ func TestUploadLocalCheckpointDir(t *testing.T) { SandboxClass: "microvm", PauseImage: testPauseImage, SnapshotFiles: []string{"data.tar"}, - Scope: ateattr.SnapshotScopeData, + Fidelity: ateattr.SnapshotFidelityVolumes, }, map[string]string{"data.tar": "data"}) _, err := s.uploadLocalCheckpointDir(ctx, validUploadPausedCheckpointRequest(), dir, uri) @@ -1640,7 +1651,7 @@ func TestUploadLocalCheckpointDir(t *testing.T) { }, map[string]string{"data.tar": "data"}) req := validUploadPausedCheckpointRequest() - req.DesiredScope = ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA + req.DesiredFidelity = ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES _, err := s.uploadLocalCheckpointDir(ctx, req, dir, uri) if got := apierror.Code(err); got != codes.FailedPrecondition { t.Fatalf("status.Code = %v (err %v), want FailedPrecondition for a scope-less manifest", got, err) @@ -1694,8 +1705,8 @@ func TestValidateUploadPausedCheckpointRequest(t *testing.T) { wantErr bool }{ {"valid", func(*ateletpb.UploadPausedCheckpointRequest) {}, false}, - {"valid data scope", func(r *ateletpb.UploadPausedCheckpointRequest) { - r.DesiredScope = ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA + {"valid volumes fidelity", func(r *ateletpb.UploadPausedCheckpointRequest) { + r.DesiredFidelity = ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES }, false}, {"invalid atespace", func(r *ateletpb.UploadPausedCheckpointRequest) { r.Atespace = "../escape" }, true}, {"golden atespace rejected", func(r *ateletpb.UploadPausedCheckpointRequest) { r.Atespace = resources.GoldenActorAtespace }, true}, @@ -1707,8 +1718,11 @@ func TestValidateUploadPausedCheckpointRequest(t *testing.T) { }, false}, {"invalid snapshot name", func(r *ateletpb.UploadPausedCheckpointRequest) { r.LocalSnapshotName = "../escape" }, true}, {"invalid snapshot uri", func(r *ateletpb.UploadPausedCheckpointRequest) { r.DestinationSnapshotUri = "not-a-uri" }, true}, - {"unspecified scope", func(r *ateletpb.UploadPausedCheckpointRequest) { - r.DesiredScope = ateletpb.SnapshotScope_SNAPSHOT_SCOPE_UNSPECIFIED + {"unspecified fidelity", func(r *ateletpb.UploadPausedCheckpointRequest) { + r.DesiredFidelity = ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED + }, true}, + {"rootfs fidelity not supported yet", func(r *ateletpb.UploadPausedCheckpointRequest) { + r.DesiredFidelity = ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS }, true}, } for _, tc := range tests { @@ -1731,14 +1745,14 @@ func TestShouldHaveSnapshots(t *testing.T) { { name: "full scope always expects snapshots", req: &ateletpb.CheckpointRequest{ - Scope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL, + Fidelity: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, }, want: true, }, { name: "data scope with durable volumes expects snapshots", req: &ateletpb.CheckpointRequest{ - Scope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA, + Fidelity: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, Spec: &ateletpb.WorkloadSpec{ Volumes: []*ateletpb.Volume{ {Name: "durable", Source: &ateletpb.Volume_DurableDir{DurableDir: &ateletpb.DurableDirVolume{}}}, @@ -1750,7 +1764,7 @@ func TestShouldHaveSnapshots(t *testing.T) { { name: "data scope with only CSI volumes does not expect snapshots", req: &ateletpb.CheckpointRequest{ - Scope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA, + Fidelity: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, Spec: &ateletpb.WorkloadSpec{ Volumes: []*ateletpb.Volume{ {Name: "csi", Source: &ateletpb.Volume_External{External: &ateletpb.ExternalVolumeSource{}}}, @@ -1762,7 +1776,7 @@ func TestShouldHaveSnapshots(t *testing.T) { { name: "data scope with both durable and CSI volumes expects snapshots", req: &ateletpb.CheckpointRequest{ - Scope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA, + Fidelity: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, Spec: &ateletpb.WorkloadSpec{ Volumes: []*ateletpb.Volume{ {Name: "durable", Source: &ateletpb.Volume_DurableDir{DurableDir: &ateletpb.DurableDirVolume{}}}, @@ -1775,8 +1789,8 @@ func TestShouldHaveSnapshots(t *testing.T) { { name: "data scope with no volumes does not expect snapshots", req: &ateletpb.CheckpointRequest{ - Scope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA, - Spec: &ateletpb.WorkloadSpec{}, + Fidelity: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, + Spec: &ateletpb.WorkloadSpec{}, }, want: false, }, diff --git a/cmd/atelet/metrics.go b/cmd/atelet/metrics.go index b3dc99ef54..b7ae516955 100644 --- a/cmd/atelet/metrics.go +++ b/cmd/atelet/metrics.go @@ -77,7 +77,7 @@ type snapshotOp struct { templateNamespace string templateName string kind string - scope string + fidelity string sandboxClass string } @@ -89,7 +89,7 @@ func (o snapshotOp) attrs() []attribute.KeyValue { attrs = append(attrs, ateattr.TemplateAtespaceKey.String(o.templateNamespace), ateattr.TemplateNameKey.String(o.templateName), - ateattr.SnapshotScopeKey.String(o.scope), + ateattr.SnapshotFidelityKey.String(o.fidelity), ) if o.kind != "" { attrs = append(attrs, ateattr.SnapshotKindKey.String(o.kind)) diff --git a/cmd/atelet/metrics_test.go b/cmd/atelet/metrics_test.go index 83d98c33d3..5c58a7ed3b 100644 --- a/cmd/atelet/metrics_test.go +++ b/cmd/atelet/metrics_test.go @@ -102,7 +102,7 @@ func TestRestoreDurationShape(t *testing.T) { templateNamespace: testTemplateNamespace, templateName: testTemplateName, kind: ateattr.SnapshotKindLatest, - scope: ateattr.SnapshotScopeData, + fidelity: ateattr.SnapshotFidelityVolumes, sandboxClass: "gvisor", } inst.recordRestore(context.Background(), op, @@ -129,7 +129,7 @@ func TestRestoreDurationShape(t *testing.T) { {ateattr.TemplateAtespaceKey, testTemplateNamespace}, {ateattr.TemplateNameKey, testTemplateName}, {ateattr.SnapshotKindKey, ateattr.SnapshotKindLatest}, - {ateattr.SnapshotScopeKey, ateattr.SnapshotScopeData}, + {ateattr.SnapshotFidelityKey, ateattr.SnapshotFidelityVolumes}, {ateattr.SandboxClassKey, "gvisor"}, } { if v := attrString(t, got, tc.key); v != tc.want { @@ -151,7 +151,7 @@ func TestCheckpointDurationShape(t *testing.T) { templateNamespace: testTemplateNamespace, templateName: testTemplateName, kind: ateattr.SnapshotKindLocal, - scope: ateattr.SnapshotScopeFull, + fidelity: ateattr.SnapshotFidelityMemory, sandboxClass: "microvm", }, phase{ateattr.SnapshotPhasePersist, time.Second}) @@ -174,7 +174,7 @@ func TestRecordPhasesSkipsZeroPhases(t *testing.T) { inst, reader := newTestInstruments(t) inst.recordRestore(context.Background(), - snapshotOp{scope: ateattr.SnapshotScopeFull}, + snapshotOp{fidelity: ateattr.SnapshotFidelityMemory}, phase{ateattr.SnapshotPhaseManifestFetch, 50 * time.Millisecond}, phase{ateattr.SnapshotPhaseDownload, 2 * time.Second}, phase{ateattr.SnapshotPhaseAteomRestore, 0}, @@ -196,7 +196,7 @@ func TestSnapshotOpAttrsOmitsUnknownDimensions(t *testing.T) { attrs := snapshotOp{ templateNamespace: testTemplateNamespace, templateName: testTemplateName, - scope: ateattr.SnapshotScopeFull, + fidelity: ateattr.SnapshotFidelityMemory, }.attrs() for _, kv := range attrs { if kv.Key == ateattr.SnapshotKindKey || kv.Key == ateattr.SandboxClassKey { diff --git a/cmd/atelet/sandbox_assets.go b/cmd/atelet/sandbox_assets.go index 0a41e16a34..0e59dd5852 100644 --- a/cmd/atelet/sandbox_assets.go +++ b/cmd/atelet/sandbox_assets.go @@ -101,14 +101,14 @@ type sandboxAssetsRecord struct { // on-node record written at Run/Restore; populated at Checkpoint. SnapshotFiles []string `json:"snapshotFiles,omitempty"` // DataSnapshotFiles is the subset of SnapshotFiles that restores the actor - // at DATA scope on its own, as reported by CheckpointWorkloadResponse. + // at VOLUMES fidelity on its own, as reported by CheckpointWorkloadResponse. // Empty when the capture holds no durable data. DataSnapshotFiles []string `json:"dataSnapshotFiles,omitempty"` - // Scope is the snapshot scope the checkpoint captured, as the shared - // ateattr label ("full" or "data"), so a snapshot's content is knowable - // from the manifest alone. Empty in the on-node record written at - // Run/Restore and in snapshot manifests written before this field existed. - Scope string `json:"scope,omitempty"` + // Fidelity is the snapshot fidelity the checkpoint captured, as the shared + // ateattr label ("memory" or "volumes"), so a snapshot's content is + // knowable from the manifest alone. Empty in the on-node record written at + // Run/Restore. + Fidelity string `json:"fidelity,omitempty"` } // recordFromRequest projects a request's per-architecture SandboxAssets onto the diff --git a/cmd/atelet/snapshotlog_test.go b/cmd/atelet/snapshotlog_test.go index a2b69424b4..0cf5276358 100644 --- a/cmd/atelet/snapshotlog_test.go +++ b/cmd/atelet/snapshotlog_test.go @@ -106,7 +106,7 @@ func TestSnapshotLogAttrsSeconds(t *testing.T) { templateNamespace: testTemplateNamespace, templateName: testTemplateName, kind: ateattr.SnapshotKindLatest, - scope: ateattr.SnapshotScopeFull, + fidelity: ateattr.SnapshotFidelityMemory, sandboxClass: "gvisor", } attrs := snapshotLogAttrs(testAttribution(), op, restoreDurationMetric, nil, @@ -125,7 +125,7 @@ func TestSnapshotLogAttrs(t *testing.T) { templateNamespace: testTemplateNamespace, templateName: testTemplateName, kind: ateattr.SnapshotKindLatest, - scope: ateattr.SnapshotScopeFull, + fidelity: ateattr.SnapshotFidelityMemory, sandboxClass: "gvisor", } @@ -157,7 +157,7 @@ func TestSnapshotLogAttrs(t *testing.T) { "ate.template.atespace": testTemplateNamespace, "ate.template.name": testTemplateName, "ate.snapshot.kind": ateattr.SnapshotKindLatest, - "ate.snapshot.scope": ateattr.SnapshotScopeFull, + "ate.snapshot.fidelity": ateattr.SnapshotFidelityMemory, "ate.sandbox.class": "gvisor", }, wantNumbers: map[string]float64{ @@ -229,7 +229,7 @@ func TestSnapshotLogAttrs(t *testing.T) { op: snapshotOp{ templateNamespace: testTemplateNamespace, templateName: testTemplateName, - scope: ateattr.SnapshotScopeFull, + fidelity: ateattr.SnapshotFidelityMemory, }, phases: []phase{{ateattr.SnapshotPhaseTotal, 700 * time.Millisecond}}, wantStrings: map[string]string{ @@ -245,7 +245,7 @@ func TestSnapshotLogAttrs(t *testing.T) { op: snapshotOp{ templateNamespace: testTemplateNamespace, templateName: testTemplateName, - scope: ateattr.SnapshotScopeFull, + fidelity: ateattr.SnapshotFidelityMemory, sandboxClass: "../../etc/passwd", }, phases: []phase{{ateattr.SnapshotPhaseTotal, time.Second}}, diff --git a/cmd/ateom-gvisor/main.go b/cmd/ateom-gvisor/main.go index c72c027f5a..663bac2c6f 100644 --- a/cmd/ateom-gvisor/main.go +++ b/cmd/ateom-gvisor/main.go @@ -545,6 +545,15 @@ func validateActorDirs(actorDirs *ateompb.ActorDirs) error { return nil } +// validateFidelity rejects a checkpoint or restore request whose fidelity +// this runtime cannot serve. +func validateFidelity(fidelity ateompb.SnapshotFidelity) error { + if errs := resources.ValidateSnapshotFidelity(fidelity, field.NewPath("fidelity")); len(errs) > 0 { + return apierror.InvalidArgument("%v", errs.ToAggregate()) + } + return nil +} + func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkloadRequest) (resp *ateompb.RunWorkloadResponse, retErr error) { if err := validateActorDirs(req.GetActorDirs()); err != nil { return nil, err @@ -677,6 +686,9 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec if err := validateActorDirs(req.GetActorDirs()); err != nil { return nil, err } + if err := validateFidelity(req.GetFidelity()); err != nil { + return nil, err + } if err := validateRunscPath(req.GetRunscPath()); err != nil { return nil, err } @@ -693,12 +705,12 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec // Per-phase timing, recorded on the way out so a failed checkpoint still // reports the phases it completed, and the failing step its elapsed time. attribution := ateomstats.ActorAttributionFromRequest(req) - scope := req.GetScope() + fidelity := req.GetFidelity() var timing checkpointTiming tLast := tStart defer func() { timing.total = time.Since(tStart) - ateomphaselog.LogSnapshotPhases(ctx, "Checkpoint timing breakdown", attribution, scope, + ateomphaselog.LogSnapshotPhases(ctx, "Checkpoint timing breakdown", attribution, fidelity, ateomphaselog.CheckpointDurationKey, err, timing.phases()) }() @@ -707,7 +719,7 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec } s.actorLogger.EmitLifecycleLog(ctx, "Actor checkpointing", attribution) - // Read before the snapshot: a Full checkpoint stops the sandbox, and its + // Read before the snapshot: a MEMORY checkpoint stops the sandbox, and its // cgroup with it. The final record waits for the teardown. hosted := s.lookupActor(req.GetActorUid()) if hosted != nil { @@ -733,14 +745,14 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec return nil, fmt.Errorf("while creating checkpoint directory: %w", err) } - // durableFiles are the durable-dir tars written below: the DATA subset. + // durableFiles are the durable-dir tars written below: the VOLUMES subset. var durableFiles []string // Always take durable-dir snapshot if at least one container has a durable-dir volume mount. // TODO(dberkov): this is a temporary workaround until gVisor supports taking durable-dir snapshots in a single request with the process snapshot. - switch scope { - case ateompb.SnapshotScope_SNAPSHOT_SCOPE_DATA: + switch fidelity { + case ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES: if !hasDurableVolumes(req.GetSpec().GetContainers()) { - return nil, fmt.Errorf("no durable-dir volumes found for DATA snapshot") + return nil, fmt.Errorf("no durable-dir volumes found for VOLUMES snapshot") } err := rcmd.cmdPause(ctx, ocispec.PauseContainer) timing.pause = lap(&tLast) @@ -763,7 +775,7 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec if tarErr != nil { return nil, fmt.Errorf("while archiving durable-dir volumes: %w", tarErr) } - case ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL: + case ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY: // Checkpoint pause container (root of the sandbox) // TODO: Consider pause -> tar -> resume -> checkpoint order for better failure handling. err := rcmd.cmdCheckpoint(ctx, ocispec.PauseContainer, checkpointPath) @@ -780,7 +792,7 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec } } default: - return nil, fmt.Errorf("unsupported snapshot scope: %v", scope) + return nil, fmt.Errorf("unsupported snapshot fidelity: %v", fidelity) } // Cleanup the containers after checkpointing. This also unhosts the actor, @@ -887,6 +899,9 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore if err := validateActorDirs(req.GetActorDirs()); err != nil { return nil, err } + if err := validateFidelity(req.GetFidelity()); err != nil { + return nil, err + } if err := validateRunscPath(req.GetRunscPath()); err != nil { return nil, err } @@ -902,13 +917,13 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore // Registered ahead of the failure cleanup below, so on failure the total // includes that cleanup; the record is marked by error.type either way. attribution := ateomstats.ActorAttributionFromRequest(req) - scope := req.GetScope() + fidelity := req.GetFidelity() containers := req.GetSpec().GetContainers() var timing restoreTiming tLast := tStart defer func() { timing.total = time.Since(tStart) - attrs := ateomphaselog.SnapshotPhaseAttrs(attribution, scope, ateomphaselog.RestoreDurationKey, retErr, timing.phases()) + attrs := ateomphaselog.SnapshotPhaseAttrs(attribution, fidelity, ateomphaselog.RestoreDurationKey, retErr, timing.phases()) attrs = append(attrs, slog.Int(containerCountKey, len(containers))) slog.LogAttrs(ctx, slog.LevelInfo, "Restore timing breakdown", attrs...) }() @@ -991,8 +1006,8 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore return nil, fmt.Errorf("while composing pause rootfs: %w", err) } - switch scope { - case ateompb.SnapshotScope_SNAPSHOT_SCOPE_DATA: + switch fidelity { + case ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES: // Create and start pause container (cold boot with durable-dir volumes restored) containersToDelete = append(containersToDelete, ocispec.PauseContainer) err := rcmd.cmdCreate(ctx, os.Stdout, ocispec.PauseContainer, nil) @@ -1005,7 +1020,7 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore if err != nil { return nil, fmt.Errorf("while starting pause container: %w", err) } - case ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL: + case ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY: // Create and restore pause container containersToDelete = append(containersToDelete, ocispec.PauseContainer) err := rcmd.cmdCreate(ctx, os.Stdout, ocispec.PauseContainer, nil) @@ -1019,7 +1034,7 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore return nil, fmt.Errorf("while restoring pause container: %w", err) } default: - return nil, fmt.Errorf("unexpected snapshot scope: %v", scope) + return nil, fmt.Errorf("unexpected snapshot fidelity: %v", fidelity) } // Create and restore each application container, each with its own log pipe so @@ -1043,21 +1058,21 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore if err != nil { return nil, fmt.Errorf("while creating %q application container: %w", ac.GetName(), err) } - switch scope { - case ateompb.SnapshotScope_SNAPSHOT_SCOPE_DATA: + switch fidelity { + case ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES: err = rcmd.cmdStart(ctx, pw, ac.GetName()) timing.appRestore += lap(&tLast) if err != nil { return nil, fmt.Errorf("while starting %q application container: %w", ac.GetName(), err) } - case ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL: + case ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY: err = rcmd.cmdRestore(ctx, pw, ac.GetName(), checkpointDir) timing.appRestore += lap(&tLast) if err != nil { return nil, fmt.Errorf("while restoring %q application container: %w", ac.GetName(), err) } default: - return nil, fmt.Errorf("unexpected snapshot scope: %v", scope) + return nil, fmt.Errorf("unexpected snapshot fidelity: %v", fidelity) } } diff --git a/cmd/ateom-gvisor/phaselog_test.go b/cmd/ateom-gvisor/phaselog_test.go index 1173c1021f..a90f81cdd3 100644 --- a/cmd/ateom-gvisor/phaselog_test.go +++ b/cmd/ateom-gvisor/phaselog_test.go @@ -65,7 +65,7 @@ func TestRestoreTimingPhases(t *testing.T) { total: 2 * time.Second, } attrs := ateomphaselog.SnapshotPhaseAttrs(resources.ActorAttribution{UID: "uid-abc"}, - ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL, ateomphaselog.RestoreDurationKey, nil, timing.phases()) + ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, ateomphaselog.RestoreDurationKey, nil, timing.phases()) attrs = append(attrs, slog.Int(containerCountKey, 2)) seen := make(map[string]bool, len(attrs)) @@ -105,8 +105,8 @@ func TestRestoreTimingPhases(t *testing.T) { if rec["ate.actor.uid"] != "uid-abc" { t.Errorf("ate.actor.uid = %v, want uid-abc", rec["ate.actor.uid"]) } - if rec["ate.snapshot.scope"] != "full" { - t.Errorf("ate.snapshot.scope = %v, want full", rec["ate.snapshot.scope"]) + if rec["ate.snapshot.fidelity"] != "memory" { + t.Errorf("ate.snapshot.fidelity = %v, want memory", rec["ate.snapshot.fidelity"]) } } @@ -124,7 +124,7 @@ func TestCheckpointTimingPhases(t *testing.T) { total: 500 * time.Millisecond, } rec := renderPhaseRecord(t, ateomphaselog.SnapshotPhaseAttrs(resources.ActorAttribution{UID: "uid-abc"}, - ateompb.SnapshotScope_SNAPSHOT_SCOPE_DATA, ateomphaselog.CheckpointDurationKey, nil, timing.phases())) + ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, ateomphaselog.CheckpointDurationKey, nil, timing.phases())) for k, want := range map[string]float64{ "ateom.actor.checkpoint.duration.prep": 0.02, @@ -144,8 +144,8 @@ func TestCheckpointTimingPhases(t *testing.T) { if v, ok := rec["ateom.actor.checkpoint.duration.checkpoint"]; ok { t.Errorf("checkpoint present with %v, want absent", v) } - if rec["ate.snapshot.scope"] != "data" { - t.Errorf("ate.snapshot.scope = %v, want data", rec["ate.snapshot.scope"]) + if rec["ate.snapshot.fidelity"] != "volumes" { + t.Errorf("ate.snapshot.fidelity = %v, want volumes", rec["ate.snapshot.fidelity"]) } } diff --git a/cmd/ateom-microvm/checkpoint.go b/cmd/ateom-microvm/checkpoint.go index fe873c30c9..a282d2df34 100644 --- a/cmd/ateom-microvm/checkpoint.go +++ b/cmd/ateom-microvm/checkpoint.go @@ -44,14 +44,14 @@ import ( // // What the snapshot holds depends on the requested scope: // -// - FULL: the whole guest. ateom drives the CH REST api-socket: pause -> snapshot +// - MEMORY: the whole guest. ateom drives the CH REST api-socket: pause -> snapshot // file:// (config.json + state.json + sparse memory-ranges) // -> tear the VMM down. Each container's rootfs is overlay(virtio-fs RO lower + // disk-backed upper): the upper is host-backed like the durable-dir volumes and // ships alongside as its own tar (see rootfsupper.go); process memory persists // via the memory snapshot. The RO lower is reconstructed from the OCI image at // restore, so it never ships. Durable-dir volumes ship alongside as per-volume tars. -// - DATA: the durable-dir volumes only, as those same tars. The guest is discarded, so +// - VOLUMES: the durable-dir volumes only, as those same tars. The guest is discarded, so // the actor cold-starts on restore with its volumes re-materialized. // // Either way the guest is paused first, which is what makes the tar coherent: the @@ -64,6 +64,9 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec if err := validateActorDirs(req.GetActorDirs()); err != nil { return nil, err } + if err := validateFidelity(req.GetFidelity()); err != nil { + return nil, err + } if !s.locks.Lock(ctx, req.GetActorUid()) { return nil, fmt.Errorf("gave up waiting for the actor's lock: %w", ctx.Err()) } @@ -81,7 +84,7 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec tStart := time.Now() var dPrep, dPause, dSnapshot, dDurable, dUpper, dTeardown time.Duration attribution := ateomstats.ActorAttributionFromRequest(req) - scope := req.GetScope() + scope := req.GetFidelity() defer func() { ateomphaselog.LogSnapshotPhases(ctx, "Checkpoint timing breakdown", attribution, scope, ateomphaselog.CheckpointDurationKey, err, []ateomphaselog.Phase{ @@ -109,20 +112,20 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec // suspended mid-flight for a call that could never have succeeded. // // Durable-dir volumes are host-backed, so they are captured the same way - // under either scope — and are the ONLY thing a Data-scope snapshot + // under either fidelity — and are the ONLY thing a VOLUMES snapshot // captures. durable := hasDurableVolumes(req.GetSpec().GetContainers()) csi := hasCsiVolumes(req.GetSpec().GetContainers()) switch scope { - case ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL: - case ateompb.SnapshotScope_SNAPSHOT_SCOPE_DATA: + case ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY: + case ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES: // TODO: Revisit handling for CSI volumes since snapshots are currently quietly ignored. if !durable && !csi { return nil, apierror.FailedPrecondition( - "no durable-dir or CSI volumes found for a Data-scope snapshot") + "no durable-dir or CSI volumes found for a VOLUMES snapshot") } default: - return nil, apierror.InvalidArgument("unsupported snapshot scope: %v", scope) + return nil, apierror.InvalidArgument("unsupported snapshot fidelity: %v", scope) } // Captured now: the checkpoint unhosts the actor, and the final record @@ -168,18 +171,18 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec // window costs the slowest of them rather than their sum (the tars scale // with the actor's data; suspend latency is the metric that matters). // - // - CH snapshot (Full only): the guest memory + VM state. A Data snapshot + // - CH snapshot (MEMORY only): the guest memory + VM state. A VOLUMES snapshot // deliberately captures no VM state — no memory image, and no base-id, // since nothing will reattach to the frozen virtio-fs lower: at restore // the actor cold-boots from the OCI image. // - Durable-dir tars (any scope, when declared): host-backed, so pausing // the write-through share makes the tars coherent. - // - Rootfs upper tars (Full only): host-backed like the durable volumes — - // the memory snapshot does not carry rootfs writes. Under Data the + // - Rootfs upper tars (MEMORY only): host-backed like the durable volumes — + // the memory snapshot does not carry rootfs writes. Under VOLUMES the // workload cold-starts on restore, discarding rootfs state. var durableFiles []string g, gctx := errgroup.WithContext(ctx) - if scope == ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL { + if scope == ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY { g.Go(func() error { t := time.Now() d, err := s.snapshotVMState(gctx, client, ra, actorUID, checkpointDir) @@ -199,7 +202,7 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec return err }) } - if scope == ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL { + if scope == ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY { g.Go(func() error { t := time.Now() err := tarRootfsUpper(gctx, rootfsUpperDir(actorDirs), checkpointDir, containerNames(req.GetSpec().GetContainers())) @@ -212,8 +215,8 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec } // Report exactly the files we wrote so atelet ships precisely this snapshot: for - // Full, the CH snapshot (config.json + state.json + memory-ranges + base-id) plus - // any durable-dir tars; for Data, those tars alone. + // MEMORY, the CH snapshot (config.json + state.json + memory-ranges + base-id) + // plus any durable-dir tars; for VOLUMES, those tars alone. snapshotFiles, err := listFiles(checkpointDir) if err != nil { return nil, fmt.Errorf("while listing snapshot files: %w", err) diff --git a/cmd/ateom-microvm/durable.go b/cmd/ateom-microvm/durable.go index 76a990c892..c27a3f4e1d 100644 --- a/cmd/ateom-microvm/durable.go +++ b/cmd/ateom-microvm/durable.go @@ -19,7 +19,7 @@ package main // Durable-dir volumes for the micro-VM runtime. // // A durable-dir volume is a directory whose contents outlive the actor's process -// state: it survives suspend/resume and, under the Data snapshot scope, is the +// state: it survives suspend/resume and, under the VOLUMES snapshot fidelity, is the // ONLY thing captured (the workload cold-starts on restore). The host side is // owned by atelet, which creates one directory per volume under // ActorDirs.durable_dir_volume_mounts_dir and wipes them when the actor's diff --git a/cmd/ateom-microvm/main.go b/cmd/ateom-microvm/main.go index 9e46395802..bf5c3b2fae 100644 --- a/cmd/ateom-microvm/main.go +++ b/cmd/ateom-microvm/main.go @@ -417,6 +417,15 @@ func validateActorDirs(actorDirs *ateompb.ActorDirs) error { return nil } +// validateFidelity rejects a checkpoint or restore request whose fidelity +// this runtime cannot serve. +func validateFidelity(fidelity ateompb.SnapshotFidelity) error { + if errs := resources.ValidateSnapshotFidelity(fidelity, field.NewPath("fidelity")); len(errs) > 0 { + return apierror.InvalidArgument("%v", errs.ToAggregate()) + } + return nil +} + // validateRuntimeAssetPaths ensures we only run assets from the static files dir func validateRuntimeAssetPaths(paths map[string]string) error { var errs field.ErrorList diff --git a/cmd/ateom-microvm/restore.go b/cmd/ateom-microvm/restore.go index fa0578a10c..ca1dbf642a 100644 --- a/cmd/ateom-microvm/restore.go +++ b/cmd/ateom-microvm/restore.go @@ -67,7 +67,7 @@ func restoreMemMode(ctx context.Context, info ch.VMMInfo) string { } // reseedGuestCRNG mixes fresh, per-restore entropy into the restored guest's kernel -// CRNG through the kata-agent (see the call site in restoreFullScope for why a restore +// CRNG through the kata-agent (see the call site in restoreMemoryFidelity for why a restore // needs this). The nonce is a throwaway; its only job is to differ between restores so // that clones of one snapshot diverge instead of producing identical randomness. The // caller owns ac: it stays open for log forwarding and guest stats. @@ -91,12 +91,13 @@ func newReseedNonce() ([]byte, error) { } // RestoreWorkload brings the actor back from a snapshot, on a possibly different -// pod. What that means depends on the scope the snapshot was taken with: +// pod. What that means depends on the fidelity the snapshot was taken with: // -// - FULL: relaunch cloud-hypervisor from the snapshot and resume the guest -// (restoreFullScope). -// - DATA: there is no guest to resume — re-materialize the durable-dir volumes and -// cold-boot the actor, which starts its containers afresh from the OCI image. +// - MEMORY: relaunch cloud-hypervisor from the snapshot and resume the guest +// (restoreMemoryFidelity). +// - VOLUMES: there is no guest to resume — re-materialize the durable-dir +// volumes and cold-boot the actor, which starts its containers afresh from +// the OCI image. // // Contract with atelet: the snapshot's files are in ActorDirs.restore_dir, // and the durable-dir volume directories re-created (empty). @@ -104,6 +105,9 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore if err := validateActorDirs(req.GetActorDirs()); err != nil { return nil, err } + if err := validateFidelity(req.GetFidelity()); err != nil { + return nil, err + } if err := validateRuntimeAssetPaths(req.GetRuntimeAssetPaths()); err != nil { return nil, err } @@ -147,10 +151,10 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore return nil, err } // Publish attribution before restore, so a stats read during it is - // attributed. A Data scope cold-boots, so its CPU counts from zero; the other + // attributed. A VOLUMES restore cold-boots, so its CPU counts from zero; the other // scopes resume the guest's counters, so theirs counts from the first // reading. - resumesGuest := req.GetScope() != ateompb.SnapshotScope_SNAPSHOT_SCOPE_DATA + resumesGuest := req.GetFidelity() != ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES if _, err := s.hostActor(ctx, attribution, resumesGuest); err != nil { return nil, err } @@ -163,8 +167,8 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore } }() - // Restore the durable-dir volumes before anything can observe them: for Full - // that means before the share's virtiofsd starts, for Data before the workload + // Restore the durable-dir volumes before anything can observe them: for MEMORY + // that means before the share's virtiofsd starts, for VOLUMES before the workload // cold-starts. if hasDurableVolumes(p.containers) { if err := untarDurableVolumes(durableDir, restoreDir, durableVolumeNames(p.containers)); err != nil { @@ -172,13 +176,13 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore } } - switch scope := req.GetScope(); scope { - case ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL: - if err := s.restoreFullScope(ctx, p, scope, restoreDir, req.GetPreserveRestoreDir(), tStart); err != nil { + switch scope := req.GetFidelity(); scope { + case ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY: + if err := s.restoreMemoryFidelity(ctx, p, scope, restoreDir, req.GetPreserveRestoreDir(), tStart); err != nil { return nil, err } - case ateompb.SnapshotScope_SNAPSHOT_SCOPE_DATA: - // A Data snapshot holds no guest state, so this is a cold boot that + case ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES: + // A VOLUMES snapshot holds no guest state, so this is a cold boot that // happens to start with the volumes already populated. wakeup probe gating comes // with the cold-boot path, so the actor is serving when we return. if err := s.coldBootActorRetrying(ctx, p); err != nil { @@ -187,19 +191,19 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore dTotal := time.Since(tStart) slog.InfoContext(ctx, "Actor restored (durable-dir volumes, cold boot)", slog.String("id", p.actorUID), slog.Duration("total", dTotal)) - // A cold boot has none of the full-scope phases, so the total is the + // A cold boot has none of the memory-restore phases, so the total is the // only observation on its record. ateomphaselog.LogSnapshotPhases(ctx, "Restore timing breakdown", attribution, scope, ateomphaselog.RestoreDurationKey, nil, []ateomphaselog.Phase{{Name: phaseTotal, D: dTotal}}) default: - return nil, apierror.InvalidArgument("unsupported snapshot scope: %v", scope) + return nil, apierror.InvalidArgument("unsupported snapshot fidelity: %v", scope) } s.actorLogger.EmitLifecycleLog(ctx, "Actor restored", attribution) return &ateompb.RestoreWorkloadResponse{}, nil } -// restoreFullScope restores a whole-guest snapshot: relaunch cloud-hypervisor +// restoreMemoryFidelity restores a whole-guest snapshot: relaunch cloud-hypervisor // directly from it and resume. // // Each container's rootfs is a host-merged overlay (image lower + host upper). Steps: @@ -211,7 +215,7 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore // and resume. Guest RAM — the actor's in-memory state and the frozen network config — // comes back from the memory snapshot; the durable-dir volumes were restored by the // caller from their tar. -func (s *AteomService) restoreFullScope(ctx context.Context, p actorBootParams, scope ateompb.SnapshotScope, restoreDir string, preserveRestoreDir bool, tStart time.Time) (retErr error) { +func (s *AteomService) restoreMemoryFidelity(ctx context.Context, p actorBootParams, scope ateompb.SnapshotFidelity, restoreDir string, preserveRestoreDir bool, tStart time.Time) (retErr error) { actorUID := p.actorUID rr := s.resolveRuntime(p.assetPaths) @@ -237,7 +241,7 @@ func (s *AteomService) restoreFullScope(ctx context.Context, p actorBootParams, } tPrep := time.Now() - // Full snapshots carry each container's upper as a tar (rootfsUpperTarFile). + // MEMORY snapshots carry each container's upper as a tar (rootfsUpperTarFile). // Start re-materializing the upper contents NOW, in the background: the // untar scales with the actor's data and is joined right before the host // overlay mounts need it, so it hides behind the bundle preparation below. diff --git a/cmd/ateom-microvm/rootfsupper.go b/cmd/ateom-microvm/rootfsupper.go index 82f528a136..11b0bbaf01 100644 --- a/cmd/ateom-microvm/rootfsupper.go +++ b/cmd/ateom-microvm/rootfsupper.go @@ -36,13 +36,13 @@ package main // removed at teardown — after CleanupSandboxState has dropped the overlay // mounts that use it. // -// Snapshots: the upper does not ride in guest memory, so a FULL snapshot +// Snapshots: the upper does not ride in guest memory, so a MEMORY snapshot // ships it as one tar per container (rootfsUpperTarFile), taken while the // guest is paused (the share is write-through, so a paused guest's completed // writes are already in the upper). Each tar holds only the contents of that // container's upperdir: ateom creates the directory layout itself on restore, // so nothing in the snapshot decides a path ateom later mounts or wipes. A -// DATA snapshot deliberately excludes rootfs state: the workload cold-starts +// VOLUMES snapshot deliberately excludes rootfs state: the workload cold-starts // on restore. import ( diff --git a/cmd/ateom-microvm/run.go b/cmd/ateom-microvm/run.go index b7b64bdd06..1500968454 100644 --- a/cmd/ateom-microvm/run.go +++ b/cmd/ateom-microvm/run.go @@ -295,7 +295,7 @@ func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkload } // actorBootParams is what a cold boot needs about an actor. It comes from a Run -// request, or from a Restore request whose snapshot scope covers only the +// request, or from a Restore request whose snapshot fidelity covers only the // durable-dir volumes (the workload itself cold-starts). type actorBootParams struct { actorRef resources.ActorRef @@ -424,8 +424,8 @@ func (s *AteomService) coldBootActor(ctx context.Context, p actorBootParams) (re // vCPUs round up; VM RAM reserves a fixed margin for the VMM + virtiofsd, which // share the pod cgroup with the guest RAM. A declared memory limit the reserve // leaves too small to boot is rejected (resolveGuestMemMiB) rather than silently - // falling back to the larger kata default. NB: a FULL-scope snapshot restore - // reuses the size baked into the snapshot (restoreFullScope), so resizing an + // falling back to the larger kata default. NB: a MEMORY snapshot restore + // reuses the size baked into the snapshot (restoreMemoryFidelity), so resizing an // existing actor takes effect on its next cold boot. sz := p.size if v := sz.VCPUs(); v > 0 { diff --git a/cmd/kubectl-ate/internal/cmd/actortemplate_test.go b/cmd/kubectl-ate/internal/cmd/actortemplate_test.go index 32f5b0ea02..b4ffe73c56 100644 --- a/cmd/kubectl-ate/internal/cmd/actortemplate_test.go +++ b/cmd/kubectl-ate/internal/cmd/actortemplate_test.go @@ -57,7 +57,7 @@ resources: - name: memory quantity: 512Mi snapshotConfig: - onCommit: SNAPSHOT_CONTENT_SCOPE_FULL + preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://ate-snapshots/ate-demo-counter/ sandboxConfig: sandboxClass: SANDBOX_CLASS_GVISOR @@ -90,8 +90,8 @@ func TestActorTemplateFromManifest(t *testing.T) { {Name: "memory", Quantity: "512Mi"}, }}, SnapshotConfig: &ateapipb.SnapshotConfig{ - OnCommit: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, - StorageLocation: "gs://ate-snapshots/ate-demo-counter/", + PreferredFidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + StorageLocation: "gs://ate-snapshots/ate-demo-counter/", }, SandboxConfig: &ateapipb.SandboxConfig{ SandboxClass: ateapipb.SandboxClass_SANDBOX_CLASS_GVISOR, diff --git a/cmd/kubectl-ate/internal/printer/printer.go b/cmd/kubectl-ate/internal/printer/printer.go index 51d3ad0299..6d32492468 100644 --- a/cmd/kubectl-ate/internal/printer/printer.go +++ b/cmd/kubectl-ate/internal/printer/printer.go @@ -335,18 +335,18 @@ func PrintTagsTo(out io.Writer, tags []*ateapipb.Tag, format string) error { return printProto(out, &ateapipb.ListTagsResponse{Tags: tags}, format) case "table": w := tabwriter.NewWriter(out, 0, 0, 3, ' ', 0) - fmt.Fprintln(w, "ATESPACE\tNAME\tSCOPE\tSTATE\tSNAPSHOT\tCONTENT SCOPE\tAGE") + fmt.Fprintln(w, "ATESPACE\tNAME\tSCOPE\tSTATE\tSNAPSHOT\tFIDELITY\tAGE") for _, tag := range tags { // A pending tag has no snapshot yet, so neither its URI nor its // content scope says anything. - snapshotURI, contentScope := "", "" + snapshotURI, fidelity := "", "" if snapshot := tag.GetStatus().GetSnapshot(); snapshot.GetSnapshotUri() != "" { snapshotURI = snapshot.GetSnapshotUri() - contentScope = snapshot.GetContentScope().String() + fidelity = snapshot.GetFidelity().String() } fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%s\n", tag.GetMetadata().GetAtespace(), tag.GetMetadata().GetName(), tag.GetScope(), - tagState(tag), snapshotURI, contentScope, + tagState(tag), snapshotURI, fidelity, formatAge(tag.GetMetadata().GetCreateTime())) } return w.Flush() diff --git a/cmd/kubectl-ate/internal/printer/printer_test.go b/cmd/kubectl-ate/internal/printer/printer_test.go index a857277d37..292cceb02f 100644 --- a/cmd/kubectl-ate/internal/printer/printer_test.go +++ b/cmd/kubectl-ate/internal/printer/printer_test.go @@ -599,7 +599,7 @@ func TestPrintTagsTo_Table(t *testing.T) { }, Scope: ateapipb.TagScope_TAG_SCOPE_PUBLISHED, Status: &ateapipb.TagStatus{ - Snapshot: &ateapipb.ExternalSnapshot{SnapshotUri: "gs://private/atespaces/team-a/tags/v2", ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL}, + Snapshot: &ateapipb.ExternalSnapshot{SnapshotUri: "gs://private/atespaces/team-a/tags/v2", Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY}, }, }, { @@ -610,7 +610,7 @@ func TestPrintTagsTo_Table(t *testing.T) { }, Scope: ateapipb.TagScope_TAG_SCOPE_ATESPACE, Status: &ateapipb.TagStatus{ - Snapshot: &ateapipb.ExternalSnapshot{SnapshotUri: "gs://private/atespaces/team-a/tags/v1", ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL}, + Snapshot: &ateapipb.ExternalSnapshot{SnapshotUri: "gs://private/atespaces/team-a/tags/v1", Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY}, }, }, { @@ -633,10 +633,10 @@ func TestPrintTagsTo_Table(t *testing.T) { } // Sorted by atespace, then name. - expected := `ATESPACE NAME SCOPE STATE SNAPSHOT CONTENT SCOPE AGE -team-a v1 TAG_SCOPE_ATESPACE Ready gs://private/atespaces/team-a/tags/v1 SNAPSHOT_CONTENT_SCOPE_FULL 5h -team-a v2 TAG_SCOPE_PUBLISHED Ready gs://private/atespaces/team-a/tags/v2 SNAPSHOT_CONTENT_SCOPE_FULL 5m -team-a v3 TAG_SCOPE_ATESPACE Pending 30s + expected := `ATESPACE NAME SCOPE STATE SNAPSHOT FIDELITY AGE +team-a v1 TAG_SCOPE_ATESPACE Ready gs://private/atespaces/team-a/tags/v1 SNAPSHOT_FIDELITY_MEMORY 5h +team-a v2 TAG_SCOPE_PUBLISHED Ready gs://private/atespaces/team-a/tags/v2 SNAPSHOT_FIDELITY_MEMORY 5m +team-a v3 TAG_SCOPE_ATESPACE Pending 30s ` if diff := cmp.Diff(expected, buf.String()); diff != "" { t.Errorf("output mismatch (-want +got):\n%s", diff) diff --git a/demos/autoscaled-workerpool/autoscaled-workerpool-template.yaml.tmpl b/demos/autoscaled-workerpool/autoscaled-workerpool-template.yaml.tmpl index 8502923d9a..dadd1e82ca 100644 --- a/demos/autoscaled-workerpool/autoscaled-workerpool-template.yaml.tmpl +++ b/demos/autoscaled-workerpool/autoscaled-workerpool-template.yaml.tmpl @@ -36,7 +36,7 @@ containers: - name: data mountPath: /home/counter snapshotConfig: - onCommit: SNAPSHOT_CONTENT_SCOPE_DATA + preferredFidelity: SNAPSHOT_FIDELITY_VOLUMES storageLocation: gs://${BUCKET_NAME}/ate-demo-autoscaled-workerpool/ sandboxConfig: sandboxClass: SANDBOX_CLASS_GVISOR diff --git a/demos/claude-code-multiplex/agent-luna-template.yaml.tmpl b/demos/claude-code-multiplex/agent-luna-template.yaml.tmpl index 3d842a0bae..2eef82044c 100644 --- a/demos/claude-code-multiplex/agent-luna-template.yaml.tmpl +++ b/demos/claude-code-multiplex/agent-luna-template.yaml.tmpl @@ -41,7 +41,7 @@ containers: - name: ANTHROPIC_API_KEY value: "${ANTHROPIC_API_KEY}" snapshotConfig: - onCommit: SNAPSHOT_CONTENT_SCOPE_FULL + preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://${BUCKET_NAME}/claude-multiplex-demo/ sandboxConfig: sandboxClass: SANDBOX_CLASS_GVISOR diff --git a/demos/claude-code-multiplex/agent-mars-template.yaml.tmpl b/demos/claude-code-multiplex/agent-mars-template.yaml.tmpl index 46baedeae4..873cc4caec 100644 --- a/demos/claude-code-multiplex/agent-mars-template.yaml.tmpl +++ b/demos/claude-code-multiplex/agent-mars-template.yaml.tmpl @@ -34,7 +34,7 @@ containers: - name: ANTHROPIC_API_KEY value: "${ANTHROPIC_API_KEY}" snapshotConfig: - onCommit: SNAPSHOT_CONTENT_SCOPE_FULL + preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://${BUCKET_NAME}/claude-multiplex-demo/ sandboxConfig: sandboxClass: SANDBOX_CLASS_GVISOR diff --git a/demos/claude-code-multiplex/agent-orion-template.yaml.tmpl b/demos/claude-code-multiplex/agent-orion-template.yaml.tmpl index f6dc7ca890..2e6c84dcf3 100644 --- a/demos/claude-code-multiplex/agent-orion-template.yaml.tmpl +++ b/demos/claude-code-multiplex/agent-orion-template.yaml.tmpl @@ -34,7 +34,7 @@ containers: - name: ANTHROPIC_API_KEY value: "${ANTHROPIC_API_KEY}" snapshotConfig: - onCommit: SNAPSHOT_CONTENT_SCOPE_FULL + preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://${BUCKET_NAME}/claude-multiplex-demo/ sandboxConfig: sandboxClass: SANDBOX_CLASS_GVISOR diff --git a/demos/counter/README.md b/demos/counter/README.md index e62db12842..87696e5fd3 100644 --- a/demos/counter/README.md +++ b/demos/counter/README.md @@ -2,7 +2,7 @@ This directory contains a demo of a stateful counter application running on Agent Substrate. -It deploys a simple Go HTTP server (`counter.go`) that increments two counters on every request — one in process memory, one in a file on a durable volume — and preserves both across suspends and resumes: the template's `Full`-scope `onCommit` snapshot captures process memory alongside the durable volumes, so the in-memory count continues from where it left off. (With a `Data`-scope snapshot policy, only the durable-volume counter would survive and the in-memory counter would restart from a cold boot.) +It deploys a simple Go HTTP server (`counter.go`) that increments two counters on every request — one in process memory, one in a file on a durable volume — and preserves both across suspends and resumes: the template's `MEMORY` `preferredFidelity` snapshot captures process memory alongside the durable volumes, so the in-memory count continues from where it left off. (With a `VOLUMES` fidelity, only the durable-volume counter would survive and the in-memory counter would restart from a cold boot.) The demo uses two kinds of resources: the `WorkerPool` is a Kubernetes CRD, while the actor template is a Substrate `ActorTemplate` resource — an `ateapipb.ActorTemplate` living in an **atespace** rather than a Kubernetes namespace, managed through the ate API with `kubectl ate`. @@ -99,7 +99,7 @@ kubectl ate suspend actor my-counter-1 -a ate-demo-counter Repeat the `curl` from step 1 and the actor resumes from its snapshot — possibly on a different worker — with **both** counters continuing from where -they left off: the memory count comes back from the `Full` snapshot's process +they left off: the memory count comes back from the `MEMORY` snapshot's process memory, the file counter from the durable volume. 4. To permanently delete the suspended actor: diff --git a/demos/counter/counter-microvm-csi-test-template.yaml b/demos/counter/counter-microvm-csi-test-template.yaml index b82924cf3d..29d65c3d14 100644 --- a/demos/counter/counter-microvm-csi-test-template.yaml +++ b/demos/counter/counter-microvm-csi-test-template.yaml @@ -33,7 +33,7 @@ containers: - name: data mountPath: /home/counter snapshotConfig: - onCommit: SNAPSHOT_CONTENT_SCOPE_DATA + preferredFidelity: SNAPSHOT_FIDELITY_VOLUMES storageLocation: gs://ate-snapshots/ate-demo-counter-microvm-csi/ sandboxConfig: sandboxClass: SANDBOX_CLASS_MICROVM diff --git a/demos/counter/counter-microvm-template.yaml.tmpl b/demos/counter/counter-microvm-template.yaml.tmpl index 34e2ed76df..ad8f5ee949 100644 --- a/demos/counter/counter-microvm-template.yaml.tmpl +++ b/demos/counter/counter-microvm-template.yaml.tmpl @@ -60,10 +60,10 @@ resources: - name: memory quantity: 512Mi snapshotConfig: - # FULL on both triggers, matching the gVisor variant: a suspend captures + # MEMORY, matching the gVisor variant: a suspend captures # guest memory, so the in-RAM count continues from where it left off — the # round-trip this demo promises. - onCommit: SNAPSHOT_CONTENT_SCOPE_FULL + preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://${BUCKET_NAME}/ate-demo-counter-microvm/ sandboxConfig: sandboxClass: SANDBOX_CLASS_MICROVM diff --git a/demos/counter/counter-template.yaml.tmpl b/demos/counter/counter-template.yaml.tmpl index 61b7765dcb..66266e93bd 100644 --- a/demos/counter/counter-template.yaml.tmpl +++ b/demos/counter/counter-template.yaml.tmpl @@ -54,12 +54,12 @@ resources: - name: memory quantity: 512Mi snapshotConfig: - # FULL, so a suspend captures process memory as well as the durable + # MEMORY, so a suspend captures process memory as well as the durable # volumes and the tutorial's in-memory counter really does continue - # across suspend/resume. DATA-scope behavior (durable volumes only, + # across suspend/resume. VOLUMES behavior (durable volumes only, # memory cold-boots) is exercised by the e2e demo suite's own # parameterized templates, not by this one. - onCommit: SNAPSHOT_CONTENT_SCOPE_FULL + preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://${BUCKET_NAME}/ate-demo-counter/ sandboxConfig: sandboxClass: SANDBOX_CLASS_GVISOR diff --git a/demos/egress/egress-microvm-template.yaml.tmpl b/demos/egress/egress-microvm-template.yaml.tmpl index 9bd813e9e7..5d0cdd4d2f 100644 --- a/demos/egress/egress-microvm-template.yaml.tmpl +++ b/demos/egress/egress-microvm-template.yaml.tmpl @@ -77,5 +77,5 @@ sandboxConfig: # explicitly makes a missing or stale install fail loudly. configName: microvm snapshotConfig: - onCommit: SNAPSHOT_CONTENT_SCOPE_FULL + preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://${BUCKET_NAME}/ate-demo-egress-microvm/ diff --git a/demos/egress/egress-template.yaml.tmpl b/demos/egress/egress-template.yaml.tmpl index 90a86b4253..fc43b8a23e 100644 --- a/demos/egress/egress-template.yaml.tmpl +++ b/demos/egress/egress-template.yaml.tmpl @@ -57,5 +57,5 @@ sandboxConfig: sandboxClass: SANDBOX_CLASS_GVISOR configName: gvisor-default snapshotConfig: - onCommit: SNAPSHOT_CONTENT_SCOPE_FULL + preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://${BUCKET_NAME}/ate-demo-egress/ diff --git a/demos/jupyter/jupyter-template.yaml.tmpl b/demos/jupyter/jupyter-template.yaml.tmpl index ff7bcb3bb6..f9ede8217b 100644 --- a/demos/jupyter/jupyter-template.yaml.tmpl +++ b/demos/jupyter/jupyter-template.yaml.tmpl @@ -40,7 +40,7 @@ containers: path: /api port: 80 snapshotConfig: - onCommit: SNAPSHOT_CONTENT_SCOPE_FULL + preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://${BUCKET_NAME}/ate-demo-jupyter/ sandboxConfig: sandboxClass: SANDBOX_CLASS_GVISOR diff --git a/demos/multi-template/counter-template.yaml.tmpl b/demos/multi-template/counter-template.yaml.tmpl index 73b2b02141..d1d634f626 100644 --- a/demos/multi-template/counter-template.yaml.tmpl +++ b/demos/multi-template/counter-template.yaml.tmpl @@ -28,7 +28,7 @@ containers: - name: counter image: ko://github.com/agent-substrate/substrate/demos/counter snapshotConfig: - onCommit: SNAPSHOT_CONTENT_SCOPE_FULL + preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://${BUCKET_NAME}/ate-demo-multi-template-counter/ sandboxConfig: sandboxClass: SANDBOX_CLASS_GVISOR diff --git a/demos/multi-template/fspersist-template.yaml.tmpl b/demos/multi-template/fspersist-template.yaml.tmpl index 6d3d36083c..53f047eb5c 100644 --- a/demos/multi-template/fspersist-template.yaml.tmpl +++ b/demos/multi-template/fspersist-template.yaml.tmpl @@ -28,7 +28,7 @@ containers: - name: fspersist image: ko://github.com/agent-substrate/substrate/demos/multi-template/fspersist snapshotConfig: - onCommit: SNAPSHOT_CONTENT_SCOPE_FULL + preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://${BUCKET_NAME}/ate-demo-multi-template-fspersist/ sandboxConfig: sandboxClass: SANDBOX_CLASS_GVISOR diff --git a/demos/parking/parking-template.yaml.tmpl b/demos/parking/parking-template.yaml.tmpl index 212311d1c9..c3606c97fb 100644 --- a/demos/parking/parking-template.yaml.tmpl +++ b/demos/parking/parking-template.yaml.tmpl @@ -27,7 +27,7 @@ containers: image: ko://github.com/agent-substrate/substrate/demos/counter command: ["/ko-app/counter"] snapshotConfig: - onCommit: SNAPSHOT_CONTENT_SCOPE_FULL + preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://${BUCKET_NAME}/ate-demo-parking/ sandboxConfig: sandboxClass: SANDBOX_CLASS_GVISOR diff --git a/demos/sandbox/manual-test-multi-template.yaml b/demos/sandbox/manual-test-multi-template.yaml index 71c9d80121..6476c31be0 100644 --- a/demos/sandbox/manual-test-multi-template.yaml +++ b/demos/sandbox/manual-test-multi-template.yaml @@ -35,7 +35,7 @@ containers: - name: vol-nfs mountPath: /nfs-data snapshotConfig: - onCommit: SNAPSHOT_CONTENT_SCOPE_FULL + preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://ate-snapshots/ate-manual-test-multi/ sandboxConfig: sandboxClass: SANDBOX_CLASS_GVISOR diff --git a/demos/sandbox/sandbox-template.yaml.tmpl b/demos/sandbox/sandbox-template.yaml.tmpl index 04c5f8a299..48d99209cf 100644 --- a/demos/sandbox/sandbox-template.yaml.tmpl +++ b/demos/sandbox/sandbox-template.yaml.tmpl @@ -38,7 +38,7 @@ resources: - name: memory quantity: 1Gi snapshotConfig: - onCommit: SNAPSHOT_CONTENT_SCOPE_FULL + preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://${BUCKET_NAME}/ate-demo-sandbox/ sandboxConfig: sandboxClass: SANDBOX_CLASS_GVISOR diff --git a/docs/api-guide.md b/docs/api-guide.md index 4d4a00787c..a976a03b6d 100644 --- a/docs/api-guide.md +++ b/docs/api-guide.md @@ -121,7 +121,7 @@ The `ActorTemplate` defines the code, environment, and state-management policies | `containers` | `[]Container` | **Required.** The workload definition — see [Container Fields](#container-fields) below. Each container may also declare an optional `wakeupProbe` HTTP probe — see [Container Wakeup Probe](#container-wakeup-probe-wakeupprobe). | | `sandboxConfig` | `SandboxConfig` | **Required.** The sandbox runtime selection: `sandboxClass` (**required**, `SANDBOX_CLASS_GVISOR` or `SANDBOX_CLASS_MICROVM`) picks the runtime family this template's actors require — only `WorkerPool`s whose `sandboxClass` matches are eligible — and `configName` (**required**) names the cluster-scoped [`SandboxConfig`](#3-sandboxconfig-the-sandbox-itself) object supplying the sandbox binaries. It must reference an existing config of the matching class; `CreateActorTemplate` rejects the template otherwise. | | `workerSelector` | `*Selector` | Optional. Gates which `WorkerPool`s actors from this template may use, by matching against each pool's labels (`matchLabels`). If unset, all pools are eligible (subject to the actor's own `worker_selector`). | -| `snapshotConfig` | `SnapshotConfig` | **Required.** The base object-storage location snapshots are written under, plus the snapshot scope (`onCommit`) used by both Pause and Suspend. See [Snapshot Storage Layout](#snapshot-storage-layout). | +| `snapshotConfig` | `SnapshotConfig` | **Required.** The base object-storage location snapshots are written under, plus the snapshot fidelity (`preferredFidelity`) every Pause and Suspend captures. See [Snapshot Storage Layout](#snapshot-storage-layout). | | `volumes` | `[]Volume` | Optional. Volumes the containers may mount, each a `durableDir`, an `externalVolumeTemplate` (see [CSI Volumes Guide](csi-volumes.md)), or a `systemInfo` volume (see [SystemInfo Volumes](#systeminfo-volumes)). Every declared volume must be mounted by at least one container. A `microvm` template may declare several `durableDir` volumes; a `gvisor` template is limited to one. | | `resources` | `*ResourceRequirements` | Optional. Declares each actor's compute size via `limits` — see [Sandbox Right-Sizing](#sandbox-right-sizing-resources). Immutable, like the rest of the template. | @@ -139,7 +139,7 @@ Unlike a Pod, an actor is sized by its **`limits`** (CPU and Memory): the size i 2. **Gate scheduling.** An actor is only placed on a `WorkerPool` whose [worker capacity](#worker-capacity-spectemplateresources) is `>=` these limits. 3. **Fall back to runtime defaults.** A zero or absent limit leaves that dimension at the runtime default: unlimited for gVisor, and 2 GiB / 1 vCPU for the micro-VM. -`requests` are not consulted today (an actor occupies its whole worker). Because the size is baked into snapshots, a **micro-VM FULL-scope restore reuses the size in the snapshot**; changing an actor's limits takes effect on its next cold boot. +`requests` are not consulted today (an actor occupies its whole worker). Because the size is baked into snapshots, a **micro-VM MEMORY-fidelity restore reuses the size in the snapshot**; changing an actor's limits takes effect on its next cold boot. Container environment variables support literal `value` entries only. Values are not interpolated (`$(VAR)` references are not expanded), and Kubernetes `envFrom`/`valueFrom` sources are not supported. @@ -375,7 +375,7 @@ An actor takes a series of snapshots over its life, so it gets a prefix of its o An owner is collected by deleting everything under its prefix, and it can delete nothing else. That is what makes a borrowed snapshot safe: an actor created from a tag points at a URI under `tags/`, which its own prefix does not cover. See [Snapshot lifetime](#snapshot-lifetime). -An `Actor` reports its current snapshot in the server-managed `status.externalSnapshot` and a `Tag` in `status.snapshot`, each an `ExternalSnapshot` carrying `snapshotUri`, `contentScope`, and `actorTemplateUid`. The URI is recorded when the snapshot is written. `actorTemplateUid` records the `ActorTemplate` whose sandbox the guest state was captured from, which is not always the template the actor points at now: an actor may be repointed while `SUSPENDED`, and the snapshot on disk still came from the old one. A resume that finds the two disagree restores the durable data only and boots the guest fresh, because memory captured under one sandbox image cannot be resumed under another. An `ActorTemplate` references its golden tag with the `ObjectRef` in `status.goldenSnapshotStatus.goldenTag`. These status fields are server-owned and ignored on input. Parse a URI only against the scheme above. +An `Actor` reports its current snapshot in the server-managed `status.externalSnapshot` and a `Tag` in `status.snapshot`, each an `ExternalSnapshot` carrying `snapshotUri`, `fidelity`, and `actorTemplateUid`. The URI is recorded when the snapshot is written. `actorTemplateUid` records the `ActorTemplate` whose sandbox the guest state was captured from, which is not always the template the actor points at now: an actor may be repointed while `SUSPENDED`, and the snapshot on disk still came from the old one. A resume that finds the two disagree restores the durable data only and boots the guest fresh, because memory captured under one sandbox image cannot be resumed under another. An `ActorTemplate` references its golden tag with the `ObjectRef` in `status.goldenSnapshotStatus.goldenTag`. These status fields are server-owned and ignored on input. Parse a URI only against the scheme above. An `ActorTemplate` belongs to one atespace, but one `storageLocation` still holds snapshots for many atespaces: the golden actor lives in the reserved `ate-golden` atespace, and a `PUBLISHED` snapshot may be cloned from other atespaces. The `` level exists so that access can be granted per tenant: an object-storage policy can only condition on an **object-name prefix**, and cannot read the identity recorded inside a snapshot's manifest. Binding a per-atespace grant on GCS looks like: diff --git a/docs/architecture.md b/docs/architecture.md index 560a4cba7f..0d063854dd 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -335,7 +335,7 @@ A `WorkerPool` selects a **sandbox class** (`spec.sandboxClass`), and each class * **gVisor** (`ateom-gvisor`, the default): Runs the workload under `runsc` for kernel-level sandboxing. Suspend and resume leverage gVisor's native checkpoint/restore of the sandboxed process tree. - * **micro-VM** (`ateom-microvm`): Runs the workload inside a [Kata Containers](https://katacontainers.io/) guest on the [Cloud Hypervisor](https://www.cloudhypervisor.org/) VMM. Suspend and resume capture a memory-only VM snapshot and restore it on-demand using `userfaultfd` memory demand-paging. Container rootfs writes are host-backed: the overlay is assembled on the host (read-only OCI image lower plus a per-actor writable upper) and served to the guest over the single virtio-fs share, so they cost reclaimable host page cache rather than guest RAM, and a `Full` snapshot ships the upper as its own tar. `DurableDir` volumes travel over that same share and are likewise shipped as a tar, so a `Data`-scope snapshot can capture them without any guest memory. Each volume is a subdirectory of the share, so an actor can have several at no extra cost in devices, which is why the micro-VM class lifts the single-`DurableDir` limit that still applies to gVisor. + * **micro-VM** (`ateom-microvm`): Runs the workload inside a [Kata Containers](https://katacontainers.io/) guest on the [Cloud Hypervisor](https://www.cloudhypervisor.org/) VMM. Suspend and resume capture a memory-only VM snapshot and restore it on-demand using `userfaultfd` memory demand-paging. Container rootfs writes are host-backed: the overlay is assembled on the host (read-only OCI image lower plus a per-actor writable upper) and served to the guest over the single virtio-fs share, so they cost reclaimable host page cache rather than guest RAM, and a `MEMORY` snapshot ships the upper as its own tar. `DurableDir` volumes travel over that same share and are likewise shipped as a tar, so a `VOLUMES` snapshot can capture them without any guest memory. Each volume is a subdirectory of the share, so an actor can have several at no extra cost in devices, which is why the micro-VM class lifts the single-`DurableDir` limit that still applies to gVisor. ### Networking Stack (`atenet` + `atunnel`) diff --git a/docs/dev/best-practices/metrics.md b/docs/dev/best-practices/metrics.md index 5560b229e8..d39232f572 100644 --- a/docs/dev/best-practices/metrics.md +++ b/docs/dev/best-practices/metrics.md @@ -159,7 +159,7 @@ ones every new metric meets: * **Normalize at the producer.** When a value arrives from the wire or from a file nothing validated, map it onto the bounded set before recording: `ateattr.NormalizeSandboxClass`, `ateattr.NormalizeOperationName`, - `ateattr.SnapshotScopeValue`. An unrecognized value reports `unknown`, never + `ateattr.SnapshotFidelityValue`. An unrecognized value reports `unknown`, never the raw string. * **Omit a label you do not know rather than emit it empty.** A resume that failed before a worker was picked has no pool; the pool keys are absent, not diff --git a/docs/glossary.md b/docs/glossary.md index 4f397ff885..ca6a17fe15 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -94,7 +94,7 @@ for etcd. ## Volumes - **DurableDir volume**: a directory mounted into one or more containers - whose contents are preserved by the [`Data` snapshot scope](#snapshots) + whose contents are preserved by the [`VOLUMES` snapshot fidelity](#snapshots) and therefore survive across Suspend/Resume independently of process memory or other rootfs writes. A volume may be mounted into multiple containers, potentially at different paths. This is the per-Actor @@ -108,27 +108,31 @@ for etcd. ## Snapshots -- **Snapshot scope**: what an `ActorTemplate`'s `SnapshotConfig` includes - in a given snapshot. Two scopes exist today: - - **`Full`**: process memory plus the rootfs delta on top of the OCI - image, and any attached `DurableDir` volumes. Used to capture - everything needed to resume hot. - - **`Data`**: only the contents of attached volumes that support - snapshots — currently `DurableDir` volumes. Process memory and the - rest of rootfs are discarded. Used to persist application data - cheaply without the cost of a full memory image. On Resume the - containers start afresh from the OCI image with the `DurableDir` - contents restored. - - Scopes describe only what a snapshot *captures*. A template sets one - scope, `onCommit`, and every snapshot of its actors uses it: the - node-local checkpoint a [Pause](#lifecycle) keeps on the node and the - snapshot a [Suspend](#lifecycle) uploads to snapshot storage. +- **Snapshot fidelity**: the state layers a snapshot holds, ordered lowest + to highest; each level includes the ones below it. + - **`VOLUMES`**: only the contents of attached volumes that support + snapshots — currently `DurableDir` volumes. Process memory and rootfs + changes are discarded. Used to persist application data cheaply + without the cost of a memory image. On Resume the containers start + afresh from the OCI image with the `DurableDir` contents restored. + - **`ROOTFS`**: volumes plus the root filesystem changes made since boot. + Defined in the API but not supported by any sandbox runtime yet; + templates that request it are rejected. + - **`MEMORY`**: volumes, rootfs changes, and process memory. Used to + capture everything needed to resume hot. + + Fidelity describes only what a snapshot *holds*. A template sets one + level, `preferredFidelity`, the ceiling for every snapshot of its actors: + the node-local checkpoint a [Pause](#lifecycle) keeps on the node and the + snapshot a [Suspend](#lifecycle) uploads to snapshot storage. It is best + effort: the system never packages more than that level, and may package + less when a layer cannot be captured or is too expensive to capture at + the time. A process resumed in place keeps its memory regardless. - **Golden Snapshot**: the initial checkpoint captured once, when an `ActorTemplate` is created, from a temporary "golden" boot of the workload. By default an Actor of that template is first restored from this shared - snapshot. It is always a `Full` capture. + snapshot. It is always a `MEMORY` capture. - **Last Snapshot**: the most recent per-Actor snapshot, written on Suspend and used to restore that specific Actor on the next Resume. diff --git a/docs/metrics/registry/metrics.yaml b/docs/metrics/registry/metrics.yaml index 70d2343141..94a5225ac9 100644 --- a/docs/metrics/registry/metrics.yaml +++ b/docs/metrics/registry/metrics.yaml @@ -301,19 +301,23 @@ groups: stability: development value: boot brief: A start from nothing. This is not a restore. It does not occur on the atelet histograms. - - id: ate.snapshot.scope + - id: ate.snapshot.fidelity stability: development - brief: The content of the snapshot. + brief: The state layers the snapshot holds. type: members: - - id: full + - id: memory stability: development - value: full - brief: All the guest state and all the data. - - id: data + value: memory + brief: Process memory, root filesystem changes, and volumes. + - id: volumes stability: development - value: data - brief: Only the data of the actor. + value: volumes + brief: Only the volumes of the actor. + - id: rootfs + stability: development + value: rootfs + brief: Volumes and root filesystem changes, without process memory. - id: unknown stability: development value: unknown @@ -793,9 +797,9 @@ groups: - ref: ate.snapshot.kind requirement_level: conditionally_required: The operation is a resume. A suspend and a pause do not restore, thus this key is absent. - - ref: ate.snapshot.scope + - ref: ate.snapshot.fidelity requirement_level: - conditionally_required: The scope is known. + conditionally_required: The fidelity is known. - ref: error.type requirement_level: conditionally_required: The operation failed. If this key is absent, the operation was correct. @@ -867,7 +871,7 @@ groups: requirement_level: required - ref: ate.template.name requirement_level: required - - ref: ate.snapshot.scope + - ref: ate.snapshot.fidelity requirement_level: required - ref: ate.snapshot.phase requirement_level: required @@ -901,7 +905,7 @@ groups: requirement_level: required - ref: ate.template.name requirement_level: required - - ref: ate.snapshot.scope + - ref: ate.snapshot.fidelity requirement_level: required - ref: ate.snapshot.phase requirement_level: required diff --git a/docs/observability.md b/docs/observability.md index bc771b3edc..fac728c27c 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -129,7 +129,7 @@ atelet's `Restore timing breakdown` and `Checkpoint timing breakdown` are the un {"time":"…","level":"INFO","msg":"Restore timing breakdown", "ate.atespace":"ate-demo-counter","ate.actor.name":"counter-1","ate.actor.uid":"8f2a…", "ate.template.atespace":"ate-demo-counter","ate.template.name":"counter", - "ate.snapshot.scope":"full","ate.snapshot.kind":"latest","ate.sandbox.class":"gvisor", + "ate.snapshot.fidelity":"memory","ate.snapshot.kind":"latest","ate.sandbox.class":"gvisor", "ate.actor.restore.duration.download":0.310, "ate.actor.restore.duration.oci_unpack":0.050, "ate.actor.restore.duration.ateom_restore":0.060, @@ -252,7 +252,7 @@ Agent Substrate emits foundational OpenTelemetry system and server metrics to mo | `ate.workerpool.workers` | ateapi | up/down counter | live worker count per pool, split by state (`idle`/`partial`/`at_capacity`/`unschedulable`) and sandbox class to provide fleet capacity and saturation at a glance | | `ate.actor.lifecycle.operation.duration` | ateapi | histogram | how long each actor operation (create/resume/suspend/pause/delete/revert) takes and whether it failed (`error.type` present = failure, absent = success); labeled by operation, template, pool (`ate.workerpool.namespace` + `ate.workerpool.name`), sandbox class, and snapshot kind and scope on resume; already-running resume no-ops are not recorded so the histogram tracks actual activations, not router traffic | | `ate.scheduler.assignment.duration` | ateapi | histogram | time of one attempt to assign a worker to an actor, with one record for each resume (ateapi retries an attempt that loses a race for a worker and records only the last attempt), with the outcome (`assigned` / `no_capacity` / `error`), the assigned pool (`ate.workerpool.namespace` + `ate.workerpool.name`) and sandbox class to catch scheduling latency and capacity starvation problems | -| `ate.actor.restore.duration` | atelet | histogram | how long each phase of a restore takes on the worker node, which is where cold-start latency actually goes once ateapi hands off (labels `ate.snapshot.phase`, `ate.snapshot.kind`, `ate.snapshot.scope`, `ate.template.atespace`, `ate.template.name`, `ate.sandbox.class`) | +| `ate.actor.restore.duration` | atelet | histogram | how long each phase of a restore takes on the worker node, which is where cold-start latency actually goes once ateapi hands off (labels `ate.snapshot.phase`, `ate.snapshot.kind`, `ate.snapshot.fidelity`, `ate.template.atespace`, `ate.template.name`, `ate.sandbox.class`) | | `ate.actor.checkpoint.duration` | atelet | histogram | the same phase breakdown for writing a snapshot, so a slow suspend can be attributed to ateom or to the upload (same labels as the restore histogram) | | `ate.imagecache.requests` | atelet | counter | image lookups in the node-local image cache, by outcome (`ate.imagecache.outcome`), with `error.type` on the `error` outcome. A miss pays for the pull and the unpack, so the hit ratio per node is a leading indicator of resume latency | @@ -277,7 +277,7 @@ For `ate.imagecache.requests`: The three snapshot labels are orthogonal and mean the same thing on every histogram that carries them: * `ate.snapshot.kind`: which snapshot the operation reads or writes. `local` (node-local, written by a pause), `latest` (the actor's own durable snapshot), `golden` (the template's image), or `boot` (from scratch, so it never appears on the atelet histograms). -* `ate.snapshot.scope`: what content it covers. `full` or `data`. +* `ate.snapshot.fidelity`: what state it holds. `volumes`, `rootfs`, or `memory`. * `ate.snapshot.phase`: which step was timed. `volume_mount`, `manifest_fetch`, `sandbox_assets`, `download`, `oci_unpack`, `ateom_restore` on restore; `sandbox_assets`, `ateom_checkpoint`, `persist` on checkpoint; `total` on both. **Phases overlap and do not sum to `total`.** The download runs concurrently with the asset fetch and OCI unpack, so each is an independent observation; use `total` as the denominator. A phase that never started is absent rather than zero. diff --git a/internal/ateattr/ateattr.go b/internal/ateattr/ateattr.go index 684889c18e..773606d367 100644 --- a/internal/ateattr/ateattr.go +++ b/internal/ateattr/ateattr.go @@ -164,7 +164,7 @@ const ( WorkerStateKey = attribute.Key("ate.worker.state") SandboxClassKey = attribute.Key("ate.sandbox.class") SnapshotKindKey = attribute.Key("ate.snapshot.kind") - SnapshotScopeKey = attribute.Key("ate.snapshot.scope") + SnapshotFidelityKey = attribute.Key("ate.snapshot.fidelity") SnapshotPhaseKey = attribute.Key("ate.snapshot.phase") ImageCacheOutcomeKey = attribute.Key("ate.imagecache.outcome") SchedulerOutcomeKey = attribute.Key("ate.scheduler.outcome") @@ -324,25 +324,28 @@ const ( SnapshotKindBoot = "boot" ) -// Values for SnapshotScopeKey, mirroring ateletpb.SnapshotScope. +// Values for SnapshotFidelityKey, mirroring ateletpb.SnapshotFidelity. const ( - SnapshotScopeFull = "full" - SnapshotScopeData = "data" - SnapshotScopeUnknown = "unknown" + SnapshotFidelityVolumes = "volumes" + SnapshotFidelityRootfs = "rootfs" + SnapshotFidelityMemory = "memory" + SnapshotFidelityUnknown = "unknown" ) -// SnapshotScopeValue maps the wire enum onto its label value, shared so ateapi -// (which sets the scope) and atelet (which receives it) cannot drift. An -// unrecognized scope reports as unknown rather than stringified, so no wire -// value can widen the label set. -func SnapshotScopeValue(scope ateletpb.SnapshotScope) string { - switch scope { - case ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL: - return SnapshotScopeFull - case ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA: - return SnapshotScopeData +// SnapshotFidelityValue maps the wire enum onto its label value, shared so +// ateapi (which sets the fidelity) and atelet (which receives it) cannot +// drift. An unrecognized fidelity reports as unknown rather than stringified, +// so no wire value can widen the label set. +func SnapshotFidelityValue(fidelity ateletpb.SnapshotFidelity) string { + switch fidelity { + case ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES: + return SnapshotFidelityVolumes + case ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS: + return SnapshotFidelityRootfs + case ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY: + return SnapshotFidelityMemory default: - return SnapshotScopeUnknown + return SnapshotFidelityUnknown } } diff --git a/internal/ateattr/ateattr_test.go b/internal/ateattr/ateattr_test.go index 7531733924..f539f03fc9 100644 --- a/internal/ateattr/ateattr_test.go +++ b/internal/ateattr/ateattr_test.go @@ -160,7 +160,7 @@ func TestKeySpellings(t *testing.T) { {WorkerStateKey, "ate.worker.state"}, {SandboxClassKey, "ate.sandbox.class"}, {SnapshotKindKey, "ate.snapshot.kind"}, - {SnapshotScopeKey, "ate.snapshot.scope"}, + {SnapshotFidelityKey, "ate.snapshot.fidelity"}, {SnapshotPhaseKey, "ate.snapshot.phase"}, {ImageCacheOutcomeKey, "ate.imagecache.outcome"}, {SchedulerOutcomeKey, "ate.scheduler.outcome"}, @@ -396,9 +396,10 @@ func TestMetricLabelValues(t *testing.T) { {SnapshotKindLocal, "local"}, {SnapshotKindBoot, "boot"}, - {SnapshotScopeFull, "full"}, - {SnapshotScopeData, "data"}, - {SnapshotScopeUnknown, "unknown"}, + {SnapshotFidelityMemory, "memory"}, + {SnapshotFidelityVolumes, "volumes"}, + {SnapshotFidelityRootfs, "rootfs"}, + {SnapshotFidelityUnknown, "unknown"}, {SnapshotPhaseVolumeMount, "volume_mount"}, {SnapshotPhaseManifestFetch, "manifest_fetch"}, @@ -568,24 +569,25 @@ func TestWorkerPoolAttributes(t *testing.T) { }) } -// TestSnapshotScopeValue pins the enum-to-label mapping ateapi and atelet share. +// TestSnapshotFidelityValue pins the enum-to-label mapping ateapi and atelet share. // An unmapped enum value must report unknown rather than its stringified form, // which would let a wire value widen the label set. -func TestSnapshotScopeValue(t *testing.T) { +func TestSnapshotFidelityValue(t *testing.T) { tests := []struct { name string - scope ateletpb.SnapshotScope + scope ateletpb.SnapshotFidelity want string }{ - {name: "full", scope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL, want: SnapshotScopeFull}, - {name: "data", scope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA, want: SnapshotScopeData}, - {name: "unspecified", scope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_UNSPECIFIED, want: SnapshotScopeUnknown}, - {name: "value outside the enum", scope: ateletpb.SnapshotScope(9999), want: SnapshotScopeUnknown}, + {name: "memory", scope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, want: SnapshotFidelityMemory}, + {name: "volumes", scope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, want: SnapshotFidelityVolumes}, + {name: "rootfs", scope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS, want: SnapshotFidelityRootfs}, + {name: "unspecified", scope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED, want: SnapshotFidelityUnknown}, + {name: "value outside the enum", scope: ateletpb.SnapshotFidelity(9999), want: SnapshotFidelityUnknown}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := SnapshotScopeValue(tt.scope); got != tt.want { - t.Errorf("SnapshotScopeValue(%v) = %q, want %q", tt.scope, got, tt.want) + if got := SnapshotFidelityValue(tt.scope); got != tt.want { + t.Errorf("SnapshotFidelityValue(%v) = %q, want %q", tt.scope, got, tt.want) } }) } diff --git a/internal/ateomphaselog/phaselog.go b/internal/ateomphaselog/phaselog.go index e74283b7c6..58929f7675 100644 --- a/internal/ateomphaselog/phaselog.go +++ b/internal/ateomphaselog/phaselog.go @@ -16,7 +16,7 @@ // ateom emits for each checkpoint and restore. Every ateom binary times its // own phases, which are implementation details of that runtime; this package // holds the shape they share so the records of all ateoms carry the same -// identity, scope and key layout, and atelet's own record joins to any of them. +// identity, fidelity and key layout, and atelet's own record joins to any of them. package ateomphaselog import ( @@ -43,7 +43,7 @@ const ( ) // Phase is one timed step of a snapshot operation. A zero duration means the -// phase never ran (a Data-scope checkpoint captures no guest) and is skipped +// phase never ran (a VOLUMES checkpoint captures no guest) and is skipped // rather than logged as instant. Name is suffixed onto the duration key; each // ateom defines its own names, with ateattr.SnapshotPhaseTotal shared so the // two layers' records agree on the denominator. @@ -52,18 +52,20 @@ type Phase struct { D time.Duration } -// ScopeLogValue maps the ateom wire enum onto the shared scope label values, -// the same way ateattr.SnapshotScopeValue does for the atelet enum. An -// unrecognized scope reports as unknown rather than stringified, so no wire -// value can widen the value set readers key on. -func ScopeLogValue(scope ateompb.SnapshotScope) string { - switch scope { - case ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL: - return ateattr.SnapshotScopeFull - case ateompb.SnapshotScope_SNAPSHOT_SCOPE_DATA: - return ateattr.SnapshotScopeData +// FidelityLogValue maps the ateom wire enum onto the shared fidelity label +// values, the same way ateattr.SnapshotFidelityValue does for the atelet +// enum. An unrecognized fidelity reports as unknown rather than stringified, +// so no wire value can widen the value set readers key on. +func FidelityLogValue(fidelity ateompb.SnapshotFidelity) string { + switch fidelity { + case ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES: + return ateattr.SnapshotFidelityVolumes + case ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS: + return ateattr.SnapshotFidelityRootfs + case ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY: + return ateattr.SnapshotFidelityMemory default: - return ateattr.SnapshotScopeUnknown + return ateattr.SnapshotFidelityUnknown } } @@ -77,9 +79,9 @@ func ScopeLogValue(scope ateompb.SnapshotScope) string { // it completed, marked with error.type (the gRPC code, context errors as // DeadlineExceeded / Canceled) so a reader can leave it out of a latency // distribution. Absence means success. -func SnapshotPhaseAttrs(a resources.ActorAttribution, scope ateompb.SnapshotScope, durationKey string, err error, phases []Phase) []slog.Attr { +func SnapshotPhaseAttrs(a resources.ActorAttribution, fidelity ateompb.SnapshotFidelity, durationKey string, err error, phases []Phase) []slog.Attr { attrs := ateattr.ActorLogAttrs(a) - attrs = append(attrs, slog.String(string(ateattr.SnapshotScopeKey), ScopeLogValue(scope))) + attrs = append(attrs, slog.String(string(ateattr.SnapshotFidelityKey), FidelityLogValue(fidelity))) if err != nil { s, ok := apierror.FromError(err) code := s.Code() @@ -101,6 +103,6 @@ func SnapshotPhaseAttrs(a resources.ActorAttribution, scope ateompb.SnapshotScop } // LogSnapshotPhases emits the SnapshotPhaseAttrs record under msg. -func LogSnapshotPhases(ctx context.Context, msg string, a resources.ActorAttribution, scope ateompb.SnapshotScope, durationKey string, err error, phases []Phase) { - slog.LogAttrs(ctx, slog.LevelInfo, msg, SnapshotPhaseAttrs(a, scope, durationKey, err, phases)...) +func LogSnapshotPhases(ctx context.Context, msg string, a resources.ActorAttribution, fidelity ateompb.SnapshotFidelity, durationKey string, err error, phases []Phase) { + slog.LogAttrs(ctx, slog.LevelInfo, msg, SnapshotPhaseAttrs(a, fidelity, durationKey, err, phases)...) } diff --git a/internal/ateomphaselog/phaselog_test.go b/internal/ateomphaselog/phaselog_test.go index 6595ab598b..7df3d0c6ca 100644 --- a/internal/ateomphaselog/phaselog_test.go +++ b/internal/ateomphaselog/phaselog_test.go @@ -53,7 +53,7 @@ func renderPhaseRecord(t *testing.T, attrs []slog.Attr) map[string]any { func TestSnapshotPhaseAttrs(t *testing.T) { t.Parallel() - attrs := SnapshotPhaseAttrs(phaseLogAttribution(), ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL, + attrs := SnapshotPhaseAttrs(phaseLogAttribution(), ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, CheckpointDurationKey, nil, []Phase{ {Name: "prep", D: 40 * time.Millisecond}, {Name: "pause", D: 3 * time.Millisecond}, @@ -81,7 +81,7 @@ func TestSnapshotPhaseAttrs(t *testing.T) { "ate.actor.uid": "uid-abc", "ate.template.atespace": "templates", "ate.template.name": "support-agent", - "ate.snapshot.scope": ateattr.SnapshotScopeFull, + "ate.snapshot.fidelity": ateattr.SnapshotFidelityMemory, } { if got, ok := rec[k]; !ok { t.Errorf("missing %s", k) @@ -140,7 +140,7 @@ func TestSnapshotPhaseAttrsFailure(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() rec := renderPhaseRecord(t, SnapshotPhaseAttrs(phaseLogAttribution(), - ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL, CheckpointDurationKey, tt.err, + ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, CheckpointDurationKey, tt.err, []Phase{ {Name: "prep", D: 40 * time.Millisecond}, {Name: "pause", D: 3 * time.Millisecond}, @@ -161,23 +161,24 @@ func TestSnapshotPhaseAttrsFailure(t *testing.T) { } } -// TestScopeLogValue pins the mapping onto the shared scope values, so the -// ateom and atelet records of one operation agree on the scope they carry. -func TestScopeLogValue(t *testing.T) { +// TestFidelityLogValue pins the mapping onto the shared fidelity values, so the +// ateom and atelet records of one operation agree on the fidelity they carry. +func TestFidelityLogValue(t *testing.T) { t.Parallel() tests := []struct { - scope ateompb.SnapshotScope + scope ateompb.SnapshotFidelity want string }{ - {ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL, ateattr.SnapshotScopeFull}, - {ateompb.SnapshotScope_SNAPSHOT_SCOPE_DATA, ateattr.SnapshotScopeData}, - {ateompb.SnapshotScope_SNAPSHOT_SCOPE_UNSPECIFIED, ateattr.SnapshotScopeUnknown}, - {ateompb.SnapshotScope(99), ateattr.SnapshotScopeUnknown}, + {ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, ateattr.SnapshotFidelityMemory}, + {ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, ateattr.SnapshotFidelityVolumes}, + {ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS, ateattr.SnapshotFidelityRootfs}, + {ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED, ateattr.SnapshotFidelityUnknown}, + {ateompb.SnapshotFidelity(99), ateattr.SnapshotFidelityUnknown}, } for _, tt := range tests { - if got := ScopeLogValue(tt.scope); got != tt.want { - t.Errorf("ScopeLogValue(%v) = %q, want %q", tt.scope, got, tt.want) + if got := FidelityLogValue(tt.scope); got != tt.want { + t.Errorf("FidelityLogValue(%v) = %q, want %q", tt.scope, got, tt.want) } } } @@ -191,7 +192,7 @@ func TestLogSnapshotPhases(t *testing.T) { t.Cleanup(func() { slog.SetDefault(prev) }) LogSnapshotPhases(context.Background(), "Restore timing breakdown", phaseLogAttribution(), - ateompb.SnapshotScope_SNAPSHOT_SCOPE_DATA, RestoreDurationKey, nil, + ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, RestoreDurationKey, nil, []Phase{{Name: ateattr.SnapshotPhaseTotal, D: 2 * time.Second}}) var rec map[string]any @@ -204,8 +205,8 @@ func TestLogSnapshotPhases(t *testing.T) { if rec["level"] != "INFO" { t.Errorf("level = %v, want INFO", rec["level"]) } - if rec["ate.snapshot.scope"] != ateattr.SnapshotScopeData { - t.Errorf("ate.snapshot.scope = %v, want %q", rec["ate.snapshot.scope"], ateattr.SnapshotScopeData) + if rec["ate.snapshot.fidelity"] != ateattr.SnapshotFidelityVolumes { + t.Errorf("ate.snapshot.fidelity = %v, want %q", rec["ate.snapshot.fidelity"], ateattr.SnapshotFidelityVolumes) } if got := rec["ateom.actor.restore.duration.total"]; got != 2.0 { t.Errorf("ateom.actor.restore.duration.total = %v, want 2", got) diff --git a/internal/e2e/suites/demo/demo_test.go b/internal/e2e/suites/demo/demo_test.go index 35052789d5..894d1bca48 100644 --- a/internal/e2e/suites/demo/demo_test.go +++ b/internal/e2e/suites/demo/demo_test.go @@ -47,7 +47,7 @@ func TestActorLifecycle(t *testing.T) { clients := e2e.GetClients() // Create actor template. - at, err := createActorTemplate(ctx, t, clients, nsObj, ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL) + at, err := createActorTemplate(ctx, t, clients, nsObj, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY) if err != nil { t.Fatalf("failed to initialize ActorTemplate: %v", err) } @@ -98,7 +98,7 @@ func TestActorSnapshotLifecycle(t *testing.T) { clients := e2e.GetClients() nsObj := e2e.CreateNamespace(t) - at, err := createActorTemplate(ctx, t, clients, nsObj, ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL) + at, err := createActorTemplate(ctx, t, clients, nsObj, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY) if err != nil { t.Fatalf("failed to initialize ActorTemplate: %v", err) } @@ -219,9 +219,9 @@ func TestDurableDirLifecycle(t *testing.T) { tc actorLifecycleTestCase }{ { - name: "onCommit:Full", + name: "preferredFidelity:MEMORY", tc: actorLifecycleTestCase{ - onCommit: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, + fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, wantMemoryAfterPause: 2, wantFileAfterPause: 2, wantMemoryAfterSuspend: 3, @@ -231,9 +231,9 @@ func TestDurableDirLifecycle(t *testing.T) { { // Pause captures the same Data scope as suspend, so memory // cold-boots on every resume. - name: "onCommit:Data", + name: "preferredFidelity:VOLUMES", tc: actorLifecycleTestCase{ - onCommit: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA, + fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, wantMemoryAfterPause: 1, wantFileAfterPause: 2, wantMemoryAfterSuspend: 1, @@ -242,28 +242,28 @@ func TestDurableDirLifecycle(t *testing.T) { }, { // Suspend from PAUSED uploads the Full local snapshot. - name: "onCommit:Full, suspend from PAUSED", + name: "preferredFidelity:MEMORY, suspend from PAUSED", tc: actorLifecycleTestCase{ - onCommit: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, - wantMemoryAfterPause: 2, - wantFileAfterPause: 2, - wantMemoryAfterSuspend: 3, - wantFileAfterSuspend: 3, - wantSnapshotContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, - suspendWhilePaused: true, + fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + wantMemoryAfterPause: 2, + wantFileAfterPause: 2, + wantMemoryAfterSuspend: 3, + wantFileAfterSuspend: 3, + wantSnapshotFidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + suspendWhilePaused: true, }, }, { // Suspend from PAUSED uploads the Data local snapshot. - name: "onCommit:Data, suspend from PAUSED", + name: "preferredFidelity:VOLUMES, suspend from PAUSED", tc: actorLifecycleTestCase{ - onCommit: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA, - wantMemoryAfterPause: 1, - wantFileAfterPause: 2, - wantMemoryAfterSuspend: 1, - wantFileAfterSuspend: 3, - wantSnapshotContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA, - suspendWhilePaused: true, + fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, + wantMemoryAfterPause: 1, + wantFileAfterPause: 2, + wantMemoryAfterSuspend: 1, + wantFileAfterSuspend: 3, + wantSnapshotFidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, + suspendWhilePaused: true, }, }, } @@ -290,9 +290,9 @@ func TestMultipleDurableDirLifecycle(t *testing.T) { tc actorLifecycleTestCase }{ { - name: "onCommit:Full", + name: "preferredFidelity:MEMORY", tc: actorLifecycleTestCase{ - onCommit: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, + fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, wantMemoryAfterPause: 2, wantFileAfterPause: 2, wantMemoryAfterSuspend: 3, @@ -301,9 +301,9 @@ func TestMultipleDurableDirLifecycle(t *testing.T) { }, }, { - name: "onCommit:Data", + name: "preferredFidelity:VOLUMES", tc: actorLifecycleTestCase{ - onCommit: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA, + fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, wantMemoryAfterPause: 1, wantFileAfterPause: 2, wantMemoryAfterSuspend: 1, @@ -332,9 +332,9 @@ func TestExternalVolumeLifecycle(t *testing.T) { tc actorLifecycleTestCase }{ { - name: "onCommit:Data", + name: "preferredFidelity:VOLUMES", tc: actorLifecycleTestCase{ - onCommit: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA, + fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, wantMemoryAfterPause: 1, wantFileAfterPause: 2, wantMemoryAfterSuspend: 1, @@ -357,7 +357,7 @@ func TestDeleteActorAnyStateWithExternalVolume(t *testing.T) { clients := e2e.GetClients() nsObj := e2e.CreateNamespace(t) - at, err := createActorTemplateWithExternalVolume(ctx, t, clients, nsObj, ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA) + at, err := createActorTemplateWithExternalVolume(ctx, t, clients, nsObj, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES) if err != nil { t.Fatalf("failed to initialize ActorTemplate: %v", err) } @@ -424,7 +424,7 @@ func TestExternalVolume_NodeMigration(t *testing.T) { clients := e2e.GetClients() nsObj := e2e.CreateNamespace(t) - at, err := createActorTemplateWithExternalVolume(ctx, t, clients, nsObj, ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA) + at, err := createActorTemplateWithExternalVolume(ctx, t, clients, nsObj, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES) if err != nil { t.Fatalf("failed to initialize ActorTemplate: %v", err) } @@ -512,7 +512,7 @@ func TestExternalVolume_NodeMigration(t *testing.T) { // 5. Suspend & Resume actor. // 6. Call to actor and validate memory and file counters. type actorLifecycleTestCase struct { - onCommit ateapipb.SnapshotContentScope + fidelity ateapipb.SnapshotFidelity wantMemoryAfterPause int wantFileAfterPause int wantMemoryAfterSuspend int @@ -524,9 +524,9 @@ type actorLifecycleTestCase struct { // wrong place, they would diverge. checkSecondFileCounter bool - // wantSnapshotContentScope, when set, asserts the content scope recorded + // wantSnapshotFidelity, when set, asserts the content scope recorded // on the ActorSnapshot the suspend produced. - wantSnapshotContentScope ateapipb.SnapshotContentScope + wantSnapshotFidelity ateapipb.SnapshotFidelity // microVMOnly skips the case outside the micro-VM environment. microVMOnly bool @@ -539,7 +539,7 @@ type actorLifecycleTestCase struct { suspendWhilePaused bool } -func runActorLifecycleTestCase(t *testing.T, prefix string, createTemplate func(context.Context, *testing.T, *e2e.Clients, *e2e.Namespace, ateapipb.SnapshotContentScope) (*ateapipb.ActorTemplate, error), tc actorLifecycleTestCase) { +func runActorLifecycleTestCase(t *testing.T, prefix string, createTemplate func(context.Context, *testing.T, *e2e.Clients, *e2e.Namespace, ateapipb.SnapshotFidelity) (*ateapipb.ActorTemplate, error), tc actorLifecycleTestCase) { // Create namespace nsObj := e2e.CreateNamespace(t) @@ -547,7 +547,7 @@ func runActorLifecycleTestCase(t *testing.T, prefix string, createTemplate func( clients := e2e.GetClients() // Create actor template. - at, err := createTemplate(ctx, t, clients, nsObj, tc.onCommit) + at, err := createTemplate(ctx, t, clients, nsObj, tc.fidelity) if err != nil { t.Fatalf("failed to initialize ActorTemplate: %v", err) } @@ -653,8 +653,8 @@ func runActorLifecycleTestCase(t *testing.T, prefix string, createTemplate func( } } - if tc.wantSnapshotContentScope != ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_UNSPECIFIED { - validateSnapshotContentScope(ctx, t, clients, actorID, tc.wantSnapshotContentScope) + if tc.wantSnapshotFidelity != ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED { + validateSnapshotFidelity(ctx, t, clients, actorID, tc.wantSnapshotFidelity) } // Resuming the actor @@ -676,9 +676,9 @@ func runActorLifecycleTestCase(t *testing.T, prefix string, createTemplate func( } } -// validateSnapshotContentScope asserts the content scope recorded on the +// validateSnapshotFidelity asserts the content scope recorded on the // suspended actor's external snapshot. -func validateSnapshotContentScope(ctx context.Context, t *testing.T, clients *e2e.Clients, actorID string, want ateapipb.SnapshotContentScope) { +func validateSnapshotFidelity(ctx context.Context, t *testing.T, clients *e2e.Clients, actorID string, want ateapipb.SnapshotFidelity) { t.Helper() actor, err := clients.SubstrateAPI.GetActor(ctx, &ateapipb.GetActorRequest{ Actor: &ateapipb.ObjectRef{Atespace: demoAtespace, Name: actorID}, @@ -689,7 +689,7 @@ func validateSnapshotContentScope(ctx context.Context, t *testing.T, clients *e2 if actor.GetStatus().GetExternalSnapshot().GetSnapshotUri() == "" { t.Fatal("suspended Actor has no external snapshot") } - if got := actor.GetStatus().GetExternalSnapshot().GetContentScope(); got != want { + if got := actor.GetStatus().GetExternalSnapshot().GetFidelity(); got != want { t.Errorf("snapshot %q content scope = %v, want %v", actor.GetStatus().GetExternalSnapshot().GetSnapshotUri(), got, want) } } @@ -1189,7 +1189,7 @@ func deletePausedActorAnyState(ctx context.Context, t *testing.T, clients *e2e.C return nil } -func createActorTemplateInternal(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj *e2e.Namespace, base string, onCommit ateapipb.SnapshotContentScope, modifyTemplate func(*ateapipb.ActorTemplate)) (*ateapipb.ActorTemplate, error) { +func createActorTemplateInternal(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj *e2e.Namespace, base string, fidelity ateapipb.SnapshotFidelity, modifyTemplate func(*ateapipb.ActorTemplate)) (*ateapipb.ActorTemplate, error) { env, err := e2e.CheckEnv("BUCKET_NAME") if err != nil { t.Fatalf("CheckEnv failed: %v", err) @@ -1206,19 +1206,19 @@ func createActorTemplateInternal(ctx context.Context, t *testing.T, clients *e2e PoolReplicas: 2, Labels: map[string]string{"demo": nsObj.Name}, SnapshotConfig: &ateapipb.SnapshotConfig{ - StorageLocation: "gs://" + env["BUCKET_NAME"] + "/ate-demo-" + name, - OnCommit: onCommit, + StorageLocation: "gs://" + env["BUCKET_NAME"] + "/ate-demo-" + name, + PreferredFidelity: fidelity, }, Modify: modifyTemplate, }) return at, nil } -func createActorTemplate(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj *e2e.Namespace, onCommit ateapipb.SnapshotContentScope) (*ateapipb.ActorTemplate, error) { - return createActorTemplateInternal(ctx, t, clients, nsObj, "counter", onCommit, nil) +func createActorTemplate(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj *e2e.Namespace, fidelity ateapipb.SnapshotFidelity) (*ateapipb.ActorTemplate, error) { + return createActorTemplateInternal(ctx, t, clients, nsObj, "counter", fidelity, nil) } -func createActorTemplateWithExternalVolume(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj *e2e.Namespace, onCommit ateapipb.SnapshotContentScope) (*ateapipb.ActorTemplate, error) { +func createActorTemplateWithExternalVolume(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj *e2e.Namespace, fidelity ateapipb.SnapshotFidelity) (*ateapipb.ActorTemplate, error) { scName := e2e.StorageClass if !hasStorageClass(ctx, clients, scName) { t.Fatalf("StorageClass %q not found in cluster; provide a valid --storage-class or install the CSI driver via --setup-csi", scName) @@ -1263,7 +1263,7 @@ func createActorTemplateWithExternalVolume(ctx context.Context, t *testing.T, cl }) } } - return createActorTemplateInternal(ctx, t, clients, nsObj, "counter-ext-vol", onCommit, modify) + return createActorTemplateInternal(ctx, t, clients, nsObj, "counter-ext-vol", fidelity, modify) } // secondDurableDirVolume is the extra durable-dir volume (and where the counter @@ -1278,7 +1278,7 @@ const ( // counter's second file counter at it, so both volumes are written on every // request. Only the micro-VM runtime accepts this: gVisor templates are still // capped at one durable-dir volume by the ActorTemplate CEL rules. -func createActorTemplateWithTwoDurableDirs(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj *e2e.Namespace, onCommit ateapipb.SnapshotContentScope) (*ateapipb.ActorTemplate, error) { +func createActorTemplateWithTwoDurableDirs(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj *e2e.Namespace, fidelity ateapipb.SnapshotFidelity) (*ateapipb.ActorTemplate, error) { modify := func(at *ateapipb.ActorTemplate) { for _, c := range at.GetContainers() { if c.GetName() != "counter" { @@ -1295,7 +1295,7 @@ func createActorTemplateWithTwoDurableDirs(ctx context.Context, t *testing.T, cl DurableDir: &ateapipb.DurableDirVolumeSource{}, }) } - return createActorTemplateInternal(ctx, t, clients, nsObj, "counter-two-durabledirs", onCommit, modify) + return createActorTemplateInternal(ctx, t, clients, nsObj, "counter-two-durabledirs", fidelity, modify) } func hasStorageClass(ctx context.Context, clients *e2e.Clients, name string) bool { @@ -1448,7 +1448,7 @@ func TestWorkerPodDeletion(t *testing.T) { clients := e2e.GetClients() // Create actor template. - at, err := createActorTemplate(ctx, t, clients, nsObj, ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL) + at, err := createActorTemplate(ctx, t, clients, nsObj, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY) if err != nil { t.Fatalf("failed to initialize ActorTemplate: %v", err) } @@ -1578,7 +1578,7 @@ func TestRevertCrashedActor(t *testing.T) { ctx := context.Background() clients := e2e.GetClients() - at, err := createActorTemplate(ctx, t, clients, nsObj, ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL) + at, err := createActorTemplate(ctx, t, clients, nsObj, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY) if err != nil { t.Fatalf("failed to initialize ActorTemplate: %v", err) } diff --git a/internal/e2e/suites/demo/termination_test.go b/internal/e2e/suites/demo/termination_test.go index 6f18ef6045..fdb7c72df9 100644 --- a/internal/e2e/suites/demo/termination_test.go +++ b/internal/e2e/suites/demo/termination_test.go @@ -42,7 +42,7 @@ func TestGracefulWorkerTermination(t *testing.T) { ctx := context.Background() clients := e2e.GetClients() - at, err := createActorTemplate(ctx, t, clients, nsObj, ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL) + at, err := createActorTemplate(ctx, t, clients, nsObj, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY) if err != nil { t.Fatalf("failed to initialize ActorTemplate: %v", err) } @@ -150,7 +150,7 @@ func TestGracefulWorkerTerminationSuspend(t *testing.T) { ctx := context.Background() clients := e2e.GetClients() - at, err := createActorTemplate(ctx, t, clients, nsObj, ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL) + at, err := createActorTemplate(ctx, t, clients, nsObj, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY) if err != nil { t.Fatalf("failed to initialize ActorTemplate: %v", err) } diff --git a/internal/e2e/suites/demo/updatetemplate_test.go b/internal/e2e/suites/demo/updatetemplate_test.go index 6da3badecf..dd507e9296 100644 --- a/internal/e2e/suites/demo/updatetemplate_test.go +++ b/internal/e2e/suites/demo/updatetemplate_test.go @@ -36,26 +36,26 @@ func TestUpdateTemplateLifecycle(t *testing.T) { t.Parallel() tests := []struct { name string - onCommit ateapipb.SnapshotContentScope + fidelity ateapipb.SnapshotFidelity }{ { - name: "onCommit:Data", - onCommit: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA, + name: "preferredFidelity:VOLUMES", + fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, }, { - name: "onCommit:Full", - onCommit: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, + name: "preferredFidelity:MEMORY", + fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { t.Parallel() - runUpdateTemplateTestCase(t, test.onCommit) + runUpdateTemplateTestCase(t, test.fidelity) }) } } -func runUpdateTemplateTestCase(t *testing.T, onCommit ateapipb.SnapshotContentScope) { +func runUpdateTemplateTestCase(t *testing.T, fidelity ateapipb.SnapshotFidelity) { nsObj := e2e.CreateNamespace(t) ctx := context.Background() clients := e2e.GetClients() @@ -70,8 +70,8 @@ func runUpdateTemplateTestCase(t *testing.T, onCommit ateapipb.SnapshotContentSc // in the durable dir is readable — the response's "file content" both // proves B's spec took effect and re-reads the preserved data. nameA, nameB := "update-a-"+nsObj.Name, "update-b-"+nsObj.Name - createdA := createUpdateTestTemplate(ctx, t, clients, nsObj, nameA, "update-a", env["BUCKET_NAME"], onCommit, nil) - createdB := createUpdateTestTemplate(ctx, t, clients, nsObj, nameB, "update-b", env["BUCKET_NAME"], onCommit, func(tmpl *ateapipb.ActorTemplate) { + createdA := createUpdateTestTemplate(ctx, t, clients, nsObj, nameA, "update-a", env["BUCKET_NAME"], fidelity, nil) + createdB := createUpdateTestTemplate(ctx, t, clients, nsObj, nameB, "update-b", env["BUCKET_NAME"], fidelity, func(tmpl *ateapipb.ActorTemplate) { ctr := tmpl.Containers[0] ctr.Command = append(ctr.Command, "--validate-existing-file-path=/home/counter/a.txt") }) @@ -192,7 +192,7 @@ func runUpdateTemplateTestCase(t *testing.T, onCommit ateapipb.SnapshotContentSc // committed snapshot from template A. Resuming from PAUSED restores the // local checkpoint (which was captured under template B, since templates // can only be updated while SUSPENDED) and preserves the in-memory counter - // when onCommit is FULL. + // when preferredFidelity is FULL. t.Logf("Pausing Actor %q under template B...", actorID) if _, err := clients.SubstrateAPI.PauseActor(ctx, &ateapipb.PauseActorRequest{ Actor: &ateapipb.ObjectRef{Atespace: demoAtespace, Name: actorID}, @@ -224,7 +224,7 @@ func runUpdateTemplateTestCase(t *testing.T, onCommit ateapipb.SnapshotContentSc t.Fatalf("failed to call actor after pause/resume under template B: %v", err) } wantMemAfterPause := 2 - if onCommit == ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA { + if fidelity == ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES { wantMemAfterPause = 1 } validateCounterResponse(t, resp, "after pause/resume under template B", wantMemAfterPause, 4) @@ -306,7 +306,7 @@ func runUpdateTemplateTestCase(t *testing.T, onCommit ateapipb.SnapshotContentSc // createUpdateTestTemplate creates a per-test WorkerPool plus a substrate // ActorTemplate copying the deployed counter fixture's resolved runtime, // capturing at the scope under test on both pause and commit. -func createUpdateTestTemplate(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj *e2e.Namespace, name, poolName, bucket string, onCommit ateapipb.SnapshotContentScope, modify func(*ateapipb.ActorTemplate)) *ateapipb.ActorTemplate { +func createUpdateTestTemplate(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj *e2e.Namespace, name, poolName, bucket string, fidelity ateapipb.SnapshotFidelity, modify func(*ateapipb.ActorTemplate)) *ateapipb.ActorTemplate { t.Helper() return e2e.CreateSubstrateCounterTemplate(ctx, t, clients, nsObj.Name, e2e.SubstrateTemplateOptions{ Atespace: demoAtespace, @@ -315,8 +315,8 @@ func createUpdateTestTemplate(ctx context.Context, t *testing.T, clients *e2e.Cl PoolReplicas: 2, Labels: map[string]string{"demo": nsObj.Name}, SnapshotConfig: &ateapipb.SnapshotConfig{ - StorageLocation: "gs://" + bucket + "/ate-demo-" + nsObj.Name, - OnCommit: onCommit, + StorageLocation: "gs://" + bucket + "/ate-demo-" + nsObj.Name, + PreferredFidelity: fidelity, }, Modify: modify, }) diff --git a/internal/e2e/suites/metrics/metrics_test.go b/internal/e2e/suites/metrics/metrics_test.go index 712148fd4d..7ab059e946 100644 --- a/internal/e2e/suites/metrics/metrics_test.go +++ b/internal/e2e/suites/metrics/metrics_test.go @@ -311,17 +311,17 @@ func validateSnapshotPhaseLabels(scrape string) error { } phase := extractLabelValue(line, "ate_snapshot_phase") kind := extractLabelValue(line, "ate_snapshot_kind") - scope := extractLabelValue(line, "ate_snapshot_scope") + fidelity := extractLabelValue(line, "ate_snapshot_fidelity") class := extractLabelValue(line, "ate_sandbox_class") if phase == "" { return fmt.Errorf("%s line is missing ate_snapshot_phase: %q", m, line) } - if kind != "" && scope != "" && class != "" { + if kind != "" && fidelity != "" && class != "" { labelled = true } } if !labelled { - return fmt.Errorf("no %s line carried all of ate_snapshot_kind, ate_snapshot_scope and ate_sandbox_class", m) + return fmt.Errorf("no %s line carried all of ate_snapshot_kind, ate_snapshot_fidelity and ate_sandbox_class", m) } } return nil diff --git a/internal/proto/ateletpb/atelet.pb.go b/internal/proto/ateletpb/atelet.pb.go index ea6f25b7bd..90b451c20d 100644 --- a/internal/proto/ateletpb/atelet.pb.go +++ b/internal/proto/ateletpb/atelet.pb.go @@ -140,58 +140,65 @@ func (CheckpointType) EnumDescriptor() ([]byte, []int) { return file_atelet_proto_rawDescGZIP(), []int{1} } -type SnapshotScope int32 +// SnapshotFidelity mirrors the public ateapi enum: the state layers a +// snapshot holds, lowest to highest, each including the ones below it. +type SnapshotFidelity int32 const ( // Not valid option; should never happen. - SnapshotScope_SNAPSHOT_SCOPE_UNSPECIFIED SnapshotScope = 0 - // Capture process memory plus the full filesystem delta on top of the OCI - // image (including any attached DurableDir volumes). - SnapshotScope_SNAPSHOT_SCOPE_FULL SnapshotScope = 1 + SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED SnapshotFidelity = 0 // Capture only the contents of attached volumes that support snapshots // (currently DurableDir-typed volumes). Memory and the rest of rootfs are // excluded. - SnapshotScope_SNAPSHOT_SCOPE_DATA SnapshotScope = 2 + SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES SnapshotFidelity = 1 + // Volumes plus the root filesystem changes made since boot. No runtime + // captures this yet; requests carrying it are rejected. + SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS SnapshotFidelity = 2 + // Capture process memory plus the full filesystem delta on top of the OCI + // image (including any attached DurableDir volumes). + SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY SnapshotFidelity = 3 ) -// Enum value maps for SnapshotScope. +// Enum value maps for SnapshotFidelity. var ( - SnapshotScope_name = map[int32]string{ - 0: "SNAPSHOT_SCOPE_UNSPECIFIED", - 1: "SNAPSHOT_SCOPE_FULL", - 2: "SNAPSHOT_SCOPE_DATA", - } - SnapshotScope_value = map[string]int32{ - "SNAPSHOT_SCOPE_UNSPECIFIED": 0, - "SNAPSHOT_SCOPE_FULL": 1, - "SNAPSHOT_SCOPE_DATA": 2, + SnapshotFidelity_name = map[int32]string{ + 0: "SNAPSHOT_FIDELITY_UNSPECIFIED", + 1: "SNAPSHOT_FIDELITY_VOLUMES", + 2: "SNAPSHOT_FIDELITY_ROOTFS", + 3: "SNAPSHOT_FIDELITY_MEMORY", + } + SnapshotFidelity_value = map[string]int32{ + "SNAPSHOT_FIDELITY_UNSPECIFIED": 0, + "SNAPSHOT_FIDELITY_VOLUMES": 1, + "SNAPSHOT_FIDELITY_ROOTFS": 2, + "SNAPSHOT_FIDELITY_MEMORY": 3, } ) -func (x SnapshotScope) Enum() *SnapshotScope { - p := new(SnapshotScope) +func (x SnapshotFidelity) Enum() *SnapshotFidelity { + p := new(SnapshotFidelity) *p = x return p } -func (x SnapshotScope) String() string { +func (x SnapshotFidelity) String() string { return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) } -func (SnapshotScope) Descriptor() protoreflect.EnumDescriptor { +func (SnapshotFidelity) Descriptor() protoreflect.EnumDescriptor { return file_atelet_proto_enumTypes[2].Descriptor() } -func (SnapshotScope) Type() protoreflect.EnumType { +func (SnapshotFidelity) Type() protoreflect.EnumType { return &file_atelet_proto_enumTypes[2] } -func (x SnapshotScope) Number() protoreflect.EnumNumber { +func (x SnapshotFidelity) Number() protoreflect.EnumNumber { return protoreflect.EnumNumber(x) } -// Deprecated: Use SnapshotScope.Descriptor instead. -func (SnapshotScope) EnumDescriptor() ([]byte, []int) { +// Deprecated: Use SnapshotFidelity.Descriptor instead. +func (SnapshotFidelity) EnumDescriptor() ([]byte, []int) { return file_atelet_proto_rawDescGZIP(), []int{2} } @@ -2522,8 +2529,8 @@ type CheckpointRequest struct { // *CheckpointRequest_LocalConfig // *CheckpointRequest_ExternalConfig Config isCheckpointRequest_Config `protobuf_oneof:"config"` - // What should be included in the checkpoint. - Scope SnapshotScope `protobuf:"varint,11,opt,name=scope,proto3,enum=atelet.SnapshotScope" json:"scope,omitempty"` + // Fidelity the checkpoint captures. + Fidelity SnapshotFidelity `protobuf:"varint,11,opt,name=fidelity,proto3,enum=atelet.SnapshotFidelity" json:"fidelity,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -2639,11 +2646,11 @@ func (x *CheckpointRequest) GetExternalConfig() *ExternalCheckpointConfiguration return nil } -func (x *CheckpointRequest) GetScope() SnapshotScope { +func (x *CheckpointRequest) GetFidelity() SnapshotFidelity { if x != nil { - return x.Scope + return x.Fidelity } - return SnapshotScope_SNAPSHOT_SCOPE_UNSPECIFIED + return SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED } type isCheckpointRequest_Config interface { @@ -2711,14 +2718,14 @@ type UploadPausedCheckpointRequest struct { LocalSnapshotName string `protobuf:"bytes,6,opt,name=local_snapshot_name,json=localSnapshotName,proto3" json:"local_snapshot_name,omitempty"` // Destination object-storage URI (the actor's in-progress snapshot URI). DestinationSnapshotUri string `protobuf:"bytes,7,opt,name=destination_snapshot_uri,json=destinationSnapshotUri,proto3" json:"destination_snapshot_uri,omitempty"` - // Scope the uploaded snapshot must have (the commit scope; FULL or DATA). - // The scope the pause checkpoint captured is not sent: atelet reads it from + // Fidelity the uploaded snapshot must have (MEMORY or VOLUMES). The + // fidelity the pause checkpoint captured is not sent: atelet reads it from // the local snapshot's own manifest, which is authoritative. When they - // differ, atelet converts where possible (a FULL capture to a DATA upload - // of the data-scope files ateom reported) and rejects otherwise. - DesiredScope SnapshotScope `protobuf:"varint,8,opt,name=desired_scope,json=desiredScope,proto3,enum=atelet.SnapshotScope" json:"desired_scope,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // differ, atelet converts where possible (a MEMORY capture to a VOLUMES + // upload of the volume files ateom reported) and rejects otherwise. + DesiredFidelity SnapshotFidelity `protobuf:"varint,8,opt,name=desired_fidelity,json=desiredFidelity,proto3,enum=atelet.SnapshotFidelity" json:"desired_fidelity,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *UploadPausedCheckpointRequest) Reset() { @@ -2800,11 +2807,11 @@ func (x *UploadPausedCheckpointRequest) GetDestinationSnapshotUri() string { return "" } -func (x *UploadPausedCheckpointRequest) GetDesiredScope() SnapshotScope { +func (x *UploadPausedCheckpointRequest) GetDesiredFidelity() SnapshotFidelity { if x != nil { - return x.DesiredScope + return x.DesiredFidelity } - return SnapshotScope_SNAPSHOT_SCOPE_UNSPECIFIED + return SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED } type UploadPausedCheckpointResponse struct { @@ -2860,13 +2867,13 @@ type RestoreRequest struct { // *RestoreRequest_LocalConfig // *RestoreRequest_ExternalConfig Config isRestoreRequest_Config `protobuf_oneof:"config"` - // What content to restore from the checkpoint. - Scope SnapshotScope `protobuf:"varint,11,opt,name=scope,proto3,enum=atelet.SnapshotScope" json:"scope,omitempty"` + // Fidelity to restore from the checkpoint. + Fidelity SnapshotFidelity `protobuf:"varint,11,opt,name=fidelity,proto3,enum=atelet.SnapshotFidelity" json:"fidelity,omitempty"` // When absent the actor has no egress: its TCP is captured and refused. EgressGateway *EgressGateway `protobuf:"bytes,13,opt,name=egress_gateway,json=egressGateway,proto3,oneof" json:"egress_gateway,omitempty"` // The actor's declared size, from the ActorTemplate's resource limits. For - // gVisor and micro-VM DATA-scope restores the sandbox is (re)sized to these; - // for a FULL micro-VM restore the size baked into the snapshot wins. Zero + // gVisor and micro-VM VOLUMES restores the sandbox is (re)sized to these; + // for a MEMORY micro-VM restore the size baked into the snapshot wins. Zero // means "unset": keep the runtime default. CpuMilli int64 `protobuf:"varint,14,opt,name=cpu_milli,json=cpuMilli,proto3" json:"cpu_milli,omitempty"` // CPU limit in millicores (1000 = one core). MemoryBytes int64 `protobuf:"varint,15,opt,name=memory_bytes,json=memoryBytes,proto3" json:"memory_bytes,omitempty"` // Memory limit in bytes. @@ -2988,11 +2995,11 @@ func (x *RestoreRequest) GetExternalConfig() *ExternalRestoreConfiguration { return nil } -func (x *RestoreRequest) GetScope() SnapshotScope { +func (x *RestoreRequest) GetFidelity() SnapshotFidelity { if x != nil { - return x.Scope + return x.Fidelity } - return SnapshotScope_SNAPSHOT_SCOPE_UNSPECIFIED + return SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED } func (x *RestoreRequest) GetEgressGateway() *EgressGateway { @@ -3240,7 +3247,7 @@ const file_atelet_proto_rawDesc = "" + "\x1fExternalCheckpointConfiguration\x12!\n" + "\fsnapshot_uri\x18\x01 \x01(\tR\vsnapshotUri\"A\n" + "\x1cExternalRestoreConfiguration\x12!\n" + - "\fsnapshot_uri\x18\x01 \x01(\tR\vsnapshotUri\"\xa9\x04\n" + + "\fsnapshot_uri\x18\x01 \x01(\tR\vsnapshotUri\"\xb2\x04\n" + "\x11CheckpointRequest\x12(\n" + "\x10target_ateom_uid\x18\x01 \x01(\tR\x0etargetAteomUid\x12\x1a\n" + "\batespace\x18\x02 \x01(\tR\batespace\x12\x1d\n" + @@ -3253,10 +3260,10 @@ const file_atelet_proto_rawDesc = "" + "\x04type\x18\b \x01(\x0e2\x16.atelet.CheckpointTypeR\x04type\x12I\n" + "\flocal_config\x18\t \x01(\v2$.atelet.LocalCheckpointConfigurationH\x00R\vlocalConfig\x12R\n" + "\x0fexternal_config\x18\n" + - " \x01(\v2'.atelet.ExternalCheckpointConfigurationH\x00R\x0eexternalConfig\x12+\n" + - "\x05scope\x18\v \x01(\x0e2\x15.atelet.SnapshotScopeR\x05scopeB\b\n" + + " \x01(\v2'.atelet.ExternalCheckpointConfigurationH\x00R\x0eexternalConfig\x124\n" + + "\bfidelity\x18\v \x01(\x0e2\x18.atelet.SnapshotFidelityR\bfidelityB\b\n" + "\x06config\"\x14\n" + - "\x12CheckpointResponse\"\x85\x03\n" + + "\x12CheckpointResponse\"\x8e\x03\n" + "\x1dUploadPausedCheckpointRequest\x12\x1a\n" + "\batespace\x18\x01 \x01(\tR\batespace\x12\x1d\n" + "\n" + @@ -3265,9 +3272,9 @@ const file_atelet_proto_rawDesc = "" + "\x17actor_template_atespace\x18\x04 \x01(\tR\x15actorTemplateAtespace\x12.\n" + "\x13actor_template_name\x18\x05 \x01(\tR\x11actorTemplateName\x12.\n" + "\x13local_snapshot_name\x18\x06 \x01(\tR\x11localSnapshotName\x128\n" + - "\x18destination_snapshot_uri\x18\a \x01(\tR\x16destinationSnapshotUri\x12:\n" + - "\rdesired_scope\x18\b \x01(\x0e2\x15.atelet.SnapshotScopeR\fdesiredScope\" \n" + - "\x1eUploadPausedCheckpointResponse\"\xf7\x05\n" + + "\x18destination_snapshot_uri\x18\a \x01(\tR\x16destinationSnapshotUri\x12C\n" + + "\x10desired_fidelity\x18\b \x01(\x0e2\x18.atelet.SnapshotFidelityR\x0fdesiredFidelity\" \n" + + "\x1eUploadPausedCheckpointResponse\"\x80\x06\n" + "\x0eRestoreRequest\x12(\n" + "\x10target_ateom_uid\x18\x01 \x01(\tR\x0etargetAteomUid\x12\x1a\n" + "\batespace\x18\x02 \x01(\tR\batespace\x12\x1d\n" + @@ -3280,8 +3287,8 @@ const file_atelet_proto_rawDesc = "" + "\x04type\x18\b \x01(\x0e2\x16.atelet.CheckpointTypeR\x04type\x12I\n" + "\flocal_config\x18\t \x01(\v2$.atelet.LocalCheckpointConfigurationH\x00R\vlocalConfig\x12O\n" + "\x0fexternal_config\x18\n" + - " \x01(\v2$.atelet.ExternalRestoreConfigurationH\x00R\x0eexternalConfig\x12+\n" + - "\x05scope\x18\v \x01(\x0e2\x15.atelet.SnapshotScopeR\x05scope\x12A\n" + + " \x01(\v2$.atelet.ExternalRestoreConfigurationH\x00R\x0eexternalConfig\x124\n" + + "\bfidelity\x18\v \x01(\x0e2\x18.atelet.SnapshotFidelityR\bfidelity\x12A\n" + "\x0eegress_gateway\x18\r \x01(\v2\x15.atelet.EgressGatewayH\x01R\regressGateway\x88\x01\x01\x12\x1b\n" + "\tcpu_milli\x18\x0e \x01(\x03R\bcpuMilli\x12!\n" + "\fmemory_bytes\x18\x0f \x01(\x03R\vmemoryBytes\x12<\n" + @@ -3297,11 +3304,12 @@ const file_atelet_proto_rawDesc = "" + "\x0eCheckpointType\x12\x1f\n" + "\x1bCHECKPOINT_TYPE_UNSPECIFIED\x10\x00\x12\x19\n" + "\x15CHECKPOINT_TYPE_LOCAL\x10\x01\x12\x1c\n" + - "\x18CHECKPOINT_TYPE_EXTERNAL\x10\x02*a\n" + - "\rSnapshotScope\x12\x1e\n" + - "\x1aSNAPSHOT_SCOPE_UNSPECIFIED\x10\x00\x12\x17\n" + - "\x13SNAPSHOT_SCOPE_FULL\x10\x01\x12\x17\n" + - "\x13SNAPSHOT_SCOPE_DATA\x10\x022\xa8\x02\n" + + "\x18CHECKPOINT_TYPE_EXTERNAL\x10\x02*\x90\x01\n" + + "\x10SnapshotFidelity\x12!\n" + + "\x1dSNAPSHOT_FIDELITY_UNSPECIFIED\x10\x00\x12\x1d\n" + + "\x19SNAPSHOT_FIDELITY_VOLUMES\x10\x01\x12\x1c\n" + + "\x18SNAPSHOT_FIDELITY_ROOTFS\x10\x02\x12\x1c\n" + + "\x18SNAPSHOT_FIDELITY_MEMORY\x10\x032\xa8\x02\n" + "\fAteomSupport\x12c\n" + "\x14MintActorCertificate\x12#.atelet.MintActorCertificateRequest\x1a$.atelet.MintActorCertificateResponse\"\x00\x12Q\n" + "\x0eRegisterWorker\x12\x1d.atelet.RegisterWorkerRequest\x1a\x1e.atelet.RegisterWorkerResponse\"\x00\x12`\n" + @@ -3331,7 +3339,7 @@ var file_atelet_proto_msgTypes = make([]protoimpl.MessageInfo, 50) var file_atelet_proto_goTypes = []any{ (ActorMetadataField)(0), // 0: atelet.ActorMetadataField (CheckpointType)(0), // 1: atelet.CheckpointType - (SnapshotScope)(0), // 2: atelet.SnapshotScope + (SnapshotFidelity)(0), // 2: atelet.SnapshotFidelity (*RegisterWorkerRequest)(nil), // 3: atelet.RegisterWorkerRequest (*HardwareIdentity)(nil), // 4: atelet.HardwareIdentity (*WorkerResources)(nil), // 5: atelet.WorkerResources @@ -3419,13 +3427,13 @@ var file_atelet_proto_depIdxs = []int32{ 1, // 32: atelet.CheckpointRequest.type:type_name -> atelet.CheckpointType 39, // 33: atelet.CheckpointRequest.local_config:type_name -> atelet.LocalCheckpointConfiguration 40, // 34: atelet.CheckpointRequest.external_config:type_name -> atelet.ExternalCheckpointConfiguration - 2, // 35: atelet.CheckpointRequest.scope:type_name -> atelet.SnapshotScope - 2, // 36: atelet.UploadPausedCheckpointRequest.desired_scope:type_name -> atelet.SnapshotScope + 2, // 35: atelet.CheckpointRequest.fidelity:type_name -> atelet.SnapshotFidelity + 2, // 36: atelet.UploadPausedCheckpointRequest.desired_fidelity:type_name -> atelet.SnapshotFidelity 20, // 37: atelet.RestoreRequest.spec:type_name -> atelet.WorkloadSpec 1, // 38: atelet.RestoreRequest.type:type_name -> atelet.CheckpointType 39, // 39: atelet.RestoreRequest.local_config:type_name -> atelet.LocalCheckpointConfiguration 41, // 40: atelet.RestoreRequest.external_config:type_name -> atelet.ExternalRestoreConfiguration - 2, // 41: atelet.RestoreRequest.scope:type_name -> atelet.SnapshotScope + 2, // 41: atelet.RestoreRequest.fidelity:type_name -> atelet.SnapshotFidelity 16, // 42: atelet.RestoreRequest.egress_gateway:type_name -> atelet.EgressGateway 19, // 43: atelet.RestoreRequest.sandbox_assets:type_name -> atelet.SandboxAssets 17, // 44: atelet.ArchAssets.FilesEntry.value:type_name -> atelet.AssetFile diff --git a/internal/proto/ateletpb/atelet.proto b/internal/proto/ateletpb/atelet.proto index 8164bc63d3..5f6f644848 100644 --- a/internal/proto/ateletpb/atelet.proto +++ b/internal/proto/ateletpb/atelet.proto @@ -449,16 +449,21 @@ enum CheckpointType { CHECKPOINT_TYPE_EXTERNAL = 2; } -enum SnapshotScope { +// SnapshotFidelity mirrors the public ateapi enum: the state layers a +// snapshot holds, lowest to highest, each including the ones below it. +enum SnapshotFidelity { // Not valid option; should never happen. - SNAPSHOT_SCOPE_UNSPECIFIED = 0; - // Capture process memory plus the full filesystem delta on top of the OCI - // image (including any attached DurableDir volumes). - SNAPSHOT_SCOPE_FULL = 1; + SNAPSHOT_FIDELITY_UNSPECIFIED = 0; // Capture only the contents of attached volumes that support snapshots // (currently DurableDir-typed volumes). Memory and the rest of rootfs are // excluded. - SNAPSHOT_SCOPE_DATA = 2; + SNAPSHOT_FIDELITY_VOLUMES = 1; + // Volumes plus the root filesystem changes made since boot. No runtime + // captures this yet; requests carrying it are rejected. + SNAPSHOT_FIDELITY_ROOTFS = 2; + // Capture process memory plus the full filesystem delta on top of the OCI + // image (including any attached DurableDir volumes). + SNAPSHOT_FIDELITY_MEMORY = 3; } // ExternalRestoreConfiguration is an external snapshot a restore reads. @@ -494,8 +499,8 @@ message CheckpointRequest { ExternalCheckpointConfiguration external_config = 10; } - // What should be included in the checkpoint. - SnapshotScope scope = 11; + // Fidelity the checkpoint captures. + SnapshotFidelity fidelity = 11; } message CheckpointResponse { @@ -517,12 +522,12 @@ message UploadPausedCheckpointRequest { // Destination object-storage URI (the actor's in-progress snapshot URI). string destination_snapshot_uri = 7; - // Scope the uploaded snapshot must have (the commit scope; FULL or DATA). - // The scope the pause checkpoint captured is not sent: atelet reads it from + // Fidelity the uploaded snapshot must have (MEMORY or VOLUMES). The + // fidelity the pause checkpoint captured is not sent: atelet reads it from // the local snapshot's own manifest, which is authoritative. When they - // differ, atelet converts where possible (a FULL capture to a DATA upload - // of the data-scope files ateom reported) and rejects otherwise. - SnapshotScope desired_scope = 8; + // differ, atelet converts where possible (a MEMORY capture to a VOLUMES + // upload of the volume files ateom reported) and rejects otherwise. + SnapshotFidelity desired_fidelity = 8; } message UploadPausedCheckpointResponse { @@ -548,15 +553,15 @@ message RestoreRequest { ExternalRestoreConfiguration external_config = 10; } - // What content to restore from the checkpoint. - SnapshotScope scope = 11; + // Fidelity to restore from the checkpoint. + SnapshotFidelity fidelity = 11; // When absent the actor has no egress: its TCP is captured and refused. optional EgressGateway egress_gateway = 13; // The actor's declared size, from the ActorTemplate's resource limits. For - // gVisor and micro-VM DATA-scope restores the sandbox is (re)sized to these; - // for a FULL micro-VM restore the size baked into the snapshot wins. Zero + // gVisor and micro-VM VOLUMES restores the sandbox is (re)sized to these; + // for a MEMORY micro-VM restore the size baked into the snapshot wins. Zero // means "unset": keep the runtime default. int64 cpu_milli = 14; // CPU limit in millicores (1000 = one core). int64 memory_bytes = 15; // Memory limit in bytes. diff --git a/internal/proto/ateompb/ateom.pb.go b/internal/proto/ateompb/ateom.pb.go index ea2b111232..d6c25de182 100644 --- a/internal/proto/ateompb/ateom.pb.go +++ b/internal/proto/ateompb/ateom.pb.go @@ -35,58 +35,65 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -type SnapshotScope int32 +// SnapshotFidelity mirrors the public ateapi enum: the state layers a +// snapshot holds, lowest to highest, each including the ones below it. +type SnapshotFidelity int32 const ( // Not valid option; should never happen. - SnapshotScope_SNAPSHOT_SCOPE_UNSPECIFIED SnapshotScope = 0 - // Capture process memory plus the full filesystem delta on top of the OCI - // image (including any attached DurableDir volumes). - SnapshotScope_SNAPSHOT_SCOPE_FULL SnapshotScope = 1 + SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED SnapshotFidelity = 0 // Capture only the contents of attached volumes that support snapshots // (currently DurableDir-typed volumes). Memory and the rest of rootfs are // excluded. - SnapshotScope_SNAPSHOT_SCOPE_DATA SnapshotScope = 2 + SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES SnapshotFidelity = 1 + // Volumes plus the root filesystem changes made since boot. No runtime + // captures this yet; requests carrying it are rejected. + SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS SnapshotFidelity = 2 + // Capture process memory plus the full filesystem delta on top of the OCI + // image (including any attached DurableDir volumes). + SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY SnapshotFidelity = 3 ) -// Enum value maps for SnapshotScope. +// Enum value maps for SnapshotFidelity. var ( - SnapshotScope_name = map[int32]string{ - 0: "SNAPSHOT_SCOPE_UNSPECIFIED", - 1: "SNAPSHOT_SCOPE_FULL", - 2: "SNAPSHOT_SCOPE_DATA", - } - SnapshotScope_value = map[string]int32{ - "SNAPSHOT_SCOPE_UNSPECIFIED": 0, - "SNAPSHOT_SCOPE_FULL": 1, - "SNAPSHOT_SCOPE_DATA": 2, + SnapshotFidelity_name = map[int32]string{ + 0: "SNAPSHOT_FIDELITY_UNSPECIFIED", + 1: "SNAPSHOT_FIDELITY_VOLUMES", + 2: "SNAPSHOT_FIDELITY_ROOTFS", + 3: "SNAPSHOT_FIDELITY_MEMORY", + } + SnapshotFidelity_value = map[string]int32{ + "SNAPSHOT_FIDELITY_UNSPECIFIED": 0, + "SNAPSHOT_FIDELITY_VOLUMES": 1, + "SNAPSHOT_FIDELITY_ROOTFS": 2, + "SNAPSHOT_FIDELITY_MEMORY": 3, } ) -func (x SnapshotScope) Enum() *SnapshotScope { - p := new(SnapshotScope) +func (x SnapshotFidelity) Enum() *SnapshotFidelity { + p := new(SnapshotFidelity) *p = x return p } -func (x SnapshotScope) String() string { +func (x SnapshotFidelity) String() string { return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) } -func (SnapshotScope) Descriptor() protoreflect.EnumDescriptor { +func (SnapshotFidelity) Descriptor() protoreflect.EnumDescriptor { return file_ateom_proto_enumTypes[0].Descriptor() } -func (SnapshotScope) Type() protoreflect.EnumType { +func (SnapshotFidelity) Type() protoreflect.EnumType { return &file_ateom_proto_enumTypes[0] } -func (x SnapshotScope) Number() protoreflect.EnumNumber { +func (x SnapshotFidelity) Number() protoreflect.EnumNumber { return protoreflect.EnumNumber(x) } -// Deprecated: Use SnapshotScope.Descriptor instead. -func (SnapshotScope) EnumDescriptor() ([]byte, []int) { +// Deprecated: Use SnapshotFidelity.Descriptor instead. +func (SnapshotFidelity) EnumDescriptor() ([]byte, []int) { return file_ateom_proto_rawDescGZIP(), []int{0} } @@ -1300,9 +1307,9 @@ type CheckpointWorkloadRequest struct { // runtime_asset_paths maps a runtime asset name to the local on-disk path // atelet fetched it to (see RunWorkloadRequest). Empty for gVisor. RuntimeAssetPaths map[string]string `protobuf:"bytes,9,rep,name=runtime_asset_paths,json=runtimeAssetPaths,proto3" json:"runtime_asset_paths,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` - // What content to include in the checkpoint. - Scope SnapshotScope `protobuf:"varint,10,opt,name=scope,proto3,enum=ateom.SnapshotScope" json:"scope,omitempty"` - ActorDirs *ActorDirs `protobuf:"bytes,11,opt,name=actor_dirs,json=actorDirs,proto3" json:"actor_dirs,omitempty"` + // Fidelity the checkpoint captures. + Fidelity SnapshotFidelity `protobuf:"varint,10,opt,name=fidelity,proto3,enum=ateom.SnapshotFidelity" json:"fidelity,omitempty"` + ActorDirs *ActorDirs `protobuf:"bytes,11,opt,name=actor_dirs,json=actorDirs,proto3" json:"actor_dirs,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -1400,11 +1407,11 @@ func (x *CheckpointWorkloadRequest) GetRuntimeAssetPaths() map[string]string { return nil } -func (x *CheckpointWorkloadRequest) GetScope() SnapshotScope { +func (x *CheckpointWorkloadRequest) GetFidelity() SnapshotFidelity { if x != nil { - return x.Scope + return x.Fidelity } - return SnapshotScope_SNAPSHOT_SCOPE_UNSPECIFIED + return SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED } func (x *CheckpointWorkloadRequest) GetActorDirs() *ActorDirs { @@ -1421,7 +1428,7 @@ type CheckpointWorkloadResponse struct { // its own set (gVisor's image files, cloud-hypervisor's snapshot set, ...). SnapshotFiles []string `protobuf:"bytes,1,rep,name=snapshot_files,json=snapshotFiles,proto3" json:"snapshot_files,omitempty"` // data_snapshot_files is the subset of snapshot_files that restores the - // actor at DATA scope on its own. Empty when the capture holds no durable data. + // actor at VOLUMES fidelity on its own. Empty when the capture holds no durable data. DataSnapshotFiles []string `protobuf:"bytes,2,rep,name=data_snapshot_files,json=dataSnapshotFiles,proto3" json:"data_snapshot_files,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache @@ -1486,13 +1493,13 @@ type RestoreWorkloadRequest struct { // runtime_asset_paths maps a runtime asset name to the local on-disk path // atelet fetched it to (see RunWorkloadRequest). Empty for gVisor. RuntimeAssetPaths map[string]string `protobuf:"bytes,9,rep,name=runtime_asset_paths,json=runtimeAssetPaths,proto3" json:"runtime_asset_paths,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` - // What content to restore from the snapshot. - Scope SnapshotScope `protobuf:"varint,10,opt,name=scope,proto3,enum=ateom.SnapshotScope" json:"scope,omitempty"` + // Fidelity to restore from the snapshot. + Fidelity SnapshotFidelity `protobuf:"varint,10,opt,name=fidelity,proto3,enum=ateom.SnapshotFidelity" json:"fidelity,omitempty"` // When absent the actor has no egress: its TCP is captured and refused. EgressGateway *EgressGateway `protobuf:"bytes,12,opt,name=egress_gateway,json=egressGateway,proto3,oneof" json:"egress_gateway,omitempty"` // The actor's declared size, from the ActorTemplate's resource limits. Used to - // (re)size the sandbox on a DATA-scope restore (fresh guest container). On a - // FULL micro-VM restore the size baked into the snapshot is authoritative and + // (re)size the sandbox on a VOLUMES restore (fresh guest container). On a + // MEMORY micro-VM restore the size baked into the snapshot is authoritative and // these are ignored. Zero means "unset": keep the runtime default. CpuMilli int64 `protobuf:"varint,14,opt,name=cpu_milli,json=cpuMilli,proto3" json:"cpu_milli,omitempty"` // CPU limit in millicores (1000 = one core). MemoryBytes int64 `protobuf:"varint,15,opt,name=memory_bytes,json=memoryBytes,proto3" json:"memory_bytes,omitempty"` // Memory limit in bytes. @@ -1597,11 +1604,11 @@ func (x *RestoreWorkloadRequest) GetRuntimeAssetPaths() map[string]string { return nil } -func (x *RestoreWorkloadRequest) GetScope() SnapshotScope { +func (x *RestoreWorkloadRequest) GetFidelity() SnapshotFidelity { if x != nil { - return x.Scope + return x.Fidelity } - return SnapshotScope_SNAPSHOT_SCOPE_UNSPECIFIED + return SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED } func (x *RestoreWorkloadRequest) GetEgressGateway() *EgressGateway { @@ -2126,7 +2133,7 @@ const file_ateom_proto_rawDesc = "" + "\rHTTPGetAction\x12\x12\n" + "\x04path\x18\x01 \x01(\tR\x04path\x12\x12\n" + "\x04port\x18\x02 \x01(\x05R\x04port\"\x15\n" + - "\x13RunWorkloadResponse\"\xd2\x04\n" + + "\x13RunWorkloadResponse\"\xdb\x04\n" + "\x19CheckpointWorkloadRequest\x12\x1a\n" + "\batespace\x18\x01 \x01(\tR\batespace\x12\x1d\n" + "\n" + @@ -2138,9 +2145,9 @@ const file_ateom_proto_rawDesc = "" + "runsc_path\x18\x06 \x01(\tR\trunscPath\x12'\n" + "\x04spec\x18\a \x01(\v2\x13.ateom.WorkloadSpecR\x04spec\x12!\n" + "\fsnapshot_uri\x18\b \x01(\tR\vsnapshotUri\x12g\n" + - "\x13runtime_asset_paths\x18\t \x03(\v27.ateom.CheckpointWorkloadRequest.RuntimeAssetPathsEntryR\x11runtimeAssetPaths\x12*\n" + - "\x05scope\x18\n" + - " \x01(\x0e2\x14.ateom.SnapshotScopeR\x05scope\x12/\n" + + "\x13runtime_asset_paths\x18\t \x03(\v27.ateom.CheckpointWorkloadRequest.RuntimeAssetPathsEntryR\x11runtimeAssetPaths\x123\n" + + "\bfidelity\x18\n" + + " \x01(\x0e2\x17.ateom.SnapshotFidelityR\bfidelity\x12/\n" + "\n" + "actor_dirs\x18\v \x01(\v2\x10.ateom.ActorDirsR\tactorDirs\x1aD\n" + "\x16RuntimeAssetPathsEntry\x12\x10\n" + @@ -2148,7 +2155,7 @@ const file_ateom_proto_rawDesc = "" + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"s\n" + "\x1aCheckpointWorkloadResponse\x12%\n" + "\x0esnapshot_files\x18\x01 \x03(\tR\rsnapshotFiles\x12.\n" + - "\x13data_snapshot_files\x18\x02 \x03(\tR\x11dataSnapshotFiles\"\x93\x06\n" + + "\x13data_snapshot_files\x18\x02 \x03(\tR\x11dataSnapshotFiles\"\x9c\x06\n" + "\x16RestoreWorkloadRequest\x12\x1a\n" + "\batespace\x18\x01 \x01(\tR\batespace\x12\x1d\n" + "\n" + @@ -2160,9 +2167,9 @@ const file_ateom_proto_rawDesc = "" + "runsc_path\x18\x06 \x01(\tR\trunscPath\x12'\n" + "\x04spec\x18\a \x01(\v2\x13.ateom.WorkloadSpecR\x04spec\x12!\n" + "\fsnapshot_uri\x18\b \x01(\tR\vsnapshotUri\x12d\n" + - "\x13runtime_asset_paths\x18\t \x03(\v24.ateom.RestoreWorkloadRequest.RuntimeAssetPathsEntryR\x11runtimeAssetPaths\x12*\n" + - "\x05scope\x18\n" + - " \x01(\x0e2\x14.ateom.SnapshotScopeR\x05scope\x12@\n" + + "\x13runtime_asset_paths\x18\t \x03(\v24.ateom.RestoreWorkloadRequest.RuntimeAssetPathsEntryR\x11runtimeAssetPaths\x123\n" + + "\bfidelity\x18\n" + + " \x01(\x0e2\x17.ateom.SnapshotFidelityR\bfidelity\x12@\n" + "\x0eegress_gateway\x18\f \x01(\v2\x14.ateom.EgressGatewayH\x00R\regressGateway\x88\x01\x01\x12\x1b\n" + "\tcpu_milli\x18\x0e \x01(\x03R\bcpuMilli\x12!\n" + "\fmemory_bytes\x18\x0f \x01(\x03R\vmemoryBytes\x12/\n" + @@ -2196,11 +2203,12 @@ const file_ateom_proto_rawDesc = "" + "\x06sample\x18\x01 \x01(\v2\x1a.ateom.WorkloadStatsSampleR\x06sample\"\x1f\n" + "\x1dGetActiveWorkloadStatsRequest\"V\n" + "\x1eGetActiveWorkloadStatsResponse\x124\n" + - "\asamples\x18\x01 \x03(\v2\x1a.ateom.WorkloadStatsSampleR\asamples*a\n" + - "\rSnapshotScope\x12\x1e\n" + - "\x1aSNAPSHOT_SCOPE_UNSPECIFIED\x10\x00\x12\x17\n" + - "\x13SNAPSHOT_SCOPE_FULL\x10\x01\x12\x17\n" + - "\x13SNAPSHOT_SCOPE_DATA\x10\x02*b\n" + + "\asamples\x18\x01 \x03(\v2\x1a.ateom.WorkloadStatsSampleR\asamples*\x90\x01\n" + + "\x10SnapshotFidelity\x12!\n" + + "\x1dSNAPSHOT_FIDELITY_UNSPECIFIED\x10\x00\x12\x1d\n" + + "\x19SNAPSHOT_FIDELITY_VOLUMES\x10\x01\x12\x1c\n" + + "\x18SNAPSHOT_FIDELITY_ROOTFS\x10\x02\x12\x1c\n" + + "\x18SNAPSHOT_FIDELITY_MEMORY\x10\x03*b\n" + "\fSandboxClass\x12\x1d\n" + "\x19SANDBOX_CLASS_UNSPECIFIED\x10\x00\x12\x18\n" + "\x14SANDBOX_CLASS_GVISOR\x10\x01\x12\x19\n" + @@ -2232,7 +2240,7 @@ func file_ateom_proto_rawDescGZIP() []byte { var file_ateom_proto_enumTypes = make([]protoimpl.EnumInfo, 3) var file_ateom_proto_msgTypes = make([]protoimpl.MessageInfo, 28) var file_ateom_proto_goTypes = []any{ - (SnapshotScope)(0), // 0: ateom.SnapshotScope + (SnapshotFidelity)(0), // 0: ateom.SnapshotFidelity (SandboxClass)(0), // 1: ateom.SandboxClass (StatsSource)(0), // 2: ateom.StatsSource (*ActorDirs)(nil), // 3: ateom.ActorDirs @@ -2282,11 +2290,11 @@ var file_ateom_proto_depIdxs = []int32{ 17, // 14: ateom.WakeupProbe.http_get:type_name -> ateom.HTTPGetAction 8, // 15: ateom.CheckpointWorkloadRequest.spec:type_name -> ateom.WorkloadSpec 29, // 16: ateom.CheckpointWorkloadRequest.runtime_asset_paths:type_name -> ateom.CheckpointWorkloadRequest.RuntimeAssetPathsEntry - 0, // 17: ateom.CheckpointWorkloadRequest.scope:type_name -> ateom.SnapshotScope + 0, // 17: ateom.CheckpointWorkloadRequest.fidelity:type_name -> ateom.SnapshotFidelity 3, // 18: ateom.CheckpointWorkloadRequest.actor_dirs:type_name -> ateom.ActorDirs 8, // 19: ateom.RestoreWorkloadRequest.spec:type_name -> ateom.WorkloadSpec 30, // 20: ateom.RestoreWorkloadRequest.runtime_asset_paths:type_name -> ateom.RestoreWorkloadRequest.RuntimeAssetPathsEntry - 0, // 21: ateom.RestoreWorkloadRequest.scope:type_name -> ateom.SnapshotScope + 0, // 21: ateom.RestoreWorkloadRequest.fidelity:type_name -> ateom.SnapshotFidelity 7, // 22: ateom.RestoreWorkloadRequest.egress_gateway:type_name -> ateom.EgressGateway 3, // 23: ateom.RestoreWorkloadRequest.actor_dirs:type_name -> ateom.ActorDirs 1, // 24: ateom.WorkloadStatsSample.sandbox_class:type_name -> ateom.SandboxClass diff --git a/internal/proto/ateompb/ateom.proto b/internal/proto/ateompb/ateom.proto index 1f5c49e1df..a742d3baaa 100644 --- a/internal/proto/ateompb/ateom.proto +++ b/internal/proto/ateompb/ateom.proto @@ -286,16 +286,21 @@ message HTTPGetAction { message RunWorkloadResponse { } -enum SnapshotScope { +// SnapshotFidelity mirrors the public ateapi enum: the state layers a +// snapshot holds, lowest to highest, each including the ones below it. +enum SnapshotFidelity { // Not valid option; should never happen. - SNAPSHOT_SCOPE_UNSPECIFIED = 0; - // Capture process memory plus the full filesystem delta on top of the OCI - // image (including any attached DurableDir volumes). - SNAPSHOT_SCOPE_FULL = 1; + SNAPSHOT_FIDELITY_UNSPECIFIED = 0; // Capture only the contents of attached volumes that support snapshots // (currently DurableDir-typed volumes). Memory and the rest of rootfs are // excluded. - SNAPSHOT_SCOPE_DATA = 2; + SNAPSHOT_FIDELITY_VOLUMES = 1; + // Volumes plus the root filesystem changes made since boot. No runtime + // captures this yet; requests carrying it are rejected. + SNAPSHOT_FIDELITY_ROOTFS = 2; + // Capture process memory plus the full filesystem delta on top of the OCI + // image (including any attached DurableDir volumes). + SNAPSHOT_FIDELITY_MEMORY = 3; } message CheckpointWorkloadRequest { @@ -325,8 +330,8 @@ message CheckpointWorkloadRequest { // atelet fetched it to (see RunWorkloadRequest). Empty for gVisor. map runtime_asset_paths = 9; - // What content to include in the checkpoint. - SnapshotScope scope = 10; + // Fidelity the checkpoint captures. + SnapshotFidelity fidelity = 10; ActorDirs actor_dirs = 11; } @@ -337,7 +342,7 @@ message CheckpointWorkloadResponse { repeated string snapshot_files = 1; // data_snapshot_files is the subset of snapshot_files that restores the - // actor at DATA scope on its own. Empty when the capture holds no durable data. + // actor at VOLUMES fidelity on its own. Empty when the capture holds no durable data. repeated string data_snapshot_files = 2; } @@ -360,15 +365,15 @@ message RestoreWorkloadRequest { // atelet fetched it to (see RunWorkloadRequest). Empty for gVisor. map runtime_asset_paths = 9; - // What content to restore from the snapshot. - SnapshotScope scope = 10; + // Fidelity to restore from the snapshot. + SnapshotFidelity fidelity = 10; // When absent the actor has no egress: its TCP is captured and refused. optional EgressGateway egress_gateway = 12; // The actor's declared size, from the ActorTemplate's resource limits. Used to - // (re)size the sandbox on a DATA-scope restore (fresh guest container). On a - // FULL micro-VM restore the size baked into the snapshot is authoritative and + // (re)size the sandbox on a VOLUMES restore (fresh guest container). On a + // MEMORY micro-VM restore the size baked into the snapshot is authoritative and // these are ignored. Zero means "unset": keep the runtime default. int64 cpu_milli = 14; // CPU limit in millicores (1000 = one core). int64 memory_bytes = 15; // Memory limit in bytes. diff --git a/internal/resources/validate.go b/internal/resources/validate.go index a5f7d3803b..855772d7b9 100644 --- a/internal/resources/validate.go +++ b/internal/resources/validate.go @@ -120,6 +120,24 @@ func ValidateActorDirs(actorDirs *ateompb.ActorDirs, fldPath *field.Path) field. return errs } +// ValidateSnapshotFidelity rejects a checkpoint or restore request whose +// fidelity no runtime can serve. ROOTFS is defined in the enum but no sandbox +// runtime captures rootfs changes without memory yet, so it is refused here +// as well as at template admission. +func ValidateSnapshotFidelity(fidelity ateompb.SnapshotFidelity, fldPath *field.Path) field.ErrorList { + switch fidelity { + case ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY: + return nil + case ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED: + return field.ErrorList{field.Required(fldPath, "")} + case ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS: + return field.ErrorList{field.Invalid(fldPath, fidelity.String(), "ROOTFS fidelity is not supported yet")} + default: + return field.ErrorList{field.NotSupported(fldPath, fidelity, + []string{ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES.String(), ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY.String()})} + } +} + func validateAbsDir(dir string, fldPath *field.Path) field.ErrorList { if dir == "" { return field.ErrorList{field.Required(fldPath, "")} diff --git a/internal/resources/validate_test.go b/internal/resources/validate_test.go index 290ded5eb7..aa14889dfa 100644 --- a/internal/resources/validate_test.go +++ b/internal/resources/validate_test.go @@ -324,6 +324,32 @@ func testActorDirs() *ateompb.ActorDirs { } } +func TestValidateSnapshotFidelity(t *testing.T) { + for _, tc := range []struct { + name string + fidelity ateompb.SnapshotFidelity + wantErr bool + }{ + {"volumes", ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, false}, + {"memory", ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, false}, + {"unspecified", ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED, true}, + {"rootfs not supported yet", ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS, true}, + {"outside the enum", ateompb.SnapshotFidelity(99), true}, + } { + t.Run(tc.name, func(t *testing.T) { + errs := ValidateSnapshotFidelity(tc.fidelity, field.NewPath("fidelity")) + if gotErr := len(errs) != 0; gotErr != tc.wantErr { + t.Fatalf("ValidateSnapshotFidelity(%v) = %v, wantErr %t", tc.fidelity, errs, tc.wantErr) + } + for _, e := range errs { + if e.Field != "fidelity" { + t.Errorf("error names field %q, want %q", e.Field, "fidelity") + } + } + }) + } +} + func TestValidateActorDirs(t *testing.T) { for _, tc := range []struct { name string diff --git a/pkg/proto/ateapipb/ateapi.pb.go b/pkg/proto/ateapipb/ateapi.pb.go index c73e198052..95fc2a114a 100644 --- a/pkg/proto/ateapipb/ateapi.pb.go +++ b/pkg/proto/ateapipb/ateapi.pb.go @@ -36,55 +36,63 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -type SnapshotContentScope int32 +// SnapshotFidelity orders the state layers a snapshot can hold, lowest to +// highest. Each level includes the ones below it. +type SnapshotFidelity int32 const ( - // Unspecified snapshot content scope. - SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_UNSPECIFIED SnapshotContentScope = 0 - // Captures process memory, root filesystem changes, and durable data. - SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL SnapshotContentScope = 1 - // Captures durable data without process memory or root filesystem changes. - SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA SnapshotContentScope = 2 // Keep this in sync with the maximums on fields of this type. + SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED SnapshotFidelity = 0 + // Durable volumes only. The actor resumes by cold-booting its containers + // from the OCI image with the volumes restored. + SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES SnapshotFidelity = 1 + // Volumes plus the root filesystem changes made since boot. Not yet + // supported by any sandbox runtime; templates may not request it. + SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS SnapshotFidelity = 2 + // Volumes, root filesystem changes, and process memory. The actor resumes + // where it left off. + SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY SnapshotFidelity = 3 // Keep this in sync with the maximums on fields of this type. ) -// Enum value maps for SnapshotContentScope. +// Enum value maps for SnapshotFidelity. var ( - SnapshotContentScope_name = map[int32]string{ - 0: "SNAPSHOT_CONTENT_SCOPE_UNSPECIFIED", - 1: "SNAPSHOT_CONTENT_SCOPE_FULL", - 2: "SNAPSHOT_CONTENT_SCOPE_DATA", - } - SnapshotContentScope_value = map[string]int32{ - "SNAPSHOT_CONTENT_SCOPE_UNSPECIFIED": 0, - "SNAPSHOT_CONTENT_SCOPE_FULL": 1, - "SNAPSHOT_CONTENT_SCOPE_DATA": 2, + SnapshotFidelity_name = map[int32]string{ + 0: "SNAPSHOT_FIDELITY_UNSPECIFIED", + 1: "SNAPSHOT_FIDELITY_VOLUMES", + 2: "SNAPSHOT_FIDELITY_ROOTFS", + 3: "SNAPSHOT_FIDELITY_MEMORY", + } + SnapshotFidelity_value = map[string]int32{ + "SNAPSHOT_FIDELITY_UNSPECIFIED": 0, + "SNAPSHOT_FIDELITY_VOLUMES": 1, + "SNAPSHOT_FIDELITY_ROOTFS": 2, + "SNAPSHOT_FIDELITY_MEMORY": 3, } ) -func (x SnapshotContentScope) Enum() *SnapshotContentScope { - p := new(SnapshotContentScope) +func (x SnapshotFidelity) Enum() *SnapshotFidelity { + p := new(SnapshotFidelity) *p = x return p } -func (x SnapshotContentScope) String() string { +func (x SnapshotFidelity) String() string { return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) } -func (SnapshotContentScope) Descriptor() protoreflect.EnumDescriptor { +func (SnapshotFidelity) Descriptor() protoreflect.EnumDescriptor { return file_ateapi_proto_enumTypes[0].Descriptor() } -func (SnapshotContentScope) Type() protoreflect.EnumType { +func (SnapshotFidelity) Type() protoreflect.EnumType { return &file_ateapi_proto_enumTypes[0] } -func (x SnapshotContentScope) Number() protoreflect.EnumNumber { +func (x SnapshotFidelity) Number() protoreflect.EnumNumber { return protoreflect.EnumNumber(x) } -// Deprecated: Use SnapshotContentScope.Descriptor instead. -func (SnapshotContentScope) EnumDescriptor() ([]byte, []int) { +// Deprecated: Use SnapshotFidelity.Descriptor instead. +func (SnapshotFidelity) EnumDescriptor() ([]byte, []int) { return file_ateapi_proto_rawDescGZIP(), []int{0} } @@ -429,12 +437,12 @@ type ExternalSnapshot struct { // +k8s:required // +k8s:maxLength=2048 # the template's storage_location bound plus the owner prefix and snapshot name SnapshotUri string `protobuf:"bytes,1,opt,name=snapshot_uri,json=snapshotUri,proto3" json:"snapshot_uri,omitempty"` - // content_scope is what the snapshot captured. + // fidelity is what the snapshot holds. // // +k8s:optional // +k8s:minimum=1 - // +k8s:maximum=2 # keep this in sync with the SnapshotContentScope enum - ContentScope SnapshotContentScope `protobuf:"varint,2,opt,name=content_scope,json=contentScope,proto3,enum=ateapi.SnapshotContentScope" json:"content_scope,omitempty"` + // +k8s:maximum=3 # keep this in sync with the SnapshotFidelity enum + Fidelity SnapshotFidelity `protobuf:"varint,2,opt,name=fidelity,proto3,enum=ateapi.SnapshotFidelity" json:"fidelity,omitempty"` // UID of the ActorTemplate whose sandbox this snapshot's guest state was // captured from. // @@ -482,11 +490,11 @@ func (x *ExternalSnapshot) GetSnapshotUri() string { return "" } -func (x *ExternalSnapshot) GetContentScope() SnapshotContentScope { +func (x *ExternalSnapshot) GetFidelity() SnapshotFidelity { if x != nil { - return x.ContentScope + return x.Fidelity } - return SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_UNSPECIFIED + return SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED } func (x *ExternalSnapshot) GetActorTemplateUid() string { @@ -505,13 +513,13 @@ type LocalSnapshot struct { // +k8s:optional // +k8s:format=k8s-short-name SnapshotName string `protobuf:"bytes,1,opt,name=snapshot_name,json=snapshotName,proto3" json:"snapshot_name,omitempty"` - // Scope the pause checkpoint captured (the template's onCommit at pause - // time). + // Fidelity the pause checkpoint captured: the template's preferredFidelity + // at pause time. // // +k8s:optional // +k8s:minimum=1 - // +k8s:maximum=2 # keep this in sync with the SnapshotContentScope enum - ContentScope SnapshotContentScope `protobuf:"varint,2,opt,name=content_scope,json=contentScope,proto3,enum=ateapi.SnapshotContentScope" json:"content_scope,omitempty"` + // +k8s:maximum=3 # keep this in sync with the SnapshotFidelity enum + Fidelity SnapshotFidelity `protobuf:"varint,2,opt,name=fidelity,proto3,enum=ateapi.SnapshotFidelity" json:"fidelity,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -553,11 +561,11 @@ func (x *LocalSnapshot) GetSnapshotName() string { return "" } -func (x *LocalSnapshot) GetContentScope() SnapshotContentScope { +func (x *LocalSnapshot) GetFidelity() SnapshotFidelity { if x != nil { - return x.ContentScope + return x.Fidelity } - return SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_UNSPECIFIED + return SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED } // Selector matches worker pools by label. @@ -2773,28 +2781,32 @@ func (x *SandboxConfig) GetConfigName() string { } // SnapshotConfig selects what actor snapshots capture and where they are -// stored. A FULL snapshot resumes from its own content. A DATA snapshot -// resumes by starting the containers afresh from the OCI image, with the -// durable-dir volumes populated from the snapshot. +// stored. type SnapshotConfig struct { state protoimpl.MessageState `protogen:"open.v1"` - // on_commit defines the scope of every snapshot taken of an actor, both - // the node-local one captured on pause and the one uploaded on suspend. - // Defaults to FULL when unset. - // - // +k8s:required - // +k8s:minimum=1 - // +k8s:maximum=2 # keep this in sync with the SnapshotContentScope enum - OnCommit SnapshotContentScope `protobuf:"varint,2,opt,name=on_commit,json=onCommit,proto3,enum=ateapi.SnapshotContentScope" json:"on_commit,omitempty"` // storage_location is the base object-storage URI snapshots of actors on // this version are stored under. Required. // // +k8s:required // +k8s:maxLength=1024 // +k8s:customValidation # Validate URI - StorageLocation string `protobuf:"bytes,4,opt,name=storage_location,json=storageLocation,proto3" json:"storage_location,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + StorageLocation string `protobuf:"bytes,1,opt,name=storage_location,json=storageLocation,proto3" json:"storage_location,omitempty"` + // preferred_fidelity is the highest fidelity the system packages when it + // snapshots the actor, on pause (node-local) and on suspend (uploaded). + // It is a best-effort ceiling, not a guarantee: the system never packages + // more than this level, and it may package less when a layer cannot be + // captured or is too expensive to capture at the time, for example memory + // under host pressure. A process resumed in place on its worker keeps its + // memory regardless of this setting. ROOTFS is not supported yet and is + // rejected. Defaults to MEMORY when unset. + // + // +k8s:required + // +k8s:minimum=1 + // +k8s:maximum=3 # keep this in sync with the SnapshotFidelity enum + // +k8s:customValidation # ROOTFS is not supported yet + PreferredFidelity SnapshotFidelity `protobuf:"varint,2,opt,name=preferred_fidelity,json=preferredFidelity,proto3,enum=ateapi.SnapshotFidelity" json:"preferred_fidelity,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *SnapshotConfig) Reset() { @@ -2827,18 +2839,18 @@ func (*SnapshotConfig) Descriptor() ([]byte, []int) { return file_ateapi_proto_rawDescGZIP(), []int{29} } -func (x *SnapshotConfig) GetOnCommit() SnapshotContentScope { +func (x *SnapshotConfig) GetStorageLocation() string { if x != nil { - return x.OnCommit + return x.StorageLocation } - return SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_UNSPECIFIED + return "" } -func (x *SnapshotConfig) GetStorageLocation() string { +func (x *SnapshotConfig) GetPreferredFidelity() SnapshotFidelity { if x != nil { - return x.StorageLocation + return x.PreferredFidelity } - return "" + return SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED } // Container is a single application container of an ActorTemplate. @@ -8024,14 +8036,14 @@ var File_ateapi_proto protoreflect.FileDescriptor const file_ateapi_proto_rawDesc = "" + "\n" + - "\fateapi.proto\x12\x06ateapi\x1a\x1fgoogle/protobuf/timestamp.proto\"\xa6\x01\n" + + "\fateapi.proto\x12\x06ateapi\x1a\x1fgoogle/protobuf/timestamp.proto\"\x99\x01\n" + "\x10ExternalSnapshot\x12!\n" + - "\fsnapshot_uri\x18\x01 \x01(\tR\vsnapshotUri\x12A\n" + - "\rcontent_scope\x18\x02 \x01(\x0e2\x1c.ateapi.SnapshotContentScopeR\fcontentScope\x12,\n" + - "\x12actor_template_uid\x18\x03 \x01(\tR\x10actorTemplateUid\"w\n" + + "\fsnapshot_uri\x18\x01 \x01(\tR\vsnapshotUri\x124\n" + + "\bfidelity\x18\x02 \x01(\x0e2\x18.ateapi.SnapshotFidelityR\bfidelity\x12,\n" + + "\x12actor_template_uid\x18\x03 \x01(\tR\x10actorTemplateUid\"j\n" + "\rLocalSnapshot\x12#\n" + - "\rsnapshot_name\x18\x01 \x01(\tR\fsnapshotName\x12A\n" + - "\rcontent_scope\x18\x02 \x01(\x0e2\x1c.ateapi.SnapshotContentScopeR\fcontentScope\"\x90\x01\n" + + "\rsnapshot_name\x18\x01 \x01(\tR\fsnapshotName\x124\n" + + "\bfidelity\x18\x02 \x01(\x0e2\x18.ateapi.SnapshotFidelityR\bfidelity\"\x90\x01\n" + "\bSelector\x12D\n" + "\fmatch_labels\x18\x01 \x03(\v2!.ateapi.Selector.MatchLabelsEntryR\vmatchLabels\x1a>\n" + "\x10MatchLabelsEntry\x12\x10\n" + @@ -8170,10 +8182,10 @@ const file_ateapi_proto_rawDesc = "" + "\rSandboxConfig\x129\n" + "\rsandbox_class\x18\x01 \x01(\x0e2\x14.ateapi.SandboxClassR\fsandboxClass\x12\x1f\n" + "\vconfig_name\x18\x02 \x01(\tR\n" + - "configName\"v\n" + - "\x0eSnapshotConfig\x129\n" + - "\ton_commit\x18\x02 \x01(\x0e2\x1c.ateapi.SnapshotContentScopeR\bonCommit\x12)\n" + - "\x10storage_location\x18\x04 \x01(\tR\x0fstorageLocation\"\xf5\x02\n" + + "configName\"\x84\x01\n" + + "\x0eSnapshotConfig\x12)\n" + + "\x10storage_location\x18\x01 \x01(\tR\x0fstorageLocation\x12G\n" + + "\x12preferred_fidelity\x18\x02 \x01(\x0e2\x18.ateapi.SnapshotFidelityR\x11preferredFidelity\"\xf5\x02\n" + "\tContainer\x12\x12\n" + "\x04name\x18\x01 \x01(\tR\x04name\x12\x14\n" + "\x05image\x18\x02 \x01(\tR\x05image\x12\x18\n" + @@ -8442,11 +8454,12 @@ const file_ateapi_proto_rawDesc = "" + "\raccess_policy\x18\x02 \x01(\v2\x14.ateapi.AccessPolicyR\faccessPolicy\"\x83\x01\n" + "!DeleteAtespaceAccessPolicyRequest\x12-\n" + "\batespace\x18\x01 \x01(\v2\x11.ateapi.ObjectRefR\batespace\x12/\n" + - "\aoptions\x18\x02 \x01(\v2\x15.ateapi.DeleteOptionsR\aoptions*\x80\x01\n" + - "\x14SnapshotContentScope\x12&\n" + - "\"SNAPSHOT_CONTENT_SCOPE_UNSPECIFIED\x10\x00\x12\x1f\n" + - "\x1bSNAPSHOT_CONTENT_SCOPE_FULL\x10\x01\x12\x1f\n" + - "\x1bSNAPSHOT_CONTENT_SCOPE_DATA\x10\x02*V\n" + + "\aoptions\x18\x02 \x01(\v2\x15.ateapi.DeleteOptionsR\aoptions*\x90\x01\n" + + "\x10SnapshotFidelity\x12!\n" + + "\x1dSNAPSHOT_FIDELITY_UNSPECIFIED\x10\x00\x12\x1d\n" + + "\x19SNAPSHOT_FIDELITY_VOLUMES\x10\x01\x12\x1c\n" + + "\x18SNAPSHOT_FIDELITY_ROOTFS\x10\x02\x12\x1c\n" + + "\x18SNAPSHOT_FIDELITY_MEMORY\x10\x03*V\n" + "\bTagScope\x12\x19\n" + "\x15TAG_SCOPE_UNSPECIFIED\x10\x00\x12\x16\n" + "\x12TAG_SCOPE_ATESPACE\x10\x01\x12\x17\n" + @@ -8541,7 +8554,7 @@ func file_ateapi_proto_rawDescGZIP() []byte { var file_ateapi_proto_enumTypes = make([]protoimpl.EnumInfo, 7) var file_ateapi_proto_msgTypes = make([]protoimpl.MessageInfo, 118) var file_ateapi_proto_goTypes = []any{ - (SnapshotContentScope)(0), // 0: ateapi.SnapshotContentScope + (SnapshotFidelity)(0), // 0: ateapi.SnapshotFidelity (TagScope)(0), // 1: ateapi.TagScope (ActorState)(0), // 2: ateapi.ActorState (SandboxClass)(0), // 3: ateapi.SandboxClass @@ -8669,8 +8682,8 @@ var file_ateapi_proto_goTypes = []any{ (*timestamppb.Timestamp)(nil), // 125: google.protobuf.Timestamp } var file_ateapi_proto_depIdxs = []int32{ - 0, // 0: ateapi.ExternalSnapshot.content_scope:type_name -> ateapi.SnapshotContentScope - 0, // 1: ateapi.LocalSnapshot.content_scope:type_name -> ateapi.SnapshotContentScope + 0, // 0: ateapi.ExternalSnapshot.fidelity:type_name -> ateapi.SnapshotFidelity + 0, // 1: ateapi.LocalSnapshot.fidelity:type_name -> ateapi.SnapshotFidelity 121, // 2: ateapi.Selector.match_labels:type_name -> ateapi.Selector.MatchLabelsEntry 125, // 3: ateapi.ResourceMetadata.create_time:type_name -> google.protobuf.Timestamp 125, // 4: ateapi.ResourceMetadata.update_time:type_name -> google.protobuf.Timestamp @@ -8721,7 +8734,7 @@ var file_ateapi_proto_depIdxs = []int32{ 125, // 49: ateapi.GoldenSnapshotStatus.take_golden_snapshot_at:type_name -> google.protobuf.Timestamp 33, // 50: ateapi.ActorTemplateStatus.golden_snapshot_status:type_name -> ateapi.GoldenSnapshotStatus 3, // 51: ateapi.SandboxConfig.sandbox_class:type_name -> ateapi.SandboxClass - 0, // 52: ateapi.SnapshotConfig.on_commit:type_name -> ateapi.SnapshotContentScope + 0, // 52: ateapi.SnapshotConfig.preferred_fidelity:type_name -> ateapi.SnapshotFidelity 40, // 53: ateapi.Container.env:type_name -> ateapi.EnvVar 41, // 54: ateapi.Container.wakeup_probe:type_name -> ateapi.ContainerWakeupProbe 52, // 55: ateapi.Container.volume_mounts:type_name -> ateapi.VolumeMount diff --git a/pkg/proto/ateapipb/ateapi.proto b/pkg/proto/ateapipb/ateapi.proto index c552b21614..8c4c52081d 100644 --- a/pkg/proto/ateapipb/ateapi.proto +++ b/pkg/proto/ateapipb/ateapi.proto @@ -179,12 +179,12 @@ message ExternalSnapshot { // +k8s:maxLength=2048 # the template's storage_location bound plus the owner prefix and snapshot name string snapshot_uri = 1; - // content_scope is what the snapshot captured. + // fidelity is what the snapshot holds. // // +k8s:optional // +k8s:minimum=1 - // +k8s:maximum=2 # keep this in sync with the SnapshotContentScope enum - SnapshotContentScope content_scope = 2; + // +k8s:maximum=3 # keep this in sync with the SnapshotFidelity enum + SnapshotFidelity fidelity = 2; // UID of the ActorTemplate whose sandbox this snapshot's guest state was // captured from. @@ -203,22 +203,28 @@ message LocalSnapshot { // +k8s:format=k8s-short-name string snapshot_name = 1; - // Scope the pause checkpoint captured (the template's onCommit at pause - // time). + // Fidelity the pause checkpoint captured: the template's preferredFidelity + // at pause time. // // +k8s:optional // +k8s:minimum=1 - // +k8s:maximum=2 # keep this in sync with the SnapshotContentScope enum - SnapshotContentScope content_scope = 2; -} - -enum SnapshotContentScope { - // Unspecified snapshot content scope. - SNAPSHOT_CONTENT_SCOPE_UNSPECIFIED = 0; - // Captures process memory, root filesystem changes, and durable data. - SNAPSHOT_CONTENT_SCOPE_FULL = 1; - // Captures durable data without process memory or root filesystem changes. - SNAPSHOT_CONTENT_SCOPE_DATA = 2; + // +k8s:maximum=3 # keep this in sync with the SnapshotFidelity enum + SnapshotFidelity fidelity = 2; +} + +// SnapshotFidelity orders the state layers a snapshot can hold, lowest to +// highest. Each level includes the ones below it. +enum SnapshotFidelity { + SNAPSHOT_FIDELITY_UNSPECIFIED = 0; + // Durable volumes only. The actor resumes by cold-booting its containers + // from the OCI image with the volumes restored. + SNAPSHOT_FIDELITY_VOLUMES = 1; + // Volumes plus the root filesystem changes made since boot. Not yet + // supported by any sandbox runtime; templates may not request it. + SNAPSHOT_FIDELITY_ROOTFS = 2; + // Volumes, root filesystem changes, and process memory. The actor resumes + // where it left off. + SNAPSHOT_FIDELITY_MEMORY = 3; // Keep this in sync with the maximums on fields of this type. } @@ -1052,26 +1058,30 @@ message SandboxConfig { } // SnapshotConfig selects what actor snapshots capture and where they are -// stored. A FULL snapshot resumes from its own content. A DATA snapshot -// resumes by starting the containers afresh from the OCI image, with the -// durable-dir volumes populated from the snapshot. +// stored. message SnapshotConfig { - // on_commit defines the scope of every snapshot taken of an actor, both - // the node-local one captured on pause and the one uploaded on suspend. - // Defaults to FULL when unset. - // - // +k8s:required - // +k8s:minimum=1 - // +k8s:maximum=2 # keep this in sync with the SnapshotContentScope enum - SnapshotContentScope on_commit = 2; - // storage_location is the base object-storage URI snapshots of actors on // this version are stored under. Required. // // +k8s:required // +k8s:maxLength=1024 // +k8s:customValidation # Validate URI - string storage_location = 4; + string storage_location = 1; + + // preferred_fidelity is the highest fidelity the system packages when it + // snapshots the actor, on pause (node-local) and on suspend (uploaded). + // It is a best-effort ceiling, not a guarantee: the system never packages + // more than this level, and it may package less when a layer cannot be + // captured or is too expensive to capture at the time, for example memory + // under host pressure. A process resumed in place on its worker keeps its + // memory regardless of this setting. ROOTFS is not supported yet and is + // rejected. Defaults to MEMORY when unset. + // + // +k8s:required + // +k8s:minimum=1 + // +k8s:maximum=3 # keep this in sync with the SnapshotFidelity enum + // +k8s:customValidation # ROOTFS is not supported yet + SnapshotFidelity preferred_fidelity = 2; } // Container is a single application container of an ActorTemplate. diff --git a/tools/apitool/exemptions.json b/tools/apitool/exemptions.json index 6d77cf51c0..804256c62f 100644 --- a/tools/apitool/exemptions.json +++ b/tools/apitool/exemptions.json @@ -324,11 +324,6 @@ "subject": "ateapi.ResumeActorResponse.actor", "message": "field has no doc comment" }, - { - "rule": "documented", - "subject": "ateapi.SnapshotContentScope", - "message": "enum has no doc comment" - }, { "rule": "documented", "subject": "ateapi.SuspendActorRequest", From 4baf18fb035a51882bb0832025b0aef2f566eb0c Mon Sep 17 00:00:00 2001 From: Zoe Zhao <35669350+zoez7@users.noreply.github.com> Date: Fri, 9 Oct 2026 01:26:38 +0000 Subject: [PATCH 21/44] Add sandboxClasses with an optional SandboxConfig ref to WorkerPool (#2332) WorkerPool now declares its sandbox class as `spec.sandboxClasses`, with an optional reference to a SandboxConfig: ```yaml spec: sandboxClasses: - name: gvisor configRef: name: gvisor-default ``` - `spec.sandboxClass` is replaced by `spec.sandboxClasses`, a list of `{name, configRef}`. The list is required and must have exactly one entry for now. `name` is `gvisor` or `microvm` and has no default. - `configRef` is optional and names a cluster-scoped SandboxConfig. Nothing reads it yet: actors still get their sandbox binaries from the ActorTemplate's SandboxConfig. - Code that needs the pool's class calls `WorkerPoolSpec.DefaultSandboxClass()`. #### Breaking change Every WorkerPool must now set `spec.sandboxClasses`. Once the CRD is updated, `spec.sandboxClass` is pruned, so re-apply each existing pool with `sandboxClasses` set (`- name: gvisor` for pools that used the old default). - [ ] Tests pass - [x] Appropriate changes to documentation are included in the PR --- .golangci-kal.yaml | 2 +- benchmarking/automation/README.md | 2 +- .../workloads/manifests/workloads.yaml.tmpl | 3 +- .../controlapi/functionaltest/common_test.go | 7 +- cmd/ateapi/internal/controlapi/metrics.go | 2 +- .../internal/controlapi/metrics_test.go | 2 +- .../internal/controllers/workerpool_apply.go | 6 +- .../controllers/workerpool_apply_test.go | 6 +- .../controllers/workerpool_controller_test.go | 5 +- .../internal/workersync/syncer.go | 6 +- .../internal/workersync/syncer_test.go | 2 +- .../isolate/fake-workersync/controller.go | 8 +-- .../fake-workersync/controller_test.go | 8 ++- .../autoscaled-workerpool.yaml.tmpl | 2 + .../claude-code-multiplex.yaml.tmpl | 2 + demos/counter/counter-microvm-csi-test.yaml | 3 +- demos/counter/counter-microvm.yaml.tmpl | 5 +- demos/counter/counter.yaml.tmpl | 4 ++ demos/egress/egress-microvm.yaml.tmpl | 3 +- demos/egress/egress.yaml.tmpl | 2 + demos/jupyter/jupyter.yaml.tmpl | 2 + demos/multi-template/multi-template.yaml.tmpl | 2 + demos/parking/parking.yaml.tmpl | 2 + demos/sandbox/manual-test-multi.yaml | 2 + demos/sandbox/sandbox.yaml.tmpl | 2 + docs/api-guide.md | 14 ++-- docs/architecture.md | 2 +- docs/csi-volumes.md | 2 + docs/upgrade.md | 12 ++-- internal/e2e/sandbox.go | 6 +- internal/e2e/sandbox_test.go | 8 +-- .../networkpolicy/networkpolicy_test.go | 5 +- internal/e2e/template.go | 6 +- .../generated/ate.dev_workerpools.yaml | 52 +++++++++++---- pkg/api/v1alpha1/workerpool_types.go | 52 ++++++++++++--- .../v1alpha1/workerpool_validation_test.go | 66 +++++++++++++++++-- pkg/api/v1alpha1/zz_generated.deepcopy.go | 42 ++++++++++++ pkg/proto/ateapipb/ateapi.pb.go | 3 +- pkg/proto/ateapipb/ateapi.proto | 3 +- 39 files changed, 281 insertions(+), 82 deletions(-) diff --git a/.golangci-kal.yaml b/.golangci-kal.yaml index fdfbdafb0b..53ab0456ff 100644 --- a/.golangci-kal.yaml +++ b/.golangci-kal.yaml @@ -114,7 +114,7 @@ linters: - path: 'pkg/api/v1alpha1/workerpool_types\.go' text: '^optionalfields: .*\bWorkerPoolPodTemplate\.PriorityClassName\b' - path: 'pkg/api/v1alpha1/workerpool_types\.go' - text: '^(optionalfields|requiredfields): .*\bWorkerPoolSpec\.(Replicas|SandboxClass|WorkerImage)\b' + text: '^(optionalfields|requiredfields): .*\bWorkerPoolSpec\.(Replicas|WorkerImage)\b' - path: 'pkg/api/v1alpha1/workerpool_types\.go' text: '^optionalfields: .*\bWorkerPoolStatus\.(ReadyReplicas|Replicas|Selector)\b' - path: 'pkg/api/v1alpha1/workerpool_types\.go' diff --git a/benchmarking/automation/README.md b/benchmarking/automation/README.md index 43270d147e..029c550cdc 100644 --- a/benchmarking/automation/README.md +++ b/benchmarking/automation/README.md @@ -45,7 +45,7 @@ the router capacity benchmark — see ## Choosing a sandbox class Each entry in `tests.yaml` may set `sandboxClass: gvisor | microvm` (default -`gvisor`). This controls both `spec.sandboxClass` on the benchmark WorkerPool +`gvisor`). This controls both `spec.sandboxClasses[0].name` on the benchmark WorkerPool and its `workerImage` (`ateom-gvisor` vs `ateom-microvm`). For `microvm` tests the target cluster must have KVM-capable nodes and the diff --git a/benchmarking/workloads/manifests/workloads.yaml.tmpl b/benchmarking/workloads/manifests/workloads.yaml.tmpl index 4309442441..022014bd10 100644 --- a/benchmarking/workloads/manifests/workloads.yaml.tmpl +++ b/benchmarking/workloads/manifests/workloads.yaml.tmpl @@ -33,7 +33,8 @@ metadata: workload: benchmark-ateom spec: replicas: ${WORKER_COUNT} - sandboxClass: ${SANDBOX_CLASS} + sandboxClasses: + - name: ${SANDBOX_CLASS} workerImage: ko://github.com/agent-substrate/substrate/cmd/ateom-${SANDBOX_CLASS} template: # Lets gvisor and microvm workers share a node tainted ate.dev/sandboxClass=shared. diff --git a/cmd/ateapi/internal/controlapi/functionaltest/common_test.go b/cmd/ateapi/internal/controlapi/functionaltest/common_test.go index ad832bf39f..8748f36261 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/common_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/common_test.go @@ -543,8 +543,9 @@ func createWorkerPool(t *testing.T, tc *testContext, ns string, name string, lab Labels: labels, }, Spec: atev1alpha1.WorkerPoolSpec{ - Replicas: 1, - WorkerImage: "ateom@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + Replicas: 1, + WorkerImage: "ateom@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + SandboxClasses: []atev1alpha1.WorkerPoolSandboxClass{{Name: atev1alpha1.SandboxClassGvisor}}, }, } _, err := tc.substrateClient.ApiV1alpha1().WorkerPools(ns).Create(context.Background(), wp, metav1.CreateOptions{}) @@ -657,7 +658,7 @@ func createWorkerPod(t *testing.T, tc *testContext, ns string, name string, node WorkerPodUid: string(createdPod.UID), Ips: []string{"127.0.0.1"}, NodeName: nodeName, - SandboxClass: string(pool.Spec.SandboxClass), + SandboxClass: string(pool.Spec.DefaultSandboxClass()), Labels: pool.GetLabels(), // Capacity is not settable here: a Worker gets it from its own // ateom's report, which the reportWorkerCapacity below stands in diff --git a/cmd/ateapi/internal/controlapi/metrics.go b/cmd/ateapi/internal/controlapi/metrics.go index a643e08ca7..8ace3106ea 100644 --- a/cmd/ateapi/internal/controlapi/metrics.go +++ b/cmd/ateapi/internal/controlapi/metrics.go @@ -87,7 +87,7 @@ func RegisterWorkerCount(meter metric.Meter, workers func() ([]*ateapipb.Worker, // list just means no seeding this cycle, not a broken observation. if pools, err := listPools(labels.Everything()); err == nil { for _, p := range pools { - class := string(p.Spec.SandboxClass) + class := string(p.Spec.DefaultSandboxClass()) if class == "" { class = string(atev1alpha1.SandboxClassGvisor) } diff --git a/cmd/ateapi/internal/controlapi/metrics_test.go b/cmd/ateapi/internal/controlapi/metrics_test.go index 21e64e9862..8f4b1f6a72 100644 --- a/cmd/ateapi/internal/controlapi/metrics_test.go +++ b/cmd/ateapi/internal/controlapi/metrics_test.go @@ -400,7 +400,7 @@ func TestSchedulerAssignmentShapeAndOutcomes(t *testing.T) { func workerPool(namespace, name string, class atev1alpha1.SandboxClass) *atev1alpha1.WorkerPool { return &atev1alpha1.WorkerPool{ ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: name}, - Spec: atev1alpha1.WorkerPoolSpec{SandboxClass: class}, + Spec: atev1alpha1.WorkerPoolSpec{SandboxClasses: []atev1alpha1.WorkerPoolSandboxClass{{Name: class}}}, } } diff --git a/cmd/atecontroller/internal/controllers/workerpool_apply.go b/cmd/atecontroller/internal/controllers/workerpool_apply.go index a11d6f2613..afa2a3edd5 100644 --- a/cmd/atecontroller/internal/controllers/workerpool_apply.go +++ b/cmd/atecontroller/internal/controllers/workerpool_apply.go @@ -164,7 +164,7 @@ func buildDeploymentApplyConfig(wp *atev1alpha1.WorkerPool, otel ateomOTelSettin WithHTTPGet(corev1ac.HTTPGetAction(). WithPath("/readyz"). WithPort(intstr.FromString("readyz")))). - WithSecurityContext(ateomSecurityContext(wp.Spec.SandboxClass)). + WithSecurityContext(ateomSecurityContext(wp.Spec.DefaultSandboxClass())). WithEnv(ateomContainerEnv(otel)...). WithVolumeMounts( corev1ac.VolumeMount(). @@ -234,8 +234,8 @@ func buildDeploymentApplyConfig(wp *atev1alpha1.WorkerPool, otel ateomOTelSettin ) applyWorkerPoolPodTemplate(podSpecAC, containerAC, wp.Spec.Template) - applySandboxClassToleration(podSpecAC, wp.Spec.SandboxClass) - maybeApplyMicroVMPodShape(podSpecAC, containerAC, wp.Spec.SandboxClass) + applySandboxClassToleration(podSpecAC, wp.Spec.DefaultSandboxClass()) + maybeApplyMicroVMPodShape(podSpecAC, containerAC, wp.Spec.DefaultSandboxClass()) podSpecAC.WithContainers(containerAC) podSpecAC.WithTerminationGracePeriodSeconds(workerTerminationGracePeriodSeconds) diff --git a/cmd/atecontroller/internal/controllers/workerpool_apply_test.go b/cmd/atecontroller/internal/controllers/workerpool_apply_test.go index 49edc1265b..d5bd9283ec 100644 --- a/cmd/atecontroller/internal/controllers/workerpool_apply_test.go +++ b/cmd/atecontroller/internal/controllers/workerpool_apply_test.go @@ -299,7 +299,7 @@ func TestSandboxClassToleration(t *testing.T) { Effect: corev1.TaintEffectNoSchedule, }}, }) - wp.Spec.SandboxClass = tt.class + wp.Spec.SandboxClasses = []atev1alpha1.WorkerPoolSandboxClass{{Name: tt.class}} ps := buildDeploymentApplyConfig(wp, ateomOTelSettings{}, installdefaults.SystemNamespace, installdefaults.AteletServiceAccount, installdefaults.RouterServiceAccount).Spec.Template.Spec var classValues []string @@ -347,7 +347,7 @@ func TestMicroVMPodShape(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { wp := testWorkerPoolApplyConfig(nil) - wp.Spec.SandboxClass = tt.class + wp.Spec.SandboxClasses = []atev1alpha1.WorkerPoolSandboxClass{{Name: tt.class}} ps := buildDeploymentApplyConfig(wp, ateomOTelSettings{}, installdefaults.SystemNamespace, installdefaults.AteletServiceAccount, installdefaults.RouterServiceAccount).Spec.Template.Spec // /dev/kvm must come from the device plugin, never a hostPath: a @@ -434,7 +434,7 @@ func TestMicroVMDeviceRequestsPreserveTemplateResources(t *testing.T) { Limits: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("2Gi")}, }, }) - wp.Spec.SandboxClass = atev1alpha1.SandboxClassMicroVM + wp.Spec.SandboxClasses = []atev1alpha1.WorkerPoolSandboxClass{{Name: atev1alpha1.SandboxClassMicroVM}} c := buildDeploymentApplyConfig(wp, ateomOTelSettings{}, installdefaults.SystemNamespace, installdefaults.AteletServiceAccount, installdefaults.RouterServiceAccount).Spec.Template.Spec.Containers[0] if got, ok := deviceLimit(c, string(corev1.ResourceMemory)); !ok || got != "2Gi" { diff --git a/cmd/atecontroller/internal/controllers/workerpool_controller_test.go b/cmd/atecontroller/internal/controllers/workerpool_controller_test.go index 7915d1501b..784f3de2f1 100644 --- a/cmd/atecontroller/internal/controllers/workerpool_controller_test.go +++ b/cmd/atecontroller/internal/controllers/workerpool_controller_test.go @@ -590,8 +590,9 @@ func makeWorkerPool(name, ns string, replicas int32, image string) *atev1alpha1. return &atev1alpha1.WorkerPool{ ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns}, Spec: atev1alpha1.WorkerPoolSpec{ - Replicas: replicas, - WorkerImage: image, + Replicas: replicas, + WorkerImage: image, + SandboxClasses: []atev1alpha1.WorkerPoolSandboxClass{{Name: atev1alpha1.SandboxClassGvisor}}, }, } } diff --git a/cmd/atecontroller/internal/workersync/syncer.go b/cmd/atecontroller/internal/workersync/syncer.go index 40f820a7af..62cf8c144d 100644 --- a/cmd/atecontroller/internal/workersync/syncer.go +++ b/cmd/atecontroller/internal/workersync/syncer.go @@ -357,7 +357,7 @@ func (s *WorkerPoolSyncer) createOrUpdateWorker(ctx context.Context, key workerK Ips: podIPs(pod), WorkerPodUid: string(pod.UID), NodeName: pod.Spec.NodeName, - SandboxClass: string(pool.Spec.SandboxClass), + SandboxClass: string(pool.Spec.DefaultSandboxClass()), Labels: pool.GetLabels(), Epoch: podEpoch(pod), // Capacity is the Worker's to report, not the syncer's to infer @@ -395,7 +395,7 @@ func (s *WorkerPoolSyncer) createOrUpdateWorker(ctx context.Context, key workerK w.Labels = pool.GetLabels() changed = true } - if w.GetSandboxClass() != string(pool.Spec.SandboxClass) { + if w.GetSandboxClass() != string(pool.Spec.DefaultSandboxClass()) { // Expected mid-rollout: sandboxClass drives the worker pod's shape, so // editing it on the pool replaces every pod rather than reclassifying // any. This pod predates that edit and is on its way out; its successor @@ -403,7 +403,7 @@ func (s *WorkerPoolSyncer) createOrUpdateWorker(ctx context.Context, key workerK // back would be rejected, and would misreport this pod's shape to the // scheduler if it were not. slog.DebugContext(ctx, "Syncer: registered worker sandbox class predates its pool", - append(key.logAttrs(), slog.String("registered", w.GetSandboxClass()), slog.String("pool", string(pool.Spec.SandboxClass)))...) + append(key.logAttrs(), slog.String("registered", w.GetSandboxClass()), slog.String("pool", string(pool.Spec.DefaultSandboxClass())))...) } if ips := podIPs(pod); !slices.Equal(w.GetIps(), ips) { // TODO: I don't think this is possible, but handling this case so we can diff --git a/cmd/atecontroller/internal/workersync/syncer_test.go b/cmd/atecontroller/internal/workersync/syncer_test.go index 6e37c99238..642e2e7eeb 100644 --- a/cmd/atecontroller/internal/workersync/syncer_test.go +++ b/cmd/atecontroller/internal/workersync/syncer_test.go @@ -84,7 +84,7 @@ func workerPod(ns, name, poolName, uid, ip string) *corev1.Pod { func workerPool(ns, name, sandboxClass string, labels map[string]string) *atev1alpha1.WorkerPool { return &atev1alpha1.WorkerPool{ ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns, Labels: labels}, - Spec: atev1alpha1.WorkerPoolSpec{SandboxClass: atev1alpha1.SandboxClass(sandboxClass)}, + Spec: atev1alpha1.WorkerPoolSpec{SandboxClasses: []atev1alpha1.WorkerPoolSandboxClass{{Name: atev1alpha1.SandboxClass(sandboxClass)}}}, } } diff --git a/cmd/benchmarking/isolate/fake-workersync/controller.go b/cmd/benchmarking/isolate/fake-workersync/controller.go index 64c6a695d1..1b41ce5aab 100644 --- a/cmd/benchmarking/isolate/fake-workersync/controller.go +++ b/cmd/benchmarking/isolate/fake-workersync/controller.go @@ -326,7 +326,7 @@ func (c *controller) ensure(ctx context.Context, pod string, d desiredWorker, w index: d.index, node: fakeworker.Node(d.index, nodes), labels: maps.Clone(d.pool.GetLabels()), - sandboxClass: string(d.pool.Spec.SandboxClass), + sandboxClass: string(d.pool.Spec.DefaultSandboxClass()), } c.mu.Lock() c.workers[w.name] = w @@ -401,7 +401,7 @@ func (c *controller) create(ctx context.Context, w *fakeWorker, wp *atev1alpha1. WorkerPodUid: name, NodeName: w.node, Ips: []string{fakeworker.IP(w.index)}, - SandboxClass: string(wp.Spec.SandboxClass), + SandboxClass: string(wp.Spec.DefaultSandboxClass()), Labels: maps.Clone(wp.GetLabels()), }}) if err != nil && status.Code(err) != codes.AlreadyExists { @@ -558,7 +558,7 @@ func templateOf(wp *atev1alpha1.WorkerPool) (template, error) { if err != nil { return template{}, err } - t := template{sandboxClass: string(wp.Spec.SandboxClass), actors: actors} + t := template{sandboxClass: string(wp.Spec.DefaultSandboxClass()), actors: actors} if wp.Spec.Template != nil && wp.Spec.Template.Resources != nil { limits := wp.Spec.Template.Resources.Limits t.cpuMilli = limits.Cpu().MilliValue() @@ -589,7 +589,7 @@ func (t template) capacity(allocatable corev1.ResourceList) *ateapipb.WorkerReso // down is replaced too. A held pool's Workers fit, since they are left as // they are. func fits(w *fakeWorker, wp *atev1alpha1.WorkerPool, allocatable map[string]corev1.ResourceList) bool { - if w.sandboxClass != string(wp.Spec.SandboxClass) { + if w.sandboxClass != string(wp.Spec.DefaultSandboxClass()) { return false } t, err := templateOf(wp) diff --git a/cmd/benchmarking/isolate/fake-workersync/controller_test.go b/cmd/benchmarking/isolate/fake-workersync/controller_test.go index 933ccbea9e..9a1a14661d 100644 --- a/cmd/benchmarking/isolate/fake-workersync/controller_test.go +++ b/cmd/benchmarking/isolate/fake-workersync/controller_test.go @@ -324,7 +324,7 @@ func (f *fakeCluster) setReplicas(pool string, n int32) { func pool(name string, replicas int32, limits corev1.ResourceList) *atev1alpha1.WorkerPool { wp := &atev1alpha1.WorkerPool{ ObjectMeta: metav1.ObjectMeta{Namespace: "benchmark-workloads", Name: name, Labels: map[string]string{"workload": name}}, - Spec: atev1alpha1.WorkerPoolSpec{Replicas: replicas, SandboxClass: atev1alpha1.SandboxClass("gvisor")}, + Spec: atev1alpha1.WorkerPoolSpec{Replicas: replicas, SandboxClasses: []atev1alpha1.WorkerPoolSandboxClass{{Name: atev1alpha1.SandboxClassGvisor}}}, } if limits != nil { wp.Spec.Template = &atev1alpha1.WorkerPoolPodTemplate{Resources: &corev1.ResourceRequirements{Limits: limits}} @@ -878,7 +878,9 @@ func TestPoolSandboxClassChangeReplacesWorkers(t *testing.T) { oldBusy := ctl.uid(busy) ctl.assignments[oldBusy] = 1 - cl.editPool("bench", func(wp *atev1alpha1.WorkerPool) { wp.Spec.SandboxClass = "microvm" }) + cl.editPool("bench", func(wp *atev1alpha1.WorkerPool) { + wp.Spec.SandboxClasses = []atev1alpha1.WorkerPoolSandboxClass{{Name: atev1alpha1.SandboxClassMicroVM}} + }) if err := reconcile(t, c); err != nil { t.Fatalf("reconcile after the class change: %v", err) } @@ -918,7 +920,7 @@ func TestFailedDrainOfAReplacedWorkerIsNoReplica(t *testing.T) { old := ctl.worker(pod) ctl.drainErr = status.Error(codes.Unavailable, "connection reset") cl.editPool("bench", func(wp *atev1alpha1.WorkerPool) { - wp.Spec.SandboxClass = "microvm" + wp.Spec.SandboxClasses = []atev1alpha1.WorkerPoolSandboxClass{{Name: atev1alpha1.SandboxClassMicroVM}} wp.Labels = map[string]string{"workload": "v2"} }) if err := reconcile(t, c); err == nil { diff --git a/demos/autoscaled-workerpool/autoscaled-workerpool.yaml.tmpl b/demos/autoscaled-workerpool/autoscaled-workerpool.yaml.tmpl index 6cde9b7938..000fd881b5 100644 --- a/demos/autoscaled-workerpool/autoscaled-workerpool.yaml.tmpl +++ b/demos/autoscaled-workerpool/autoscaled-workerpool.yaml.tmpl @@ -38,3 +38,5 @@ metadata: spec: replicas: 5 workerImage: ko://github.com/agent-substrate/substrate/cmd/ateom-gvisor + sandboxClasses: + - name: gvisor diff --git a/demos/claude-code-multiplex/claude-code-multiplex.yaml.tmpl b/demos/claude-code-multiplex/claude-code-multiplex.yaml.tmpl index e76789be87..ef1360fdcc 100644 --- a/demos/claude-code-multiplex/claude-code-multiplex.yaml.tmpl +++ b/demos/claude-code-multiplex/claude-code-multiplex.yaml.tmpl @@ -35,3 +35,5 @@ metadata: spec: replicas: 2 workerImage: ko://github.com/agent-substrate/substrate/cmd/ateom-gvisor + sandboxClasses: + - name: gvisor diff --git a/demos/counter/counter-microvm-csi-test.yaml b/demos/counter/counter-microvm-csi-test.yaml index e6e470a86a..3306226a75 100644 --- a/demos/counter/counter-microvm-csi-test.yaml +++ b/demos/counter/counter-microvm-csi-test.yaml @@ -32,5 +32,6 @@ metadata: workload: counter-microvm-csi spec: replicas: 1 - sandboxClass: microvm + sandboxClasses: + - name: microvm workerImage: ko://github.com/agent-substrate/substrate/cmd/ateom-microvm diff --git a/demos/counter/counter-microvm.yaml.tmpl b/demos/counter/counter-microvm.yaml.tmpl index edacc2ab29..aa68bdd1f0 100644 --- a/demos/counter/counter-microvm.yaml.tmpl +++ b/demos/counter/counter-microvm.yaml.tmpl @@ -39,7 +39,10 @@ spec: # Two workers: one for the demo actor, one spare for the golden actor the # template reconciler runs while building the golden snapshot. replicas: 2 - sandboxClass: microvm + sandboxClasses: + - name: microvm + configRef: + name: microvm workerImage: ko://github.com/agent-substrate/substrate/cmd/ateom-microvm template: nodeSelector: diff --git a/demos/counter/counter.yaml.tmpl b/demos/counter/counter.yaml.tmpl index 5e53430f3f..61bf5e0ae3 100644 --- a/demos/counter/counter.yaml.tmpl +++ b/demos/counter/counter.yaml.tmpl @@ -38,6 +38,10 @@ spec: # deploy blocks on a rollout that never completes. replicas: 3 workerImage: ko://github.com/agent-substrate/substrate/cmd/ateom-gvisor + sandboxClasses: + - name: gvisor + configRef: + name: gvisor-default template: nodeSelector: ate.dev/substrate-version: "${SUBSTRATE_VERSION}" diff --git a/demos/egress/egress-microvm.yaml.tmpl b/demos/egress/egress-microvm.yaml.tmpl index 3b0766bf54..3facf92508 100644 --- a/demos/egress/egress-microvm.yaml.tmpl +++ b/demos/egress/egress-microvm.yaml.tmpl @@ -36,7 +36,8 @@ metadata: workload: egress-microvm spec: replicas: 2 - sandboxClass: microvm + sandboxClasses: + - name: microvm workerImage: ko://github.com/agent-substrate/substrate/cmd/ateom-microvm # No template.resources, unlike the counter demo's micro-VM variant: an # unlimited ateom container reports no capacity, which the scheduler reads as diff --git a/demos/egress/egress.yaml.tmpl b/demos/egress/egress.yaml.tmpl index 1d57a3e2b7..28265fda39 100644 --- a/demos/egress/egress.yaml.tmpl +++ b/demos/egress/egress.yaml.tmpl @@ -34,3 +34,5 @@ metadata: spec: replicas: 2 workerImage: ko://github.com/agent-substrate/substrate/cmd/ateom-gvisor + sandboxClasses: + - name: gvisor diff --git a/demos/jupyter/jupyter.yaml.tmpl b/demos/jupyter/jupyter.yaml.tmpl index e9494da0b0..9c49c176a6 100644 --- a/demos/jupyter/jupyter.yaml.tmpl +++ b/demos/jupyter/jupyter.yaml.tmpl @@ -33,6 +33,8 @@ metadata: spec: replicas: 1 workerImage: ko://github.com/agent-substrate/substrate/cmd/ateom-gvisor + sandboxClasses: + - name: gvisor --- apiVersion: v1 diff --git a/demos/multi-template/multi-template.yaml.tmpl b/demos/multi-template/multi-template.yaml.tmpl index b9cd1ca3b4..1f8824ad75 100644 --- a/demos/multi-template/multi-template.yaml.tmpl +++ b/demos/multi-template/multi-template.yaml.tmpl @@ -37,3 +37,5 @@ metadata: spec: replicas: 3 workerImage: ko://github.com/agent-substrate/substrate/cmd/ateom-gvisor + sandboxClasses: + - name: gvisor diff --git a/demos/parking/parking.yaml.tmpl b/demos/parking/parking.yaml.tmpl index b1ac48598e..bc09c25271 100644 --- a/demos/parking/parking.yaml.tmpl +++ b/demos/parking/parking.yaml.tmpl @@ -44,3 +44,5 @@ spec: # the pool and exercise the parking path. replicas: 2 workerImage: ko://github.com/agent-substrate/substrate/cmd/ateom-gvisor + sandboxClasses: + - name: gvisor diff --git a/demos/sandbox/manual-test-multi.yaml b/demos/sandbox/manual-test-multi.yaml index 8cb3dbdbbb..74cab46198 100644 --- a/demos/sandbox/manual-test-multi.yaml +++ b/demos/sandbox/manual-test-multi.yaml @@ -31,3 +31,5 @@ metadata: spec: replicas: 1 workerImage: ko://github.com/agent-substrate/substrate/cmd/ateom-gvisor + sandboxClasses: + - name: gvisor diff --git a/demos/sandbox/sandbox.yaml.tmpl b/demos/sandbox/sandbox.yaml.tmpl index 0d32300cdf..3f7614b424 100644 --- a/demos/sandbox/sandbox.yaml.tmpl +++ b/demos/sandbox/sandbox.yaml.tmpl @@ -31,6 +31,8 @@ metadata: spec: replicas: 2 workerImage: ko://github.com/agent-substrate/substrate/cmd/ateom-gvisor + sandboxClasses: + - name: gvisor template: # Per-worker resources: these size the worker POD and advertise its # scheduling capacity (the per-actor ceiling). The sandbox itself is sized by diff --git a/docs/api-guide.md b/docs/api-guide.md index a976a03b6d..3c07ad6e16 100644 --- a/docs/api-guide.md +++ b/docs/api-guide.md @@ -12,7 +12,7 @@ The `WorkerPool` defines the pool of physical "warm" compute capacity. It manage | :--- | :--- | :--- | | `replicas` | `int32` | **Required.** Number of physical standby pods to maintain in the cluster. | | `workerImage` | `string` | **Required.** The container image for the `ateom` herder process (e.g. `ko://github.com/agent-substrate/substrate/cmd/ateom-gvisor`). | -| `sandboxClass` | `string` | Optional. The sandbox runtime family for the pool: `gvisor` (default) or `microvm`. Drives the worker pod shape (e.g. KVM device mounts, node placement). The sandbox binaries themselves come from the [`SandboxConfig`](#3-sandboxconfig-the-sandbox-itself) each `ActorTemplate` selects. | +| `sandboxClasses` | `[]WorkerPoolSandboxClass` | **Required.** The sandbox runtime families the pool runs; exactly one entry today. Each entry has `name` (**required**, `gvisor` or `microvm`), which drives the worker pod shape (e.g. KVM device mounts, node placement), and `configRef.name` (optional), which names a cluster-scoped [`SandboxConfig`](#3-sandboxconfig-the-sandbox-itself) of the same class. `configRef` is not consumed yet: the sandbox binaries still come from the `SandboxConfig` each `ActorTemplate` selects. | | `template` | `WorkerPoolPodTemplate` | **Optional.** Metadata, scheduling, and resource settings for worker workloads. | #### `WorkerPoolPodTemplate` (`spec.template`) @@ -81,13 +81,15 @@ metadata: spec: replicas: 10 workerImage: ko://github.com/agent-substrate/substrate/cmd/ateom-gvisor + sandboxClasses: + - name: gvisor + configRef: + name: gvisor-default template: labels: project: agent-platform annotations: policy.example.com/exemption: sandbox-host - # sandboxClass defaults to gvisor. The sandbox binaries come from the - # SandboxConfig each ActorTemplate selects, not from the pool. ``` ### Devices (GPUs) — temporarily unsupported @@ -119,7 +121,7 @@ The `ActorTemplate` defines the code, environment, and state-management policies | Field | Type | Description | | :--- | :--- | :--- | | `containers` | `[]Container` | **Required.** The workload definition — see [Container Fields](#container-fields) below. Each container may also declare an optional `wakeupProbe` HTTP probe — see [Container Wakeup Probe](#container-wakeup-probe-wakeupprobe). | -| `sandboxConfig` | `SandboxConfig` | **Required.** The sandbox runtime selection: `sandboxClass` (**required**, `SANDBOX_CLASS_GVISOR` or `SANDBOX_CLASS_MICROVM`) picks the runtime family this template's actors require — only `WorkerPool`s whose `sandboxClass` matches are eligible — and `configName` (**required**) names the cluster-scoped [`SandboxConfig`](#3-sandboxconfig-the-sandbox-itself) object supplying the sandbox binaries. It must reference an existing config of the matching class; `CreateActorTemplate` rejects the template otherwise. | +| `sandboxConfig` | `SandboxConfig` | **Required.** The sandbox runtime selection: `sandboxClass` (**required**, `SANDBOX_CLASS_GVISOR` or `SANDBOX_CLASS_MICROVM`) picks the runtime family this template's actors require — only `WorkerPool`s whose `sandboxClasses` entry matches are eligible — and `configName` (**required**) names the cluster-scoped [`SandboxConfig`](#3-sandboxconfig-the-sandbox-itself) object supplying the sandbox binaries. It must reference an existing config of the matching class; `CreateActorTemplate` rejects the template otherwise. | | `workerSelector` | `*Selector` | Optional. Gates which `WorkerPool`s actors from this template may use, by matching against each pool's labels (`matchLabels`). If unset, all pools are eligible (subject to the actor's own `worker_selector`). | | `snapshotConfig` | `SnapshotConfig` | **Required.** The base object-storage location snapshots are written under, plus the snapshot fidelity (`preferredFidelity`) every Pause and Suspend captures. See [Snapshot Storage Layout](#snapshot-storage-layout). | | `volumes` | `[]Volume` | Optional. Volumes the containers may mount, each a `durableDir`, an `externalVolumeTemplate` (see [CSI Volumes Guide](csi-volumes.md)), or a `systemInfo` volume (see [SystemInfo Volumes](#systeminfo-volumes)). Every declared volume must be mounted by at least one container. A `microvm` template may declare several `durableDir` volumes; a `gvisor` template is limited to one. | @@ -127,7 +129,7 @@ The `ActorTemplate` defines the code, environment, and state-management policies The sandbox itself — the binaries (e.g. the gVisor `runsc` binary) and the `pauseImage` holding the sandbox's namespaces — comes from the cluster-scoped [`SandboxConfig`](#3-sandboxconfig-the-sandbox-itself) object the template names via `sandboxConfig.configName`. An actor always resolves the config from its current template — repointing the actor at another template requires the same config. -Because a snapshot is not restorable across sandbox runtimes, `sandboxClass` is a **hard scheduling gate**: an actor is only ever placed on a `WorkerPool` of the matching class. It is AND'd with `workerSelector` (and the actor's `worker_selector`), which can only narrow the eligible pools further. It has no default — `sandboxConfig` is required and its `sandboxClass` must be set — and, like the rest of the template, is immutable, so each template's class is fixed at creation. +Because a snapshot is not restorable across sandbox runtimes, `sandboxClass` is a **hard scheduling gate**: an actor is only ever placed on a `WorkerPool` whose `sandboxClasses` entry matches. It is AND'd with `workerSelector` (and the actor's `worker_selector`), which can only narrow the eligible pools further. It has no default — `sandboxConfig` is required and its `sandboxClass` must be set — and, like the rest of the template, is immutable, so each template's class is fixed at creation. ### Sandbox Right-Sizing (`resources`) @@ -472,7 +474,7 @@ Once a template is `Ready`, creating an actor logically (via `kubectl ate create ## 5. Best Practices * **Startup Logic:** Place expensive initialization (loading large models, establishing baseline connections) in your application's entry point. These will be captured in the Golden Snapshot and won't need to be repeated on every resumption. -* **Placement:** Ensure your `ActorTemplate`'s `sandboxClass` matches your `WorkerPool`'s, and use the template's `workerSelector` to target specific pools — pool selection is by label match, not by namespace or RBAC. +* **Placement:** Ensure your `ActorTemplate`'s `sandboxClass` matches your `WorkerPool`'s `sandboxClasses[].name`, and use the template's `workerSelector` to target specific pools — pool selection is by label match, not by namespace or RBAC. * **Version Management:** When updating code, create a new `ActorTemplate` (e.g. `v2`). Substrate treats each template as an immutable state root. * **Eviction:** When its worker pod is evicted, an actor gets `SIGTERM` and 30 minutes to be suspended. After that it is killed and moves to `ACTOR_STATE_CRASHED`, and everything since its last snapshot is lost. So an actor that runs for more than 30 minutes without a suspend can lose data. A `CRASHED` actor can be recovered back to `ACTOR_STATE_SUSPENDED` at its last external snapshot using `RevertActor` (`kubectl ate revert`). diff --git a/docs/architecture.md b/docs/architecture.md index 0d063854dd..1034ad5808 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -331,7 +331,7 @@ The node-level subsystem manages the physical execution of sandboxes and the mov ### Sandbox Classes -A `WorkerPool` selects a **sandbox class** (`spec.sandboxClass`), and each class has a matching `ateom` herder image. The sandbox binaries themselves are not baked into the worker image — they, and the pause image holding the sandbox's namespaces, come at runtime from a cluster-scoped [`SandboxConfig`](api-guide.md#3-sandboxconfig-the-sandbox-itself) the `ActorTemplate` names in its sandbox config (naming one is currently required; per-class cluster defaults are planned) and are pinned into each snapshot's manifest so restores stay reproducible across runtime upgrades. +A `WorkerPool` selects a **sandbox class** (`spec.sandboxClasses[].name`), and each class has a matching `ateom` herder image. The sandbox binaries themselves are not baked into the worker image — they, and the pause image holding the sandbox's namespaces, come at runtime from a cluster-scoped [`SandboxConfig`](api-guide.md#3-sandboxconfig-the-sandbox-itself) the `ActorTemplate` names in its sandbox config (naming one is currently required; per-class cluster defaults are planned) and are pinned into each snapshot's manifest so restores stay reproducible across runtime upgrades. * **gVisor** (`ateom-gvisor`, the default): Runs the workload under `runsc` for kernel-level sandboxing. Suspend and resume leverage gVisor's native checkpoint/restore of the sandboxed process tree. diff --git a/docs/csi-volumes.md b/docs/csi-volumes.md index d33bb83c08..ae170b3f25 100644 --- a/docs/csi-volumes.md +++ b/docs/csi-volumes.md @@ -154,6 +154,8 @@ metadata: spec: replicas: 5 workerImage: ko://github.com/agent-substrate/substrate/cmd/ateom-gvisor + sandboxClasses: + - name: gvisor ``` The `ActorTemplate` is a protojson-shaped `ateapipb.ActorTemplate`, created diff --git a/docs/upgrade.md b/docs/upgrade.md index 70b5e22388..6fc20f5fad 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -346,7 +346,7 @@ version did not change (see Checkout and environment) and the command rolled the running atelet in place. Last, build and push the new worker images. The refs print together -at the end. The one for your pool's `sandboxClass` goes into the clone +at the end. The one for your pool's `sandboxClasses[].name` goes into the clone in step 4: ```bash @@ -354,7 +354,7 @@ go run ./cmd/ate-setup publish worker-images ``` If you are using prebuilt images there is nothing to publish. The new worker image -is the release's `ateom-` image under the same repo and +is the release's `ateom-` image under the same repo and tag as the control plane, pinned by digest: ```bash @@ -376,15 +376,19 @@ else, including the `metadata.labels` the scheduler matches actors by, carries over as is. ```bash -NEW_IMAGE= +NEW_IMAGE= +# Only read when the old pool predates spec.sandboxClasses. +SANDBOX_CLASS= kubectl -n $NS get workerpool $OLD_WORKERPOOL -o json \ - | jq --arg name "$NEW_WORKERPOOL" --arg image "$NEW_IMAGE" --arg version "$NEW_VERSION" ' + | jq --arg name "$NEW_WORKERPOOL" --arg image "$NEW_IMAGE" --arg version "$NEW_VERSION" \ + --arg class "$SANDBOX_CLASS" ' {apiVersion, kind, metadata: {name: $name, namespace: .metadata.namespace, labels: .metadata.labels}, spec: (.spec + {workerImage: $image})} | .spec.template.nodeSelector["ate.dev/substrate-version"] = $version + | .spec.sandboxClasses //= [{name: $class}] ' \ | kubectl apply -f - ``` diff --git a/internal/e2e/sandbox.go b/internal/e2e/sandbox.go index 68e4835cb6..db5eb8b89d 100644 --- a/internal/e2e/sandbox.go +++ b/internal/e2e/sandbox.go @@ -21,7 +21,7 @@ import ( ) // SandboxClassMicroVM is the kata + cloud-hypervisor runtime, spelled as the -// WorkerPool/ActorTemplate spec.sandboxClass field spells it. +// WorkerPool spec.sandboxClasses[].name and ActorTemplate sandboxClass spell it. const SandboxClassMicroVM = "microvm" // sandboxClassEnv selects which runtime's fixtures the suites build actors @@ -184,7 +184,7 @@ func fixtureSubstitutions(bucket, name string) (inline, blocks map[string]string "${FIXTURE_SUFFIX}": "-" + name, } blocks = map[string]string{ - "${WORKERPOOL_RUNTIME}": "", + "${WORKERPOOL_RUNTIME}": " sandboxClasses:\n - name: gvisor", "${TEMPLATE_SANDBOX_CLASS}": "", "${TEMPLATE_RESOURCES}": "", // Off unless the caller opts in; see WithTrustBundle. @@ -199,7 +199,7 @@ func fixtureSubstitutions(bucket, name string) (inline, blocks map[string]string // The micro-VM ActorTemplates name the cluster-wide SandboxConfig // hack/install-microvm-deps.sh installs (configName: microvm), so a // missing or stale one fails loudly. The pool only selects the class. - blocks["${WORKERPOOL_RUNTIME}"] = " sandboxClass: microvm" + blocks["${WORKERPOOL_RUNTIME}"] = " sandboxClasses:\n - name: microvm" // Must match the WorkerPool's: a snapshot is not portable across sandbox // classes, so only same-class pools are eligible to run these actors. blocks["${TEMPLATE_SANDBOX_CLASS}"] = " sandboxClass: microvm" diff --git a/internal/e2e/sandbox_test.go b/internal/e2e/sandbox_test.go index ac443fd8ec..1d63f07355 100644 --- a/internal/e2e/sandbox_test.go +++ b/internal/e2e/sandbox_test.go @@ -132,8 +132,8 @@ func TestRenderSubstrateFixtures_GVisor(t *testing.T) { if !strings.HasSuffix(pool.Spec.WorkerImage, "/cmd/ateom-gvisor") { t.Errorf("WorkerPool workerImage = %q, want the gVisor ateom", pool.Spec.WorkerImage) } - if pool.Spec.SandboxClass != "" { - t.Errorf("WorkerPool carries micro-VM runtime fields: class=%q", pool.Spec.SandboxClass) + if pool.Spec.DefaultSandboxClass() != v1alpha1.SandboxClassGvisor { + t.Errorf("WorkerPool runtime = class %q, want gvisor", pool.Spec.DefaultSandboxClass()) } templates := renderTemplates(t, fixture.manifests.Template) @@ -181,8 +181,8 @@ func TestRenderSubstrateFixtures_MicroVM(t *testing.T) { if !strings.HasSuffix(pool.Spec.WorkerImage, "/cmd/ateom-microvm") { t.Errorf("WorkerPool workerImage = %q, want the micro-VM ateom", pool.Spec.WorkerImage) } - if pool.Spec.SandboxClass != SandboxClassMicroVM { - t.Errorf("WorkerPool runtime = class %q, want microvm", pool.Spec.SandboxClass) + if pool.Spec.DefaultSandboxClass() != SandboxClassMicroVM { + t.Errorf("WorkerPool runtime = class %q, want microvm", pool.Spec.DefaultSandboxClass()) } templates := renderTemplates(t, fixture.manifests.Template) diff --git a/internal/e2e/suites/networkpolicy/networkpolicy_test.go b/internal/e2e/suites/networkpolicy/networkpolicy_test.go index c2cf26b9d4..ff021982c7 100644 --- a/internal/e2e/suites/networkpolicy/networkpolicy_test.go +++ b/internal/e2e/suites/networkpolicy/networkpolicy_test.go @@ -51,8 +51,9 @@ func TestNetworkPolicyLifecycleAndReconciliation(t *testing.T) { Namespace: nsObj.Name, }, Spec: v1alpha1.WorkerPoolSpec{ - Replicas: 1, - WorkerImage: "ateom:v1", + Replicas: 1, + WorkerImage: "ateom:v1", + SandboxClasses: []v1alpha1.WorkerPoolSandboxClass{{Name: v1alpha1.SandboxClassGvisor}}, }, } diff --git a/internal/e2e/template.go b/internal/e2e/template.go index 7a31a614bc..9f90201a3a 100644 --- a/internal/e2e/template.go +++ b/internal/e2e/template.go @@ -85,9 +85,9 @@ func CreateSubstrateTemplateFrom(ctx context.Context, t *testing.T, clients *Cli Labels: opts.Labels, }, Spec: v1alpha1.WorkerPoolSpec{ - Replicas: opts.PoolReplicas, - WorkerImage: existingWp.Spec.WorkerImage, - SandboxClass: existingWp.Spec.SandboxClass, + Replicas: opts.PoolReplicas, + WorkerImage: existingWp.Spec.WorkerImage, + SandboxClasses: existingWp.Spec.SandboxClasses, }, } if _, err := clients.SubstrateK8s.ApiV1alpha1().WorkerPools(namespace).Create(ctx, wp, metav1.CreateOptions{}); err != nil { diff --git a/manifests/ate-install/generated/ate.dev_workerpools.yaml b/manifests/ate-install/generated/ate.dev_workerpools.yaml index e6981dd899..cad0eb1319 100644 --- a/manifests/ate-install/generated/ate.dev_workerpools.yaml +++ b/manifests/ate-install/generated/ate.dev_workerpools.yaml @@ -73,20 +73,47 @@ spec: format: int32 minimum: 0 type: integer - sandboxClass: - default: gvisor + sandboxClasses: description: |- - SandboxClass selects the sandbox runtime family for this pool, which drives - the worker pod shape (KVM/vhost device mounts and node placement). The - concrete binary is still selected by WorkerImage. Defaults to gvisor. - The sandbox binaries themselves come from the SandboxConfig each - ActorTemplate names (required). + SandboxClasses lists the sandbox runtime families this pool runs. Exactly + one entry is allowed today. + items: + description: WorkerPoolSandboxClass is one sandbox runtime family + a WorkerPool runs. + properties: + configRef: + description: |- + ConfigRef names a cluster-scoped SandboxConfig for this sandbox class. + The referenced config's SandboxClass must match Name. Not consumed yet. + properties: + name: + description: Name is the SandboxConfig's metadata.name. + minLength: 1 + type: string + required: + - name + type: object + name: + description: |- + Name selects the sandbox runtime family, which drives the worker pod + shape (KVM/vhost device mounts and node placement). The concrete binary + is still selected by WorkerImage. The sandbox binaries themselves come + from the SandboxConfig each ActorTemplate names (required). - See Also: TODOs in ActorTemplate SandboxClass - enum: - - gvisor - - microvm - type: string + See Also: TODOs in ActorTemplate SandboxClass + enum: + - gvisor + - microvm + type: string + required: + - name + type: object + maxItems: 1 + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map template: description: Template holds optional metadata, scheduling, and resource settings for worker workloads. @@ -449,6 +476,7 @@ spec: type: string required: - replicas + - sandboxClasses - workerImage type: object status: diff --git a/pkg/api/v1alpha1/workerpool_types.go b/pkg/api/v1alpha1/workerpool_types.go index 0582acde26..65d7686231 100644 --- a/pkg/api/v1alpha1/workerpool_types.go +++ b/pkg/api/v1alpha1/workerpool_types.go @@ -101,18 +101,54 @@ type WorkerPoolSpec struct { // +optional Template *WorkerPoolPodTemplate `json:"template,omitempty"` - // SandboxClass selects the sandbox runtime family for this pool, which drives - // the worker pod shape (KVM/vhost device mounts and node placement). The - // concrete binary is still selected by WorkerImage. Defaults to gvisor. - // The sandbox binaries themselves come from the SandboxConfig each - // ActorTemplate names (required). + // SandboxClasses lists the sandbox runtime families this pool runs. Exactly + // one entry is allowed today. + // + // +required + // +kubebuilder:validation:MinItems=1 + // +kubebuilder:validation:MaxItems=1 + // +listType=map + // +listMapKey=name + SandboxClasses []WorkerPoolSandboxClass `json:"sandboxClasses,omitempty"` +} + +// WorkerPoolSandboxClass is one sandbox runtime family a WorkerPool runs. +type WorkerPoolSandboxClass struct { + // Name selects the sandbox runtime family, which drives the worker pod + // shape (KVM/vhost device mounts and node placement). The concrete binary + // is still selected by WorkerImage. The sandbox binaries themselves come + // from the SandboxConfig each ActorTemplate names (required). // // See Also: TODOs in ActorTemplate SandboxClass // - // +optional + // +required // +kubebuilder:validation:Enum=gvisor;microvm - // +kubebuilder:default=gvisor - SandboxClass SandboxClass `json:"sandboxClass,omitempty"` + Name SandboxClass `json:"name,omitempty"` + + // ConfigRef names a cluster-scoped SandboxConfig for this sandbox class. + // The referenced config's SandboxClass must match Name. Not consumed yet. + // + // +optional + ConfigRef *SandboxConfigReference `json:"configRef,omitempty"` +} + +// SandboxConfigReference names a cluster-scoped SandboxConfig. +type SandboxConfigReference struct { + // Name is the SandboxConfig's metadata.name. + // + // +required + // +kubebuilder:validation:MinLength=1 + Name string `json:"name,omitempty"` +} + +// DefaultSandboxClass returns the sandbox class of the pool's first +// SandboxClasses entry, or "" if there is none. +// Currently SandboxClasses is always a list of length 1. +func (s *WorkerPoolSpec) DefaultSandboxClass() SandboxClass { + if len(s.SandboxClasses) == 0 { + return "" + } + return s.SandboxClasses[0].Name } type WorkerPoolStatus struct { diff --git a/pkg/api/v1alpha1/workerpool_validation_test.go b/pkg/api/v1alpha1/workerpool_validation_test.go index 617e68d787..99d47c889c 100644 --- a/pkg/api/v1alpha1/workerpool_validation_test.go +++ b/pkg/api/v1alpha1/workerpool_validation_test.go @@ -42,8 +42,9 @@ func TestWorkerPoolValidation(t *testing.T) { Namespace: "default", }, Spec: WorkerPoolSpec{ - Replicas: 1, - WorkerImage: "ateom:latest", + Replicas: 1, + WorkerImage: "ateom:latest", + SandboxClasses: []WorkerPoolSandboxClass{{Name: SandboxClassGvisor}}, }, } @@ -70,6 +71,60 @@ func TestWorkerPoolValidation(t *testing.T) { }, wantErr: true, errMsg: "spec.workerImage: Invalid value: \"\": spec.workerImage in body should be at least 1 chars long", + }, { + name: "missing sandboxClasses", + mutate: func(wp *WorkerPool) { + wp.Spec.SandboxClasses = nil + }, + wantErr: true, + errMsg: "spec.sandboxClasses: Required value", + }, { + name: "empty sandboxClasses", + mutate: func(wp *WorkerPool) { + wp.Spec.SandboxClasses = []WorkerPoolSandboxClass{} + }, + wantErr: true, + errMsg: "spec.sandboxClasses", + }, { + name: "sandboxClass with empty name", + mutate: func(wp *WorkerPool) { + wp.Spec.SandboxClasses = []WorkerPoolSandboxClass{{}} + }, + wantErr: true, + errMsg: "spec.sandboxClasses[0].name", + }, { + name: "more than one sandboxClass", + mutate: func(wp *WorkerPool) { + wp.Spec.SandboxClasses = []WorkerPoolSandboxClass{{Name: SandboxClassGvisor}, {Name: SandboxClassMicroVM}} + }, + wantErr: true, + errMsg: "spec.sandboxClasses: Too many", + }, { + name: "unknown sandboxClass", + mutate: func(wp *WorkerPool) { + wp.Spec.SandboxClasses = []WorkerPoolSandboxClass{{Name: "kata"}} + }, + wantErr: true, + errMsg: "spec.sandboxClasses[0].name: Unsupported value", + }, { + name: "sandboxClass with configRef", + mutate: func(wp *WorkerPool) { + wp.Spec.SandboxClasses = []WorkerPoolSandboxClass{{ + Name: SandboxClassGvisor, + ConfigRef: &SandboxConfigReference{Name: "gvisor-default"}, + }} + }, + wantErr: false, + }, { + name: "configRef with empty name", + mutate: func(wp *WorkerPool) { + wp.Spec.SandboxClasses = []WorkerPoolSandboxClass{{ + Name: SandboxClassGvisor, + ConfigRef: &SandboxConfigReference{}, + }} + }, + wantErr: true, + errMsg: "spec.sandboxClasses[0].configRef.name", }, { name: "valid template", mutate: func(wp *WorkerPool) { @@ -190,7 +245,7 @@ func TestWorkerPoolValidation(t *testing.T) { // Devices are no longer interpreted: the limit only places the pod. name: "extended resource on a pool is not interpreted", mutate: func(wp *WorkerPool) { - wp.Spec.SandboxClass = SandboxClassMicroVM + wp.Spec.SandboxClasses = []WorkerPoolSandboxClass{{Name: SandboxClassMicroVM}} wp.Spec.Template = gpuTemplate(corev1.ResourceList{gpuResourceName: resource.MustParse("1")}, nil) }, wantErr: false, @@ -227,8 +282,9 @@ func TestWorkerPoolReservedMetadataUpdate(t *testing.T) { Namespace: "default", }, Spec: WorkerPoolSpec{ - Replicas: 1, - WorkerImage: "example.com/ateom:latest", + Replicas: 1, + WorkerImage: "example.com/ateom:latest", + SandboxClasses: []WorkerPoolSandboxClass{{Name: SandboxClassGvisor}}, }, } if err := k8sClient.Create(ctx, wp); err != nil { diff --git a/pkg/api/v1alpha1/zz_generated.deepcopy.go b/pkg/api/v1alpha1/zz_generated.deepcopy.go index e9e19cc475..3cacd34349 100644 --- a/pkg/api/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/api/v1alpha1/zz_generated.deepcopy.go @@ -189,6 +189,21 @@ func (in *SandboxConfigList) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SandboxConfigReference) DeepCopyInto(out *SandboxConfigReference) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SandboxConfigReference. +func (in *SandboxConfigReference) DeepCopy() *SandboxConfigReference { + if in == nil { + return nil + } + out := new(SandboxConfigReference) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SandboxConfigSpec) DeepCopyInto(out *SandboxConfigSpec) { *out = *in @@ -366,6 +381,26 @@ func (in *WorkerPoolPodTemplate) DeepCopy() *WorkerPoolPodTemplate { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *WorkerPoolSandboxClass) DeepCopyInto(out *WorkerPoolSandboxClass) { + *out = *in + if in.ConfigRef != nil { + in, out := &in.ConfigRef, &out.ConfigRef + *out = new(SandboxConfigReference) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WorkerPoolSandboxClass. +func (in *WorkerPoolSandboxClass) DeepCopy() *WorkerPoolSandboxClass { + if in == nil { + return nil + } + out := new(WorkerPoolSandboxClass) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *WorkerPoolSpec) DeepCopyInto(out *WorkerPoolSpec) { *out = *in @@ -374,6 +409,13 @@ func (in *WorkerPoolSpec) DeepCopyInto(out *WorkerPoolSpec) { *out = new(WorkerPoolPodTemplate) (*in).DeepCopyInto(*out) } + if in.SandboxClasses != nil { + in, out := &in.SandboxClasses, &out.SandboxClasses + *out = make([]WorkerPoolSandboxClass, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WorkerPoolSpec. diff --git a/pkg/proto/ateapipb/ateapi.pb.go b/pkg/proto/ateapipb/ateapi.pb.go index 95fc2a114a..10bd750fae 100644 --- a/pkg/proto/ateapipb/ateapi.pb.go +++ b/pkg/proto/ateapipb/ateapi.pb.go @@ -6699,8 +6699,7 @@ type Worker struct { // +k8s:customValidation # until `format=k8s-ip` is supported // +k8s:immutable Ips []string `protobuf:"bytes,7,rep,name=ips,proto3" json:"ips,omitempty"` - // sandbox_class mirrors the WorkerPool's sandboxClass; its values are the - // CRD's own vocabulary, so it is only bounded, not validated. + // sandbox_class mirrors the WorkerPool's default sandbox class // // +k8s:optional // +k8s:maxLength=63 diff --git a/pkg/proto/ateapipb/ateapi.proto b/pkg/proto/ateapipb/ateapi.proto index 8c4c52081d..4f51e88cfa 100644 --- a/pkg/proto/ateapipb/ateapi.proto +++ b/pkg/proto/ateapipb/ateapi.proto @@ -2070,8 +2070,7 @@ message Worker { // +k8s:immutable repeated string ips = 7; - // sandbox_class mirrors the WorkerPool's sandboxClass; its values are the - // CRD's own vocabulary, so it is only bounded, not validated. + // sandbox_class mirrors the WorkerPool's default sandbox class // // +k8s:optional // +k8s:maxLength=63 From 288694ef2297bb5d6fab30eca328ddaa89015f91 Mon Sep 17 00:00:00 2001 From: Anish Ramasekar Date: Fri, 9 Oct 2026 03:02:17 +0000 Subject: [PATCH 22/44] Fail snapshot plugin calls with Unavailable after 30s when the plugin is down (#2380) Calls to the snapshot plugin used `WaitForReady(true)`, so once the sidecar went down after startup a call waited out the caller's deadline (10m in ate-api-server, unbounded without one). - Calls wait at most 30s for the connection, then fail with `Unavailable`. Calls already running aren't cut short, and a sidecar restart within the window is ridden through. - atelet and ate-api-server now return a plugin `Unavailable` as `Unavailable` instead of `Internal`, so ate-api-server retries instead of crashing the actor. - The upload right after `CheckpointWorkload` still waits, since the workload is already gone and failing there would lose state. Tested against a real Unix-socket plugin: a 40s outage fails at 30s, a 12s outage succeeds. - [x] Tests pass --------- Signed-off-by: Anish Ramasekar --- cmd/ateapi/internal/controlapi/workflow.go | 5 +- .../workflow_snapshot_plugin_test.go | 72 +++++ cmd/ateapi/main.go | 6 +- cmd/atelet/main.go | 29 +- cmd/atelet/snapshot_transfer.go | 20 +- cmd/atelet/snapshot_transfer_test.go | 230 +++++++++++++++ cmd/snapshot-plugin/main.go | 2 +- internal/objectstoreplugin/calls_test.go | 41 +++ internal/objectstoreplugin/plugin_test.go | 8 +- internal/objectstoreplugin/socket.go | 103 ++++++- internal/objectstoreplugin/socket_test.go | 268 ++++++++++++++++++ 11 files changed, 749 insertions(+), 35 deletions(-) create mode 100644 cmd/ateapi/internal/controlapi/workflow_snapshot_plugin_test.go create mode 100644 internal/objectstoreplugin/calls_test.go create mode 100644 internal/objectstoreplugin/socket_test.go diff --git a/cmd/ateapi/internal/controlapi/workflow.go b/cmd/ateapi/internal/controlapi/workflow.go index 810eabe24f..552b97c7a0 100644 --- a/cmd/ateapi/internal/controlapi/workflow.go +++ b/cmd/ateapi/internal/controlapi/workflow.go @@ -26,6 +26,7 @@ import ( "github.com/agent-substrate/substrate/internal/actorevent" "github.com/agent-substrate/substrate/internal/apierror" "github.com/agent-substrate/substrate/internal/ateattr" + "github.com/agent-substrate/substrate/internal/objectstoreplugin" "github.com/agent-substrate/substrate/internal/resources" listersv1alpha1 "github.com/agent-substrate/substrate/pkg/client/listers/api/v1alpha1" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" @@ -146,14 +147,14 @@ func NewActorWorkflow( // snapshot plugin. func (w *ActorWorkflow) cleanupSnapshot(ctx context.Context, prefix resources.StoragePrefix) error { _, err := w.snapshotPlugin.CleanupSnapshot(ctx, &objectstoresnapshotv1.CleanupSnapshotRequest{SnapshotUri: prefix.String()}) - return err + return objectstoreplugin.CallError(err) } // copySnapshot copies every object under src to dst through the control // snapshot plugin. func (w *ActorWorkflow) copySnapshot(ctx context.Context, src, dst resources.StoragePrefix) error { _, err := w.snapshotPlugin.CopySnapshot(ctx, &objectstoresnapshotv1.CopySnapshotRequest{SrcUri: src.String(), DstUri: dst.String()}) - return err + return objectstoreplugin.CallError(err) } // actorWorkflowStore enumerates the exact storage methods needed by diff --git a/cmd/ateapi/internal/controlapi/workflow_snapshot_plugin_test.go b/cmd/ateapi/internal/controlapi/workflow_snapshot_plugin_test.go new file mode 100644 index 0000000000..a70602307c --- /dev/null +++ b/cmd/ateapi/internal/controlapi/workflow_snapshot_plugin_test.go @@ -0,0 +1,72 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlapi + +import ( + "context" + "testing" + + "github.com/agent-substrate/substrate/internal/ateinterceptors" + "github.com/agent-substrate/substrate/internal/resources" + objectstoresnapshotv1 "github.com/agent-substrate/substrate/pkg/proto/objectstoresnapshotpb/v1" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// failingControlPlugin fails every call with err. +type failingControlPlugin struct { + err error +} + +func (p failingControlPlugin) CleanupSnapshot(context.Context, *objectstoresnapshotv1.CleanupSnapshotRequest, ...grpc.CallOption) (*objectstoresnapshotv1.CleanupSnapshotResponse, error) { + return nil, p.err +} + +func (p failingControlPlugin) CopySnapshot(context.Context, *objectstoresnapshotv1.CopySnapshotRequest, ...grpc.CallOption) (*objectstoresnapshotv1.CopySnapshotResponse, error) { + return nil, p.err +} + +// A control snapshot plugin that cannot be reached fails the request with +// Unavailable, which a client retries; other plugin errors stay Internal. +func TestSnapshotPluginErrorCodeToClient(t *testing.T) { + prefix, err := resources.ParseStoragePrefix("gs://bucket/root/atespaces/team-a/actors/uid1/snapshots/snap1") + if err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + pluginCode codes.Code + want codes.Code + }{ + {codes.Unavailable, codes.Unavailable}, + {codes.Internal, codes.Internal}, + {codes.FailedPrecondition, codes.Internal}, + } { + t.Run(tc.pluginCode.String(), func(t *testing.T) { + w := &ActorWorkflow{snapshotPlugin: failingControlPlugin{err: status.Error(tc.pluginCode, "plugin failed")}} + for name, call := range map[string]func(context.Context) error{ + "cleanup": func(ctx context.Context) error { return w.cleanupSnapshot(ctx, prefix) }, + "copy": func(ctx context.Context) error { return w.copySnapshot(ctx, prefix, prefix) }, + } { + // The public server interceptor decides the code the client sees. + _, err := ateinterceptors.ServerUnaryInterceptor(context.Background(), nil, &grpc.UnaryServerInfo{FullMethod: "/test"}, + func(ctx context.Context, _ any) (any, error) { return nil, call(ctx) }) + if got := status.Code(err); got != tc.want { + t.Errorf("%s: code = %s (%v), want %s", name, got, err, tc.want) + } + } + }) + } +} diff --git a/cmd/ateapi/main.go b/cmd/ateapi/main.go index 4bd1553336..3b3e383d66 100644 --- a/cmd/ateapi/main.go +++ b/cmd/ateapi/main.go @@ -261,13 +261,13 @@ func main() { serverboot.Fatal(ctx, "Failed to create metric instruments", err) } - snapshotPluginConn, err := objectstoreplugin.Dial(*snapshotPluginSocket) + snapshotPluginConn, err := objectstoreplugin.Dial(*snapshotPluginSocket, objectstoreplugin.ReadyWait) if err != nil { serverboot.Fatal(ctx, "Failed to set up the snapshot plugin client", err) } defer snapshotPluginConn.Close() - // Plugin calls wait for the connection to become ready, so a plugin - // that never serves would hang them. Fail at startup instead. + // A plugin that never serves would fail every call. Fail at startup + // instead, giving the sidecar time to come up. readyCtx, cancelReady := context.WithTimeout(ctx, time.Minute) err = objectstoreplugin.WaitReady(readyCtx, snapshotPluginConn) cancelReady() diff --git a/cmd/atelet/main.go b/cmd/atelet/main.go index 461acd247f..6ac28da6c9 100644 --- a/cmd/atelet/main.go +++ b/cmd/atelet/main.go @@ -290,13 +290,13 @@ func main() { go trustBundles.Informer().Run(stopCh) cache.WaitForCacheSync(stopCh, trustBundles.Informer().HasSynced) - snapshotPluginConn, err := objectstoreplugin.Dial(*snapshotPluginSocket) + snapshotPluginConn, err := objectstoreplugin.Dial(*snapshotPluginSocket, objectstoreplugin.ReadyWait) if err != nil { serverboot.Fatal(ctx, "Failed to set up the snapshot plugin client", err) } defer snapshotPluginConn.Close() - // Transfers wait for the connection to become ready, so a plugin that - // never serves would hang them. Fail at startup instead. + // A plugin that never serves would fail every call. Fail at startup + // instead, giving the sidecar time to come up. readyCtx, cancelReady := context.WithTimeout(ctx, time.Minute) err = objectstoreplugin.WaitReady(readyCtx, snapshotPluginConn) cancelReady() @@ -753,6 +753,10 @@ func (s *AteomHerder) Checkpoint(ctx context.Context, req *ateletpb.CheckpointRe tPersist := time.Now() switch req.GetType() { case ateletpb.CheckpointType_CHECKPOINT_TYPE_EXTERNAL: + // CheckpointWorkload has already deleted the workload, so a retried + // Checkpoint cannot redo this upload. It waits for the snapshot plugin + // as long as ctx allows, and a plugin error reaches the control plane as + // Internal, which crashes the actor. // TODO(#362): Because we do not cache the external snapshot files when upload fails, we have to mark the Actor as CRASHED. if err := s.uploadExternalCheckpoint(ctx, req, checkpointDir, sandboxRec); err != nil { dPersist = time.Since(tPersist) @@ -875,12 +879,16 @@ func shouldHaveSnapshots(req *ateletpb.CheckpointRequest) bool { return false } +// uploadExternalCheckpoint uploads the checkpoint CheckpointWorkload just +// took. Its plugin calls pass grpc.WaitForReady, so they wait out a plugin +// outage for as long as ctx allows rather than only objectstoreplugin's ready +// wait, and their errors are not reported as retryable: see Checkpoint. func (s *AteomHerder) uploadExternalCheckpoint(ctx context.Context, req *ateletpb.CheckpointRequest, checkpointDir string, rec *sandboxAssetsRecord) error { uri, err := resources.ParseSnapshotURI(req.GetExternalConfig().GetSnapshotUri()) if err != nil { return err } - return s.uploadSnapshot(ctx, uri, checkpointDir, rec, req.GetActorTemplateAtespace(), req.GetActorTemplateName()) + return s.uploadSnapshot(ctx, uri, checkpointDir, rec, req.GetActorTemplateAtespace(), req.GetActorTemplateName(), grpc.WaitForReady(true)) } // uploadSnapshot uploads rec's snapshot files from srcDir to uri through the @@ -888,8 +896,9 @@ func (s *AteomHerder) uploadExternalCheckpoint(ctx context.Context, req *ateletp // own call: its presence is the commit marker — readers assume every file it // lists is already present. A crash mid-upload thus leaves only orphaned // files, never a manifest pointing at files that never landed; retries -// overwrite the deterministic object names. -func (s *AteomHerder) uploadSnapshot(ctx context.Context, uri resources.SnapshotURI, srcDir string, rec *sandboxAssetsRecord, templateAtespace, templateName string) error { +// overwrite the deterministic object names. opts are passed to every plugin +// call, and plugin errors are returned as uploadSnapshotFiles returns them. +func (s *AteomHerder) uploadSnapshot(ctx context.Context, uri resources.SnapshotURI, srcDir string, rec *sandboxAssetsRecord, templateAtespace, templateName string, opts ...grpc.CallOption) error { root, err := os.OpenRoot(srcDir) if err != nil { return fmt.Errorf("while opening snapshot directory: %w", err) @@ -903,7 +912,7 @@ func (s *AteomHerder) uploadSnapshot(ctx context.Context, uri resources.Snapshot } recordSnapshotSize(ctx, fileName, allocatedBytes(info), templateAtespace, templateName) } - if err := s.uploadSnapshotFiles(ctx, uri.String(), srcDir, rec.SnapshotFiles); err != nil { + if err := s.uploadSnapshotFiles(ctx, uri.String(), srcDir, rec.SnapshotFiles, opts...); err != nil { return fmt.Errorf("while uploading snapshot files: %w", err) } @@ -911,7 +920,7 @@ func (s *AteomHerder) uploadSnapshot(ctx context.Context, uri resources.Snapshot if err != nil { return fmt.Errorf("while marshaling snapshot manifest: %w", err) } - if err := s.uploadManifest(ctx, uri.String(), manifest); err != nil { + if err := s.uploadManifest(ctx, uri.String(), manifest, opts...); err != nil { return fmt.Errorf("while uploading snapshot manifest: %w", err) } return nil @@ -1015,7 +1024,9 @@ func (s *AteomHerder) uploadLocalCheckpointDir(ctx context.Context, req *ateletp } } - return rec.SandboxClass, s.uploadSnapshot(ctx, uri, localDir, rec, req.GetActorTemplateAtespace(), req.GetActorTemplateName()) + // The local snapshot stays until the upload succeeds, so a retry can + // upload it again. + return rec.SandboxClass, objectstoreplugin.CallError(s.uploadSnapshot(ctx, uri, localDir, rec, req.GetActorTemplateAtespace(), req.GetActorTemplateName())) } func readSnapshotManifest(dir string) ([]byte, error) { diff --git a/cmd/atelet/snapshot_transfer.go b/cmd/atelet/snapshot_transfer.go index f6b2df59d2..0cbc579c49 100644 --- a/cmd/atelet/snapshot_transfer.go +++ b/cmd/atelet/snapshot_transfer.go @@ -20,7 +20,9 @@ import ( "os" "path/filepath" + "github.com/agent-substrate/substrate/internal/objectstoreplugin" objectstoresnapshotv1 "github.com/agent-substrate/substrate/pkg/proto/objectstoresnapshotpb/v1" + "google.golang.org/grpc" ) // External snapshots move between the node and storage only through the @@ -37,11 +39,14 @@ func (s *AteomHerder) fetchSnapshotFiles(ctx context.Context, snapshotURI, dstDi WritePath: dstDir, Files: files, }) - return err + return objectstoreplugin.CallError(err) } -// uploadSnapshotFiles uploads the named files in srcDir to the snapshot. -func (s *AteomHerder) uploadSnapshotFiles(ctx context.Context, snapshotURI, srcDir string, files []string) error { +// uploadSnapshotFiles uploads the named files in srcDir to the snapshot, +// passing opts to the plugin call. It returns the plugin's error as is: an +// upload that can be retried reports it through objectstoreplugin.CallError, +// one that cannot does not. +func (s *AteomHerder) uploadSnapshotFiles(ctx context.Context, snapshotURI, srcDir string, files []string, opts ...grpc.CallOption) error { if len(files) == 0 { return nil } @@ -49,7 +54,7 @@ func (s *AteomHerder) uploadSnapshotFiles(ctx context.Context, snapshotURI, srcD SnapshotUri: snapshotURI, LocalPath: srcDir, Files: files, - }) + }, opts...) return err } @@ -67,8 +72,9 @@ func (s *AteomHerder) fetchManifest(ctx context.Context, snapshotURI string) ([] return os.ReadFile(filepath.Join(dir, sandboxManifestName)) } -// uploadManifest uploads manifest as the snapshot's manifest. -func (s *AteomHerder) uploadManifest(ctx context.Context, snapshotURI string, manifest []byte) error { +// uploadManifest uploads manifest as the snapshot's manifest, as +// uploadSnapshotFiles does. +func (s *AteomHerder) uploadManifest(ctx context.Context, snapshotURI string, manifest []byte, opts ...grpc.CallOption) error { dir, err := s.snapshotScratch() if err != nil { return err @@ -77,7 +83,7 @@ func (s *AteomHerder) uploadManifest(ctx context.Context, snapshotURI string, ma if err := os.WriteFile(filepath.Join(dir, sandboxManifestName), manifest, 0o600); err != nil { return fmt.Errorf("while staging snapshot manifest: %w", err) } - return s.uploadSnapshotFiles(ctx, snapshotURI, dir, []string{sandboxManifestName}) + return s.uploadSnapshotFiles(ctx, snapshotURI, dir, []string{sandboxManifestName}, opts...) } // snapshotScratch creates a fresh directory for one manifest transfer. The diff --git a/cmd/atelet/snapshot_transfer_test.go b/cmd/atelet/snapshot_transfer_test.go index e9b8a14e76..a13873e789 100644 --- a/cmd/atelet/snapshot_transfer_test.go +++ b/cmd/atelet/snapshot_transfer_test.go @@ -20,15 +20,27 @@ import ( "fmt" "io" "os" + "path/filepath" "slices" "strings" "sync" "testing" + "time" + "github.com/agent-substrate/substrate/internal/apierror" + "github.com/agent-substrate/substrate/internal/ateattr" + "github.com/agent-substrate/substrate/internal/ateinterceptors" "github.com/agent-substrate/substrate/internal/objectstoreplugin" "github.com/agent-substrate/substrate/internal/objectstoreplugin/objectstoreplugintest" "github.com/agent-substrate/substrate/internal/resources" "github.com/agent-substrate/substrate/pkg/objectstorage" + objectstoresnapshotv1 "github.com/agent-substrate/substrate/pkg/proto/objectstoresnapshotpb/v1" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/health" + healthpb "google.golang.org/grpc/health/grpc_health_v1" + "google.golang.org/grpc/status" ) // newPluginHerder returns an AteomHerder whose snapshots go through the @@ -136,3 +148,221 @@ func TestSnapshotTransferSkipsEmptyFileLists(t *testing.T) { t.Errorf("objects stored = %v, want none", store.puts) } } + +// failingNodePlugin fails every call with err. +type failingNodePlugin struct { + err error +} + +func (p failingNodePlugin) FetchSnapshot(context.Context, *objectstoresnapshotv1.FetchSnapshotRequest, ...grpc.CallOption) (*objectstoresnapshotv1.FetchSnapshotResponse, error) { + return nil, p.err +} + +func (p failingNodePlugin) UploadSnapshot(context.Context, *objectstoresnapshotv1.UploadSnapshotRequest, ...grpc.CallOption) (*objectstoresnapshotv1.UploadSnapshotResponse, error) { + return nil, p.err +} + +// A snapshot plugin that cannot be reached reaches ate-api-server as +// Unavailable, which it retries, rather than Internal, which crashes the +// actor. The upload after CheckpointWorkload is the exception: a retried +// Checkpoint cannot redo it, so it stays Internal. Other plugin errors reach +// ate-api-server as Internal. +func TestSnapshotPluginErrorCodeThroughAtelet(t *testing.T) { + uri, err := resources.ParseSnapshotURI(pausedSnapshotURI) + if err != nil { + t.Fatal(err) + } + rec := sandboxAssetsRecord{ + SandboxClass: "gvisor", + PauseImage: testPauseImage, + SnapshotFiles: []string{"a"}, + Fidelity: ateattr.SnapshotFidelityMemory, + } + transfers := []struct { + name string + retryable bool + run func(ctx context.Context, s *AteomHerder) error + }{ + {"download", true, func(ctx context.Context, s *AteomHerder) error { + return s.downloadExternalCheckpoint(ctx, pausedSnapshotURI, t.TempDir(), []string{"a"}) + }}, + {"paused upload", true, func(ctx context.Context, s *AteomHerder) error { + dir := t.TempDir() + writeLocalSnapshot(t, dir, rec, map[string]string{"a": "a"}) + _, err := s.uploadLocalCheckpointDir(ctx, validUploadPausedCheckpointRequest(), dir, uri) + return err + }}, + {"checkpoint upload", false, func(ctx context.Context, s *AteomHerder) error { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "a"), []byte("a"), 0o600); err != nil { + return err + } + return s.uploadExternalCheckpoint(ctx, validCheckpointRequest(), dir, &rec) + }}, + } + for _, pluginCode := range []codes.Code{codes.Unavailable, codes.Internal, codes.NotFound} { + for _, tr := range transfers { + t.Run(pluginCode.String()+"/"+tr.name, func(t *testing.T) { + want := codes.Internal + if pluginCode == codes.Unavailable && tr.retryable { + want = codes.Unavailable + } + s := &AteomHerder{ + snapshotPlugin: failingNodePlugin{err: status.Error(pluginCode, "plugin failed")}, + snapshotScratchDir: t.TempDir(), + } + // atelet's server interceptor decides the code ate-api-server sees. + _, err := ateinterceptors.InternalServerUnaryInterceptor(context.Background(), nil, &grpc.UnaryServerInfo{FullMethod: "/test"}, + func(ctx context.Context, _ any) (any, error) { return nil, tr.run(ctx, s) }) + if got := status.Code(err); got != want { + t.Errorf("code = %s (%v), want %s", got, err, want) + } + }) + } + } +} + +// pluginSidecar serves a NodePlugin on a Unix socket and can be stopped and +// started again on the same path, as kubelet restarts a sidecar container. +type pluginSidecar struct { + t *testing.T + path string + plugin *objectstoreplugin.NodePlugin + + mu sync.Mutex + srv *grpc.Server +} + +func newPluginSidecar(t *testing.T, store objectstorage.ObjectStorage) *pluginSidecar { + t.Helper() + plugin, err := objectstoreplugin.NewNodePlugin(store, "/") + if err != nil { + t.Fatal(err) + } + // Unix socket paths are length-limited; t.TempDir can exceed that on macOS. + dir, err := os.MkdirTemp("", "snapplug") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { os.RemoveAll(dir) }) + s := &pluginSidecar{t: t, path: filepath.Join(dir, "plugin.sock"), plugin: plugin} + t.Cleanup(s.stop) + return s +} + +// start may run on any goroutine, so it reports failures with Error. +func (s *pluginSidecar) start() { + lis, err := objectstoreplugin.Listen(s.path) + if err != nil { + s.t.Errorf("Listen: %v", err) + return + } + srv := grpc.NewServer() + objectstoresnapshotv1.RegisterNodeProviderServer(srv, s.plugin) + healthpb.RegisterHealthServer(srv, health.NewServer()) + go srv.Serve(lis) + s.mu.Lock() + s.srv = srv + s.mu.Unlock() +} + +// stop ends the server and every connection to it at once, as a crashed +// sidecar does. +func (s *pluginSidecar) stop() { + s.mu.Lock() + srv := s.srv + s.srv = nil + s.mu.Unlock() + if srv != nil { + srv.Stop() + } +} + +// The upload after CheckpointWorkload waits out a plugin outage longer than +// the ready wait, since a retried Checkpoint cannot redo it. The paused +// upload, which a retry can redo, still fails within the ready wait. +func TestCheckpointUploadWaitsOutPluginOutage(t *testing.T) { + const wait = 200 * time.Millisecond + store := &orderedObjectStorage{} + sidecar := newPluginSidecar(t, store) + sidecar.start() + conn, err := objectstoreplugin.Dial(sidecar.path, wait) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + if err := objectstoreplugin.WaitReady(ctx, conn); err != nil { + t.Fatalf("WaitReady = %v", err) + } + s := &AteomHerder{ + snapshotPlugin: objectstoresnapshotv1.NewNodeProviderClient(conn), + snapshotScratchDir: t.TempDir(), + } + rec := sandboxAssetsRecord{ + SandboxClass: "gvisor", + PauseImage: testPauseImage, + SnapshotFiles: []string{"a"}, + Fidelity: ateattr.SnapshotFidelityMemory, + } + pausedDir := t.TempDir() + writeLocalSnapshot(t, pausedDir, rec, map[string]string{"a": "a"}) + checkpointDir := t.TempDir() + if err := os.WriteFile(filepath.Join(checkpointDir, "a"), []byte("a"), 0o600); err != nil { + t.Fatal(err) + } + uri, err := resources.ParseSnapshotURI(pausedSnapshotURI) + if err != nil { + t.Fatal(err) + } + + // Lose the plugin, and wait until the connection has seen it go so the + // uploads start on a lost connection rather than racing the loss. + sidecar.stop() + for conn.GetState() == connectivity.Ready { + if !conn.WaitForStateChange(ctx, connectivity.Ready) { + t.Fatal("connection still ready after the plugin stopped") + } + } + + if _, err := s.uploadLocalCheckpointDir(ctx, validUploadPausedCheckpointRequest(), pausedDir, uri); apierror.Code(err) != codes.Unavailable { + t.Fatalf("paused upload during the outage = %v, want %s", err, codes.Unavailable) + } + + const outage = 10 * wait + restarted := make(chan struct{}) + go func() { + defer close(restarted) + time.Sleep(outage) + sidecar.start() + }() + defer func() { <-restarted }() + + start := time.Now() + if err := s.uploadExternalCheckpoint(ctx, validCheckpointRequest(), checkpointDir, &rec); err != nil { + t.Fatalf("checkpoint upload across a plugin outage = %v, want success", err) + } + if elapsed := time.Since(start); elapsed < outage/2 { + t.Errorf("checkpoint upload succeeded after %s, want it to have waited out the %s outage", elapsed, outage) + } + store.mu.Lock() + defer store.mu.Unlock() + object := strings.TrimPrefix(testSnapshotPath, "bucket/") + if want := []string{object + "/a.zstd", object + "/" + sandboxManifestName}; !slices.Equal(store.puts, want) { + t.Errorf("objects stored = %v, want %v", store.puts, want) + } +} + +// fetchManifest keeps the plugin's NotFound readable, which the paused +// snapshot upload uses to tell a missing snapshot from a failed probe. +func TestFetchManifestKeepsNotFound(t *testing.T) { + s := &AteomHerder{ + snapshotPlugin: failingNodePlugin{err: status.Error(codes.NotFound, "no manifest")}, + snapshotScratchDir: t.TempDir(), + } + _, err := s.fetchManifest(context.Background(), pausedSnapshotURI) + if status.Code(err) != codes.NotFound { + t.Errorf("fetchManifest = %v, want %s", err, codes.NotFound) + } +} diff --git a/cmd/snapshot-plugin/main.go b/cmd/snapshot-plugin/main.go index 4f8d8c82c5..d62391e416 100644 --- a/cmd/snapshot-plugin/main.go +++ b/cmd/snapshot-plugin/main.go @@ -149,7 +149,7 @@ func newObjectStore(ctx context.Context) (objectstore.Store, error) { // healthcheck dials the plugin at socket and waits up to timeout for it to // report that it is serving. func healthcheck(ctx context.Context, socket string, timeout time.Duration) error { - conn, err := objectstoreplugin.Dial(socket) + conn, err := objectstoreplugin.Dial(socket, objectstoreplugin.ReadyWait) if err != nil { return err } diff --git a/internal/objectstoreplugin/calls_test.go b/internal/objectstoreplugin/calls_test.go new file mode 100644 index 0000000000..dd272f9919 --- /dev/null +++ b/internal/objectstoreplugin/calls_test.go @@ -0,0 +1,41 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package objectstoreplugin + +import ( + "errors" + "testing" + + "github.com/agent-substrate/substrate/internal/apierror" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func TestCallError(t *testing.T) { + if err := CallError(nil); err != nil { + t.Errorf("CallError(nil) = %v, want nil", err) + } + unavailable := status.Error(codes.Unavailable, "connection refused") + if got := apierror.Code(CallError(unavailable)); got != codes.Unavailable { + t.Errorf("apierror.Code(CallError(Unavailable)) = %s, want %s", got, codes.Unavailable) + } + for _, code := range []codes.Code{codes.NotFound, codes.Internal, codes.FailedPrecondition} { + in := status.Error(code, "plugin error") + got := CallError(in) + if !errors.Is(got, in) || status.Code(got) != code { + t.Errorf("CallError(%s) = %v, want the plugin's status unchanged", code, got) + } + } +} diff --git a/internal/objectstoreplugin/plugin_test.go b/internal/objectstoreplugin/plugin_test.go index d3e6ac5a43..9857f8ba80 100644 --- a/internal/objectstoreplugin/plugin_test.go +++ b/internal/objectstoreplugin/plugin_test.go @@ -134,7 +134,7 @@ func serve(t *testing.T, backend *memObjects, root string) *grpc.ClientConn { go srv.Serve(lis) t.Cleanup(srv.Stop) - conn, err := Dial(filepath.Join(sockDir, "plugin.sock")) + conn, err := Dial(filepath.Join(sockDir, "plugin.sock"), ReadyWait) if err != nil { t.Fatal(err) } @@ -149,9 +149,9 @@ func TestWaitReady(t *testing.T) { t.Errorf("WaitReady against a serving plugin = %v", err) } - // A socket nobody listens on must fail when ctx ends, not hang: Dial - // makes every call wait for the plugin. - conn, err := Dial(filepath.Join(t.TempDir(), "missing.sock")) + // A socket nobody listens on must fail when ctx ends, not hang: WaitReady + // waits for the plugin as long as ctx allows. + conn, err := Dial(filepath.Join(t.TempDir(), "missing.sock"), ReadyWait) if err != nil { t.Fatal(err) } diff --git a/internal/objectstoreplugin/socket.go b/internal/objectstoreplugin/socket.go index 14138dc8de..a5881aea04 100644 --- a/internal/objectstoreplugin/socket.go +++ b/internal/objectstoreplugin/socket.go @@ -22,13 +22,37 @@ import ( "net" "os" "path/filepath" + "time" + "github.com/agent-substrate/substrate/internal/apierror" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "google.golang.org/grpc" + "google.golang.org/grpc/backoff" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" "google.golang.org/grpc/credentials/insecure" healthpb "google.golang.org/grpc/health/grpc_health_v1" + "google.golang.org/grpc/status" ) +// ReadyWait is the ready wait servers pass to Dial. It rides through a sidecar +// restart, including the first 10s step of kubelet's crash-loop back-off, and +// fails a plugin that stays down well before a caller's deadline would. +const ReadyWait = 30 * time.Second + +// reconnectParams keeps reconnect attempts to the local socket frequent, so a +// plugin that comes back is reached within the ready wait; gRPC's default back-off +// grows to two minutes. +var reconnectParams = grpc.ConnectParams{ + Backoff: backoff.Config{ + BaseDelay: 100 * time.Millisecond, + Multiplier: 1.6, + Jitter: 0.2, + MaxDelay: time.Second, + }, + MinConnectTimeout: 5 * time.Second, +} + // Listen removes any stale socket at path and listens on a fresh one that // only the socket's owner can connect to. func Listen(path string) (net.Listener, error) { @@ -49,17 +73,28 @@ func Listen(path string) (net.Listener, error) { return lis, nil } -// Dial returns a long-lived client connection to a plugin socket. Calls wait -// for the plugin to come up rather than failing while it starts, so a caller -// that must not hang on a plugin that never comes up checks WaitReady first. +// Dial returns a long-lived client connection to a plugin socket. A unary +// call waits up to readyWait for the connection to become ready, then fails +// with codes.Unavailable, so a plugin that is lost after startup fails calls +// promptly while a brief restart does not. A call that starts on a ready +// connection is not bounded by readyWait, and one in flight when the plugin +// goes fails with codes.Unavailable at once. A caller that passes +// grpc.WaitForReady itself opts out of the bound and waits as long as its +// context allows; WaitReady does so at startup. +// +// The bound covers unary calls only. A streaming call gets gRPC's plain +// fail-fast and fails at once while the plugin restarts, so the first +// streaming RPC on this connection needs a grpc.WithStreamInterceptor that +// applies the same wait. // // The connection is unauthenticated: the socket is reachable only from // inside the pod, and only by its owner. -func Dial(path string) (*grpc.ClientConn, error) { +func Dial(path string, readyWait time.Duration) (*grpc.ClientConn, error) { conn, err := grpc.NewClient("unix://"+path, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithStatsHandler(otelgrpc.NewClientHandler()), - grpc.WithDefaultCallOptions(grpc.WaitForReady(true)), + grpc.WithConnectParams(reconnectParams), + grpc.WithUnaryInterceptor(boundedWaitForReady(readyWait)), ) if err != nil { return nil, fmt.Errorf("while dialing snapshot plugin at %s: %w", path, err) @@ -67,12 +102,51 @@ func Dial(path string) (*grpc.ClientConn, error) { return conn, nil } +// boundedWaitForReady waits up to wait for cc to become ready before issuing +// a call that sets no grpc.WaitForReady option, then issues it fail-fast. +func boundedWaitForReady(wait time.Duration) grpc.UnaryClientInterceptor { + return func(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { + for _, opt := range opts { + if _, ok := opt.(grpc.FailFastCallOption); ok { + return invoker(ctx, method, req, reply, cc, opts...) + } + } + if err := awaitReady(ctx, cc, wait); err != nil { + return err + } + return invoker(ctx, method, req, reply, cc, opts...) + } +} + +// awaitReady returns nil once cc is ready or shut down, the latter left for +// the call itself to report. It returns codes.Unavailable if cc is not ready +// within wait, and ctx's error as a status if ctx ends first. +func awaitReady(ctx context.Context, cc *grpc.ClientConn, wait time.Duration) error { + waitCtx, cancel := context.WithTimeout(ctx, wait) + defer cancel() + for { + state := cc.GetState() + switch state { + case connectivity.Ready, connectivity.Shutdown: + return nil + case connectivity.Idle: + cc.Connect() + } + if !cc.WaitForStateChange(waitCtx, state) { + if ctx.Err() != nil { + return status.FromContextError(ctx.Err()).Err() + } + return status.Errorf(codes.Unavailable, "snapshot plugin at %s is not ready after %s (connection %s)", cc.Target(), wait, state) + } + } +} + // WaitReady blocks until the plugin behind conn reports that it is serving, -// or ctx ends. Bounding ctx is what turns a plugin that never comes up (a -// wrong socket path, a crash loop) into an error at startup rather than a -// transfer that hangs. +// or ctx ends, however long the plugin takes to come up. Bounding ctx is what +// turns a plugin that never comes up (a wrong socket path, a crash loop) into +// an error at startup. func WaitReady(ctx context.Context, conn *grpc.ClientConn) error { - resp, err := healthpb.NewHealthClient(conn).Check(ctx, &healthpb.HealthCheckRequest{}) + resp, err := healthpb.NewHealthClient(conn).Check(ctx, &healthpb.HealthCheckRequest{}, grpc.WaitForReady(true)) if err != nil { return fmt.Errorf("while waiting for the snapshot plugin at %s: %w", conn.Target(), err) } @@ -81,3 +155,14 @@ func WaitReady(ctx context.Context, conn *grpc.ClientConn) error { } return nil } + +// CallError returns the error a server reports to its own caller for a failed +// plugin call. codes.Unavailable, which a call reports when the plugin cannot +// be reached, becomes an apierror.Unavailable so the server's caller retries +// rather than treating it as Internal. Any other error is returned unchanged. +func CallError(err error) error { + if status.Code(err) == codes.Unavailable { + return apierror.Unavailable("snapshot plugin: %w", err) + } + return err +} diff --git a/internal/objectstoreplugin/socket_test.go b/internal/objectstoreplugin/socket_test.go new file mode 100644 index 0000000000..b849b82713 --- /dev/null +++ b/internal/objectstoreplugin/socket_test.go @@ -0,0 +1,268 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package objectstoreplugin + +import ( + "context" + "os" + "path/filepath" + "sync" + "testing" + "time" + + objectstoresnapshotv1 "github.com/agent-substrate/substrate/pkg/proto/objectstoresnapshotpb/v1" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/health" + healthpb "google.golang.org/grpc/health/grpc_health_v1" + "google.golang.org/grpc/status" +) + +// slowControl is a ControlProvider whose CleanupSnapshot takes delay, and +// closes started when a call arrives. +type slowControl struct { + objectstoresnapshotv1.UnimplementedControlProviderServer + + delay time.Duration + once sync.Once + started chan struct{} +} + +func newSlowControl(delay time.Duration) *slowControl { + return &slowControl{delay: delay, started: make(chan struct{})} +} + +func (c *slowControl) CleanupSnapshot(ctx context.Context, _ *objectstoresnapshotv1.CleanupSnapshotRequest) (*objectstoresnapshotv1.CleanupSnapshotResponse, error) { + c.once.Do(func() { close(c.started) }) + select { + case <-time.After(c.delay): + return &objectstoresnapshotv1.CleanupSnapshotResponse{}, nil + case <-ctx.Done(): + return nil, status.FromContextError(ctx.Err()).Err() + } +} + +// sidecar serves a ControlProvider on a Unix socket and can be stopped and +// started again on the same path, as kubelet restarts a sidecar container. +type sidecar struct { + t *testing.T + path string + control objectstoresnapshotv1.ControlProviderServer + + mu sync.Mutex + srv *grpc.Server +} + +func newSidecar(t *testing.T, control objectstoresnapshotv1.ControlProviderServer) *sidecar { + t.Helper() + // Unix socket paths are length-limited; t.TempDir can exceed that on macOS. + dir, err := os.MkdirTemp("", "snapplug") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { os.RemoveAll(dir) }) + s := &sidecar{t: t, path: filepath.Join(dir, "plugin.sock"), control: control} + t.Cleanup(s.stop) + return s +} + +// start may run on any goroutine, so it reports failures with Error. +func (s *sidecar) start() { + lis, err := Listen(s.path) + if err != nil { + s.t.Errorf("Listen: %v", err) + return + } + srv := grpc.NewServer() + objectstoresnapshotv1.RegisterControlProviderServer(srv, s.control) + healthpb.RegisterHealthServer(srv, health.NewServer()) + go srv.Serve(lis) + s.mu.Lock() + s.srv = srv + s.mu.Unlock() +} + +// stop ends the server and every connection to it at once, as a crashed +// sidecar does. +func (s *sidecar) stop() { + s.mu.Lock() + srv := s.srv + s.srv = nil + s.mu.Unlock() + if srv != nil { + srv.Stop() + } +} + +// dialReady connects to s with the given wait and passes the startup +// readiness check. +func (s *sidecar) dialReady(wait time.Duration) (*grpc.ClientConn, objectstoresnapshotv1.ControlProviderClient) { + s.t.Helper() + conn, err := Dial(s.path, wait) + if err != nil { + s.t.Fatal(err) + } + s.t.Cleanup(func() { conn.Close() }) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if err := WaitReady(ctx, conn); err != nil { + s.t.Fatalf("WaitReady = %v", err) + } + return conn, objectstoresnapshotv1.NewControlProviderClient(conn) +} + +// lose stops s and waits until conn has seen the plugin go, so the next call +// starts on a lost connection rather than racing the loss. +func (s *sidecar) lose(conn *grpc.ClientConn) { + s.t.Helper() + s.stop() + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + for conn.GetState() == connectivity.Ready { + if !conn.WaitForStateChange(ctx, connectivity.Ready) { + s.t.Fatal("connection still ready 10s after the plugin stopped") + } + } +} + +var cleanupReq = &objectstoresnapshotv1.CleanupSnapshotRequest{SnapshotUri: testURI} + +// A plugin lost after startup fails a call with Unavailable once the ready +// wait runs out, not when the caller's deadline does. +func TestCallFailsPromptlyAfterPluginLoss(t *testing.T) { + const wait = 300 * time.Millisecond + s := newSidecar(t, newSlowControl(0)) + s.start() + conn, client := s.dialReady(wait) + s.lose(conn) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + start := time.Now() + _, err := client.CleanupSnapshot(ctx, cleanupReq) + elapsed := time.Since(start) + if status.Code(err) != codes.Unavailable { + t.Fatalf("CleanupSnapshot after plugin loss = %v, want %s", err, codes.Unavailable) + } + if elapsed > 5*time.Second { + t.Errorf("CleanupSnapshot failed after %s, want about %s, well before the caller's 10s deadline", elapsed, wait) + } +} + +// A call that ends with the caller's context while waiting reports the +// context's error, not Unavailable. +func TestCallWaitEndsWithCallerContext(t *testing.T) { + s := newSidecar(t, newSlowControl(0)) + s.start() + conn, client := s.dialReady(time.Minute) + s.lose(conn) + + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + _, err := client.CleanupSnapshot(ctx, cleanupReq) + if status.Code(err) != codes.DeadlineExceeded { + t.Errorf("CleanupSnapshot = %v, want %s", err, codes.DeadlineExceeded) + } +} + +// A call that starts on a ready connection runs as long as the plugin takes, +// however short the ready wait. +func TestSlowCallOutlastsReadyWait(t *testing.T) { + const wait = 100 * time.Millisecond + s := newSidecar(t, newSlowControl(10*wait)) + s.start() + _, client := s.dialReady(wait) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if _, err := client.CleanupSnapshot(ctx, cleanupReq); err != nil { + t.Errorf("CleanupSnapshot on a slow, healthy plugin = %v, want success", err) + } +} + +// A plugin that restarts within the ready wait serves the call. +func TestCallRidesThroughPluginRestart(t *testing.T) { + s := newSidecar(t, newSlowControl(0)) + s.start() + conn, client := s.dialReady(10 * time.Second) + s.lose(conn) + + restarted := make(chan struct{}) + go func() { + defer close(restarted) + time.Sleep(time.Second) + s.start() + }() + defer func() { <-restarted }() + + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + if _, err := client.CleanupSnapshot(ctx, cleanupReq); err != nil { + t.Errorf("CleanupSnapshot across a plugin restart = %v, want success", err) + } +} + +// A call in flight when the plugin dies fails with Unavailable right away. +func TestInFlightCallFailsWhenPluginDies(t *testing.T) { + control := newSlowControl(time.Minute) + s := newSidecar(t, control) + s.start() + _, client := s.dialReady(time.Minute) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + errc := make(chan error, 1) + go func() { + _, err := client.CleanupSnapshot(ctx, cleanupReq) + errc <- err + }() + <-control.started + start := time.Now() + s.stop() + err := <-errc + if status.Code(err) != codes.Unavailable { + t.Fatalf("in-flight CleanupSnapshot = %v, want %s", err, codes.Unavailable) + } + if elapsed := time.Since(start); elapsed > 5*time.Second { + t.Errorf("in-flight CleanupSnapshot failed %s after the plugin died, want promptly", elapsed) + } +} + +// WaitReady at startup waits for the plugin as long as its context allows, +// not only for the per-call ready wait. +func TestWaitReadyOutlastsReadyWait(t *testing.T) { + const wait = 100 * time.Millisecond + s := newSidecar(t, newSlowControl(0)) + conn, err := Dial(s.path, wait) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + + started := make(chan struct{}) + go func() { + defer close(started) + time.Sleep(10 * wait) + s.start() + }() + defer func() { <-started }() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if err := WaitReady(ctx, conn); err != nil { + t.Errorf("WaitReady for a plugin that starts after the ready wait = %v, want success", err) + } +} From 59c5b2a540cd875375c82747b24b52f6cb013797 Mon Sep 17 00:00:00 2001 From: ktsoator Date: Fri, 9 Oct 2026 09:08:28 +0000 Subject: [PATCH 23/44] docs(api): fix systeminfo volume examples (#2346) Remove the extra `spec` wrapper from the SystemInfo volume examples and use the `ACTOR_METADATA_FIELD_*` enum names, so the examples can be parsed by `kubectl ate create actor-template`. To reproduce, copy the `actorMetadata` YAML example from the SystemInfo Volumes section of [docs/api-guide.md](https://github.com/agent-substrate/substrate/blob/760e577a38e66c0de51d05b3dad9a4db7dd0ff51/docs/api-guide.md#L188) into `actor-template.yaml`, then run from the repository root: ```bash go run ./cmd/kubectl-ate create actor-template -f actor-template.yaml ``` This fails with `unknown field "spec"`. Remove the top-level `spec` key and unindent its contents, then run the same command again. It now fails with `invalid value for enum field field: "name"`. --- docs/api-guide.md | 64 +++++++++++++++++++++++------------------------ 1 file changed, 31 insertions(+), 33 deletions(-) diff --git a/docs/api-guide.md b/docs/api-guide.md index 3c07ad6e16..58dfab4b69 100644 --- a/docs/api-guide.md +++ b/docs/api-guide.md @@ -185,28 +185,27 @@ To deliver identity information, including credentials, to a running actor, you Available information sources: #### actorMetadata -The actorMetadata data source projects the actor's identity fields to files, one per item, analogous to the [Kubernetes downwardAPI volume](https://kubernetes.io/docs/concepts/storage/volumes/#downwardapi). Each item selects a `field` — `name` (unique within an atespace), `atespace` (together with the name, the actor's full identity), or `uid` (server-generated, distinguishes incarnations of the same name) — and the `path` the value is written to, raw with no trailing newline. `path` is a clean relative path from the root of the volume (no leading `/`, no `.` or `..` segments, at most 16 segments) and must not repeat another path projected into the same volume. +The actorMetadata data source projects the actor's identity fields to files, one per item, analogous to the [Kubernetes downwardAPI volume](https://kubernetes.io/docs/concepts/storage/volumes/#downwardapi). Each item selects a `field` — `ACTOR_METADATA_FIELD_NAME` (unique within an atespace), `ACTOR_METADATA_FIELD_ATESPACE` (together with the name, the actor's full identity), or `ACTOR_METADATA_FIELD_UID` (server-generated, distinguishes incarnations of the same name) — and the `path` the value is written to, raw with no trailing newline. `path` is a clean relative path from the root of the volume (no leading `/`, no `.` or `..` segments, at most 16 segments) and must not repeat another path projected into the same volume. ```yaml -spec: - volumes: +volumes: +- name: system-info + systemInfo: + dataSources: + - actorMetadata: + items: + - field: ACTOR_METADATA_FIELD_NAME + path: actor-name + - field: ACTOR_METADATA_FIELD_ATESPACE + path: atespace + - field: ACTOR_METADATA_FIELD_UID + path: actor-uid +containers: +- name: main + # ... + volumeMounts: - name: system-info - systemInfo: - dataSources: - - actorMetadata: - items: - - field: name - path: actor-name - - field: atespace - path: atespace - - field: uid - path: actor-uid - containers: - - name: main - # ... - volumeMounts: - - name: system-info - mountPath: /run/ate # the actor reads e.g. /run/ate/actor-name + mountPath: /run/ate # the actor reads e.g. /run/ate/actor-name ``` The values are delivered as files on a read-only per-actor bind mount, not environment variables, precisely so they carry the correct values after a resume from a shared snapshot — an env var (or a file baked into the image) would be frozen at the snapshot-source actor's values, since it lives in the checkpointed process memory, and would therefore be identical for every actor restored from that snapshot. The metadata fields themselves are fixed for the actor's lifetime, so workloads may cache them; future data sources that rotate (identity tokens and certificates) must be re-read at time of use. @@ -222,21 +221,20 @@ Supported names are allowlisted: shipped on Debian (consumed via the distroless-static base image). ```yaml -spec: - volumes: +volumes: +- name: trust + systemInfo: + dataSources: + - trustBundle: + names: + - egress-mitm.ate.dev + path: ca.pem +containers: +- name: main + # ... + volumeMounts: - name: trust - systemInfo: - dataSources: - - trustBundle: - names: - - egress-mitm.ate.dev - path: ca.pem - containers: - - name: main - # ... - volumeMounts: - - name: trust - mountPath: /run/substrate/certs # the actor reads /run/substrate/certs/ca.pem + mountPath: /run/substrate/certs # the actor reads /run/substrate/certs/ca.pem ``` atelet resolves the bundle on the node when the actor starts, reading the backing object through a cluster-wide watch (the same informer that drives live refresh) and sanitizing it the way kubelet does for projections: only `CERTIFICATE` PEM blocks are kept, deduplicated across all the named bundles, with block headers stripped and the anchors deliberately shuffled, so consumers must not depend on their order. The actor itself never talks to any bundle backend. Starting the actor fails, with an error naming the bundle, if any name is not on the allowlist, the bundle's backend is unavailable in this deployment, or the resolved bundle is missing, empty, or contains no certificates. From 632e66221fc841e03ae12e155c0e5c55fe89ec91 Mon Sep 17 00:00:00 2001 From: Tim Bai Date: Fri, 9 Oct 2026 14:37:30 +0000 Subject: [PATCH 24/44] atelet: drop its actor usage records and count each activation's CPU once (#1984) The ateoms now write the actor usage records and serve their latest sample (#2194, #2195), so atelet stops writing its own and counts CPU per activation. - **atelet CPU.** Baselines are keyed by actor and epoch. A new activation counts in full if it began after atelet started, and a worker that does not answer keeps its baselines. This fixes the undercount on micro-VM restores and the double charge when a silent worker's baselines were dropped. - **atelet records removed.** The ateoms' `ate.actor.usage_sampled` records replace them, ending the period with two usage records per actor. - **Timeout.** `GetActiveWorkloadStats` answers from memory, so atelet's `statsRPCTimeout` drops from 55s to 5s. The 50s poll floor stays. - **Cleanup.** `UsageEmitter.EmitFinal` and its flush, atelet's stdout `SyncedWriter`, and the exported `actorlog.LabelsKey` go; nothing uses them now. - **Docs.** The proto comments describe the cached discovery read. The registry and the guide say when the final record is written: whenever the ateom tears an activation down (checkpoint, terminate, graceful shutdown, or a micro-VM re-host), read just before the teardown. #### Known limitation and future work atelet's CPU counter only sees samples of actors still hosted, so it loses each activation's tail: the CPU from the last sample atelet read to the teardown, up to one poll interval plus one sample interval. An activation shorter than that, the common checkpoint-when-idle case, can lose most or all of its CPU in the metric. The usage records keep it, through the final record. To fix it later, the ateoms can own the counter: each ateom adds an activation's CPU increase at every reading, initial, periodic, and final, and exports it through the existing OTLP relay. No gRPC change is needed. atelet then drops the baseline map, its start time, and the silent-worker rule; this also removes the baselines a crashed worker's leftover directory keeps. The cost is cardinality: the series become one per worker pod per label set instead of one per node, still bounded by pods rather than actors, plus a short-lived series per replaced pod. A collector can drop the pod attributes if that matters. #### Breaking change The stdout usage record moves from atelet to the ateoms, with flat `ate.*` keys and the message `Actor usage sampled`. Log filters on atelet's `Actor usage sample` record stop matching: filter on `jsonPayload.msg="Actor usage sampled"` and the `ate.*` keys instead. Identity is no longer in the label group, so on GKE it does not land in `LogEntry.labels`. Part of #1748 (steps 4 to 6). - [x] Tests pass - [x] Appropriate changes to documentation are included in the PR --- cmd/atelet/main.go | 11 +- cmd/atelet/statsevents.go | 184 -------- cmd/atelet/statsevents_test.go | 307 ------------- cmd/atelet/statspoller.go | 183 ++++---- cmd/atelet/statspoller_test.go | 430 ++++++++---------- docs/dev/best-practices/otel-collector.md | 2 +- docs/metrics/registry/events.yaml | 16 +- docs/metrics/registry/metrics.yaml | 28 +- docs/metrics/substrate.yaml | 9 + docs/observability.md | 42 +- internal/actorlog/logger.go | 11 +- internal/actorlog/logger_test.go | 12 +- internal/ateomstats/usage.go | 28 +- internal/ateomstats/usage_test.go | 41 +- internal/proto/ateompb/ateom.pb.go | 11 +- internal/proto/ateompb/ateom.proto | 36 +- internal/proto/ateompb/ateom_grpc.pb.go | 50 +- .../ate-install/kind/ate-otel-config.yaml | 8 +- 18 files changed, 410 insertions(+), 999 deletions(-) delete mode 100644 cmd/atelet/statsevents.go delete mode 100644 cmd/atelet/statsevents_test.go diff --git a/cmd/atelet/main.go b/cmd/atelet/main.go index 6ac28da6c9..085ad698ce 100644 --- a/cmd/atelet/main.go +++ b/cmd/atelet/main.go @@ -34,7 +34,6 @@ import ( "github.com/agent-substrate/substrate/cmd/atelet/internal/credentialprovider" "github.com/agent-substrate/substrate/cmd/atelet/internal/sparsefile" "github.com/agent-substrate/substrate/cmd/atelet/internal/trustbundle" - "github.com/agent-substrate/substrate/internal/actorlog" "github.com/agent-substrate/substrate/internal/apierror" "github.com/agent-substrate/substrate/internal/ateapiauth" "github.com/agent-substrate/substrate/internal/ateattr" @@ -127,13 +126,7 @@ func main() { return } ctx := context.Background() - // One synchronized writer in front of stdout, shared by the runtime - // logger and the usage-event drain (see startStatsPoller): uncoordinated - // writers stay tear-free only while every record fits a pipe's - // atomic-write size -- an accident of field sizes, not a contract. Same - // pattern as the ateoms' actor-log forwarders. - logSink := actorlog.NewSyncedWriter(os.Stdout) - serverboot.InitLoggerWithWriter(logSink) + serverboot.InitLogger() if err := serverboot.SetLogLevel(*logLevelFlag); err != nil { serverboot.Fatal(ctx, "Invalid --log-level", err) } @@ -261,7 +254,7 @@ func main() { // crash-looping every actor operation on the node. slog.ErrorContext(ctx, "Actor stats sampling disabled: failed to create instruments", slog.Any("err", err)) } else { - startStatsPoller(ctx, interval, statsInst, k8sClient, logSink) + startStatsPoller(ctx, interval, statsInst, k8sClient) } } diff --git a/cmd/atelet/statsevents.go b/cmd/atelet/statsevents.go deleted file mode 100644 index fd2672ec1e..0000000000 --- a/cmd/atelet/statsevents.go +++ /dev/null @@ -1,184 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "context" - "io" - "log/slog" - "sync" - "sync/atomic" - - "cloud.google.com/go/compute/metadata" - - "github.com/agent-substrate/substrate/internal/actorlog" - "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/contextlogging" - "github.com/agent-substrate/substrate/internal/proto/ateompb" - "github.com/agent-substrate/substrate/internal/resources" -) - -// The events channel is the per-actor half of the usage telemetry split: the -// metrics (statspoller.go) aggregate to the bounded template-level label set, -// and everything with actor or atespace identity travels here instead, as -// structured log events -- the same stream and label vocabulary as actorlog's -// lifecycle events, never a TSDB series. - -// usageSampleMsg is the message every usage event carries; consumers filter on -// it plus the "kind" field. -const usageSampleMsg = "Actor usage sample" - -// eventKindPeriodic marks the events riding the poller's sweep -- today the -// only kind, carried on the wire so future kinds (lifecycle brackets, say) -// can join without reshaping the record. -const eventKindPeriodic = "periodic" - -// defaultLabelsKey resolves the actor-identity label group's spelling -- -// actorlog's, so usage events and lifecycle events promote into Cloud -// Logging labels the same way. metadata.OnGCE probes the metadata server, so -// startStatsPoller warms this off the boot path on a throwaway goroutine; -// sync.OnceValue makes an emit that arrives first wait for the in-flight -// probe. That wait is first-tick-only and bounded: milliseconds on GCE, the -// transport's 2s dial timeout off GCE, a 5s cap on the one pathological -// branch (SMBIOS says GCE, probes disagree) -- and no request path ever -// touches it. -var defaultLabelsKey = sync.OnceValue(func() string { - return actorlog.LabelsKey(metadata.OnGCE()) -}) - -// statsEventEmitter writes per-actor usage events to the process log stream. -// A nil emitter is a valid no-op, so call sites need no guard. -type statsEventEmitter struct { - log *slog.Logger - labelsKey func() string -} - -// newStatsEventEmitter builds an emitter over its own fixed-level handler on -// w rather than the serverboot logger: these records are a data feed, not -// leveled diagnostics, and quieting the node with --log-level=warn must not -// silently sever them -- the subsystem's one off-switch is -// --actor-stats-poll-interval=0. The records still carry level INFO on the -// wire, so nothing downstream changes. In production w is an asyncWriter -// over the process's synchronized stdout writer (see startStatsPoller). -func newStatsEventEmitter(w io.Writer, labelsKey func() string) *statsEventEmitter { - return &statsEventEmitter{ - log: slog.New(contextlogging.NewHandler(slog.NewJSONHandler(w, nil))), - labelsKey: labelsKey, - } -} - -// usageEventQueueDepth is the asyncWriter's buffer, in records: about one -// tick of a packed node. A healthy drain outruns the RPC-paced producers by -// orders of magnitude, so the queue only fills once the pipe is dead -- -// where any finite depth drops, and the choice is cosmetic. -const usageEventQueueDepth = 256 - -// asyncWriter decouples event emission from the log consumer's health. A -// write to a full stdout pipe blocks forever, and emit runs inside the -// sweep's errgroup, so one stalled log consumer (wedged rotation, disk-full -// fallout) would otherwise park a probe, wedge the sweep, and silently -// freeze the metrics channel that shares it. Writes land in a bounded queue -// drained by one goroutine; a full queue drops the record and counts it. -// Dropping is safe: the samples are point-in-time readings the next healthy -// tick repairs, and when stdout is dead the events are lost either way -- -// the choice is whether the metrics die with them. -type asyncWriter struct { - w io.Writer - ch chan []byte - - // report carries the drop warning over the same fixed-level pipeline as - // the records: the loss signal is part of the feed's integrity, so it - // must be exactly as unkillable by --log-level as the feed itself. - report *slog.Logger - - dropped atomic.Int64 -} - -func newAsyncWriter(ctx context.Context, w io.Writer, depth int) *asyncWriter { - aw := &asyncWriter{ - w: w, - ch: make(chan []byte, depth), - report: slog.New(contextlogging.NewHandler(slog.NewJSONHandler(w, nil))), - } - go aw.run(ctx) - return aw -} - -func (aw *asyncWriter) run(ctx context.Context) { - for { - select { - case <-ctx.Done(): - return - case b := <-aw.ch: - // Best-effort: a failed write has nobody to report to. - _, _ = aw.w.Write(b) - // Report drops only after a successful write, when the stream - // can carry the warning -- written directly, so the drain does - // not queue behind itself. - if n := aw.dropped.Swap(0); n > 0 { - aw.report.WarnContext(ctx, "Usage events dropped while the log stream stalled", slog.Int64("count", n)) - } - } - } -} - -// Write queues one record without ever blocking. It reports full success -// even on a drop: the emitter has no recovery to offer, and the drop is -// already counted for the writer goroutine to report. -func (aw *asyncWriter) Write(p []byte) (int, error) { - // The slog handler reuses its buffer after Write returns, so the queue - // must own a copy. - b := append([]byte(nil), p...) - select { - case aw.ch <- b: - default: - aw.dropped.Add(1) - } - return len(p), nil -} - -// emit writes one usage event. The identity comes solely from the sample's -// echo, per the stats RPCs' attribution contract. pool is the caller's -// pod-to-pool resolution -- the same enrichment the metric labels carry, so a -// pool-level metric spike can pivot to the actors behind it. A zero-valued -// pool omits the label pair rather than emitting empty strings, following the -// metric channel's rule. -func (e *statsEventEmitter) emit(ctx context.Context, kind string, s *ateompb.WorkloadStatsSample, pool workerPoolRef) { - if e == nil || s == nil { - return - } - a := resources.ActorAttribution{ - Ref: resources.ActorRef{Atespace: s.GetAtespace(), Name: s.GetActorName()}, - UID: s.GetActorUid(), - TemplateAtespace: s.GetActorTemplateAtespace(), - TemplateName: s.GetActorTemplateName(), - } - labels := ateattr.ActorLogLabels(a, "") - if pool != (workerPoolRef{}) { - labels[string(ateattr.WorkerPoolNamespaceKey)] = pool.namespace - labels[string(ateattr.WorkerPoolNameKey)] = pool.name - } - e.log.LogAttrs(ctx, slog.LevelInfo, usageSampleMsg, - slog.Any(e.labelsKey(), labels), - slog.String("kind", kind), - slog.String("sandbox_class", sandboxClassLabel(s.GetSandboxClass())), - slog.String("source", statsSourceLabel(s.GetSource())), - slog.Uint64("memory_current_bytes", s.GetMemoryCurrentBytes()), - slog.Uint64("memory_peak_bytes", s.GetMemoryPeakBytes()), - slog.Uint64("memory_working_set_bytes", s.GetMemoryWorkingSetBytes()), - slog.Uint64("cpu_usage_usec", s.GetCpuUsageUsec()), - slog.Int64("observed_at_unix_nano", s.GetObservedAtUnixNano()), - ) -} diff --git a/cmd/atelet/statsevents_test.go b/cmd/atelet/statsevents_test.go deleted file mode 100644 index cffcfd362c..0000000000 --- a/cmd/atelet/statsevents_test.go +++ /dev/null @@ -1,307 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "bytes" - "context" - "encoding/json" - "sync" - "testing" - "time" - - "github.com/agent-substrate/substrate/internal/actorlog" - "github.com/agent-substrate/substrate/internal/proto/ateompb" - "github.com/agent-substrate/substrate/internal/serverboot" -) - -// eventSample is the fully-populated sample the event tests emit; distinct -// values per field so a crossed wire is visible in the JSON. -func eventSample() *ateompb.WorkloadStatsSample { - return &ateompb.WorkloadStatsSample{ - Atespace: "space-a", - ActorName: "actor-a", - ActorUid: "uid-a", - ActorTemplateAtespace: "ns-a", - ActorTemplateName: "template-a", - SandboxClass: ateompb.SandboxClass_SANDBOX_CLASS_MICROVM, - Source: ateompb.StatsSource_STATS_SOURCE_GUEST_AGENT, - MemoryCurrentBytes: 1000, - MemoryPeakBytes: 2000, - MemoryWorkingSetBytes: 700, - CpuUsageUsec: 1234, - ObservedAtUnixNano: 42, - } -} - -// syncBuffer is a mutex-guarded buffer, so tests stay valid if an emitter -// call site ever moves onto a goroutine. -type syncBuffer struct { - mu sync.Mutex - buf bytes.Buffer -} - -func (b *syncBuffer) Write(p []byte) (int, error) { - b.mu.Lock() - defer b.mu.Unlock() - return b.buf.Write(p) -} - -func (b *syncBuffer) Bytes() []byte { - b.mu.Lock() - defer b.mu.Unlock() - return append([]byte(nil), b.buf.Bytes()...) -} - -func (b *syncBuffer) Len() int { - b.mu.Lock() - defer b.mu.Unlock() - return b.buf.Len() -} - -func (b *syncBuffer) String() string { return string(b.Bytes()) } - -// newBufferEmitter returns an emitter writing JSON records into buf. The -// labels-key closure stands in for defaultLabelsKey, whose metadata-server -// probe has no place in a unit test. -func newBufferEmitter(buf *syncBuffer, isOnGCE bool) *statsEventEmitter { - return newStatsEventEmitter(buf, func() string { return actorlog.LabelsKey(isOnGCE) }) -} - -func TestStatsEventEmitterEmit(t *testing.T) { - var buf syncBuffer - e := newBufferEmitter(&buf, false) - - e.emit(context.Background(), eventKindPeriodic, eventSample(), workerPoolRef{}) - - var rec map[string]any - if err := json.Unmarshal(buf.Bytes(), &rec); err != nil { - t.Fatalf("event is not one JSON record: %v (%q)", err, buf.String()) - } - if got := rec["msg"]; got != usageSampleMsg { - t.Errorf("msg = %v, want %q", got, usageSampleMsg) - } - if got := rec["kind"]; got != "periodic" { - t.Errorf("kind = %v, want periodic", got) - } - // The identity travels as the actorlog-style label group, so usage events - // join lifecycle events and container logs under the same keys. - labels, ok := rec["labels"].(map[string]any) - if !ok { - t.Fatalf("labels group missing or wrong shape: %v", rec["labels"]) - } - for key, want := range map[string]string{ - "ate.atespace": "space-a", - "ate.actor.name": "actor-a", - "ate.actor.uid": "uid-a", - "ate.template.atespace": "ns-a", - "ate.template.name": "template-a", - } { - if got := labels[key]; got != want { - t.Errorf("labels[%q] = %v, want %q", key, got, want) - } - } - for key, want := range map[string]float64{ - "memory_current_bytes": 1000, - "memory_peak_bytes": 2000, - "memory_working_set_bytes": 700, - "cpu_usage_usec": 1234, - "observed_at_unix_nano": 42, - } { - if got := rec[key]; got != want { - t.Errorf("%s = %v, want %v", key, got, want) - } - } - if got := rec["sandbox_class"]; got != "microvm" { - t.Errorf("sandbox_class = %v, want microvm", got) - } - if got := rec["source"]; got != "guest-agent" { - t.Errorf("source = %v, want guest-agent", got) - } - // An unresolved pool omits the pair rather than emitting empty strings. - for _, key := range []string{"ate.workerpool.namespace", "ate.workerpool.name"} { - if _, ok := labels[key]; ok { - t.Errorf("zero pool ref emitted label %q: %v", key, labels[key]) - } - } -} - -// TestStatsEventEmitterPoolLabels pins the pool enrichment on events: a -// resolved pod's events carry the same ate.workerpool label pair as the -// metric channel, inside the promoted label group. -func TestStatsEventEmitterPoolLabels(t *testing.T) { - var buf syncBuffer - e := newBufferEmitter(&buf, false) - - e.emit(context.Background(), eventKindPeriodic, eventSample(), workerPoolRef{namespace: "pool-ns", name: "pool-a"}) - - var rec map[string]any - if err := json.Unmarshal(buf.Bytes(), &rec); err != nil { - t.Fatalf("unmarshal: %v", err) - } - labels, ok := rec["labels"].(map[string]any) - if !ok { - t.Fatalf("labels group missing or wrong shape: %v", rec["labels"]) - } - if got := labels["ate.workerpool.namespace"]; got != "pool-ns" { - t.Errorf("labels[ate.workerpool.namespace] = %v, want pool-ns", got) - } - if got := labels["ate.workerpool.name"]; got != "pool-a" { - t.Errorf("labels[ate.workerpool.name] = %v, want pool-a", got) - } -} - -// TestStatsEventEmitterGCELabelsKey pins the label-group spelling split: on -// GCE the group must sit under the key Cloud Logging promotes. -func TestStatsEventEmitterGCELabelsKey(t *testing.T) { - var buf syncBuffer - e := newBufferEmitter(&buf, true) - - e.emit(context.Background(), eventKindPeriodic, eventSample(), workerPoolRef{}) - - var rec map[string]any - if err := json.Unmarshal(buf.Bytes(), &rec); err != nil { - t.Fatalf("unmarshal: %v", err) - } - if _, ok := rec["logging.googleapis.com/labels"]; !ok { - t.Errorf("GCE emitter did not use the promoted labels key; record keys: %v", buf.String()) - } -} - -// TestStatsEventEmitterIgnoresLogLevel pins the emitter's independence from -// the serverboot verbosity knob: usage events are a data feed, and quieting -// the node with --log-level=warn (or error) must not silently sever them. -func TestStatsEventEmitterIgnoresLogLevel(t *testing.T) { - if err := serverboot.SetLogLevel("error"); err != nil { - t.Fatal(err) - } - t.Cleanup(func() { - if err := serverboot.SetLogLevel("info"); err != nil { - t.Fatal(err) - } - }) - - var buf syncBuffer - newBufferEmitter(&buf, false).emit(context.Background(), eventKindPeriodic, eventSample(), workerPoolRef{}) - - if buf.Len() == 0 { - t.Error("raising the serverboot log level suppressed a usage event") - } -} - -// TestStatsEventEmitterNil: a nil emitter and a nil sample are both valid -// no-ops, so call sites stay unconditional. -func TestStatsEventEmitterNil(t *testing.T) { - var e *statsEventEmitter - e.emit(context.Background(), eventKindPeriodic, eventSample(), workerPoolRef{}) // must not panic - - var buf syncBuffer - newBufferEmitter(&buf, false).emit(context.Background(), eventKindPeriodic, nil, workerPoolRef{}) - if buf.Len() != 0 { - t.Errorf("nil sample emitted a record: %q", buf.String()) - } -} - -// stallableWriter blocks every Write until released -- the stalled log -// consumer the asyncWriter exists for. -type stallableWriter struct { - gate chan struct{} - wrote syncBuffer -} - -func (w *stallableWriter) Write(p []byte) (int, error) { - <-w.gate - return w.wrote.Write(p) -} - -// TestAsyncWriterNeverBlocks pins the contract: with the underlying -// writer wedged, every Write returns immediately -- overflow drops and -// counts instead of blocking -- and releasing the wedge drains what the -// queue held. -func TestAsyncWriterNeverBlocks(t *testing.T) { - under := &stallableWriter{gate: make(chan struct{})} - ctx, cancel := context.WithCancel(context.Background()) - t.Cleanup(cancel) - const depth = 4 - aw := newAsyncWriter(ctx, under, depth) - - // The writer goroutine dequeues one record and wedges on it; the queue - // holds depth more. Everything past that must drop, not block. The - // in-flight handoff is asynchronous, so tolerate one record of slack. - const writes = depth + 8 - done := make(chan struct{}) - go func() { - defer close(done) - for i := 0; i < writes; i++ { - if _, err := aw.Write([]byte("x")); err != nil { - t.Errorf("Write returned %v, want nil", err) - } - } - }() - select { - case <-done: - case <-time.After(5 * time.Second): - t.Fatal("Write blocked on a wedged log stream") - } - dropped := int(aw.dropped.Load()) - if dropped < writes-depth-1 || dropped > writes-depth { - t.Errorf("dropped = %d, want %d or %d", dropped, writes-depth-1, writes-depth) - } - - // Conservation: everything not dropped -- the wedged in-flight record - // plus the queue -- drains once the stream recovers, and the drop report - // arrives over the same sink, unkillable by any verbosity knob. - close(under.gate) - wantWritten := writes - dropped - waitFor(t, func() bool { return bytes.Count(under.wrote.Bytes(), []byte("x")) >= wantWritten }) - if got := bytes.Count(under.wrote.Bytes(), []byte("x")); got != wantWritten { - t.Errorf("drained %d records, want %d", got, wantWritten) - } - waitFor(t, func() bool { return bytes.Contains(under.wrote.Bytes(), []byte("Usage events dropped")) }) -} - -// TestAsyncWriterCopies: the slog handler reuses its buffer after -// Write returns, so the queue must hold copies, not aliases. -func TestAsyncWriterCopies(t *testing.T) { - under := &stallableWriter{gate: make(chan struct{})} - ctx, cancel := context.WithCancel(context.Background()) - t.Cleanup(cancel) - aw := newAsyncWriter(ctx, under, 4) - - p := []byte("original") - if _, err := aw.Write(p); err != nil { - t.Fatal(err) - } - copy(p, "clobber!") - - close(under.gate) - waitFor(t, func() bool { return under.wrote.Len() > 0 }) - if got := under.wrote.String(); got != "original" { - t.Errorf("record = %q, want the copy taken at Write time", got) - } -} - -// waitFor polls until cond returns true or the deadline passes. -func waitFor(t *testing.T, cond func() bool) { - t.Helper() - deadline := time.Now().Add(5 * time.Second) - for time.Now().Before(deadline) { - if cond() { - return - } - time.Sleep(5 * time.Millisecond) - } - t.Fatal("condition not reached before deadline") -} diff --git a/cmd/atelet/statspoller.go b/cmd/atelet/statspoller.go index b60c1b86a7..cdd9aade76 100644 --- a/cmd/atelet/statspoller.go +++ b/cmd/atelet/statspoller.go @@ -37,6 +37,7 @@ import ( "k8s.io/client-go/kubernetes" "github.com/agent-substrate/substrate/internal/ateattr" + "github.com/agent-substrate/substrate/internal/ateomstats" "github.com/agent-substrate/substrate/internal/nodepath" "github.com/agent-substrate/substrate/internal/proto/ateompb" ) @@ -48,15 +49,16 @@ import ( const workerPoolLabel = "ate.dev/worker-pool" // minActorStatsPollInterval is the floor a configured poll interval is clamped -// to: one actor's worst-case micro-VM read, maxActorContainers (25) containers -// at statsCallTimeout (2s) each. Sweeps never overlap, so it bounds -// guest-agent load, not correctness. +// to. The ateoms serve GetActiveWorkloadStats from their own sampling timer, so +// a poll faster than their sample interval only reads the same sample again. +// The floor also bounds the worker pod list each sweep sends the apiserver +// (resolveWorkerPools), so do not lower it to cut lag. const minActorStatsPollInterval = 50 * time.Second -// statsRPCTimeout bounds one ateom's GetActiveWorkloadStats call. The micro-VM -// ateom sets its statsSweepBudget (45s) below this timeout and reports guests -// it did not reach as pending, so the call does not grow with actor count. -const statsRPCTimeout = 55 * time.Second +// statsRPCTimeout bounds one ateom's GetActiveWorkloadStats call. The ateom +// answers from memory in under a millisecond, whatever its actor count or +// guest speed, so the timeout is only for a stuck socket. +const statsRPCTimeout = 5 * time.Second // statsSweepConcurrency bounds how many ateoms one sweep probes at once, one // call per ateom whatever it hosts. It caps how many stuck sockets can hold a @@ -91,9 +93,10 @@ type activeStatsClient interface { // It holds no worker-to-actor mapping and never asks the control plane: every // ateom registers itself on disk by creating its socket directory at boot (the // same sockets the lifecycle RPCs dial), so one readdir plus one probe per -// socket is complete discovery, and an atelet restart loses nothing because -// nothing was held. Attribution comes solely from the identity echoed inside -// each sample, per the RPC's contract. +// socket is complete discovery, and discovery survives an atelet restart. +// The CPU used between the old atelet's last sweep and the new one's first is +// not charged. Attribution comes solely from the identity echoed inside each +// sample, per the RPC's contract. type statsPoller struct { // interval between the end of one sweep and the start of the next. Sweeps // never overlap: a slow sweep delays the next tick rather than stacking a @@ -122,37 +125,52 @@ type statsPoller struct { inst *statsInstruments - // eventEmitter receives one usage event per sample per sweep -- the - // per-actor channel the aggregates deliberately erase identity from. - // Nil disables emission; emit is nil-safe. - eventEmitter *statsEventEmitter - // cachedPools carries pool resolutions across sweeps, so one failed pod // list cannot re-home a tick's samples -- and the CPU counter's // increments, which can never be re-attributed -- onto a pool-less label // set. Safe because a pod's pool is immutable for the pod's lifetime: an // entry can be stale, never wrong. Only the sweep loop touches it; // resolveWorkerPools prunes it to the pods whose ateom directories still - // exist, the same bound lastCPU keeps. + // exist. cachedPools map[string]workerPoolRef - // lastCPU is the last measured cpu_usage_usec per actor uid, the baseline + // lastCPU is the last measured cpu_usage_usec per activation, the baseline // for the next delta; read-only during a sweep and replaced whole at the - // end. Pending actors keep their entry, unseen actors are dropped. - lastCPU map[string]uint64 + // end. An activation keeps its entry while its worker reports it, pending + // included, and while its worker does not answer; it is dropped once that + // worker answers without it or the worker's directory is gone. A worker + // that dies without removing its directory keeps its entries until atelet + // restarts. + lastCPU map[cpuKey]cpuBaseline + + // startedAt is when this atelet started, in unix nanoseconds. An activation + // that began later was never charged, so its first sample counts in full. + startedAt int64 +} + +// cpuKey is one activation of one actor, the span a CPU counter counts within. +type cpuKey struct { + actorUID string + epoch int64 +} + +// cpuBaseline is an activation's last CPU reading and the worker that sent it. +type cpuBaseline struct { + usec uint64 + podUID string } // templateAggregate is one tick's sums for one templateKey group: the // bounded label set #174 permits on a TSDB series. Actor and atespace -// identity deliberately never reach a metric label; per-actor detail is the -// events channel's job, not this one's. +// identity deliberately never reach a metric label; per-actor detail is on +// the ateoms' usage records. // // The memory fields are point-in-time sums the gauges observe. cpuDeltaUsec is -// different: cpu_usage_usec is a cumulative per-epoch counter per actor, so -// summing the raw values across a churning actor set would be meaningless to -// rate() -- instead the poller tracks each actor's last seen value and this -// carries the sweep's INCREASE, which tick adds onto a monotonic counter. -// Counter semantics survive actors joining, leaving, and resetting epochs by +// different: cpu_usage_usec is a cumulative counter per activation, so summing +// the raw values across a churning actor set would be meaningless to rate() -- +// instead the poller tracks each activation's last seen value and this carries +// the sweep's INCREASE, which tick adds onto a monotonic counter. Counter +// semantics survive actors joining, leaving, and starting new activations by // construction. type templateAggregate struct { sampledActors int64 @@ -233,8 +251,8 @@ func (p *statsPoller) tick(ctx context.Context) { // their directory but not yet listened, and workers torn down mid-sweep. The // no-sample answers are equally routine: an empty samples list is an idle // worker, and a pending entry (source UNSPECIFIED) is a workload the ateom -// could not measure (boot, restore, teardown, or an unreached guest), which -// adds nothing but keeps its CPU baseline. +// has not measured (boot, restore, or an unreached guest), which adds nothing +// but keeps its CPU baseline. func (p *statsPoller) collect(ctx context.Context) map[templateKey]*templateAggregate { entries, err := os.ReadDir(p.ateomsDir) if err != nil { @@ -245,9 +263,11 @@ func (p *statsPoller) collect(ctx context.Context) map[templateKey]*templateAggr } podUIDs := make([]string, 0, len(entries)) + present := make(map[string]bool, len(entries)) for _, e := range entries { if e.IsDir() { podUIDs = append(podUIDs, e.Name()) + present[e.Name()] = true } } pools := p.resolveWorkerPools(ctx, podUIDs) @@ -255,8 +275,10 @@ func (p *statsPoller) collect(ctx context.Context) map[templateKey]*templateAggr var ( mu sync.Mutex aggs = make(map[templateKey]*templateAggregate) - seenCPU = make(map[string]uint64) - g errgroup.Group + seenCPU = make(map[cpuKey]cpuBaseline) + // answered is the workers whose probe succeeded this sweep. + answered = make(map[string]bool) + g errgroup.Group ) g.SetLimit(statsSweepConcurrency) for _, podUID := range podUIDs { @@ -282,29 +304,29 @@ func (p *statsPoller) collect(ctx context.Context) map[templateKey]*templateAggr return nil } + mu.Lock() + answered[podUID] = true + mu.Unlock() + // One entry per workload the ateom is hosting; empty when it is // available. for _, sample := range resp.GetSamples() { + cpuAt := cpuKey{actorUID: sample.GetActorUid(), epoch: sample.GetEpochUnixNano()} if sample.GetSource() == ateompb.StatsSource_STATS_SOURCE_UNSPECIFIED { // Pending: hosted but not measured, so it adds nothing - // and its CPU baseline carries forward. A measured value - // from another worker this sweep (restore in flight) wins. - if last, ok := p.lastCPU[sample.GetActorUid()]; ok { + // and its CPU baseline carries forward. + if last, ok := p.lastCPU[cpuAt]; ok { mu.Lock() - if _, measured := seenCPU[sample.GetActorUid()]; !measured { - seenCPU[sample.GetActorUid()] = last - } + seenCPU[cpuAt] = last mu.Unlock() } continue } - p.eventEmitter.emit(ctx, eventKindPeriodic, sample, pools[podUID]) - key := templateKey{ templateNamespace: sample.GetActorTemplateAtespace(), templateName: sample.GetActorTemplateName(), - sandboxClass: sandboxClassLabel(sample.GetSandboxClass()), - source: statsSourceLabel(sample.GetSource()), + sandboxClass: ateomstats.SandboxClassLabel(sample.GetSandboxClass()), + source: ateomstats.StatsSourceLabel(sample.GetSource()), workerPool: pools[podUID], } mu.Lock() @@ -317,22 +339,21 @@ func (p *statsPoller) collect(ctx context.Context) map[templateKey]*templateAggr agg.memoryCurrentBytes = addSat(agg.memoryCurrentBytes, sample.GetMemoryCurrentBytes()) agg.memoryWorkingSetBytes = addSat(agg.memoryWorkingSetBytes, sample.GetMemoryWorkingSetBytes()) - // The counter increase since the baseline. On a decrease, a - // cgroup counter restarted at zero, so the new value is the - // usage since then. A guest-agent decrease is ambiguous (a - // resume at another guest's value, a restart at zero, or a - // container exit), so it charges nothing. A sample with no - // baseline charges nothing: atelet cannot tell a new actor - // from its own restart. + // The counter increase since the activation's baseline; the + // ateom never lets it decrease within an activation. An + // activation with no baseline that began after this atelet + // started was never charged, so its value counts in full. One + // that began earlier may have been charged by the atelet before + // this one, so it only sets the baseline. Both times come from + // this node's clock: atelet reads only the ateoms on its node. cpu := sample.GetCpuUsageUsec() - seenCPU[sample.GetActorUid()] = cpu - if last, ok := p.lastCPU[sample.GetActorUid()]; ok { - switch { - case last <= cpu: - agg.cpuDeltaUsec = addSat(agg.cpuDeltaUsec, cpu-last) - case sample.GetSource() == ateompb.StatsSource_STATS_SOURCE_CGROUP: - agg.cpuDeltaUsec = addSat(agg.cpuDeltaUsec, cpu) + seenCPU[cpuAt] = cpuBaseline{usec: cpu, podUID: podUID} + if last, ok := p.lastCPU[cpuAt]; ok { + if cpu > last.usec { + agg.cpuDeltaUsec = addSat(agg.cpuDeltaUsec, cpu-last.usec) } + } else if cpuAt.epoch > p.startedAt { + agg.cpuDeltaUsec = addSat(agg.cpuDeltaUsec, cpu) } mu.Unlock() } @@ -342,8 +363,15 @@ func (p *statsPoller) collect(ctx context.Context) map[templateKey]*templateAggr // The tasks only ever return nil: a probe that fails is "not a target this // tick", never a failed sweep. _ = g.Wait() - // Replacing (not merging) the baselines drops actors this sweep did not - // see, so lastCPU cannot grow with actor churn. + // A worker that did not answer keeps its activations' baselines: dropping + // one would charge the activation in full again once the worker answers. + // Everything else this sweep did not see is dropped, so lastCPU does not + // grow with actor churn on workers that answer. + for k, b := range p.lastCPU { + if _, seen := seenCPU[k]; !seen && present[b.podUID] && !answered[b.podUID] { + seenCPU[k] = b + } + } p.lastCPU = seenCPU return aggs } @@ -365,31 +393,6 @@ func addSat(agg int64, v uint64) int64 { return agg + int64(v) } -// sandboxClassLabel maps the wire enum to the ate.sandbox.class label values -// the rest of the system uses. -func sandboxClassLabel(c ateompb.SandboxClass) string { - switch c { - case ateompb.SandboxClass_SANDBOX_CLASS_GVISOR: - return "gvisor" - case ateompb.SandboxClass_SANDBOX_CLASS_MICROVM: - return "microvm" - default: - return ateattr.SandboxClassUnknown - } -} - -// statsSourceLabel maps the wire enum to the ate.stats.source label values. -func statsSourceLabel(s ateompb.StatsSource) string { - switch s { - case ateompb.StatsSource_STATS_SOURCE_CGROUP: - return ateattr.StatsSourceCgroup - case ateompb.StatsSource_STATS_SOURCE_GUEST_AGENT: - return ateattr.StatsSourceGuestAgent - default: - return ateattr.StatsSourceUnspecified - } -} - // resolveWorkerPools returns this sweep's pod-UID-to-pool map: fresh // resolutions win, a failed or partial list falls back to cachedPools, and // the result -- rebuilt restricted to the pods whose ateom directories exist @@ -573,21 +576,13 @@ func (i *statsInstruments) addCPU(ctx context.Context, aggs map[templateKey]*tem } } -// startStatsPoller assembles the sampling subsystem -- the metrics poller and -// the periodic events channel -- and starts the poller. Split from main's -// boot sequence so the subsystem has one obvious entry point. +// startStatsPoller assembles the metrics poller and starts it. Split from +// main's boot sequence so the subsystem has one obvious entry point. // // The poller dials its own short-lived connection per probe (see // dialAteomStats) and takes no AteomDialer: the isolation from the lifecycle // RPCs' connection cache is structural, not just behavioral. -// logSink is the process's synchronized stdout writer, shared with the -// runtime logger so the event drain and slog can never tear each other's -// records. -func startStatsPoller(ctx context.Context, interval time.Duration, inst *statsInstruments, k8sClient kubernetes.Interface, logSink io.Writer) { - // Warm the labels-key resolution so the first emit does not pay the - // metadata probe either; see defaultLabelsKey. - go defaultLabelsKey() - +func startStatsPoller(ctx context.Context, interval time.Duration, inst *statsInstruments, k8sClient kubernetes.Interface) { poller := &statsPoller{ interval: interval, ateomsDir: nodepath.AteomsDir(), @@ -598,8 +593,8 @@ func startStatsPoller(ctx context.Context, interval time.Duration, inst *statsIn } return ateompb.NewAteomClient(conn), closer, nil }, - inst: inst, - eventEmitter: newStatsEventEmitter(newAsyncWriter(ctx, logSink, usageEventQueueDepth), defaultLabelsKey), + inst: inst, + startedAt: time.Now().UnixNano(), } // NODE_NAME comes from the Downward API; without it the samples still // flow, just grouped without pool labels. diff --git a/cmd/atelet/statspoller_test.go b/cmd/atelet/statspoller_test.go index a90cc3665f..f6f6f9636f 100644 --- a/cmd/atelet/statspoller_test.go +++ b/cmd/atelet/statspoller_test.go @@ -15,15 +15,12 @@ package main import ( - "bytes" "context" - "encoding/json" "errors" "io" "math" "os" "path/filepath" - "sort" "sync" "testing" "time" @@ -53,19 +50,9 @@ type fakeStatsAteom struct { // sawDeadline records whether the probe's context carried one, pinning the // per-call timeout. sawDeadline bool - // gate, when set, is received from before answering, so a test can order - // this probe after another worker's response has been folded. - gate <-chan struct{} } func (f *fakeStatsAteom) GetActiveWorkloadStats(ctx context.Context, req *ateompb.GetActiveWorkloadStatsRequest, opts ...grpc.CallOption) (*ateompb.GetActiveWorkloadStatsResponse, error) { - if f.gate != nil { - select { - case <-f.gate: - case <-ctx.Done(): - return nil, ctx.Err() // a misconfigured gate fails the probe, not the run - } - } f.mu.Lock() f.calls++ _, f.sawDeadline = ctx.Deadline() @@ -112,6 +99,22 @@ func pendingSample(actorUID, templateNS, templateName string) *ateompb.WorkloadS } } +// testStartedAt is when the fixture's atelet started. Samples default to epoch +// zero, an activation that began before it; later epochs began after. +const testStartedAt = 1000 + +// atEpoch sets the activation a sample belongs to. +func atEpoch(s *ateompb.WorkloadStatsSample, epoch int64) *ateompb.WorkloadStatsSample { + s.EpochUnixNano = epoch + return s +} + +// baseline is the CPU baseline of one activation, and whether there is one. +func baseline(p *statsPoller, actorUID string, epoch int64) (uint64, bool) { + b, ok := p.lastCPU[cpuKey{actorUID: actorUID, epoch: epoch}] + return b.usec, ok +} + // availableResponse is an idle ateom's answer: the empty list. func availableResponse() *ateompb.GetActiveWorkloadStatsResponse { return &ateompb.GetActiveWorkloadStatsResponse{} @@ -131,21 +134,15 @@ func pendingResponse(actorUID string) *ateompb.GetActiveWorkloadStatsResponse { } // closeRecorder counts Close calls, standing in for a probe's connection. -// collect closes it after folding the response, so onClose is the point at -// which that worker's entries are in the aggregates. type closeRecorder struct { - mu sync.Mutex - closes int - onClose func() + mu sync.Mutex + closes int } func (c *closeRecorder) Close() error { c.mu.Lock() c.closes++ c.mu.Unlock() - if c.onClose != nil { - c.onClose() - } return nil } @@ -166,6 +163,7 @@ func newPollerFixture(t *testing.T, fakes map[string]*fakeStatsAteom) (*statsPol } return &statsPoller{ ateomsDir: dir, + startedAt: testStartedAt, dial: func(_ context.Context, podUID string) (activeStatsClient, io.Closer, error) { f, ok := fakes[podUID] if !ok || f == nil { @@ -353,19 +351,18 @@ func cpuResponse(actorUID string, cpuUsec uint64) *ateompb.GetActiveWorkloadStat } // TestStatsPollerCPUDeltas pins the increase computation across sweeps: the -// first sight of an actor establishes a baseline and charges nothing (atelet -// cannot tell a new actor from its own restart, and re-charging an epoch the -// previous atelet counted would spike the counter), a later sweep charges -// only the increase, a cgroup decrease is an epoch reset whose new value is -// the usage since the reset, and an actor that disappears stops contributing and -// is dropped from the baselines. +// first sight of an activation that began before atelet started establishes a +// baseline and charges nothing (the previous atelet may have counted it), a +// later sweep charges only the increase, a new activation that began after +// atelet started counts in full, and an actor that disappears stops +// contributing and is dropped from the baselines. func TestStatsPollerCPUDeltas(t *testing.T) { key := templateKey{templateNamespace: "ns-a", templateName: "tmpl-a", sandboxClass: "gvisor", source: "cgroup"} fake := &fakeStatsAteom{resp: cpuResponse("uid-a", 1000)} p, _ := newPollerFixture(t, map[string]*fakeStatsAteom{"uid-1": fake}) if got := p.collect(context.Background())[key].cpuDeltaUsec; got != 0 { - t.Errorf("first sweep delta = %d, want 0 (baseline only on first sight)", got) + t.Errorf("first sweep delta = %d, want 0 (an activation from before atelet started only sets its baseline)", got) } fake.resp = cpuResponse("uid-a", 1600) @@ -373,11 +370,13 @@ func TestStatsPollerCPUDeltas(t *testing.T) { t.Errorf("second sweep delta = %d, want 600 (the increase)", got) } - // Epoch reset: the counter went backwards, so the new value is the usage - // since the reset. - fake.resp = cpuResponse("uid-a", 250) + // A new activation that began after atelet started: its value is all + // usage atelet has not charged. + fake.resp = &ateompb.GetActiveWorkloadStatsResponse{Samples: []*ateompb.WorkloadStatsSample{ + atEpoch(cpuResponse("uid-a", 250).Samples[0], testStartedAt+1), + }} if got := p.collect(context.Background())[key].cpuDeltaUsec; got != 250 { - t.Errorf("post-reset sweep delta = %d, want 250", got) + t.Errorf("new activation sweep delta = %d, want 250", got) } // The actor leaves: nothing to contribute, and its baseline must be @@ -419,68 +418,6 @@ func TestStatsPollerWorkerPoolLabels(t *testing.T) { } } -// TestStatsPollerPeriodicEvents pins the events channel: one event per -// executing sample per sweep, none for idle or mid-boot ateoms, identity -// taken from the echo, pool labels from the sweep's own resolution. -func TestStatsPollerPeriodicEvents(t *testing.T) { - fakes := map[string]*fakeStatsAteom{ - "uid-1": {resp: executingResponse("ns-a", "tmpl-a", ateompb.SandboxClass_SANDBOX_CLASS_GVISOR, ateompb.StatsSource_STATS_SOURCE_CGROUP, 1000, 700)}, - "uid-2": {resp: availableResponse()}, - } - p, _ := newPollerFixture(t, fakes) - var buf syncBuffer - p.eventEmitter = newBufferEmitter(&buf, false) - p.fetchWorkerPools = func(context.Context) map[string]workerPoolRef { - return map[string]workerPoolRef{"uid-1": {namespace: "pool-ns", name: "pool-a"}} - } - - p.collect(context.Background()) - - lines := bytes.Count(bytes.TrimSpace(buf.Bytes()), []byte("\n")) + 1 - if buf.Len() == 0 { - t.Fatal("no periodic event emitted for the executing ateom") - } - if lines != 1 { - t.Fatalf("emitted %d events, want 1 (idle ateoms emit nothing): %q", lines, buf.String()) - } - var rec map[string]any - if err := json.Unmarshal(bytes.TrimSpace(buf.Bytes()), &rec); err != nil { - t.Fatalf("unmarshal: %v", err) - } - if got := rec["kind"]; got != "periodic" { - t.Errorf("kind = %v, want periodic", got) - } - labels, _ := rec["labels"].(map[string]any) - if got := labels["ate.workerpool.name"]; got != "pool-a" { - t.Errorf("labels[ate.workerpool.name] = %v, want pool-a", got) - } -} - -func TestAddSat(t *testing.T) { - tests := []struct { - name string - agg int64 - v uint64 - want int64 - }{ - {name: "normal add", agg: 100, v: 50, want: 150}, - {name: "zero add", agg: 100, v: 0, want: 100}, - // A wire value above MaxInt64 -- a corrupt or hostile guest reading -- - // must pin at the ceiling, not wrap the aggregate negative. - {name: "value above MaxInt64 saturates", agg: 0, v: math.MaxUint64, want: math.MaxInt64}, - // The addition itself can also overflow once inputs are clamped. - {name: "sum overflow saturates", agg: math.MaxInt64 - 10, v: 100, want: math.MaxInt64}, - {name: "exactly at ceiling", agg: math.MaxInt64 - 5, v: 5, want: math.MaxInt64}, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - if got := addSat(tc.agg, tc.v); got != tc.want { - t.Errorf("addSat(%d, %d) = %d, want %d", tc.agg, tc.v, tc.want, got) - } - }) - } -} - // TestStatsPollerCollectSaturatesCorruptSamples pins the end-to-end behavior: // one guest reporting absurd counters must not flip a template's aggregates // negative -- a negative gauge misreads as "no memory", and a negative CPU @@ -731,7 +668,7 @@ func TestStatsPollerMultiActorCPUBaselines(t *testing.T) { if got := p.collect(context.Background())[key].cpuDeltaUsec; got != 0 { t.Fatalf("first sweep delta = %d, want 0 (baselines only)", got) } - if _, ok := p.lastCPU["actor-3"]; ok { + if _, ok := baseline(p, "actor-3", 0); ok { t.Error("pending actor was given a CPU baseline") } @@ -745,11 +682,11 @@ func TestStatsPollerMultiActorCPUBaselines(t *testing.T) { t.Errorf("second sweep delta = %d, want 850 (600 + 250)", got) } - // actor-2 resets its epoch; actor-1 keeps advancing. The reset charges - // the new value, not a negative delta. + // actor-2 starts a new activation; actor-1 keeps advancing. The new + // activation charges its value, not a negative delta. fake.resp = &ateompb.GetActiveWorkloadStatsResponse{Samples: []*ateompb.WorkloadStatsSample{ measuredSample("actor-1", "ns-a", "tmpl-a", 1, 1, 1700), - measuredSample("actor-2", "ns-a", "tmpl-a", 1, 1, 40), + atEpoch(measuredSample("actor-2", "ns-a", "tmpl-a", 1, 1, 40), testStartedAt+1), }} if got := p.collect(context.Background())[key].cpuDeltaUsec; got != 140 { t.Errorf("reset sweep delta = %d, want 140 (100 + 40)", got) @@ -759,38 +696,6 @@ func TestStatsPollerMultiActorCPUBaselines(t *testing.T) { } } -// TestStatsPollerMultiActorEvents pins one usage event per measured entry -// and none for a pending one. -func TestStatsPollerMultiActorEvents(t *testing.T) { - fakes := map[string]*fakeStatsAteom{ - "uid-w1": {resp: &ateompb.GetActiveWorkloadStatsResponse{Samples: []*ateompb.WorkloadStatsSample{ - measuredSample("actor-1", "ns-a", "tmpl-a", 1000, 700, 10), - measuredSample("actor-2", "ns-a", "tmpl-a", 500, 300, 20), - pendingSample("actor-3", "ns-a", "tmpl-a"), - }}}, - } - p, _ := newPollerFixture(t, fakes) - var buf syncBuffer - p.eventEmitter = newBufferEmitter(&buf, false) - - p.collect(context.Background()) - - var uids []string - for _, line := range bytes.Split(bytes.TrimSpace(buf.Bytes()), []byte("\n")) { - var rec map[string]any - if err := json.Unmarshal(line, &rec); err != nil { - t.Fatalf("event line is not JSON: %v: %s", err, line) - } - labels, _ := rec["labels"].(map[string]any) - uid, _ := labels["ate.actor.uid"].(string) - uids = append(uids, uid) - } - sort.Strings(uids) - if want := []string{"actor-1", "actor-2"}; !cmp.Equal(want, uids) { - t.Errorf("event actor uids = %v, want %v (one per measured entry, none for pending)", uids, want) - } -} - // TestStatsPollerPendingKeepsCPUBaseline pins that a sweep in which an actor // reports pending carries its baseline forward: the interval is charged when // the actor is measured again, not lost. @@ -809,7 +714,7 @@ func TestStatsPollerPendingKeepsCPUBaseline(t *testing.T) { if got := p.collect(context.Background()); len(got) != 0 { t.Fatalf("pending sweep produced aggregates %v, want none", got) } - if got, ok := p.lastCPU["actor-1"]; !ok || got != 1000 { + if got, ok := baseline(p, "actor-1", 0); !ok || got != 1000 { t.Fatalf("baseline after pending sweep = %d (present=%v), want 1000 kept", got, ok) } @@ -821,130 +726,177 @@ func TestStatsPollerPendingKeepsCPUBaseline(t *testing.T) { } } -// TestStatsPollerCPUDecreaseBySource pins the decrease rule, with and without -// a pending sweep between the samples: a cgroup decrease charges the new value, -// since that counter restarts at zero, and a guest-agent decrease charges -// nothing, since that counter can resume at another guest's value. -func TestStatsPollerCPUDecreaseBySource(t *testing.T) { - guestAgent := func(s *ateompb.WorkloadStatsSample) *ateompb.WorkloadStatsSample { - s.SandboxClass = ateompb.SandboxClass_SANDBOX_CLASS_MICROVM - s.Source = ateompb.StatsSource_STATS_SOURCE_GUEST_AGENT - return s +// TestStatsPollerCPUEpochs pins the charge rule for activations: an increase +// within one charges the increase, a decrease charges nothing, and a new one +// charges its first value in full only if it began after atelet started. +func TestStatsPollerCPUEpochs(t *testing.T) { + key := templateKey{templateNamespace: "ns-a", templateName: "tmpl-a", sandboxClass: "gvisor", source: "cgroup"} + sample := func(epoch int64, cpu uint64) *ateompb.WorkloadStatsSample { + return atEpoch(measuredSample("actor-1", "ns-a", "tmpl-a", 1, 1, cpu), epoch) } for _, tc := range []struct { name string - sample func(cpu uint64) *ateompb.WorkloadStatsSample - key templateKey - pending bool - wantDelta int64 + sweeps []*ateompb.WorkloadStatsSample // nil is a pending sweep + wantDelta int64 // on the last sweep }{ - { - name: "cgroup", - sample: func(cpu uint64) *ateompb.WorkloadStatsSample { - return measuredSample("actor-1", "ns-a", "tmpl-a", 1, 1, cpu) - }, - key: templateKey{templateNamespace: "ns-a", templateName: "tmpl-a", sandboxClass: "gvisor", source: "cgroup"}, - wantDelta: 400, - }, - { - name: "cgroup across pending", - sample: func(cpu uint64) *ateompb.WorkloadStatsSample { - return measuredSample("actor-1", "ns-a", "tmpl-a", 1, 1, cpu) - }, - key: templateKey{templateNamespace: "ns-a", templateName: "tmpl-a", sandboxClass: "gvisor", source: "cgroup"}, - pending: true, - wantDelta: 400, - }, - { - name: "guest-agent", - sample: func(cpu uint64) *ateompb.WorkloadStatsSample { - return guestAgent(measuredSample("actor-1", "ns-a", "tmpl-a", 1, 1, cpu)) - }, - key: templateKey{templateNamespace: "ns-a", templateName: "tmpl-a", sandboxClass: "microvm", source: "guest-agent"}, - wantDelta: 0, - }, - { - name: "guest-agent across pending", - sample: func(cpu uint64) *ateompb.WorkloadStatsSample { - return guestAgent(measuredSample("actor-1", "ns-a", "tmpl-a", 1, 1, cpu)) - }, - key: templateKey{templateNamespace: "ns-a", templateName: "tmpl-a", sandboxClass: "microvm", source: "guest-agent"}, - pending: true, - wantDelta: 0, - }, + {"increase within an activation", []*ateompb.WorkloadStatsSample{sample(2000, 1000), sample(2000, 1400)}, 400}, + {"decrease within an activation", []*ateompb.WorkloadStatsSample{sample(2000, 1000), sample(2000, 400)}, 0}, + {"new activation after start", []*ateompb.WorkloadStatsSample{sample(2000, 1000), sample(3000, 400)}, 400}, + {"new activation across a pending sweep", []*ateompb.WorkloadStatsSample{sample(2000, 1000), nil, sample(3000, 400)}, 400}, + {"first sight of an activation after start", []*ateompb.WorkloadStatsSample{sample(2000, 700)}, 700}, + {"first sight of an activation at start", []*ateompb.WorkloadStatsSample{sample(testStartedAt, 700)}, 0}, + {"first sight of an activation before start", []*ateompb.WorkloadStatsSample{sample(testStartedAt-1, 700)}, 0}, } { t.Run(tc.name, func(t *testing.T) { fake := &fakeStatsAteom{} p, _ := newPollerFixture(t, map[string]*fakeStatsAteom{"uid-w1": fake}) - sweep := func(sample *ateompb.WorkloadStatsSample) map[templateKey]*templateAggregate { - fake.resp = &ateompb.GetActiveWorkloadStatsResponse{Samples: []*ateompb.WorkloadStatsSample{sample}} - return p.collect(context.Background()) - } - - sweep(tc.sample(1000)) // baseline 1000 - if tc.pending { - sweep(pendingSample("actor-1", "ns-a", "tmpl-a")) - } - if got := sweep(tc.sample(400))[tc.key].cpuDeltaUsec; got != tc.wantDelta { - t.Errorf("delta for a decrease 1000 -> 400 = %d, want %d", got, tc.wantDelta) + var got int64 + for _, sw := range tc.sweeps { + if sw == nil { + sw = atEpoch(pendingSample("actor-1", "ns-a", "tmpl-a"), 3000) + } + fake.resp = &ateompb.GetActiveWorkloadStatsResponse{Samples: []*ateompb.WorkloadStatsSample{sw}} + got = 0 + if agg := p.collect(context.Background())[key]; agg != nil { + got = agg.cpuDeltaUsec + } } - // Either way 400 is the new baseline. - if got := sweep(tc.sample(500))[tc.key].cpuDeltaUsec; got != 100 { - t.Errorf("delta after the decrease = %d, want 100", got) + if got != tc.wantDelta { + t.Errorf("last sweep delta = %d, want %d", got, tc.wantDelta) } }) } } -// TestStatsPollerRestoreInFlightMeasuredWins pins that when one worker -// measures an actor and another reports it pending in the same sweep, the -// measured value is the baseline in either fold order. -func TestStatsPollerRestoreInFlightMeasuredWins(t *testing.T) { +// TestStatsPollerRestoreChargesEachActivationOnce pins a restore across two +// workers: the source's activation and the destination's are separate +// baselines, so nothing is charged twice and the source's is dropped once it +// leaves. +func TestStatsPollerRestoreChargesEachActivationOnce(t *testing.T) { key := templateKey{templateNamespace: "ns-a", templateName: "tmpl-a", sandboxClass: "gvisor", source: "cgroup"} - for _, order := range []string{"measured first", "pending first", "unordered"} { - t.Run(order, func(t *testing.T) { - src := &fakeStatsAteom{resp: &ateompb.GetActiveWorkloadStatsResponse{Samples: []*ateompb.WorkloadStatsSample{ - measuredSample("actor-1", "ns-a", "tmpl-a", 1, 1, 1000), - }}} - dst := &fakeStatsAteom{resp: availableResponse()} - p, closers := newPollerFixture(t, map[string]*fakeStatsAteom{"uid-src": src, "uid-dst": dst}) - - p.collect(context.Background()) // baseline C0 = 1000 - - // Sweep N: source measured at C1 = 1300; destination pending. - src.resp = &ateompb.GetActiveWorkloadStatsResponse{Samples: []*ateompb.WorkloadStatsSample{ - measuredSample("actor-1", "ns-a", "tmpl-a", 1, 1, 1300), - }} - dst.resp = &ateompb.GetActiveWorkloadStatsResponse{Samples: []*ateompb.WorkloadStatsSample{ - pendingSample("actor-1", "ns-a", "tmpl-a"), - }} - // The gated probe answers only after the other worker's response has - // been folded: collect closes a probe's connection after the fold. - gate := make(chan struct{}) - release := sync.OnceFunc(func() { close(gate) }) - switch order { - case "measured first": - dst.gate = gate - closers["uid-src"].onClose = release - case "pending first": - src.gate = gate - closers["uid-dst"].onClose = release - } - if got := p.collect(context.Background())[key].cpuDeltaUsec; got != 300 { - t.Fatalf("sweep N delta = %d, want 300", got) - } - if got := p.lastCPU["actor-1"]; got != 1300 { - t.Fatalf("baseline after sweep N = %d, want the measured 1300, not the carried 1000", got) - } + const before, after = 2000, 3000 + src := &fakeStatsAteom{resp: &ateompb.GetActiveWorkloadStatsResponse{Samples: []*ateompb.WorkloadStatsSample{ + atEpoch(measuredSample("actor-1", "ns-a", "tmpl-a", 1, 1, 1000), before), + }}} + dst := &fakeStatsAteom{resp: availableResponse()} + p, _ := newPollerFixture(t, map[string]*fakeStatsAteom{"uid-src": src, "uid-dst": dst}) + p.collect(context.Background()) - // Sweep N+1: only the destination hosts it now, measured at C2 = 1500. - closers["uid-src"].onClose, closers["uid-dst"].onClose = nil, nil - src.resp, src.gate = availableResponse(), nil - dst.resp, dst.gate = &ateompb.GetActiveWorkloadStatsResponse{Samples: []*ateompb.WorkloadStatsSample{ - measuredSample("actor-1", "ns-a", "tmpl-a", 1, 1, 1500), - }}, nil - if got := p.collect(context.Background())[key].cpuDeltaUsec; got != 200 { - t.Errorf("sweep N+1 delta = %d, want 200 (C2-C1); 500 would double-charge C1-C0", got) + // The source still measures its activation; the destination hosts the new + // one, not yet measured. + src.resp = &ateompb.GetActiveWorkloadStatsResponse{Samples: []*ateompb.WorkloadStatsSample{ + atEpoch(measuredSample("actor-1", "ns-a", "tmpl-a", 1, 1, 1300), before), + }} + dst.resp = &ateompb.GetActiveWorkloadStatsResponse{Samples: []*ateompb.WorkloadStatsSample{ + atEpoch(pendingSample("actor-1", "ns-a", "tmpl-a"), after), + }} + if got := p.collect(context.Background())[key].cpuDeltaUsec; got != 300 { + t.Fatalf("restore sweep delta = %d, want 300", got) + } + + // Only the destination hosts it now, 200 into its activation. + src.resp = availableResponse() + dst.resp = &ateompb.GetActiveWorkloadStatsResponse{Samples: []*ateompb.WorkloadStatsSample{ + atEpoch(measuredSample("actor-1", "ns-a", "tmpl-a", 1, 1, 200), after), + }} + if got := p.collect(context.Background())[key].cpuDeltaUsec; got != 200 { + t.Errorf("post-restore delta = %d, want 200", got) + } + if _, ok := baseline(p, "actor-1", before); ok { + t.Error("the source activation's baseline survived its worker answering without it") + } +} + +// TestStatsPollerKeepsBaselineOfSilentWorker pins that a worker that does not +// answer keeps its activations' baselines, so the next answer charges the +// increase rather than the activation in full. +func TestStatsPollerKeepsBaselineOfSilentWorker(t *testing.T) { + key := templateKey{templateNamespace: "ns-a", templateName: "tmpl-a", sandboxClass: "gvisor", source: "cgroup"} + fake := &fakeStatsAteom{resp: &ateompb.GetActiveWorkloadStatsResponse{Samples: []*ateompb.WorkloadStatsSample{ + atEpoch(measuredSample("actor-1", "ns-a", "tmpl-a", 1, 1, 1000), 2000), + }}} + p, _ := newPollerFixture(t, map[string]*fakeStatsAteom{"uid-w1": fake}) + p.collect(context.Background()) + + fake.err = errors.New("deadline exceeded") + p.collect(context.Background()) + if got, ok := baseline(p, "actor-1", 2000); !ok || got != 1000 { + t.Fatalf("baseline after a silent sweep = %d (present %v), want 1000 kept", got, ok) + } + + fake.err = nil + fake.resp = &ateompb.GetActiveWorkloadStatsResponse{Samples: []*ateompb.WorkloadStatsSample{ + atEpoch(measuredSample("actor-1", "ns-a", "tmpl-a", 1, 1, 1600), 2000), + }} + if got := p.collect(context.Background())[key].cpuDeltaUsec; got != 600 { + t.Errorf("delta after the silent sweep = %d, want 600, not the full 1600", got) + } +} + +// TestStatsPollerSilentWorkerKeepsBaselinesUntilDirGone pins that a worker +// that stops answering for good, as one killed without cleanup, keeps its +// baselines for as long as its directory stays. +func TestStatsPollerSilentWorkerKeepsBaselinesUntilDirGone(t *testing.T) { + fake := &fakeStatsAteom{resp: &ateompb.GetActiveWorkloadStatsResponse{Samples: []*ateompb.WorkloadStatsSample{ + atEpoch(measuredSample("actor-1", "ns-a", "tmpl-a", 1, 1, 1000), 2000), + }}} + p, _ := newPollerFixture(t, map[string]*fakeStatsAteom{"uid-w1": fake}) + p.collect(context.Background()) + + fake.err = errors.New("connection refused") + for range 3 { + p.collect(context.Background()) + } + if _, ok := baseline(p, "actor-1", 2000); !ok { + t.Fatal("baseline dropped while the silent worker's directory stays") + } + if err := os.Remove(filepath.Join(p.ateomsDir, "uid-w1")); err != nil { + t.Fatal(err) + } + p.collect(context.Background()) + if len(p.lastCPU) != 0 { + t.Errorf("baselines after the directory went = %v, want none", p.lastCPU) + } +} + +// TestStatsPollerDropsBaselineOfGoneWorker pins that a worker whose +// directory is gone takes its baselines with it. +func TestStatsPollerDropsBaselineOfGoneWorker(t *testing.T) { + fake := &fakeStatsAteom{resp: &ateompb.GetActiveWorkloadStatsResponse{Samples: []*ateompb.WorkloadStatsSample{ + atEpoch(measuredSample("actor-1", "ns-a", "tmpl-a", 1, 1, 1000), 2000), + }}} + p, _ := newPollerFixture(t, map[string]*fakeStatsAteom{"uid-w1": fake}) + p.collect(context.Background()) + + if err := os.Remove(filepath.Join(p.ateomsDir, "uid-w1")); err != nil { + t.Fatal(err) + } + p.collect(context.Background()) + if len(p.lastCPU) != 0 { + t.Errorf("baselines after the worker left = %v, want none", p.lastCPU) + } +} + +func TestAddSat(t *testing.T) { + tests := []struct { + name string + agg int64 + v uint64 + want int64 + }{ + {name: "normal add", agg: 100, v: 50, want: 150}, + {name: "zero add", agg: 100, v: 0, want: 100}, + // A wire value above MaxInt64 -- a corrupt or hostile guest reading -- + // must pin at the ceiling, not wrap the aggregate negative. + {name: "value above MaxInt64 saturates", agg: 0, v: math.MaxUint64, want: math.MaxInt64}, + // The addition itself can also overflow once inputs are clamped. + {name: "sum overflow saturates", agg: math.MaxInt64 - 10, v: 100, want: math.MaxInt64}, + {name: "exactly at ceiling", agg: math.MaxInt64 - 5, v: 5, want: math.MaxInt64}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := addSat(tc.agg, tc.v); got != tc.want { + t.Errorf("addSat(%d, %d) = %d, want %d", tc.agg, tc.v, tc.want, got) } }) } diff --git a/docs/dev/best-practices/otel-collector.md b/docs/dev/best-practices/otel-collector.md index 0a3f4a2523..37abf47576 100644 --- a/docs/dev/best-practices/otel-collector.md +++ b/docs/dev/best-practices/otel-collector.md @@ -443,7 +443,7 @@ instead of stdout, and `otlp,console` to both; only the kind overlay sets it, to Everything else is stdout. `serverboot.InitLogger` writes structured JSON there, and `ateom` wraps actor container output with the `ate.*` metadata labels described in [Actor Observability](../../observability.md). The ateom relay -carries logs, traces, and metrics; worker pods do not emit log records yet. +carries logs, traces, and metrics. Those labels sit in a nested group (`labels`, or `logging.googleapis.com/labels` on GKE, where the key promotes the group into `LogEntry.labels`). A filelog diff --git a/docs/metrics/registry/events.yaml b/docs/metrics/registry/events.yaml index d649e2a34d..5b4a1ee694 100644 --- a/docs/metrics/registry/events.yaml +++ b/docs/metrics/registry/events.yaml @@ -108,7 +108,19 @@ groups: note: > The ateom that runs the actor writes one record per actor on a timer, plus an initial and a final record per activation. ate.stats.kind tells - them apart. + them apart. The records of an activation are written in that order: the + initial one, then the periodic ones, then the final one. Each is dated + when its sample was read, thus a final record that repeats an older + sample can carry an earlier timestamp than the records before it. The + initial record is absent when the first reading fails. The final record + is written when the ateom tears the activation down: a checkpoint, a + terminate, a graceful shutdown of the worker, or a re-host on a + micro-VM. It is read just before the teardown and repeats the newest + sample when that read fails or times out, or is pending when none was + measured. It is absent when the activation ends otherwise, as by a + crash, a lost worker, or a re-host on gVisor. Thus take the highest + ate.stats.cpu.time per ate.actor.uid and ate.actor.epoch rather than + wait for the final record. The timestamp is when the measurement was read. The measurements are absent, not zero, while the actor is not measurable: the source is @@ -125,7 +137,7 @@ groups: annotations: substrate: emitted_by: [ateom-gvisor, ateom-microvm] - code_anchor: internal/actorevent/actorevent.go + code_anchor: internal/ateomstats/usage.go cuj: How much CPU and memory does this actor use, and since when? attributes: - ref: ate.atespace diff --git a/docs/metrics/registry/metrics.yaml b/docs/metrics/registry/metrics.yaml index 94a5225ac9..1c0b6a23d8 100644 --- a/docs/metrics/registry/metrics.yaml +++ b/docs/metrics/registry/metrics.yaml @@ -603,10 +603,10 @@ groups: type: double brief: The CPU time used since the activation began, in seconds. note: > - Logs only. Restarts at zero with each ate.actor.epoch. The lifetime - figure is the sum over the epochs of the highest value in each. The - unit is that of the instrument ate.actor.stats.cpu.time, thus one - query serves both signals. + Logs only. Restarts at zero with each ate.actor.epoch and never + decreases within one. The lifetime figure is the sum over the epochs + of the highest value in each. The unit is that of the instrument + ate.actor.stats.cpu.time, thus one query serves both signals. examples: [1.5] - id: registry.ate.deviation @@ -951,11 +951,21 @@ groups: stability: development brief: The total CPU time in seconds of the actors that are in operation. note: > - This is a usual counter. The other three stats instruments use a callback. - atelet adds the increase of each actor to this counter. Thus the value - always increases. If a template leaves the node, its series keeps its last - value. It does not disappear. This is the behaviour that a rate() query - needs. + This is a usual counter. The other three stats instruments use a + callback. atelet adds the increase of each activation to this counter. + Thus the value always increases. If a template leaves the node, its + series keeps its last value. It does not disappear. This is the + behaviour that a rate() query needs. + + It does not count usage twice. It can count less: the end of an + activation after the last sample atelet read of it, which is up to one + poll interval plus one ateom sample interval plus the sweep's own time + (up to 45s on a micro-VM), so an activation shorter than that can lose + most or all of its CPU here, while its usage records keep it; the time + around an atelet restart; and, in a + micro-VM restored from a full snapshot, a container's use before its + first successful reading. A node clock that steps back across an atelet + restart can count one activation in full a second time. annotations: substrate: emitted_by: [atelet] diff --git a/docs/metrics/substrate.yaml b/docs/metrics/substrate.yaml index 63a40343de..ffef2b775c 100644 --- a/docs/metrics/substrate.yaml +++ b/docs/metrics/substrate.yaml @@ -234,3 +234,12 @@ blind_spots: - area: router shutdown outcome code: cmd/atenet/internal/router/drain.go brief: The code has two TODOs. Nothing counts a drain that ends with an error. + - area: CPU of short activations + code: cmd/atelet/statspoller.go + brief: > + ate.actor.stats.cpu.time counts only the samples atelet reads while an + actor is hosted. It loses the end of each activation, up to one poll + interval plus one ateom sample interval. Thus an activation shorter than + that, as a checkpoint-when-idle agent, can show little or no CPU here. + Its ate.actor.usage_sampled records keep it. Issue #2362 moves the + counter to the ateoms. diff --git a/docs/observability.md b/docs/observability.md index fac728c27c..9ff57400c4 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -179,7 +179,7 @@ The counter carries no actor identity, so this record is the only way to attribu #### The same records over OTLP -With `OTEL_LOGS_EXPORTER=otlp` these records go out as OTLP log events instead of stdout, so a collector reads them without knowing substrate's stdout envelope. Unset or `none`, which is what every environment but kind uses today, keeps them on stdout. `otlp,console` writes both, as kind does for debugging. `console` here is the component's own JSON log line, not the OTel SDK's console exporter format. The value is a comma-separated list: names are trimmed, lowercased, and counted once, and an unknown or empty item logs a warning and is skipped while the rest apply. `none` with an exporter, or a value that names no known exporter, logs a warning and keeps the default, `none`. ateapi is the only emitter today. The ateoms have a LoggerProvider on the same switch and export through [the ateom relay](#the-ateom-otlp-relay), which carries logs, traces, and metrics. atecontroller copies `OTEL_LOGS_EXPORTER` into each worker pod, so the kind ConfigMap sets the ateoms to `otlp,console` too. +With `OTEL_LOGS_EXPORTER=otlp` these records go out as OTLP log events instead of stdout, so a collector reads them without knowing substrate's stdout envelope. Unset or `none`, which is what every environment but kind uses today, keeps them on stdout. `otlp,console` writes both, as kind does for debugging. `console` here is the component's own JSON log line, not the OTel SDK's console exporter format. The value is a comma-separated list: names are trimmed, lowercased, and counted once, and an unknown or empty item logs a warning and is skipped while the rest apply. `none` with an exporter, or a value that names no known exporter, logs a warning and keeps the default, `none`. ateapi emits the lifecycle events and the ateoms the usage samples, on the same switch; the ateoms export through [the ateom relay](#the-ateom-otlp-relay), which carries logs, traces, and metrics. atecontroller copies `OTEL_LOGS_EXPORTER` into each worker pod, so the kind ConfigMap sets the ateoms to `otlp,console` too. Three `event.name` values, which is the OTLP LogRecord's own field rather than an attribute: @@ -189,7 +189,7 @@ Three `event.name` values, which is the OTLP LogRecord's own field rather than a | `ate.actor.crashed` | `Actor crashed` | 17 | the same keys | | `ate.actor.usage_sampled` | `Actor usage sampled` | 9 | the five identity keys, `ate.workerpool.*`, `ate.sandbox.class`, `ate.stats.*`, `ate.actor.epoch` | -`ate.actor.usage_sampled` is the ateoms' record: one per actor per sampling period, plus an `initial` and a `final` per activation, told apart by `ate.stats.kind`. Its timestamp is when the measurement was read. Its measurements are named after the `ate.actor.stats.*` instruments and share their units, so `ate.stats.cpu.time` is seconds. They are absent, not zero, while the actor is not measurable, which the record says with `ate.stats.source` unspecified. `ate.stats.cpu.time` restarts at zero with each `ate.actor.epoch`, the unix-nano time the activation began, so a lifetime figure is the sum over epochs of each epoch's highest value; `ate.stats.memory.usage` and `ate.stats.memory.working_set` are absolute, and `ate.stats.memory.peak` is as the source reports it. The same measurements ride `WorkloadStatsSample` on the stats RPCs, with the epoch beside them. +`ate.actor.usage_sampled` is the ateoms' record: one per actor per sampling interval, plus an `initial` record per activation and a `final` one when the ateom tears it down, told apart by `ate.stats.kind`. Its timestamp is when the measurement was read. Its measurements are named after the `ate.actor.stats.*` instruments and share their units, so `ate.stats.cpu.time` is seconds. They are absent, not zero, while the actor is not measurable, which the record says with `ate.stats.source` unspecified. `ate.stats.cpu.time` restarts at zero with each `ate.actor.epoch`, the unix-nano time the activation began, so a lifetime figure is the sum over epochs of each epoch's highest value; `ate.stats.memory.usage` and `ate.stats.memory.working_set` are absolute, and `ate.stats.memory.peak` is as the source reports it. The same measurements ride `WorkloadStatsSample` on the stats RPCs, with the epoch beside them. A crash is its own name because an event name promises a set of attributes and a crash has a different severity and shape. There is no name per state: `ate.actor.state` already says which transition happened, so a consumer still selects on that one attribute and needs no map from a name to a state. All three names are in [`docs/metrics/registry/events.yaml`](metrics/registry/events.yaml), which `make verify` checks. @@ -203,31 +203,31 @@ The attributes are the same flat `ate.*` keys as the stdout record, so they arri ### Per-Actor Usage Events -atelet emits one usage record per **executing** actor per sampling tick, from the same sweep that feeds the [`ate.actor.stats.*` metrics](#the-metric-registry). The two are the halves of one split: the metrics aggregate to the bounded template/pool label set a TSDB can hold, and everything carrying actor identity travels here, on the log stream, where cardinality is free. An idle fleet is silent by design — a worker with no executing actor emits nothing. +Each ateom samples its actors on its own timer and writes [`ate.actor.usage_sampled`](#the-same-records-over-otlp) for each one, in this order: an `initial` record when an activation becomes measurable, a `periodic` record per interval, and a `final` record when the ateom tears the activation down: a checkpoint, a terminate, a graceful shutdown, or a re-host on a micro-VM. The sampler reads an actor only after its initial reading and until its final record; until then the discovery read reports it as pending. A failed first reading writes no `initial` record, and periodic records start anyway, so an unreachable guest is still reported. These records and the [`ate.actor.stats.*` metrics](#the-metric-registry) are the halves of one split: the metrics aggregate to the bounded template and pool label set a TSDB can hold, and everything carrying actor identity travels here, on the log stream, where cardinality is free. + +The stdout record carries the same flat `ate.*` keys as the OTLP event, dated when the measurement was read: ```json -{"time":"…","level":"INFO","msg":"Actor usage sample", - "logging.googleapis.com/labels":{ - "ate.atespace":"ate-demo-counter","ate.actor.name":"counter-1","ate.actor.uid":"8f2a…", - "ate.template.atespace":"ate-demo-counter","ate.template.name":"counter", - "ate.workerpool.namespace":"ate-demo-counter","ate.workerpool.name":"counter-pool"}, - "kind":"periodic","sandbox_class":"gvisor","source":"cgroup", - "memory_current_bytes":39845888,"memory_peak_bytes":52428800, - "memory_working_set_bytes":31457280,"cpu_usage_usec":196776, - "observed_at_unix_nano":1788903245000000000} +{"time":"…","level":"INFO","msg":"Actor usage sampled", + "ate.atespace":"ate-demo-counter","ate.actor.name":"counter-1","ate.actor.uid":"8f2a…", + "ate.template.atespace":"ate-demo-counter","ate.template.name":"counter", + "ate.workerpool.namespace":"ate-demo-counter","ate.workerpool.name":"counter-pool", + "ate.sandbox.class":"gvisor","ate.stats.source":"cgroup","ate.stats.kind":"periodic", + "ate.actor.epoch":1788903185000000000, + "ate.stats.memory.usage":39845888,"ate.stats.memory.working_set":31457280, + "ate.stats.cpu.time":0.196776,"ate.stats.memory.peak":52428800} ``` -* **Identity** rides in the same label group as the actor lifecycle events and container logs (`labels` off GCE, `logging.googleapis.com/labels` on GCE, which Cloud Logging promotes into `LogEntry.labels`), so every [query dimension above](#centralized-logging-backends-multi-dimensional-aggregation) applies unchanged, and one filter returns an actor's output, transitions, and usage interleaved. The pool pair is present when the worker pod resolved to a WorkerPool and absent otherwise, the same rule as on the metrics. -* **Consumers filter on** `msg` plus `kind`. `kind` is `periodic` today; future kinds (lifecycle brackets) will join on the same record shape: +* **Consumers filter on** the message and the keys, which sit at the top level rather than in the label group: ```text -labels."ate.actor.uid"="8f2a…" AND jsonPayload.msg="Actor usage sample" +jsonPayload.msg="Actor usage sampled" AND jsonPayload."ate.actor.uid"="8f2a…" ``` -* **The measurements** mirror the wire sample. `memory_current_bytes` and `memory_working_set_bytes` are point-in-time (current includes reclaimable page cache; the working set does not, and is the figure to compare against a memory limit). `memory_peak_bytes` and `cpu_usage_usec` **accumulate within the current epoch** (peak reads the cgroup's `memory.peak`, so it is zero on kernels older than 5.19 — unmeasured, not measured-as-zero), whose boundary depends on `source`: the `cgroup` source (host cgroup, gVisor) restarts both at zero on every restore, while the `guest-agent` source (inside the micro-VM) keeps counting across restores. Either way, per-actor CPU over a window is the increase between two samples, never a sum of raw values — and a decrease means the counter reset, not negative usage. -* **Cadence and switch**: `--actor-stats-poll-interval` on atelet governs the sweep (default `1m`, floor `50s`, `0` disables the metrics and the events together). -* **No trace context**: the sweep serves no request, so there is no span to join — the [Joining Logs to Traces](#joining-logs-to-traces) fields are absent on these records. -* **Delivery is best-effort, decoupled from node health**: the records ride atelet's stdout behind a bounded queue, so a stalled log consumer costs events (with a `Usage events dropped` warning on recovery), never the sweep or the metrics. The feed does not honor atelet's `--log-level` — a node quieted to `warn` keeps emitting usage data; the interval flag is the only switch. +* **The measurements** follow [the rules above](#the-same-records-over-otlp): absent while the actor is pending, CPU in seconds since the activation began, memory usage and working set absolute. An activation that ends any other way, as by a crash, has no `final` record, so take the highest CPU per epoch rather than wait for one. The `final` reading is taken just before the teardown, bounded on a micro-VM to 10ms. When it fails or times out, the `final` record repeats the newest measured sample, dated when that sample was read, or is pending when none was measured. +* **Cadence**: `--usage-sample-interval` on the ateom sets the timer (default `1m`, at least `50s`). Each sweep writes one periodic record per actor, and `GetActiveWorkloadStats` serves the latest sample rather than reading again. +* **Trace context**: `initial` records come from the Run or Restore call (on a micro-VM, from a goroutine the call starts, so the record can land just before or after the call returns) and `final` records from the call that tears the activation down, a Checkpoint, a Terminate, or, on a micro-VM, the Run or Restore that re-hosts it; both carry that call's span, like any record in [Joining Logs to Traces](#joining-logs-to-traces). `periodic` records come from the sampler, and a shutdown's `final` records from the drain, which serve no request, so they carry none. +* **Delivery is best-effort**: the stdout record ignores `--log-level` and sits behind a bounded queue, so a stalled log consumer costs records, with a warning, never the sampler. **Do not put actor identity on a log-based metric.** Aggregating these events in the log store is what they are for, but a log-based metric built over them must label only by the bounded set (template, sandbox class, source, pool) — promoting `ate.actor.uid` or `ate.actor.name` into a metric label reintroduces exactly the per-actor cardinality this split keeps out of the TSDB. @@ -237,7 +237,7 @@ labels."ate.actor.uid"="8f2a…" AND jsonPayload.msg="Actor usage sample" Agent Substrate emits foundational OpenTelemetry system and server metrics to monitor the overall health and performance of the control plane services. Every metric below is emitted by a service binary over OTLP and is **independent of the deployment** — a Kind dev cluster gets the same instruments as production; only the backend differs (see [Where Telemetry Goes](#4-where-telemetry-goes)). -> [`docs/metrics/registry/metrics.yaml`](metrics/registry/metrics.yaml) defines each instrument. Read it when you need all the labels, the bucket limits, or the permitted values of a label. The table below does not have each instrument. The request-parking instruments and the actor resource-usage instruments (`ate.actor.stats.*`) are in the registry only. Refer to [The metric registry](#the-metric-registry); per-actor usage detail is the [events channel](#per-actor-usage-events)'s job. +> [`docs/metrics/registry/metrics.yaml`](metrics/registry/metrics.yaml) defines each instrument. Read it when you need all the labels, the bucket limits, or the permitted values of a label. The table below does not have each instrument. The request-parking instruments and the actor resource-usage instruments (`ate.actor.stats.*`) are in the registry only. Refer to [The metric registry](#the-metric-registry); per-actor usage detail is on the [usage records](#per-actor-usage-events). | Metric | Emitted by | Type | Measures | |--------|------------|------|----------| @@ -404,7 +404,7 @@ Telemetry is emitted the same way everywhere; only the backend differs between a | Path | service → in-cluster `opentelemetry-collector` | service → Google Managed Prometheus (GMP) | | Metrics | collector Prometheus exporter on `:8889` | Google Cloud Monitoring | | Traces | Jaeger UI | Google Cloud Trace | -| Logs | pod stdout; ateapi's [actor lifecycle events](#the-same-records-over-otlp) also to the collector's `debug` exporter | pod stdout. No OTLP logs: `OTEL_LOGS_EXPORTER` is unset | +| Logs | pod stdout; the [actor events](#the-same-records-over-otlp), ateapi's lifecycle records and the ateoms' usage samples, also to the collector's `debug` exporter | pod stdout. No OTLP logs: `OTEL_LOGS_EXPORTER` is unset | | Dashboards | Not supported | Google Cloud Monitoring (see [Dashboards](#5-dashboards)) | > In Kind, `ateapi`, `atelet`, `ate-controller`, and `atenet-router` are pointed at the in-cluster collector, and the controller propagates the endpoint to the ateom worker pods it creates, so all component telemetry lands locally. diff --git a/internal/actorlog/logger.go b/internal/actorlog/logger.go index 7ebc7f1a79..47885520c9 100644 --- a/internal/actorlog/logger.go +++ b/internal/actorlog/logger.go @@ -73,12 +73,9 @@ const ( labelsKeyGCE = "logging.googleapis.com/labels" ) -// LabelsKey returns the label group's spelling for this environment. Every -// emitter of the actor-identity label group (container logs, lifecycle -// events, usage events) must pick its key here, so they all promote into -// Cloud Logging the same way -- and so going vendor-neutral later means -// changing one function. -func LabelsKey(isOnGCE bool) string { +// labelsKeyFor returns the label group's spelling for this environment, so +// going vendor-neutral later means changing one function. +func labelsKeyFor(isOnGCE bool) string { if isOnGCE { return labelsKeyGCE } @@ -89,7 +86,7 @@ func LabelsKey(isOnGCE bool) string { func NewActorLogger(w io.Writer, isOnGCE bool) *ActorLogger { return &ActorLogger{ writer: w, - labelsKey: LabelsKey(isOnGCE), + labelsKey: labelsKeyFor(isOnGCE), } } diff --git a/internal/actorlog/logger_test.go b/internal/actorlog/logger_test.go index 7b6bf9d9a8..8d86caaa49 100644 --- a/internal/actorlog/logger_test.go +++ b/internal/actorlog/logger_test.go @@ -451,13 +451,13 @@ func mustSpanID(t *testing.T, s string) trace.SpanID { return id } -// TestLabelsKey pins the one place the label group's spelling is chosen: the +// TestLabelsKeyFor pins the one place the label group's spelling is chosen: the // GCE spelling is the key Cloud Logging promotes into LogEntry.labels. -func TestLabelsKey(t *testing.T) { - if got := LabelsKey(false); got != "labels" { - t.Errorf("LabelsKey(false) = %q, want labels", got) +func TestLabelsKeyFor(t *testing.T) { + if got := labelsKeyFor(false); got != "labels" { + t.Errorf("labelsKeyFor(false) = %q, want labels", got) } - if got := LabelsKey(true); got != "logging.googleapis.com/labels" { - t.Errorf("LabelsKey(true) = %q, want logging.googleapis.com/labels", got) + if got := labelsKeyFor(true); got != "logging.googleapis.com/labels" { + t.Errorf("labelsKeyFor(true) = %q, want logging.googleapis.com/labels", got) } } diff --git a/internal/ateomstats/usage.go b/internal/ateomstats/usage.go index 17bd09c08f..b3801fd961 100644 --- a/internal/ateomstats/usage.go +++ b/internal/ateomstats/usage.go @@ -31,10 +31,6 @@ import ( "github.com/agent-substrate/substrate/internal/resources" ) -// finalFlushTimeout bounds the flush after a final record. The ateom stays up -// after a checkpoint, so the flush buys promptness, not delivery. -const finalFlushTimeout = 500 * time.Millisecond - // Pool is the WorkerPool of the ateom's pod. type Pool struct { Namespace string @@ -46,9 +42,6 @@ type Pool struct { type UsageEmitter struct { pool Pool events *actorevent.Emitter - // flush pushes queued OTLP records out after a final record; nil when OTLP - // is off. - flush func(context.Context) error } // NewUsageEmitter writes the records over OTLP through lp, the provider @@ -57,7 +50,7 @@ func NewUsageEmitter(lp *sdklog.LoggerProvider, stdout slog.Handler, pool Pool) if lp == nil { return &UsageEmitter{pool: pool, events: actorevent.NewEmitterTo(noop.NewLoggerProvider(), stdout)} } - return &UsageEmitter{pool: pool, events: actorevent.NewEmitterTo(lp, stdout), flush: lp.ForceFlush} + return &UsageEmitter{pool: pool, events: actorevent.NewEmitterTo(lp, stdout)} } // Emit writes one record for s, dated when s was read. kind is one of the @@ -70,25 +63,6 @@ func (e *UsageEmitter) Emit(ctx context.Context, kind string, s *ateompb.Workloa e.events.LogAt(ctx, actorevent.UsageSampled, at, UsageAttrs(e.pool, kind, s)) } -// EmitFinal writes the final record of an activation and flushes it in the -// background, off the checkpoint's path. -func (e *UsageEmitter) EmitFinal(ctx context.Context, s *ateompb.WorkloadStatsSample) { - if e == nil { - return - } - e.Emit(ctx, ateattr.StatsKindFinal, s) - if e.flush == nil { - return - } - go func() { - flushCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), finalFlushTimeout) - defer cancel() - if err := e.flush(flushCtx); err != nil { - slog.DebugContext(ctx, "Usage records not flushed after the final sample", slog.Any("err", err)) - } - }() -} - // UsageAttrs is the attribute set of ate.actor.usage_sampled for s. The // measurements are absent while s has no source, and the peak also when the // source reports none. diff --git a/internal/ateomstats/usage_test.go b/internal/ateomstats/usage_test.go index 339a3cc6ed..a6efc61d7c 100644 --- a/internal/ateomstats/usage_test.go +++ b/internal/ateomstats/usage_test.go @@ -145,16 +145,14 @@ func (r *recorder) len() int { return len(r.records) } -func newTestEmitter(flush func(context.Context) error) (*UsageEmitter, *recorder) { +func newTestEmitter() (*UsageEmitter, *recorder) { rec := &recorder{} - e := NewUsageEmitter(nil, rec, testPool) - e.flush = flush - return e, rec + return NewUsageEmitter(nil, rec, testPool), rec } func TestEmitDatesRecordAtReadTime(t *testing.T) { t.Parallel() - e, rec := newTestEmitter(nil) + e, rec := newTestEmitter() s := measuredSample() e.Emit(context.Background(), ateattr.StatsKindInitial, s) if rec.len() != 1 { @@ -169,39 +167,6 @@ func TestNilEmitterWritesNothing(t *testing.T) { t.Parallel() var e *UsageEmitter e.Emit(context.Background(), ateattr.StatsKindInitial, measuredSample()) - e.EmitFinal(context.Background(), measuredSample()) -} - -func TestEmitFinalFlushes(t *testing.T) { - t.Parallel() - deadlines := make(chan time.Time, 1) - e, rec := newTestEmitter(func(ctx context.Context) error { - d, _ := ctx.Deadline() - deadlines <- d - return nil - }) - start := time.Now() - e.EmitFinal(context.Background(), measuredSample()) - if rec.len() != 1 { - t.Fatalf("wrote %d records, want 1", rec.len()) - } - if kind := attrMap(recordAttrs(rec.records[0]))[string(ateattr.StatsKindKey)].String(); kind != ateattr.StatsKindFinal { - t.Errorf("kind = %q, want final", kind) - } - select { - case d := <-deadlines: - if d.IsZero() || d.Sub(start) > finalFlushTimeout+time.Second { - t.Errorf("flush deadline = %v, want about %v after the record", d, finalFlushTimeout) - } - case <-time.After(5 * time.Second): - t.Fatal("final record was not flushed") - } -} - -func recordAttrs(r slog.Record) []slog.Attr { - var attrs []slog.Attr - r.Attrs(func(a slog.Attr) bool { attrs = append(attrs, a); return true }) - return attrs } // TestStartSamplerSweepsOnSchedule runs six intervals and stops: six sweeps, diff --git a/internal/proto/ateompb/ateom.pb.go b/internal/proto/ateompb/ateom.pb.go index d6c25de182..23f5a590c2 100644 --- a/internal/proto/ateompb/ateom.pb.go +++ b/internal/proto/ateompb/ateom.pb.go @@ -1768,15 +1768,14 @@ type WorkloadStatsSample struct { // drifts upward with cache the kernel would drop for free under pressure. MemoryWorkingSetBytes uint64 `protobuf:"varint,10,opt,name=memory_working_set_bytes,json=memoryWorkingSetBytes,proto3" json:"memory_working_set_bytes,omitempty"` // Cumulative CPU time since the activation began (epoch_unix_nano), for - // every source. The cgroup source restarts on its own; the guest-agent - // source is rebased on its first read after a restore, since the guest's - // counters survive in guest RAM. An ateom that leaves epoch_unix_nano at zero - // also sends the raw guest counter. A lifetime figure is the sum over epochs - // of each epoch's highest value. + // every source, and never decreasing within it. The cgroup source restarts on + // its own; the guest-agent source counts from its first read after a restore + // that resumes the guest, since the guest's counters survive in guest RAM. A + // lifetime figure is the sum over epochs of each epoch's highest value. CpuUsageUsec uint64 `protobuf:"varint,11,opt,name=cpu_usage_usec,json=cpuUsageUsec,proto3" json:"cpu_usage_usec,omitempty"` ObservedAtUnixNano int64 `protobuf:"varint,12,opt,name=observed_at_unix_nano,json=observedAtUnixNano,proto3" json:"observed_at_unix_nano,omitempty"` // The activation this sample belongs to, as the unix-nano time it began. A - // Run or Restore starts one. Zero from an ateom that does not set it. + // Run or Restore starts one. EpochUnixNano int64 `protobuf:"varint,13,opt,name=epoch_unix_nano,json=epochUnixNano,proto3" json:"epoch_unix_nano,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache diff --git a/internal/proto/ateompb/ateom.proto b/internal/proto/ateompb/ateom.proto index a742d3baaa..89755de037 100644 --- a/internal/proto/ateompb/ateom.proto +++ b/internal/proto/ateompb/ateom.proto @@ -78,27 +78,26 @@ service Ateom { // that dies during boot is attributable rather than anonymous. rpc GetWorkloadStats(GetWorkloadStatsRequest) returns (GetWorkloadStatsResponse) {} - // GetActiveWorkloadStats samples whatever this ateom is currently - // executing, without asserting an identity. It is the discovery read for a + // GetActiveWorkloadStats reports every actor this ateom hosts, without + // asserting an identity. It is the discovery read for a // scraper that enumerates ateoms and holds no worker-to-actor mapping; // GetWorkloadStats above is the verified read for a caller that must be // answered about a specific actor. // - // Every state a blind caller can find is a normal answer here, never an - // error: an "available" ateom answers an empty samples list, and a workload - // with nothing to measure YET (a poll landing in a boot or a restore) - // appears as a sample carrying its attribution with - // source = STATS_SOURCE_UNSPECIFIED -- "not measured", per the sample's own - // contract -- so even a workload that dies during boot is attributable, not - // anonymous. Error codes are reserved for real failures reading a sandbox - // that should be measurable. This is deliberately unlike GetWorkloadStats, - // whose caller asserts knowledge the codes then answer. + // It serves each actor's latest sample from the ateom's own sampling timer, + // so a poll reads no sandbox and has no error path: an "available" ateom + // answers an empty samples list, and a workload with nothing measured yet + // (a boot, a restore, or a guest the last sweep did not reach) appears as a + // sample carrying its attribution with source = STATS_SOURCE_UNSPECIFIED -- + // "not measured", per the sample's own contract -- so even a workload that + // dies during boot is attributable, not anonymous. A sample is up to one + // sampling interval old. This is deliberately unlike GetWorkloadStats, whose + // caller asserts knowledge the codes then answer. // // Consumers MUST attribute each sample solely from the identity echoed // inside it, never from a mapping they hold: without an asserted uid, the // response is the only statement of who was measured. Like GetWorkloadStats - // it is a pure read, safe on a timer, and does not touch the lifecycle - // mutex. + // it is safe on a timer and does not touch the lifecycle mutex. rpc GetActiveWorkloadStats(GetActiveWorkloadStatsRequest) returns (GetActiveWorkloadStatsResponse) {} // TerminateWorkload stops and deletes container workloads and cleans up @@ -463,17 +462,16 @@ message WorkloadStatsSample { // drifts upward with cache the kernel would drop for free under pressure. uint64 memory_working_set_bytes = 10; // Cumulative CPU time since the activation began (epoch_unix_nano), for - // every source. The cgroup source restarts on its own; the guest-agent - // source is rebased on its first read after a restore, since the guest's - // counters survive in guest RAM. An ateom that leaves epoch_unix_nano at zero - // also sends the raw guest counter. A lifetime figure is the sum over epochs - // of each epoch's highest value. + // every source, and never decreasing within it. The cgroup source restarts on + // its own; the guest-agent source counts from its first read after a restore + // that resumes the guest, since the guest's counters survive in guest RAM. A + // lifetime figure is the sum over epochs of each epoch's highest value. uint64 cpu_usage_usec = 11; int64 observed_at_unix_nano = 12; // The activation this sample belongs to, as the unix-nano time it began. A - // Run or Restore starts one. Zero from an ateom that does not set it. + // Run or Restore starts one. int64 epoch_unix_nano = 13; } diff --git a/internal/proto/ateompb/ateom_grpc.pb.go b/internal/proto/ateompb/ateom_grpc.pb.go index 5e05361a83..572fee2b79 100644 --- a/internal/proto/ateompb/ateom_grpc.pb.go +++ b/internal/proto/ateompb/ateom_grpc.pb.go @@ -101,27 +101,26 @@ type AteomClient interface { // "booting" distinguishable from "not here" at all, and it means a workload // that dies during boot is attributable rather than anonymous. GetWorkloadStats(ctx context.Context, in *GetWorkloadStatsRequest, opts ...grpc.CallOption) (*GetWorkloadStatsResponse, error) - // GetActiveWorkloadStats samples whatever this ateom is currently - // executing, without asserting an identity. It is the discovery read for a + // GetActiveWorkloadStats reports every actor this ateom hosts, without + // asserting an identity. It is the discovery read for a // scraper that enumerates ateoms and holds no worker-to-actor mapping; // GetWorkloadStats above is the verified read for a caller that must be // answered about a specific actor. // - // Every state a blind caller can find is a normal answer here, never an - // error: an "available" ateom answers an empty samples list, and a workload - // with nothing to measure YET (a poll landing in a boot or a restore) - // appears as a sample carrying its attribution with - // source = STATS_SOURCE_UNSPECIFIED -- "not measured", per the sample's own - // contract -- so even a workload that dies during boot is attributable, not - // anonymous. Error codes are reserved for real failures reading a sandbox - // that should be measurable. This is deliberately unlike GetWorkloadStats, - // whose caller asserts knowledge the codes then answer. + // It serves each actor's latest sample from the ateom's own sampling timer, + // so a poll reads no sandbox and has no error path: an "available" ateom + // answers an empty samples list, and a workload with nothing measured yet + // (a boot, a restore, or a guest the last sweep did not reach) appears as a + // sample carrying its attribution with source = STATS_SOURCE_UNSPECIFIED -- + // "not measured", per the sample's own contract -- so even a workload that + // dies during boot is attributable, not anonymous. A sample is up to one + // sampling interval old. This is deliberately unlike GetWorkloadStats, whose + // caller asserts knowledge the codes then answer. // // Consumers MUST attribute each sample solely from the identity echoed // inside it, never from a mapping they hold: without an asserted uid, the // response is the only statement of who was measured. Like GetWorkloadStats - // it is a pure read, safe on a timer, and does not touch the lifecycle - // mutex. + // it is safe on a timer and does not touch the lifecycle mutex. GetActiveWorkloadStats(ctx context.Context, in *GetActiveWorkloadStatsRequest, opts ...grpc.CallOption) (*GetActiveWorkloadStatsResponse, error) // TerminateWorkload stops and deletes container workloads and cleans up // network and bundle overlays on ateom. @@ -256,27 +255,26 @@ type AteomServer interface { // "booting" distinguishable from "not here" at all, and it means a workload // that dies during boot is attributable rather than anonymous. GetWorkloadStats(context.Context, *GetWorkloadStatsRequest) (*GetWorkloadStatsResponse, error) - // GetActiveWorkloadStats samples whatever this ateom is currently - // executing, without asserting an identity. It is the discovery read for a + // GetActiveWorkloadStats reports every actor this ateom hosts, without + // asserting an identity. It is the discovery read for a // scraper that enumerates ateoms and holds no worker-to-actor mapping; // GetWorkloadStats above is the verified read for a caller that must be // answered about a specific actor. // - // Every state a blind caller can find is a normal answer here, never an - // error: an "available" ateom answers an empty samples list, and a workload - // with nothing to measure YET (a poll landing in a boot or a restore) - // appears as a sample carrying its attribution with - // source = STATS_SOURCE_UNSPECIFIED -- "not measured", per the sample's own - // contract -- so even a workload that dies during boot is attributable, not - // anonymous. Error codes are reserved for real failures reading a sandbox - // that should be measurable. This is deliberately unlike GetWorkloadStats, - // whose caller asserts knowledge the codes then answer. + // It serves each actor's latest sample from the ateom's own sampling timer, + // so a poll reads no sandbox and has no error path: an "available" ateom + // answers an empty samples list, and a workload with nothing measured yet + // (a boot, a restore, or a guest the last sweep did not reach) appears as a + // sample carrying its attribution with source = STATS_SOURCE_UNSPECIFIED -- + // "not measured", per the sample's own contract -- so even a workload that + // dies during boot is attributable, not anonymous. A sample is up to one + // sampling interval old. This is deliberately unlike GetWorkloadStats, whose + // caller asserts knowledge the codes then answer. // // Consumers MUST attribute each sample solely from the identity echoed // inside it, never from a mapping they hold: without an asserted uid, the // response is the only statement of who was measured. Like GetWorkloadStats - // it is a pure read, safe on a timer, and does not touch the lifecycle - // mutex. + // it is safe on a timer and does not touch the lifecycle mutex. GetActiveWorkloadStats(context.Context, *GetActiveWorkloadStatsRequest) (*GetActiveWorkloadStatsResponse, error) // TerminateWorkload stops and deletes container workloads and cleans up // network and bundle overlays on ateom. diff --git a/manifests/ate-install/kind/ate-otel-config.yaml b/manifests/ate-install/kind/ate-otel-config.yaml index 52467280ba..ed2c8b9334 100644 --- a/manifests/ate-install/kind/ate-otel-config.yaml +++ b/manifests/ate-install/kind/ate-otel-config.yaml @@ -30,10 +30,10 @@ # the ateom worker pods it creates, which have no other way to receive them. # Kind only; production keeps the SDK defaults. # -# OTEL_LOGS_EXPORTER=otlp sends the actor events over OTLP instead of stdout; -# kind adds console to keep them in kubectl logs too. serverboot defaults it to -# none, so the base ConfigMap leaves it unset and every other environment keeps -# them on stdout. +# OTEL_LOGS_EXPORTER=otlp sends the actor events, ateapi's lifecycle records +# and the ateoms' usage samples, over OTLP instead of stdout; kind adds console +# to keep them in kubectl logs too. serverboot defaults it to none, so the base +# ConfigMap leaves it unset and every other environment keeps them on stdout. apiVersion: v1 kind: ConfigMap metadata: From f6a1dc72a35d9875302e2998b2aa23b67dbc732a Mon Sep 17 00:00:00 2001 From: Da Huang <127548902+git286@users.noreply.github.com> Date: Fri, 9 Oct 2026 15:23:31 +0000 Subject: [PATCH 25/44] ateom-gvisor logs a warning when a running actor's sandbox dies (#2258) Part of #2212 (o11y side of #2211). If the gVisor sandbox dies after start (sentry crash, kill, OOM kill), nothing notices today. The actor stays RUNNING and its usage samples keep reading like an idle actor's, because the pause cgroup leaf and its counters outlive the processes in it. This makes the death visible: - `cgroupstats` also reads `populated` from `cgroup.events` and `oom_kill` from `memory.events`. If either file is missing, it never reads as a dead sandbox. - The ateom's usage sweep (`sweepUsage`, every `--usage-sample-interval`, 1m by default) logs one WARN per activation when the leaf is empty. Since #2194, `GetActiveWorkloadStats` serves the sweep's cached sample and reads no cgroup, so the check lives in the sweep, the only periodic cgroup read; `measure` hands back the raw `cgroupstats.Sample` for it. The record uses `ateattr.ActorLogAttrs` plus `ate.sandbox.oom_kills`, so filtering on `ate.actor.uid` finds it with the rest of the actor's logs. - Actors with a lifecycle RPC holding or waiting on their lock are skipped, since checkpoint and terminate empty the cgroup on purpose. `actorlock` gets a non-blocking `Busy` for this. An activation whose final record is written (checkpoint, terminate, drain) is no longer swept at all. - `readKeyedUint` walks lines in place instead of using a `bufio.Scanner`, which allocated 4 KiB per file. With the two extra reads, reading 1000 actors' cgroups takes about 9% longer than before and allocates half the bytes. It does not change actor state, RPC responses or the stats samples. Moving the actor to CRASHED is #2211. It also does not catch an app that exits while the sentry stays up. Tested on kind and on GKE (substrate-relay) with the branch image on the egress pool, before the rebase onto #2194 (the check then ran in `GetActiveWorkloadStats`). Unit tests pass at the rebased head; live re-verification on kind is pending: | Case | kind | GKE | |---|---|---| | OOM kill via `memory.max` | 1 WARN, `oom_kills=3` | 1 WARN, `oom_kills=1` | | `kill -9` the sentry | 1 WARN, `oom_kills=0` | 1 WARN, `oom_kills=0` | | Normal suspend | none | none | | Suspend then resume | none | none | | Idle actors | none (5) | none (3) | Each warning showed up on the first sweep after the fault and was not repeated. On GKE it lands in Cloud Logging as severity WARNING with the `ate.*` fields. The stats pipeline was unchanged. One caveat: `oom_kill` counts over the cgroup's lifetime, so a leaf left behind by an earlier activation of the same actor (#2107) carries its count forward. - [x] Tests pass - [x] Appropriate changes to documentation are included in the PR --- cmd/ateom-gvisor/hosted.go | 3 + .../internal/cgroupstats/cgroupstats.go | 37 +++- .../internal/cgroupstats/cgroupstats_test.go | 41 ++++ cmd/ateom-gvisor/stats.go | 66 +++++-- cmd/ateom-gvisor/stats_test.go | 177 ++++++++++++++++++ docs/metrics/registry/metrics.yaml | 18 +- docs/observability.md | 11 ++ internal/actorlock/actorlock.go | 11 ++ internal/actorlock/actorlock_test.go | 86 +++++++++ internal/ateattr/ateattr.go | 5 + 10 files changed, 427 insertions(+), 28 deletions(-) diff --git a/cmd/ateom-gvisor/hosted.go b/cmd/ateom-gvisor/hosted.go index 890763ab9d..45b06e35e3 100644 --- a/cmd/ateom-gvisor/hosted.go +++ b/cmd/ateom-gvisor/hosted.go @@ -21,6 +21,7 @@ import ( "fmt" "log/slog" "net" + "sync/atomic" "time" "github.com/agent-substrate/substrate/internal/apierror" @@ -42,6 +43,8 @@ type hostedActor struct { session *workloadSession // usage is this activation's epoch, CPU baseline, and latest sample. usage *ateomstats.Activation + // deadReported keeps the usage sweep to one warning per activation. + deadReported atomic.Bool } // admitActor reserves capacity before network setup. An actor that is already diff --git a/cmd/ateom-gvisor/internal/cgroupstats/cgroupstats.go b/cmd/ateom-gvisor/internal/cgroupstats/cgroupstats.go index 8ca586ce09..7ed8eb5124 100644 --- a/cmd/ateom-gvisor/internal/cgroupstats/cgroupstats.go +++ b/cmd/ateom-gvisor/internal/cgroupstats/cgroupstats.go @@ -21,11 +21,10 @@ // Every read is scoped to a caller-supplied directory rather than a hardcoded // /sys/fs/cgroup path, which keeps the parsing testable from a fixture tree // without root or a live sandbox. The package deliberately does not know what a -// sandbox is; it reads four numbers out of five files. +// sandbox is; it reads what the kernel reports in a handful of files. package cgroupstats import ( - "bufio" "bytes" "errors" "fmt" @@ -65,6 +64,16 @@ type Sample struct { // to this cgroup (see ateomcgroup.Delegate, which enables controllers // best-effort and carries on when one cannot be enabled). CPUUsageUsec uint64 + + // OOMKills is memory.events' oom_kill: processes in the cgroup the kernel's + // OOM killer has killed over the cgroup's lifetime. Zero if the file is + // absent. + OOMKills uint64 + + // Empty is true when cgroup.events reports populated 0: no process is left + // in the cgroup. It reads false when the file is absent, so a kernel that + // does not say is never taken for a dead sandbox. + Empty bool } // Read returns a Sample for the cgroup v2 directory at dir. @@ -80,6 +89,7 @@ type Sample struct { // memory controller's full accounting, and cpu.stat is absent when the cpu // controller was not delegated. Failing the sample for any of them would mean // reporting no memory numbers because the node could not report CPU. +// memory.events and cgroup.events are optional in the same way. func Read(dir string) (Sample, error) { current, ok, err := readUint(filepath.Join(dir, "memory.current")) if err != nil { @@ -95,6 +105,8 @@ func Read(dir string) (Sample, error) { peak, _, _ := readUint(filepath.Join(dir, "memory.peak")) inactiveFile, haveInactiveFile, _ := readKeyedUint(filepath.Join(dir, "memory.stat"), "inactive_file") cpuUsage, _, _ := readKeyedUint(filepath.Join(dir, "cpu.stat"), "usage_usec") + oomKills, _, _ := readKeyedUint(filepath.Join(dir, "memory.events"), "oom_kill") + populated, havePopulated, _ := readKeyedUint(filepath.Join(dir, "cgroup.events"), "populated") // Without inactive_file there is nothing to subtract, so the working set // collapses to memory.current. That over-reports by however much reclaimable @@ -117,6 +129,8 @@ func Read(dir string) (Sample, error) { MemoryPeakBytes: peak, MemoryWorkingSetBytes: workingSet, CPUUsageUsec: cpuUsage, + OOMKills: oomKills, + Empty: havePopulated && populated == 0, }, nil } @@ -150,23 +164,26 @@ func readKeyedUint(path, key string) (uint64, bool, error) { return 0, false, fmt.Errorf("reading %q: %w", path, err) } - sc := bufio.NewScanner(bytes.NewReader(b)) - for sc.Scan() { + // The files are a few hundred bytes and read for every actor on every + // sweep, so lines are walked in place, and only one that starts with the + // key is split. + for line := range bytes.Lines(b) { + line = bytes.TrimLeft(line, " \t") + if !bytes.HasPrefix(line, []byte(key)) { + continue + } // Fields, then a length check, rather than indexing straight into the // split: a blank or single-token line is not worth a panic in an RPC // handler, and these files are read on a timer for the life of a workload. - f := strings.Fields(sc.Text()) - if len(f) != 2 || f[0] != key { + f := bytes.Fields(line) + if len(f) != 2 || string(f[0]) != key { continue } - v, err := strconv.ParseUint(f[1], 10, 64) + v, err := strconv.ParseUint(string(f[1]), 10, 64) if err != nil { return 0, false, fmt.Errorf("parsing %q of %q: %w", key, path, err) } return v, true, nil } - if err := sc.Err(); err != nil { - return 0, false, fmt.Errorf("scanning %q: %w", path, err) - } return 0, false, nil } diff --git a/cmd/ateom-gvisor/internal/cgroupstats/cgroupstats_test.go b/cmd/ateom-gvisor/internal/cgroupstats/cgroupstats_test.go index 47d5dd365a..8b5198a295 100644 --- a/cmd/ateom-gvisor/internal/cgroupstats/cgroupstats_test.go +++ b/cmd/ateom-gvisor/internal/cgroupstats/cgroupstats_test.go @@ -167,6 +167,44 @@ func TestRead(t *testing.T) { }, want: Sample{}, }, + { + name: "live cgroup is not empty", + files: map[string]string{ + "memory.current": "1000\n", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0\n", + "cgroup.events": "populated 1\nfrozen 0\n", + }, + want: Sample{ + MemoryCurrentBytes: 1000, + MemoryWorkingSetBytes: 1000, + }, + }, + { + // What a sandbox leaf looked like on kind after the kernel OOM-killed + // the sentry: the leaf and its counters outlive every process in it. + name: "OOM-killed sandbox is empty", + files: map[string]string{ + "memory.current": "16384\n", + "memory.events": "low 0\nhigh 0\nmax 0\noom 5334\noom_kill 3\noom_group_kill 0\n", + "cgroup.events": "populated 0\nfrozen 0\n", + }, + want: Sample{ + MemoryCurrentBytes: 16384, + MemoryWorkingSetBytes: 16384, + OOMKills: 3, + Empty: true, + }, + }, + { + name: "no cgroup.events is not taken for empty", + files: map[string]string{ + "memory.current": "1000\n", + }, + want: Sample{ + MemoryCurrentBytes: 1000, + MemoryWorkingSetBytes: 1000, + }, + }, { // A short, blank, or over-long line must not panic: this runs in an RPC // handler on a timer, and grpc-go does not recover handler panics. @@ -190,6 +228,9 @@ func TestRead(t *testing.T) { "memory.current": "157286400\n", "memory.peak": "not-a-number\n", "cpu.stat": "usage_usec eleventy\n", + "memory.events": "oom_kill lots\n", + // Garbage here must not read as a dead sandbox. + "cgroup.events": "populated no\n", }, want: Sample{ MemoryCurrentBytes: 157286400, diff --git a/cmd/ateom-gvisor/stats.go b/cmd/ateom-gvisor/stats.go index b8c80285ac..c044cce546 100644 --- a/cmd/ateom-gvisor/stats.go +++ b/cmd/ateom-gvisor/stats.go @@ -104,7 +104,9 @@ func (s *AteomService) GetWorkloadStats(ctx context.Context, req *ateompb.GetWor if hosted == nil { return nil, apierror.NotFound("ateom is not executing actor %q", req.GetActorUid()) } - sample, err := s.measure(ctx, hosted) + // The raw reading is dropped: the dead-sandbox check runs in the usage + // sweep, which reads every hosted actor. + sample, _, err := s.measure(ctx, hosted) if err != nil { // The requested actor is the active one but its cgroup is not there. // Most often that is a poll landing in the boot: the ateom retains the @@ -162,10 +164,11 @@ func (s *AteomService) sweepUsage(ctx context.Context) { // An actor with no numbers is reported as pending, so a failed read for // one does not lose the others. Past the initial reading, a missing // cgroup means the sandbox is going away. - sample, err := s.measure(ctx, h) + sample, raw, err := s.measure(ctx, h) if err != nil { if !errors.Is(err, fs.ErrNotExist) { - slog.WarnContext(ctx, "Failed to read sandbox cgroup", slog.String("actorUID", h.attribution.UID), slog.Any("err", err)) + attrs := append(ateattr.ActorLogAttrs(h.attribution), slog.Any("err", err)) + slog.LogAttrs(ctx, slog.LevelWarn, "Failed to read sandbox cgroup", attrs...) } sample = h.usage.WithEpoch(pendingSample(&h.attribution)) } @@ -176,6 +179,9 @@ func (s *AteomService) sweepUsage(ctx context.Context) { if s.lookupActor(h.attribution.UID) != h { continue } + if raw.Empty { // a failed read returns a zero Sample + s.warnDeadSandbox(ctx, h, raw) + } h.usage.Periodic(sample, func() { s.usage.Emit(ctx, ateattr.StatsKindPeriodic, sample) }) } } @@ -183,7 +189,7 @@ func (s *AteomService) sweepUsage(ctx context.Context) { // recordInitial samples a new activation once its sandbox is up and writes its // initial record. func (s *AteomService) recordInitial(ctx context.Context, h *hostedActor) { - sample, err := s.measure(ctx, h) + sample, _, err := s.measure(ctx, h) if err != nil { slog.WarnContext(ctx, "No initial usage sample", slog.String(string(ateattr.ActorUIDKey), h.attribution.UID), slog.Any("err", err)) sample = nil @@ -195,7 +201,7 @@ func (s *AteomService) recordInitial(ctx context.Context, h *hostedActor) { // record a successful checkpoint or terminate writes. A failed read leaves the // newest measured sample to stand in. func (s *AteomService) readFinal(ctx context.Context, h *hostedActor) { - if sample, err := s.measure(ctx, h); err == nil { + if sample, _, err := s.measure(ctx, h); err == nil { h.usage.Store(sample) } } @@ -239,21 +245,26 @@ func pendingSample(active *resources.ActorAttribution) *ateompb.WorkloadStatsSam } } -// measure reads h's sandbox cgroup as a reading of its activation. -func (s *AteomService) measure(ctx context.Context, h *hostedActor) (*ateompb.WorkloadStatsSample, error) { - return h.usage.Measure(ctx, func() (*ateompb.WorkloadStatsSample, map[string]uint64, error) { - sample, err := s.sampleSandbox(&h.attribution) +// measure reads h's sandbox cgroup as a reading of its activation, returning +// the raw cgroup reading alongside it for the dead-sandbox check. +func (s *AteomService) measure(ctx context.Context, h *hostedActor) (*ateompb.WorkloadStatsSample, cgroupstats.Sample, error) { + var raw cgroupstats.Sample + sample, err := h.usage.Measure(ctx, func() (*ateompb.WorkloadStatsSample, map[string]uint64, error) { + sample, r, err := s.sampleSandbox(&h.attribution) + raw = r return sample, nil, err }) + return sample, raw, err } // sampleSandbox reads the sandbox cgroup and builds the sample attributed to -// active. Errors come back raw -- notably fs.ErrNotExist for a cgroup that is -// not there yet -- because the callers disagree on what that means: an error -// code for the keyed read, a pending sample for the sweep. +// active, returning the raw reading alongside it. Errors come back raw -- +// notably fs.ErrNotExist for a cgroup that is not there yet -- because the +// callers disagree on what that means: an error code for the keyed read, a +// pending sample for the sweep. // The read holds no lifecycle lock, so the keyed caller re-checks the actor // record it loaded after this returns. -func (s *AteomService) sampleSandbox(active *resources.ActorAttribution) (*ateompb.WorkloadStatsSample, error) { +func (s *AteomService) sampleSandbox(active *resources.ActorAttribution) (*ateompb.WorkloadStatsSample, cgroupstats.Sample, error) { read := s.readSandboxCgroup if read == nil { read = cgroupstats.Read @@ -261,7 +272,7 @@ func (s *AteomService) sampleSandbox(active *resources.ActorAttribution) (*ateom observedAt := time.Now() sample, err := read(filepath.Join(s.cgroupRoot, ocispec.GVisorCgroupLeaf(active.UID, sandboxCgroupContainer))) if err != nil { - return nil, err + return nil, cgroupstats.Sample{}, err } return &ateompb.WorkloadStatsSample{ @@ -280,5 +291,30 @@ func (s *AteomService) sampleSandbox(active *resources.ActorAttribution) (*ateom CpuUsageUsec: sample.CPUUsageUsec, ObservedAtUnixNano: observedAt.UnixNano(), - }, nil + }, sample, nil +} + +// warnDeadSandbox logs, once per activation, that h's sandbox cgroup has no +// process left. Nothing else notices this today: the actor stays RUNNING and +// its stats read like an idle actor's (#2211). +// +// A lifecycle RPC empties the cgroup on purpose when it tears the sandbox down, +// and holds the actor's lock while it does, so a busy actor is skipped. The +// cgroup was read before the lock was checked, and h is looked up again after, +// so a teardown that starts or finishes in between is never reported. +// +// oom_kill counts over the cgroup's lifetime. A leaf left behind by an earlier +// activation of the same actor carries its count forward. +func (s *AteomService) warnDeadSandbox(ctx context.Context, h *hostedActor, raw cgroupstats.Sample) { + if h.deadReported.Load() { + return + } + if s.locks.Busy(h.attribution.UID) || s.lookupActor(h.attribution.UID) != h { + return + } + if !h.deadReported.CompareAndSwap(false, true) { + return + } + attrs := append(ateattr.ActorLogAttrs(h.attribution), slog.Uint64(string(ateattr.SandboxOOMKillsKey), raw.OOMKills)) + slog.LogAttrs(ctx, slog.LevelWarn, "Sandbox has no processes left while the actor is hosted", attrs...) } diff --git a/cmd/ateom-gvisor/stats_test.go b/cmd/ateom-gvisor/stats_test.go index 4d0452f46a..42f80e92da 100644 --- a/cmd/ateom-gvisor/stats_test.go +++ b/cmd/ateom-gvisor/stats_test.go @@ -17,10 +17,14 @@ package main import ( + "bytes" "context" "github.com/agent-substrate/substrate/internal/actorlock" + "log/slog" "os" "path/filepath" + "strings" + "sync" "testing" "time" @@ -497,3 +501,176 @@ func sweepAndList(s *AteomService) (*ateompb.GetActiveWorkloadStatsResponse, err s.sweepUsage(context.Background()) return s.GetActiveWorkloadStats(context.Background(), &ateompb.GetActiveWorkloadStatsRequest{}) } + +// deadCgroup is a sandbox leaf after the kernel OOM-killed the sentry: the leaf +// and its counters remain, the processes do not. +var deadCgroup = map[string]string{ + "memory.current": "16384\n", + "memory.events": "oom 5334\noom_kill 3\n", + "cgroup.events": "populated 0\nfrozen 0\n", +} + +// captureWarnings routes the default logger to a buffer for the test. +func captureWarnings(t *testing.T) *bytes.Buffer { + t.Helper() + var buf bytes.Buffer + prev := slog.Default() + slog.SetDefault(slog.New(slog.NewJSONHandler(&buf, &slog.HandlerOptions{Level: slog.LevelWarn}))) + t.Cleanup(func() { slog.SetDefault(prev) }) + return &buf +} + +const deadSandboxMsg = "Sandbox has no processes left while the actor is hosted" + +// TestSweepUsageDeadSandbox: a hosted actor whose sandbox cgroup +// is empty is warned about once per activation, and not at all while a +// lifecycle RPC is tearing it down. +func TestSweepUsageDeadSandbox(t *testing.T) { + for _, tc := range []struct { + name string + files map[string]string + busy bool + wantWarns int + }{ + {name: "dead sandbox warns once", files: deadCgroup, wantWarns: 1}, + {name: "teardown in progress is skipped", files: deadCgroup, busy: true}, + {name: "live sandbox does not warn", files: healthyCgroup}, + {name: "booting sandbox with no cgroup yet does not warn", files: nil}, + } { + t.Run(tc.name, func(t *testing.T) { + logs := captureWarnings(t) + s := newStatsService(t, tc.files) + setHostedActor(s, &testActor) + if tc.busy { + if !s.locks.Lock(context.Background(), testActor.UID) { + t.Fatal("could not take the actor lock") + } + defer s.locks.Unlock(testActor.UID) + } + + // Two sweeps: the second must not repeat the warning. + for range 2 { + got, err := sweepAndList(s) + if err != nil { + t.Fatalf("sweepAndList() error = %v, want nil", err) + } + if len(got.GetSamples()) != 1 { + t.Fatalf("sweepAndList() = %v, want one sample", got) + } + } + + if n := strings.Count(logs.String(), deadSandboxMsg); n != tc.wantWarns { + t.Fatalf("got %d dead-sandbox warnings, want %d; logs:\n%s", n, tc.wantWarns, logs) + } + if tc.wantWarns > 0 { + for _, want := range []string{`"ate.actor.uid":"uid-a"`, `"ate.actor.name":"actor-a"`, `"ate.atespace":"space-a"`, `"ate.template.name":"template-a"`, `"ate.sandbox.oom_kills":3`} { + if !strings.Contains(logs.String(), want) { + t.Errorf("warning missing %s; logs:\n%s", want, logs) + } + } + } + }) + } +} + +// A record that is no longer the hosted one -- the actor was unhosted or +// re-hosted after the cgroup was read -- is never reported. +func TestWarnDeadSandboxStaleRecord(t *testing.T) { + logs := captureWarnings(t) + s := newStatsService(t, deadCgroup) + setHostedActor(s, &testActor) + stale := &hostedActor{attribution: testActor} + + s.warnDeadSandbox(context.Background(), stale, cgroupstats.Sample{Empty: true}) + + if strings.Contains(logs.String(), deadSandboxMsg) { + t.Fatalf("warned about a stale record; logs:\n%s", logs) + } + if stale.deadReported.Load() { + t.Error("stale record marked as reported") + } +} + +// The once-per-activation guarantee is per hostedActor: a new activation of +// the same actor that dies again is reported again. +func TestSweepUsageDeadSandboxNewActivation(t *testing.T) { + logs := captureWarnings(t) + s := newStatsService(t, deadCgroup) + for range 2 { + setHostedActor(s, &testActor) // a fresh hostedActor, as hostActor makes + s.sweepUsage(context.Background()) + } + if n := strings.Count(logs.String(), deadSandboxMsg); n != 2 { + t.Fatalf("got %d dead-sandbox warnings over two activations, want 2; logs:\n%s", n, logs) + } +} + +// Concurrent sweeps of one dead sandbox still warn once. +func TestSweepUsageDeadSandboxConcurrentSweeps(t *testing.T) { + logs := captureWarnings(t) + s := newStatsService(t, deadCgroup) + setHostedActor(s, &testActor) + + var wg sync.WaitGroup + for range 16 { + wg.Go(func() { + s.sweepUsage(context.Background()) + }) + } + wg.Wait() + if n := strings.Count(logs.String(), deadSandboxMsg); n != 1 { + t.Fatalf("got %d dead-sandbox warnings from concurrent sweeps, want 1; logs:\n%s", n, logs) + } +} + +// drainedCgroup is the sandbox leaf after a drain has killed the app +// containers: the sentry and gofers stay in the pause leaf, so it is still +// populated. +var drainedCgroup = map[string]string{ + "memory.current": "31457280\n", + "memory.events": "oom 0\noom_kill 0\n", + "cgroup.events": "populated 1\nfrozen 0\n", +} + +// TestDeadSandboxCheckDuringDrain pins what the check does while +// gracefulShutdown runs. The drain kills the app containers without taking +// actor locks or unhosting, but it never stops the sandbox, so the leaf stays +// populated and nothing is logged. A sandbox that dies during the drain is +// still reported. +func TestDeadSandboxCheckDuringDrain(t *testing.T) { + for _, tc := range []struct { + name string + files map[string]string + wantWarns int + }{ + {name: "app containers killed, sentry alive", files: drainedCgroup}, + {name: "sandbox died during the drain", files: deadCgroup, wantWarns: 1}, + } { + t.Run(tc.name, func(t *testing.T) { + logs := captureWarnings(t) + s := newStatsService(t, tc.files) + setHostedActor(s, &testActor) + s.shuttingDown.Store(true) + + s.sweepUsage(context.Background()) + if n := strings.Count(logs.String(), deadSandboxMsg); n != tc.wantWarns { + t.Fatalf("got %d dead-sandbox warnings, want %d; logs:\n%s", n, tc.wantWarns, logs) + } + }) + } +} + +// TestDrainNeverKillsPauseContainer: gracefulShutdown kills the names +// containerNames takes from the spec. atelet rejects a spec that names the +// pause container, so the drain cannot stop the sandbox itself. +func TestDrainNeverKillsPauseContainer(t *testing.T) { + if err := resources.ValidateContainerNames([]string{ocispec.PauseContainer}); err == nil { + t.Fatalf("ValidateContainerNames accepted %q; a spec could then put it on the drain's kill list", ocispec.PauseContainer) + } + spec := []*ateompb.Container{{Name: "app"}, {Name: "sidecar"}} + for _, name := range containerNames(spec) { + if name == ocispec.PauseContainer { + t.Fatalf("containerNames(%v) includes %q", spec, ocispec.PauseContainer) + } + } +} diff --git a/docs/metrics/registry/metrics.yaml b/docs/metrics/registry/metrics.yaml index 1c0b6a23d8..b74d03a2c5 100644 --- a/docs/metrics/registry/metrics.yaml +++ b/docs/metrics/registry/metrics.yaml @@ -246,9 +246,9 @@ groups: - id: registry.ate.sandbox type: attribute_group brief: > - The class of the sandbox. The class comes from the template. Thus it adds - no series to the template labels. It lets a dashboard group the data by - class without a list of the template names. + The sandbox that hosts an actor. The class comes from the template. Thus + it adds no series to the template labels. It lets a dashboard group the + data by class without a list of the template names. attributes: - id: ate.sandbox.class stability: development @@ -272,6 +272,18 @@ groups: ateapi reports it for a Worker that CreateWorker accepted with no class, and for an ActorTemplate with no sandbox config. atelet reports it for a snapshot manifest with no class. + - id: ate.sandbox.oom_kills + stability: development + type: int + brief: > + The processes in the sandbox cgroup leaf that the kernel OOM killer + has killed, over the life of the leaf (memory.events oom_kill). + note: > + Logs only. It is only ever recorded beside actor identity, which no + metric may carry. ateom-gvisor sets it on the record it writes when a + hosted actor's sandbox has no process left. A leaf left behind by an + earlier activation of the same actor carries its count forward. + examples: [0, 3] - id: registry.ate.snapshot type: attribute_group diff --git a/docs/observability.md b/docs/observability.md index 9ff57400c4..12e63a3665 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -177,6 +177,17 @@ Creating an actor counts as a change. A new actor is born suspended, so it gets The counter carries no actor identity, so this record is the only way to attribute a crash to one agent. The decision-point line that precedes it (`Setting Actor to crashed due to error`) carries only `ate.atespace` and `ate.actor.name`: it is written before the Actor is loaded, so no uid exists yet. +ateom-gvisor's `Sandbox has no processes left while the actor is hosted` is a WARN written when a running actor's gVisor sandbox has died: the sentry crashed, was killed, or was OOM-killed. The ateom's usage sweep finds it, so it shows up within one sweep (`--usage-sample-interval`, a minute by default), once per activation. `ate.sandbox.oom_kills` is the cgroup's `oom_kill` count, so a non-zero value points at the kernel OOM killer: + +```json +{"time":"…","level":"WARN","msg":"Sandbox has no processes left while the actor is hosted", + "ate.atespace":"ate-demo-counter","ate.actor.name":"counter-1","ate.actor.uid":"8f2a…", + "ate.template.atespace":"ate-demo-counter","ate.template.name":"counter", + "ate.sandbox.oom_kills":3} +``` + +The actor still reports `RUNNING` after this record; nothing moves it to `CRASHED` yet ([#2211](https://github.com/agent-substrate/substrate/issues/2211)). It does not fire when an application container exits while the sentry is still up, or during a checkpoint or terminate. The count covers the life of the cgroup leaf, so a leaf left behind by an earlier activation of the same actor carries its count forward. + #### The same records over OTLP With `OTEL_LOGS_EXPORTER=otlp` these records go out as OTLP log events instead of stdout, so a collector reads them without knowing substrate's stdout envelope. Unset or `none`, which is what every environment but kind uses today, keeps them on stdout. `otlp,console` writes both, as kind does for debugging. `console` here is the component's own JSON log line, not the OTel SDK's console exporter format. The value is a comma-separated list: names are trimmed, lowercased, and counted once, and an unknown or empty item logs a warning and is skipped while the rest apply. `none` with an exporter, or a value that names no known exporter, logs a warning and keeps the default, `none`. ateapi emits the lifecycle events and the ateoms the usage samples, on the same switch; the ateoms export through [the ateom relay](#the-ateom-otlp-relay), which carries logs, traces, and metrics. atecontroller copies `OTEL_LOGS_EXPORTER` into each worker pod, so the kind ConfigMap sets the ateoms to `otlp,console` too. diff --git a/internal/actorlock/actorlock.go b/internal/actorlock/actorlock.go index 96312c6862..b822f6559e 100644 --- a/internal/actorlock/actorlock.go +++ b/internal/actorlock/actorlock.go @@ -83,6 +83,17 @@ func (l *Locks) Lock(ctx context.Context, actorUID string) bool { return false } +// Busy reports whether a lifecycle RPC holds or is waiting for the named +// actor's lock. It never blocks, so a reader that must not take the lock can +// still tell whether the actor is mid-operation. The answer may be stale by the +// time it returns: use it for advisory checks, not to decide lifecycle actions. +func (l *Locks) Busy(actorUID string) bool { + l.mu.Lock() + defer l.mu.Unlock() + _, ok := l.held[actorUID] + return ok +} + // Unlock releases the named actor's lock. func (l *Locks) Unlock(actorUID string) { l.release(actorUID, true) diff --git a/internal/actorlock/actorlock_test.go b/internal/actorlock/actorlock_test.go index 527a3c083a..726d13b8f4 100644 --- a/internal/actorlock/actorlock_test.go +++ b/internal/actorlock/actorlock_test.go @@ -60,6 +60,92 @@ func TestActorLocksSerializeTheSameActor(t *testing.T) { locks.Unlock("actor-a") } +// Busy is true from Lock to Unlock and only for the actor that holds it. +func TestActorLocksBusy(t *testing.T) { + locks := New() + if locks.Busy("actor-a") { + t.Fatal("Busy(actor-a) = true before any Lock") + } + if !locks.Lock(context.Background(), "actor-a") { + t.Fatal("could not take actor-a's lock") + } + if !locks.Busy("actor-a") { + t.Error("Busy(actor-a) = false while held") + } + if locks.Busy("actor-b") { + t.Error("Busy(actor-b) = true while only actor-a is held") + } + locks.Unlock("actor-a") + if locks.Busy("actor-a") { + t.Error("Busy(actor-a) = true after Unlock") + } +} + +// Busy counts waiters as well as the holder. A waiter that gives up, or that +// takes the lock over when the holder releases it, never leaves the actor +// looking idle while a lifecycle RPC still holds the lock. +func TestActorLocksBusyWaiter(t *testing.T) { + // startWaiter blocks a second Lock behind the holder and returns once it + // has registered, which is when the entry holds two references. + startWaiter := func(t *testing.T, locks *Locks, ctx context.Context) <-chan bool { + t.Helper() + got := make(chan bool, 1) + go func() { got <- locks.Lock(ctx, "actor-a") }() + deadline := time.Now().Add(5 * time.Second) + for { + locks.mu.Lock() + refs := locks.held["actor-a"].refs + locks.mu.Unlock() + if refs == 2 { + return got + } + if time.Now().After(deadline) { + t.Fatal("the second Lock never started waiting") + } + time.Sleep(time.Millisecond) + } + } + + t.Run("waiter gives up", func(t *testing.T) { + locks := New() + if !locks.Lock(context.Background(), "actor-a") { + t.Fatal("could not take actor-a's lock") + } + ctx, cancel := context.WithCancel(context.Background()) + got := startWaiter(t, locks, ctx) + cancel() + if <-got { + t.Fatal("the cancelled waiter took the lock") + } + if !locks.Busy("actor-a") { + t.Error("Busy(actor-a) = false after the waiter gave up, while the holder still holds it") + } + locks.Unlock("actor-a") + if locks.Busy("actor-a") { + t.Error("Busy(actor-a) = true after the holder released it") + } + }) + + t.Run("waiter takes over", func(t *testing.T) { + locks := New() + if !locks.Lock(context.Background(), "actor-a") { + t.Fatal("could not take actor-a's lock") + } + got := startWaiter(t, locks, context.Background()) + locks.Unlock("actor-a") + if !<-got { + t.Fatal("the waiter did not take the lock") + } + if !locks.Busy("actor-a") { + t.Error("Busy(actor-a) = false while the former waiter holds it") + } + locks.Unlock("actor-a") + if locks.Busy("actor-a") { + t.Error("Busy(actor-a) = true after the last holder released it") + } + }) +} + // The map must not grow by one entry per actor the worker has ever hosted. func TestActorLocksForgetIdleActors(t *testing.T) { locks := New() diff --git a/internal/ateattr/ateattr.go b/internal/ateattr/ateattr.go index 773606d367..3d3531b864 100644 --- a/internal/ateattr/ateattr.go +++ b/internal/ateattr/ateattr.go @@ -191,6 +191,11 @@ const ( StatsCPUTimeKey = attribute.Key("ate.stats.cpu.time") ) +// SandboxOOMKillsKey is memory.events' oom_kill for the sandbox cgroup leaf, on +// the record ateom-gvisor writes when a hosted actor's sandbox has no process +// left. Logs only: it is only ever recorded beside actor identity. +const SandboxOOMKillsKey = attribute.Key("ate.sandbox.oom_kills") + // Values for StatsKindKey. An initial or final sample brackets an activation; a // periodic one is the timer's. const ( From fa3fc5bac321e8204abbf64166778edbaffc02aa Mon Sep 17 00:00:00 2001 From: Michelle Au Date: Fri, 9 Oct 2026 15:38:16 +0000 Subject: [PATCH 26/44] Update Snapshots API to support Actor Lifecycle v2 (#2289) Partially implements the proposed API changes in: https://docs.google.com/document/d/1lQ6fhrRb2sLCeFrvhsfytaoKVbA8X4w0TEwj84pr8uU/edit?tab=t.0 Depends on (and temporily includes) https://github.com/agent-substrate/substrate/pull/2222, review that one first. Part of #2069 This PR tries not to make major changes to existing semantics. They will be done as a followup. Other things in the proposal that are not implemented in this PR (tracked separately) - hardware information (https://github.com/agent-substrate/substrate/issues/2246) - sandbox information - combining pause/resume and introducing detach (https://github.com/agent-substrate/substrate/issues/798) #### Breaking change The Snapshots API has been completely rewritten and replaces - ActorStatus in_progress_snapshot_uri, external_snapshot, local_snapshot, in_progress_local_snapshot_name fields. - TagStatus snapshot, storage_location fields ---- - [ ] Tests pass - [ ] Appropriate changes to documentation are included in the PR --- .../internal/apivalidation/actor_test.go | 189 +- .../internal/apivalidation/common_test.go | 4 - cmd/ateapi/internal/apivalidation/tag_test.go | 3 +- .../internal/apivalidation/validation_test.go | 272 ++- .../apivalidation/zz_generated.validation.go | 728 +++--- cmd/ateapi/internal/controlapi/actor.go | 22 +- .../controlapi/actor_template_test.go | 7 +- cmd/ateapi/internal/controlapi/actor_test.go | 21 +- cmd/ateapi/internal/controlapi/crash.go | 9 +- cmd/ateapi/internal/controlapi/crash_test.go | 16 +- .../functionaltest/actor_template_test.go | 4 +- .../controlapi/functionaltest/actor_test.go | 165 +- .../controlapi/functionaltest/common_test.go | 83 +- .../controlapi/functionaltest/tag_test.go | 45 +- .../internal/controlapi/snapshot_helpers.go | 186 ++ .../controlapi/snapshot_helpers_test.go | 230 ++ cmd/ateapi/internal/controlapi/tag.go | 2 +- cmd/ateapi/internal/controlapi/tag_test.go | 18 +- .../controlapi/template_reconciler.go | 7 +- .../controlapi/template_reconciler_test.go | 29 +- .../internal/controlapi/workflow_delete.go | 8 +- .../controlapi/workflow_delete_test.go | 36 +- .../internal/controlapi/workflow_pause.go | 40 +- .../controlapi/workflow_pause_test.go | 70 +- .../workflow_reconcile_assignments_test.go | 9 +- .../internal/controlapi/workflow_resume.go | 21 +- .../controlapi/workflow_resume_test.go | 96 +- .../internal/controlapi/workflow_revert.go | 13 +- .../controlapi/workflow_revert_test.go | 76 +- .../internal/controlapi/workflow_suspend.go | 118 +- .../controlapi/workflow_suspend_test.go | 159 +- .../internal/controlapi/workflow_tag.go | 77 +- .../internal/controlapi/workflow_tag_test.go | 31 +- .../internal/controlapi/workflow_test.go | 9 +- .../controlapi/workflow_worker_delete.go | 4 - .../controlapi/workflow_worker_delete_test.go | 25 +- cmd/ateapi/internal/store/atepg/tag.go | 21 +- cmd/ateapi/internal/store/atepg/tag_test.go | 15 +- .../internal/store/storecontract/contract.go | 109 +- cmd/kubectl-ate/internal/cmd/tag_test.go | 9 +- cmd/kubectl-ate/internal/printer/printer.go | 21 +- .../internal/printer/printer_test.go | 27 +- docs/api-guide.md | 14 +- docs/architecture.md | 2 +- internal/e2e/suites/demo/demo_test.go | 84 +- .../e2e/suites/demo/updatetemplate_test.go | 16 +- .../e2e/suites/security/crossatespace_test.go | 26 +- pkg/proto/ateapipb/ateapi.pb.go | 2061 ++++++++++------- pkg/proto/ateapipb/ateapi.proto | 191 +- 49 files changed, 3562 insertions(+), 1866 deletions(-) create mode 100644 cmd/ateapi/internal/controlapi/snapshot_helpers.go create mode 100644 cmd/ateapi/internal/controlapi/snapshot_helpers_test.go diff --git a/cmd/ateapi/internal/apivalidation/actor_test.go b/cmd/ateapi/internal/apivalidation/actor_test.go index 4019e32999..d1cd9fbf1f 100644 --- a/cmd/ateapi/internal/apivalidation/actor_test.go +++ b/cmd/ateapi/internal/apivalidation/actor_test.go @@ -435,88 +435,210 @@ func TestValidateActorUpdate(t *testing.T) { })), field.ErrorList{field.Invalid(field.NewPath("status", "assigned_node"), nil, "").WithOrigin("format=k8s-long-name")}, }, { - "valid actor.status.in_progress_snapshot_uri", + "valid actor.status.last_assigned_generation", validInput(), validOutput(withStatus(func(s *ateapipb.ActorStatus) { - s.InProgressSnapshotUri = "gs://private/atespaces/as/actors/" + someActorUID + "/snapshots/snap-1" + s.LastAssignedGeneration = 1 })), nil, }, { - "invalid actor.status.in_progress_snapshot_uri: too long", + "increasing actor.status.last_assigned_generation", + validInput(withStatus(func(s *ateapipb.ActorStatus) { + s.LastAssignedGeneration = 2 + })), + validOutput(withStatus(func(s *ateapipb.ActorStatus) { + s.LastAssignedGeneration = 3 + })), + nil, + }, { + "unchanged actor.status.last_assigned_generation", + validInput(withStatus(func(s *ateapipb.ActorStatus) { + s.LastAssignedGeneration = 3 + })), + validOutput(withStatus(func(s *ateapipb.ActorStatus) { + s.LastAssignedGeneration = 3 + })), + nil, + }, { + "decreasing actor.status.last_assigned_generation", + validInput(withStatus(func(s *ateapipb.ActorStatus) { + s.LastAssignedGeneration = 3 + })), + validOutput(withStatus(func(s *ateapipb.ActorStatus) { + s.LastAssignedGeneration = 1 + })), + field.ErrorList{field.Invalid(field.NewPath("status", "last_assigned_generation"), nil, "").WithOrigin("monotonic")}, + }, { + "unsetting actor.status.last_assigned_generation", + validInput(withStatus(func(s *ateapipb.ActorStatus) { + s.LastAssignedGeneration = 3 + })), + validOutput(withStatus(func(s *ateapipb.ActorStatus) { + s.LastAssignedGeneration = 0 + })), + field.ErrorList{field.Invalid(field.NewPath("status", "last_assigned_generation"), nil, "").WithOrigin("update")}, + }, { + "negative actor.status.last_assigned_generation", validInput(), validOutput(withStatus(func(s *ateapipb.ActorStatus) { - s.InProgressSnapshotUri = "gs://" + strings.Repeat("x", 2044) + s.LastAssignedGeneration = -1 })), - field.ErrorList{field.TooLong(field.NewPath("status", "in_progress_snapshot_uri"), nil, 2048).WithOrigin("maxLength")}, + field.ErrorList{field.Invalid(field.NewPath("status", "last_assigned_generation"), nil, "").WithOrigin("minimum")}, }, { - "valid actor.status.external_snapshot", + "valid actor.status.snapshots with durable object snapshot", validInput(), validOutput(withStatus(func(s *ateapipb.ActorStatus) { - s.ExternalSnapshot = &ateapipb.ExternalSnapshot{ - SnapshotUri: "gs://private/atespaces/as/actors/" + someActorUID + "/snapshots/snap-1", - Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, - } + s.Snapshots = []*ateapipb.Snapshot{validSnapshot()} })), nil, }, { - "valid actor.status.external_snapshot.actor_template_uid", + "valid actor.status.snapshots actor_template_uid", validInput(), validOutput(withStatus(func(s *ateapipb.ActorStatus) { - s.ExternalSnapshot = &ateapipb.ExternalSnapshot{SnapshotUri: "gs://private/atespaces/as/actors/" + someActorUID + "/snapshots/snap-1", ActorTemplateUid: "01234567-89ab-cdef-0123-456789abcdef"} + s.Snapshots = []*ateapipb.Snapshot{validSnapshot(func(snap *ateapipb.Snapshot) { + snap.ActorTemplateUid = "01234567-89ab-cdef-0123-456789abcdef" + })} })), nil, }, { // Each suspend restamps the UID of the template the snapshot was captured under. - "changing actor.status.external_snapshot.actor_template_uid is allowed", + "changing actor.status.snapshots.actor_template_uid is allowed", validInput(withStatus(func(s *ateapipb.ActorStatus) { - s.ExternalSnapshot = &ateapipb.ExternalSnapshot{SnapshotUri: "gs://private/atespaces/as/actors/" + someActorUID + "/snapshots/snap-1", ActorTemplateUid: "01234567-89ab-cdef-0123-456789abcdef"} + s.Snapshots = []*ateapipb.Snapshot{validSnapshot(func(snap *ateapipb.Snapshot) { + snap.ActorTemplateUid = "01234567-89ab-cdef-0123-456789abcdef" + })} })), validOutput(withStatus(func(s *ateapipb.ActorStatus) { - s.ExternalSnapshot = &ateapipb.ExternalSnapshot{SnapshotUri: "gs://private/atespaces/as/actors/" + someActorUID + "/snapshots/snap-1", ActorTemplateUid: "fedcba98-7654-3210-fedc-ba9876543210"} + s.Snapshots = []*ateapipb.Snapshot{validSnapshot(func(snap *ateapipb.Snapshot) { + snap.ActorTemplateUid = "fedcba98-7654-3210-fedc-ba9876543210" + })} })), nil, }, { - "invalid actor.status.external_snapshot.actor_template_uid", + "invalid actor.status.snapshots.actor_template_uid", validInput(), validOutput(withStatus(func(s *ateapipb.ActorStatus) { - s.ExternalSnapshot = &ateapipb.ExternalSnapshot{SnapshotUri: "gs://private/atespaces/as/actors/" + someActorUID + "/snapshots/snap-1", ActorTemplateUid: "not-a-uuid"} + s.Snapshots = []*ateapipb.Snapshot{validSnapshot(func(snap *ateapipb.Snapshot) { + snap.ActorTemplateUid = "not-a-uuid" + })} })), - field.ErrorList{field.Invalid(field.NewPath("status", "external_snapshot", "actor_template_uid"), nil, "").WithOrigin("format=k8s-uuid")}, + field.ErrorList{field.Invalid(field.NewPath("status", "snapshots").Index(0).Child("actor_template_uid"), nil, "").WithOrigin("format=k8s-uuid")}, }, { - "valid actor.status.local_snapshot.snapshot_name", + "valid actor.status.snapshots local snapshot_name", validInput(), validOutput(withStatus(func(s *ateapipb.ActorStatus) { - s.LocalSnapshot = &ateapipb.LocalSnapshot{SnapshotName: "snap-1"} + s.Snapshots = []*ateapipb.Snapshot{validSnapshot(func(snap *ateapipb.Snapshot) { + snap.Storage = []*ateapipb.SnapshotStorage{{ + Durability: ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL, + Status: ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED, + Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + Local: &ateapipb.LocalSnapshot{SnapshotName: "snap-1"}, + }} + })} })), nil, }, { - "invalid actor.status.local_snapshot.snapshot_name", + "invalid actor.status.snapshots local snapshot_name", validInput(), validOutput(withStatus(func(s *ateapipb.ActorStatus) { - s.LocalSnapshot = &ateapipb.LocalSnapshot{SnapshotName: "SNAP 1"} + s.Snapshots = []*ateapipb.Snapshot{validSnapshot(func(snap *ateapipb.Snapshot) { + snap.Storage = []*ateapipb.SnapshotStorage{{ + Durability: ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL, + Status: ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED, + Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + Local: &ateapipb.LocalSnapshot{SnapshotName: "SNAP 1"}, + }} + })} })), - field.ErrorList{field.Invalid(field.NewPath("status", "local_snapshot", "snapshot_name"), nil, "").WithOrigin("format=k8s-short-name")}, + field.ErrorList{field.Invalid(field.NewPath("status", "snapshots").Index(0).Child("storage").Index(0).Child("local", "snapshot_name"), nil, "").WithOrigin("format=k8s-short-name")}, }, { - "valid actor.status.local_snapshot.fidelity", + "duplicate actor.status.snapshots generation", validInput(), validOutput(withStatus(func(s *ateapipb.ActorStatus) { - s.LocalSnapshot = &ateapipb.LocalSnapshot{Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES} + s.Snapshots = []*ateapipb.Snapshot{ + validSnapshot(func(snap *ateapipb.Snapshot) { snap.Generation = 1 }), + validSnapshot(func(snap *ateapipb.Snapshot) { snap.Generation = 1 }), + } + })), + field.ErrorList{field.Duplicate(field.NewPath("status", "snapshots").Index(1), nil)}, + }, { + "duplicate actor.status.snapshots.storage durability", + validInput(), + validOutput(withStatus(func(s *ateapipb.ActorStatus) { + s.Snapshots = []*ateapipb.Snapshot{validSnapshot(func(snap *ateapipb.Snapshot) { + snap.Storage = []*ateapipb.SnapshotStorage{ + validSnapshotStorage(), + validSnapshotStorage(), + } + })} + })), + field.ErrorList{field.Duplicate(field.NewPath("status", "snapshots").Index(0).Child("storage").Index(1), nil)}, + }, { + "too many actor.status.snapshots.storage entries", + validInput(), + validOutput(withStatus(func(s *ateapipb.ActorStatus) { + s.Snapshots = []*ateapipb.Snapshot{validSnapshot(func(snap *ateapipb.Snapshot) { + snap.Storage = []*ateapipb.SnapshotStorage{ + validSnapshotStorage(func(ss *ateapipb.SnapshotStorage) { + ss.Durability = ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL + ss.Object = nil + ss.Local = &ateapipb.LocalSnapshot{SnapshotName: "snap-1"} + }), + validSnapshotStorage(func(ss *ateapipb.SnapshotStorage) { + ss.Durability = ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE + }), + validSnapshotStorage(func(ss *ateapipb.SnapshotStorage) { + ss.Durability = ateapipb.SnapshotDurability(3) + }), + } + })} + })), + field.ErrorList{field.TooMany(field.NewPath("status", "snapshots").Index(0).Child("storage"), 3, 2).WithOrigin("maxItems")}, + }, { + "invalid actor.status.snapshots.storage.durability", + validInput(), + validOutput(withStatus(func(s *ateapipb.ActorStatus) { + s.Snapshots = []*ateapipb.Snapshot{validSnapshot(func(snap *ateapipb.Snapshot) { + snap.Storage[0].Durability = ateapipb.SnapshotDurability(3) + })} + })), + field.ErrorList{field.Invalid(field.NewPath("status", "snapshots").Index(0).Child("storage").Index(0).Child("durability"), nil, "").WithOrigin("maximum")}, + }, { + "valid actor.status.snapshots.storage.fidelity", + validInput(), + validOutput(withStatus(func(s *ateapipb.ActorStatus) { + s.Snapshots = []*ateapipb.Snapshot{validSnapshot(func(snap *ateapipb.Snapshot) { + snap.Storage[0].Fidelity = ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES + })} })), nil, }, { - "negative actor.status.local_snapshot.fidelity", + "missing actor.status.snapshots.storage.fidelity", validInput(), validOutput(withStatus(func(s *ateapipb.ActorStatus) { - s.LocalSnapshot = &ateapipb.LocalSnapshot{Fidelity: ateapipb.SnapshotFidelity(-1)} + s.Snapshots = []*ateapipb.Snapshot{validSnapshot(func(snap *ateapipb.Snapshot) { + snap.Storage[0].Fidelity = ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED + })} })), - field.ErrorList{field.Invalid(field.NewPath("status", "local_snapshot", "fidelity"), nil, "").WithOrigin("minimum")}, + field.ErrorList{field.Required(field.NewPath("status", "snapshots").Index(0).Child("storage").Index(0).Child("fidelity"), "")}, }, { - "invalid actor.status.local_snapshot.fidelity", + "negative actor.status.snapshots.storage.fidelity", validInput(), validOutput(withStatus(func(s *ateapipb.ActorStatus) { - s.LocalSnapshot = &ateapipb.LocalSnapshot{Fidelity: ateapipb.SnapshotFidelity(4)} + s.Snapshots = []*ateapipb.Snapshot{validSnapshot(func(snap *ateapipb.Snapshot) { + snap.Storage[0].Fidelity = ateapipb.SnapshotFidelity(-1) + })} })), - field.ErrorList{field.Invalid(field.NewPath("status", "local_snapshot", "fidelity"), nil, "").WithOrigin("maximum")}, + field.ErrorList{field.Invalid(field.NewPath("status", "snapshots").Index(0).Child("storage").Index(0).Child("fidelity"), nil, "").WithOrigin("minimum")}, + }, { + "invalid actor.status.snapshots.storage.fidelity", + validInput(), + validOutput(withStatus(func(s *ateapipb.ActorStatus) { + s.Snapshots = []*ateapipb.Snapshot{validSnapshot(func(snap *ateapipb.Snapshot) { + snap.Storage[0].Fidelity = ateapipb.SnapshotFidelity(4) + })} + })), + field.ErrorList{field.Invalid(field.NewPath("status", "snapshots").Index(0).Child("storage").Index(0).Child("fidelity"), nil, "").WithOrigin("maximum")}, }, { "too many actor_volumes", validInput(), @@ -562,11 +684,6 @@ func TestValidateActorUpdate(t *testing.T) { }} })), nil, - }, { - "invalid actor.status.in_progress_local_snapshot_name", - validInput(), - validOutput(withStatus(func(s *ateapipb.ActorStatus) { s.InProgressLocalSnapshotName = "BAD NAME" })), - field.ErrorList{field.Invalid(field.NewPath("status", "in_progress_local_snapshot_name"), nil, "").WithOrigin("format=k8s-short-name")}, }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/cmd/ateapi/internal/apivalidation/common_test.go b/cmd/ateapi/internal/apivalidation/common_test.go index b7e5039d81..5df329e104 100644 --- a/cmd/ateapi/internal/apivalidation/common_test.go +++ b/cmd/ateapi/internal/apivalidation/common_test.go @@ -26,10 +26,6 @@ const ( ) const ( - // testStorageLocation is the snapshot_config.storage_location the tests - // build snapshot URIs under. - testStorageLocation = "gs://bucket/root" - // someActorUID stands in for the UID the store assigns an Actor, for tests // that need a well-formed snapshot URI but never exercise who owns it. Those // seed their Actor in a single call, before a real UID exists. diff --git a/cmd/ateapi/internal/apivalidation/tag_test.go b/cmd/ateapi/internal/apivalidation/tag_test.go index ec53badf30..23bbd43958 100644 --- a/cmd/ateapi/internal/apivalidation/tag_test.go +++ b/cmd/ateapi/internal/apivalidation/tag_test.go @@ -54,9 +54,8 @@ func TestValidateCreateTagRequest(t *testing.T) { req: &ateapipb.CreateTagRequest{ Tag: validTag(func(tag *ateapipb.Tag) { tag.Status = &ateapipb.TagStatus{ - Snapshot: validExternalSnapshot(), + Snapshot: validSnapshot(), ActorTemplateUid: someActorUID, - StorageLocation: testStorageLocation, } }), }, diff --git a/cmd/ateapi/internal/apivalidation/validation_test.go b/cmd/ateapi/internal/apivalidation/validation_test.go index 3b2e0e4461..afb373f233 100644 --- a/cmd/ateapi/internal/apivalidation/validation_test.go +++ b/cmd/ateapi/internal/apivalidation/validation_test.go @@ -1142,10 +1142,9 @@ func TestValidateDeleteOptions(t *testing.T) { } } -func validExternalSnapshot(mutate ...func(*ateapipb.ExternalSnapshot)) *ateapipb.ExternalSnapshot { - s := &ateapipb.ExternalSnapshot{ +func validObjectSnapshot(mutate ...func(*ateapipb.ObjectSnapshot)) *ateapipb.ObjectSnapshot { + s := &ateapipb.ObjectSnapshot{ SnapshotUri: "gs://private/atespaces/as/actors/" + someActorUID + "/snapshots/snap-1", - Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, } for _, m := range mutate { m(s) @@ -1153,24 +1152,56 @@ func validExternalSnapshot(mutate ...func(*ateapipb.ExternalSnapshot)) *ateapipb return s } -// badExternalSnapshot violates both of ExternalSnapshot's rules at once, so a -// caller can assert that a containing type reaches every field of it. -func badExternalSnapshot(mutate ...func(*ateapipb.ExternalSnapshot)) *ateapipb.ExternalSnapshot { - breakIt := func(s *ateapipb.ExternalSnapshot) { - s.SnapshotUri = "" - s.Fidelity = ateapipb.SnapshotFidelity(4) +func validSnapshotStorage(mutate ...func(*ateapipb.SnapshotStorage)) *ateapipb.SnapshotStorage { + ss := &ateapipb.SnapshotStorage{ + Durability: ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, + Status: ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED, + Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + Object: validObjectSnapshot(), } - return validExternalSnapshot(append([]func(*ateapipb.ExternalSnapshot){breakIt}, mutate...)...) + for _, m := range mutate { + m(ss) + } + return ss +} + +func validSnapshot(mutate ...func(*ateapipb.Snapshot)) *ateapipb.Snapshot { + s := &ateapipb.Snapshot{ + Generation: 1, + Owner: ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, + Storage: []*ateapipb.SnapshotStorage{validSnapshotStorage()}, + } + for _, m := range mutate { + m(s) + } + return s +} + +// badSnapshot violates Snapshot's rules at once, so a caller can assert that a +// containing type reaches every field of it. +func badSnapshot(mutate ...func(*ateapipb.Snapshot)) *ateapipb.Snapshot { + breakIt := func(s *ateapipb.Snapshot) { + s.Storage = []*ateapipb.SnapshotStorage{{ + Durability: ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, + Status: ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED, + Fidelity: ateapipb.SnapshotFidelity(4), + Object: &ateapipb.ObjectSnapshot{SnapshotUri: ""}, + }} + } + return validSnapshot(append([]func(*ateapipb.Snapshot){breakIt}, mutate...)...) } -func TestValidateExternalSnapshot(t *testing.T) { - valid := validExternalSnapshot - uriPath := field.NewPath("snapshot_uri") - scopePath := field.NewPath("fidelity") +func TestValidateSnapshot(t *testing.T) { + valid := validSnapshot + genPath := field.NewPath("generation") + ownerPath := field.NewPath("owner") + storagePath := field.NewPath("storage") + fidelityPath := storagePath.Index(0).Child("fidelity") + uriPath := storagePath.Index(0).Child("object", "snapshot_uri") tests := []struct { name string - obj *ateapipb.ExternalSnapshot + obj *ateapipb.Snapshot want field.ErrorList }{ { @@ -1178,67 +1209,146 @@ func TestValidateExternalSnapshot(t *testing.T) { obj: valid(), }, { - name: "valid fidelity: data", - obj: valid(func(s *ateapipb.ExternalSnapshot) { - s.Fidelity = ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES + name: "missing generation", + obj: valid(func(s *ateapipb.Snapshot) { s.Generation = 0 }), + want: field.ErrorList{field.Required(genPath, "")}, + }, + { + name: "negative generation", + obj: valid(func(s *ateapipb.Snapshot) { s.Generation = -1 }), + want: field.ErrorList{field.Invalid(genPath, nil, "").WithOrigin("minimum")}, + }, + { + name: "missing owner", + obj: valid(func(s *ateapipb.Snapshot) { s.Owner = ateapipb.SnapshotOwner_SNAPSHOT_OWNER_UNSPECIFIED }), + want: field.ErrorList{field.Required(ownerPath, "")}, + }, + { + name: "owner above the enum", + obj: valid(func(s *ateapipb.Snapshot) { s.Owner = ateapipb.SnapshotOwner(3) }), + want: field.ErrorList{field.Invalid(ownerPath, nil, "").WithOrigin("maximum")}, + }, + { + name: "valid fidelity: volumes", + obj: valid(func(s *ateapipb.Snapshot) { + s.Storage[0].Fidelity = ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES + }), + }, + { + name: "missing fidelity", + obj: valid(func(s *ateapipb.Snapshot) { + s.Storage[0].Fidelity = ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED }), + want: field.ErrorList{field.Required(fidelityPath, "")}, + }, + { + name: "missing storage", + obj: valid(func(s *ateapipb.Snapshot) { s.Storage = nil }), + want: field.ErrorList{field.Required(storagePath, "")}, + }, + { + name: "too many storage entries", + obj: valid(func(s *ateapipb.Snapshot) { + s.Storage = []*ateapipb.SnapshotStorage{ + validSnapshotStorage(func(ss *ateapipb.SnapshotStorage) { + ss.Durability = ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL + ss.Object = nil + ss.Local = &ateapipb.LocalSnapshot{SnapshotName: "snap-1"} + }), + validSnapshotStorage(func(ss *ateapipb.SnapshotStorage) { + ss.Durability = ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE + }), + validSnapshotStorage(func(ss *ateapipb.SnapshotStorage) { + ss.Durability = ateapipb.SnapshotDurability(3) + }), + } + }), + want: field.ErrorList{field.TooMany(storagePath, 3, 2).WithOrigin("maxItems")}, + }, + { + name: "missing storage durability", + obj: valid(func(s *ateapipb.Snapshot) { + s.Storage[0].Durability = ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_UNSPECIFIED + }), + want: field.ErrorList{field.Required(storagePath.Index(0).Child("durability"), "")}, + }, + { + name: "negative storage durability", + obj: valid(func(s *ateapipb.Snapshot) { + s.Storage[0].Durability = ateapipb.SnapshotDurability(-1) + }), + want: field.ErrorList{field.Invalid(storagePath.Index(0).Child("durability"), nil, "").WithOrigin("minimum")}, + }, + { + name: "storage durability above the enum", + obj: valid(func(s *ateapipb.Snapshot) { + s.Storage[0].Durability = ateapipb.SnapshotDurability(3) + }), + want: field.ErrorList{field.Invalid(storagePath.Index(0).Child("durability"), nil, "").WithOrigin("maximum")}, }, { - // UNSPECIFIED reads as FULL, so optional lets the zero value skip - // the bounds rather than failing the minimum. - name: "valid fidelity: unspecified", - obj: valid(func(s *ateapipb.ExternalSnapshot) { - s.Fidelity = ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED + name: "storage union missing both local and object", + obj: valid(func(s *ateapipb.Snapshot) { + s.Storage[0].Object = nil + s.Storage[0].Local = nil }), + want: field.ErrorList{field.Invalid(storagePath.Index(0), nil, "one of").WithOrigin("union")}, + }, + { + name: "storage union has both local and object", + obj: valid(func(s *ateapipb.Snapshot) { + s.Storage[0].Local = &ateapipb.LocalSnapshot{SnapshotName: "snap-1"} + }), + want: field.ErrorList{field.Invalid(storagePath.Index(0), nil, "one of").WithOrigin("union")}, }, { name: "missing snapshot_uri", - obj: valid(func(s *ateapipb.ExternalSnapshot) { s.SnapshotUri = "" }), + obj: valid(func(s *ateapipb.Snapshot) { s.Storage[0].Object.SnapshotUri = "" }), want: field.ErrorList{field.Required(uriPath, "")}, }, { name: "snapshot_uri too long", - obj: valid(func(s *ateapipb.ExternalSnapshot) { - s.SnapshotUri = "gs://" + strings.Repeat("x", 2044) + obj: valid(func(s *ateapipb.Snapshot) { + s.Storage[0].Object.SnapshotUri = "gs://" + strings.Repeat("x", 2044) }), want: field.ErrorList{field.TooLong(uriPath, nil, 2048).WithOrigin("maxLength")}, }, { name: "fidelity above the enum", - obj: valid(func(s *ateapipb.ExternalSnapshot) { s.Fidelity = ateapipb.SnapshotFidelity(4) }), - want: field.ErrorList{field.Invalid(scopePath, nil, "").WithOrigin("maximum")}, + obj: valid(func(s *ateapipb.Snapshot) { s.Storage[0].Fidelity = ateapipb.SnapshotFidelity(4) }), + want: field.ErrorList{field.Invalid(fidelityPath, nil, "").WithOrigin("maximum")}, }, { name: "negative fidelity", - obj: valid(func(s *ateapipb.ExternalSnapshot) { s.Fidelity = ateapipb.SnapshotFidelity(-1) }), - want: field.ErrorList{field.Invalid(scopePath, nil, "").WithOrigin("minimum")}, + obj: valid(func(s *ateapipb.Snapshot) { s.Storage[0].Fidelity = ateapipb.SnapshotFidelity(-1) }), + want: field.ErrorList{field.Invalid(fidelityPath, nil, "").WithOrigin("minimum")}, }, { name: "every field invalid", - obj: badExternalSnapshot(), + obj: badSnapshot(), want: field.ErrorList{ field.Required(uriPath, ""), - field.Invalid(scopePath, nil, "").WithOrigin("maximum"), + field.Invalid(fidelityPath, nil, "").WithOrigin("maximum"), }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { op := operation.Operation{Type: operation.Create} - assertValidateErr(t, Validate_ExternalSnapshot(context.Background(), op, nil, tt.obj, nil), tt.want) + assertValidateErr(t, Validate_Snapshot(context.Background(), op, nil, tt.obj, nil), tt.want) }) } } -func TestValidateExternalSnapshotUpdate(t *testing.T) { - valid := validExternalSnapshot - uriPath := field.NewPath("snapshot_uri") - scopePath := field.NewPath("fidelity") +func TestValidateSnapshotUpdate(t *testing.T) { + valid := validSnapshot + fidelityPath := field.NewPath("storage").Index(0).Child("fidelity") + uriPath := field.NewPath("storage").Index(0).Child("object", "snapshot_uri") tests := []struct { name string - oldObj *ateapipb.ExternalSnapshot - newObj *ateapipb.ExternalSnapshot + oldObj *ateapipb.Snapshot + newObj *ateapipb.Snapshot want field.ErrorList }{ { @@ -1251,84 +1361,92 @@ func TestValidateExternalSnapshotUpdate(t *testing.T) { // before these rules existed does not block updates to the rest of // the object. name: "unchanged invalid fields are not revalidated", - oldObj: badExternalSnapshot(), - newObj: badExternalSnapshot(), + oldObj: badSnapshot(), + newObj: badSnapshot(), }, { name: "fidelity changed to a valid value", oldObj: valid(), - newObj: valid(func(s *ateapipb.ExternalSnapshot) { - s.Fidelity = ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES + newObj: valid(func(s *ateapipb.Snapshot) { + s.Storage[0].Fidelity = ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES }), }, + { + name: "fidelity cleared", + oldObj: valid(), + newObj: valid(func(s *ateapipb.Snapshot) { + s.Storage[0].Fidelity = ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED + }), + want: field.ErrorList{field.Required(fidelityPath, "")}, + }, { name: "fidelity changed to a value outside the enum", oldObj: valid(), - newObj: valid(func(s *ateapipb.ExternalSnapshot) { s.Fidelity = ateapipb.SnapshotFidelity(4) }), - want: field.ErrorList{field.Invalid(scopePath, nil, "").WithOrigin("maximum")}, + newObj: valid(func(s *ateapipb.Snapshot) { s.Storage[0].Fidelity = ateapipb.SnapshotFidelity(4) }), + want: field.ErrorList{field.Invalid(fidelityPath, nil, "").WithOrigin("maximum")}, }, { name: "snapshot_uri cleared", oldObj: valid(), - newObj: valid(func(s *ateapipb.ExternalSnapshot) { s.SnapshotUri = "" }), + newObj: valid(func(s *ateapipb.Snapshot) { s.Storage[0].Object.SnapshotUri = "" }), want: field.ErrorList{field.Required(uriPath, "")}, }, { // The other side of the ratchet: a row that predates these rules // can still be repaired, one field at a time. name: "fidelity repaired", - oldObj: badExternalSnapshot(), - newObj: badExternalSnapshot(func(s *ateapipb.ExternalSnapshot) { - s.Fidelity = ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY + oldObj: badSnapshot(), + newObj: badSnapshot(func(s *ateapipb.Snapshot) { + s.Storage[0].Fidelity = ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY }), }, { name: "snapshot_uri changed to a value that is too long", oldObj: valid(), - newObj: valid(func(s *ateapipb.ExternalSnapshot) { - s.SnapshotUri = "gs://" + strings.Repeat("x", 2044) + newObj: valid(func(s *ateapipb.Snapshot) { + s.Storage[0].Object.SnapshotUri = "gs://" + strings.Repeat("x", 2044) }), want: field.ErrorList{field.TooLong(uriPath, nil, 2048).WithOrigin("maxLength")}, }, { name: "snapshot_uri repaired", - oldObj: badExternalSnapshot(), - newObj: badExternalSnapshot(func(s *ateapipb.ExternalSnapshot) { s.SnapshotUri = valid().SnapshotUri }), + oldObj: badSnapshot(), + newObj: badSnapshot(func(s *ateapipb.Snapshot) { s.Storage[0].Object.SnapshotUri = validObjectSnapshot().SnapshotUri }), }, { name: "every field repaired", - oldObj: badExternalSnapshot(), + oldObj: badSnapshot(), newObj: valid(), }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { op := operation.Operation{Type: operation.Update} - assertValidateErr(t, Validate_ExternalSnapshot(context.Background(), op, nil, tt.newObj, tt.oldObj), tt.want) + assertValidateErr(t, Validate_Snapshot(context.Background(), op, nil, tt.newObj, tt.oldObj), tt.want) }) } } -// TestValidateNestedExternalSnapshot checks that every type that -// holds ExternalSnapshot has to descend into it, and report under +// TestValidateNestedSnapshot checks that every type that +// holds Snapshot has to descend into it, and report under // the holder's own path. -func TestValidateNestedExternalSnapshot(t *testing.T) { +func TestValidateNestedSnapshot(t *testing.T) { tests := []struct { name string - // path is where the offending ExternalSnapshot sits in the holder. + // path is where the offending Snapshot sits in the holder. path *field.Path validate func(ctx context.Context) field.ErrorList }{ { - name: "actor.status.external_snapshot", - path: field.NewPath("status", "external_snapshot"), + name: "actor.status.snapshots", + path: field.NewPath("status", "snapshots").Index(0), validate: func(ctx context.Context) field.ErrorList { // The live path: the server validates the Actor it is about to // write, as an update against the stored one. op := operation.Operation{Type: operation.Update} oldVal := validActor(withActorStatus()) newVal := validActor(withActorStatus(func(s *ateapipb.ActorStatus) { - s.ExternalSnapshot = badExternalSnapshot() + s.Snapshots = []*ateapipb.Snapshot{badSnapshot()} })) return Validate_Actor(ctx, op, nil, newVal, oldVal) }, @@ -1339,7 +1457,7 @@ func TestValidateNestedExternalSnapshot(t *testing.T) { validate: func(ctx context.Context) field.ErrorList { op := operation.Operation{Type: operation.Create} obj := validTag(func(tag *ateapipb.Tag) { - tag.Status.Snapshot = badExternalSnapshot() + tag.Status.Snapshot = badSnapshot() }) return Validate_Tag(ctx, op, nil, obj, nil) }, @@ -1348,8 +1466,8 @@ func TestValidateNestedExternalSnapshot(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { want := field.ErrorList{ - field.Required(tt.path.Child("snapshot_uri"), ""), - field.Invalid(tt.path.Child("fidelity"), nil, "").WithOrigin("maximum"), + field.Required(tt.path.Child("storage").Index(0).Child("object", "snapshot_uri"), ""), + field.Invalid(tt.path.Child("storage").Index(0).Child("fidelity"), nil, "").WithOrigin("maximum"), } assertValidateErr(t, tt.validate(context.Background()), want) }) @@ -1360,9 +1478,8 @@ func validTag(mutate ...func(*ateapipb.Tag)) *ateapipb.Tag { tag := &ateapipb.Tag{ Metadata: validResourceMetadata(), Status: &ateapipb.TagStatus{ - Snapshot: validExternalSnapshot(), + Snapshot: validSnapshot(func(s *ateapipb.Snapshot) { s.Owner = ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG }), ActorTemplateUid: someActorUID, - StorageLocation: testStorageLocation, }, Scope: ateapipb.TagScope_TAG_SCOPE_ATESPACE, SourceActor: &ateapipb.ObjectRef{Atespace: "as", Name: "nm"}, @@ -1454,16 +1571,9 @@ func TestValidateTag(t *testing.T) { want: field.ErrorList{field.Invalid(statusPath.Child("actor_template_uid"), nil, "").WithOrigin("format=k8s-uuid")}, }, { - name: "missing status.storage_location", - obj: valid(func(tag *ateapipb.Tag) { tag.Status.StorageLocation = "" }), - want: field.ErrorList{field.Required(statusPath.Child("storage_location"), "")}, - }, - { - name: "status.storage_location too long", - obj: valid(func(tag *ateapipb.Tag) { - tag.Status.StorageLocation = "gs://" + strings.Repeat("x", 1020) - }), - want: field.ErrorList{field.TooLong(statusPath.Child("storage_location"), nil, 1024).WithOrigin("maxLength")}, + name: "missing status.snapshot", + obj: valid(func(tag *ateapipb.Tag) { tag.Status.Snapshot = nil }), + want: field.ErrorList{field.Required(statusPath.Child("snapshot"), "")}, }, } for _, tt := range tests { @@ -1551,13 +1661,13 @@ func TestValidateTagRequestPayloads(t *testing.T) { name: "create: invalid nested snapshot", validate: func(ctx context.Context, op operation.Operation) field.ErrorList { tag := validTag() - tag.Status.Snapshot = badExternalSnapshot() + tag.Status.Snapshot = badSnapshot() req := &ateapipb.CreateTagRequest{Tag: tag} return Validate_CreateTagRequest(ctx, op, nil, req, nil) }, want: field.ErrorList{ - field.Required(tagPath.Child("status", "snapshot", "snapshot_uri"), ""), - field.Invalid(tagPath.Child("status", "snapshot", "fidelity"), nil, "").WithOrigin("maximum"), + field.Required(tagPath.Child("status", "snapshot", "storage").Index(0).Child("object", "snapshot_uri"), ""), + field.Invalid(tagPath.Child("status", "snapshot", "storage").Index(0).Child("fidelity"), nil, "").WithOrigin("maximum"), }, }, { @@ -1581,7 +1691,7 @@ func TestValidateTagRequestPayloads(t *testing.T) { name: "update: nested snapshot is not descended into", validate: func(ctx context.Context, op operation.Operation) field.ErrorList { tag := validTag() - tag.Status.Snapshot = badExternalSnapshot() + tag.Status.Snapshot = badSnapshot() req := &ateapipb.UpdateTagRequest{Tag: tag} return Validate_UpdateTagRequest(ctx, op, nil, req, nil) }, diff --git a/cmd/ateapi/internal/apivalidation/zz_generated.validation.go b/cmd/ateapi/internal/apivalidation/zz_generated.validation.go index 422aacb52a..691e4416e7 100644 --- a/cmd/ateapi/internal/apivalidation/zz_generated.validation.go +++ b/cmd/ateapi/internal/apivalidation/zz_generated.validation.go @@ -706,41 +706,56 @@ func Validate_ActorStatus( errs = append(errs, fn(fldPath.Child("worker_assignment"), obj.WorkerAssignment, oldVal, oldObj != nil)...) } - { // field ateapipb.ActorStatus.InProgressSnapshotUri + { // field ateapipb.ActorStatus.ActorVolumes fn := func( fldPath *field.Path, - obj, oldObj *string, + obj, oldObj []*ateapipb.ExternalVolume, oldValueCorrelated bool) (errs field.ErrorList) { // don't revalidate unchanged data if oldValueCorrelated && op.Type == operation.Update { - if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + if ateDeepEqual(obj, oldObj) { return nil } } // call field-attached validations earlyReturn := false - if e := validate.OptionalValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + if e := validate.PtrSliceNoNils[ateapipb.ExternalVolume](ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.MaxItems(ctx, op, fldPath, obj, oldObj, 32).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.OptionalSlice(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { earlyReturn = true } if earlyReturn { return // do not proceed } - if e := validate.MaxLength(ctx, op, fldPath, obj, oldObj, 2048); len(e) != 0 { + // lists with map semantics require unique keys + if e := validate.PtrSliceUnique(ctx, op, fldPath, obj, oldObj, + func(a *ateapipb.ExternalVolume, b *ateapipb.ExternalVolume) bool { return a.VolumeName == b.VolumeName }); len(e) != 0 { + errs = append(errs, e...) + } + // iterate the list and call the type's validation function + if e := validate.EachPtrSliceVal(ctx, op, fldPath, obj, oldObj, + func(a *ateapipb.ExternalVolume, b *ateapipb.ExternalVolume) bool { return a.VolumeName == b.VolumeName }, ateDeepEqual, Validate_ExternalVolume); len(e) != 0 { errs = append(errs, e...) } return } oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.ActorStatus) *string { - return &oldObj.InProgressSnapshotUri + func(oldObj *ateapipb.ActorStatus) []*ateapipb.ExternalVolume { + return oldObj.ActorVolumes }) - errs = append(errs, fn(fldPath.Child("in_progress_snapshot_uri"), &obj.InProgressSnapshotUri, oldVal, oldObj != nil)...) + errs = append(errs, fn(fldPath.Child("actor_volumes"), obj.ActorVolumes, oldVal, oldObj != nil)...) } - { // field ateapipb.ActorStatus.ExternalSnapshot + { // field ateapipb.ActorStatus.Crash fn := func( fldPath *field.Path, - obj, oldObj *ateapipb.ExternalSnapshot, + obj, oldObj *ateapipb.ActorCrash, oldValueCorrelated bool) (errs field.ErrorList) { // don't revalidate unchanged data if oldValueCorrelated && op.Type == operation.Update { @@ -757,50 +772,51 @@ func Validate_ActorStatus( return // do not proceed } // call the type's validation function - errs = append(errs, Validate_ExternalSnapshot(ctx, op, fldPath, obj, oldObj)...) + errs = append(errs, Validate_ActorCrash(ctx, op, fldPath, obj, oldObj)...) return } oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.ActorStatus) *ateapipb.ExternalSnapshot { - return oldObj.ExternalSnapshot + func(oldObj *ateapipb.ActorStatus) *ateapipb.ActorCrash { + return oldObj.Crash }) - errs = append(errs, fn(fldPath.Child("external_snapshot"), obj.ExternalSnapshot, oldVal, oldObj != nil)...) + errs = append(errs, fn(fldPath.Child("crash"), obj.Crash, oldVal, oldObj != nil)...) } - { // field ateapipb.ActorStatus.LocalSnapshot + { // field ateapipb.ActorStatus.AssignedNode fn := func( fldPath *field.Path, - obj, oldObj *ateapipb.LocalSnapshot, + obj, oldObj *string, oldValueCorrelated bool) (errs field.ErrorList) { // don't revalidate unchanged data if oldValueCorrelated && op.Type == operation.Update { - if ateDeepEqual(obj, oldObj) { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { return nil } } // call field-attached validations earlyReturn := false - if e := validate.OptionalPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + if e := validate.OptionalValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { earlyReturn = true } if earlyReturn { return // do not proceed } - // call the type's validation function - errs = append(errs, Validate_LocalSnapshot(ctx, op, fldPath, obj, oldObj)...) + if e := validate.LongName(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + errs = append(errs, e...) + } return } oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.ActorStatus) *ateapipb.LocalSnapshot { - return oldObj.LocalSnapshot + func(oldObj *ateapipb.ActorStatus) *string { + return &oldObj.AssignedNode }) - errs = append(errs, fn(fldPath.Child("local_snapshot"), obj.LocalSnapshot, oldVal, oldObj != nil)...) + errs = append(errs, fn(fldPath.Child("assigned_node"), &obj.AssignedNode, oldVal, oldObj != nil)...) } - { // field ateapipb.ActorStatus.ActorVolumes + { // field ateapipb.ActorStatus.Snapshots fn := func( fldPath *field.Path, - obj, oldObj []*ateapipb.ExternalVolume, + obj, oldObj []*ateapipb.Snapshot, oldValueCorrelated bool) (errs field.ErrorList) { // don't revalidate unchanged data if oldValueCorrelated && op.Type == operation.Update { @@ -810,11 +826,11 @@ func Validate_ActorStatus( } // call field-attached validations earlyReturn := false - if e := validate.PtrSliceNoNils[ateapipb.ExternalVolume](ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + if e := validate.PtrSliceNoNils[ateapipb.Snapshot](ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { errs = append(errs, e...) earlyReturn = true } - if e := validate.MaxItems(ctx, op, fldPath, obj, oldObj, 32).MarkShortCircuit(); len(e) != 0 { + if e := validate.MaxItems(ctx, op, fldPath, obj, oldObj, 16).MarkShortCircuit(); len(e) != 0 { errs = append(errs, e...) earlyReturn = true } @@ -826,27 +842,27 @@ func Validate_ActorStatus( } // lists with map semantics require unique keys if e := validate.PtrSliceUnique(ctx, op, fldPath, obj, oldObj, - func(a *ateapipb.ExternalVolume, b *ateapipb.ExternalVolume) bool { return a.VolumeName == b.VolumeName }); len(e) != 0 { + func(a *ateapipb.Snapshot, b *ateapipb.Snapshot) bool { return a.Generation == b.Generation }); len(e) != 0 { errs = append(errs, e...) } // iterate the list and call the type's validation function if e := validate.EachPtrSliceVal(ctx, op, fldPath, obj, oldObj, - func(a *ateapipb.ExternalVolume, b *ateapipb.ExternalVolume) bool { return a.VolumeName == b.VolumeName }, ateDeepEqual, Validate_ExternalVolume); len(e) != 0 { + func(a *ateapipb.Snapshot, b *ateapipb.Snapshot) bool { return a.Generation == b.Generation }, ateDeepEqual, Validate_Snapshot); len(e) != 0 { errs = append(errs, e...) } return } oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.ActorStatus) []*ateapipb.ExternalVolume { - return oldObj.ActorVolumes + func(oldObj *ateapipb.ActorStatus) []*ateapipb.Snapshot { + return oldObj.Snapshots }) - errs = append(errs, fn(fldPath.Child("actor_volumes"), obj.ActorVolumes, oldVal, oldObj != nil)...) + errs = append(errs, fn(fldPath.Child("snapshots"), obj.Snapshots, oldVal, oldObj != nil)...) } - { // field ateapipb.ActorStatus.InProgressLocalSnapshotName + { // field ateapipb.ActorStatus.LastAssignedGeneration fn := func( fldPath *field.Path, - obj, oldObj *string, + obj, oldObj *int32, oldValueCorrelated bool) (errs field.ErrorList) { // don't revalidate unchanged data if oldValueCorrelated && op.Type == operation.Update { @@ -859,80 +875,27 @@ func Validate_ActorStatus( if e := validate.OptionalValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { earlyReturn = true } - if earlyReturn { - return // do not proceed - } - if e := validate.ShortName(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + if e := validate.UpdateValue(ctx, op, fldPath, obj, oldObj, + func(a int32, b int32) bool { return a == b }, validate.NoUnset).MarkShortCircuit(); len(e) != 0 { errs = append(errs, e...) - } - return - } - oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.ActorStatus) *string { - return &oldObj.InProgressLocalSnapshotName - }) - errs = append(errs, fn(fldPath.Child("in_progress_local_snapshot_name"), &obj.InProgressLocalSnapshotName, oldVal, oldObj != nil)...) - } - - { // field ateapipb.ActorStatus.Crash - fn := func( - fldPath *field.Path, - obj, oldObj *ateapipb.ActorCrash, - oldValueCorrelated bool) (errs field.ErrorList) { - // don't revalidate unchanged data - if oldValueCorrelated && op.Type == operation.Update { - if ateDeepEqual(obj, oldObj) { - return nil - } - } - // call field-attached validations - earlyReturn := false - if e := validate.OptionalPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { earlyReturn = true } if earlyReturn { return // do not proceed } - // call the type's validation function - errs = append(errs, Validate_ActorCrash(ctx, op, fldPath, obj, oldObj)...) - return - } - oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.ActorStatus) *ateapipb.ActorCrash { - return oldObj.Crash - }) - errs = append(errs, fn(fldPath.Child("crash"), obj.Crash, oldVal, oldObj != nil)...) - } - - { // field ateapipb.ActorStatus.AssignedNode - fn := func( - fldPath *field.Path, - obj, oldObj *string, - oldValueCorrelated bool) (errs field.ErrorList) { - // don't revalidate unchanged data - if oldValueCorrelated && op.Type == operation.Update { - if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { - return nil - } - } - // call field-attached validations - earlyReturn := false - if e := validate.OptionalValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { - earlyReturn = true - } - if earlyReturn { - return // do not proceed + if e := validate.Monotonic(ctx, op, fldPath, obj, oldObj).MarkAlpha(); len(e) != 0 { + errs = append(errs, e...) } - if e := validate.LongName(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + if e := validate.Minimum(ctx, op, fldPath, obj, oldObj, 0); len(e) != 0 { errs = append(errs, e...) } return } oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.ActorStatus) *string { - return &oldObj.AssignedNode + func(oldObj *ateapipb.ActorStatus) *int32 { + return &oldObj.LastAssignedGeneration }) - errs = append(errs, fn(fldPath.Child("assigned_node"), &obj.AssignedNode, oldVal, oldObj != nil)...) + errs = append(errs, fn(fldPath.Child("last_assigned_generation"), &obj.LastAssignedGeneration, oldVal, oldObj != nil)...) } return errs @@ -3587,112 +3550,6 @@ func Validate_EnvVar( return errs } -// Validate_ExternalSnapshot validates an instance of ExternalSnapshot according -// to declarative validation rules in the API schema. -func Validate_ExternalSnapshot( - ctx context.Context, op operation.Operation, fldPath *field.Path, - obj, oldObj *ateapipb.ExternalSnapshot) (errs field.ErrorList) { - - { // field ateapipb.ExternalSnapshot.SnapshotUri - fn := func( - fldPath *field.Path, - obj, oldObj *string, - oldValueCorrelated bool) (errs field.ErrorList) { - // don't revalidate unchanged data - if oldValueCorrelated && op.Type == operation.Update { - if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { - return nil - } - } - // call field-attached validations - earlyReturn := false - if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { - errs = append(errs, e...) - earlyReturn = true - } - if earlyReturn { - return // do not proceed - } - if e := validate.MaxLength(ctx, op, fldPath, obj, oldObj, 2048); len(e) != 0 { - errs = append(errs, e...) - } - return - } - oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.ExternalSnapshot) *string { - return &oldObj.SnapshotUri - }) - errs = append(errs, fn(fldPath.Child("snapshot_uri"), &obj.SnapshotUri, oldVal, oldObj != nil)...) - } - - { // field ateapipb.ExternalSnapshot.Fidelity - fn := func( - fldPath *field.Path, - obj, oldObj *ateapipb.SnapshotFidelity, - oldValueCorrelated bool) (errs field.ErrorList) { - // don't revalidate unchanged data - if oldValueCorrelated && op.Type == operation.Update { - if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { - return nil - } - } - // call field-attached validations - earlyReturn := false - if e := validate.OptionalValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { - earlyReturn = true - } - if earlyReturn { - return // do not proceed - } - if e := validate.Maximum(ctx, op, fldPath, obj, oldObj, 3); len(e) != 0 { - errs = append(errs, e...) - } - if e := validate.Minimum(ctx, op, fldPath, obj, oldObj, 1); len(e) != 0 { - errs = append(errs, e...) - } - return - } - oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.ExternalSnapshot) *ateapipb.SnapshotFidelity { - return &oldObj.Fidelity - }) - errs = append(errs, fn(fldPath.Child("fidelity"), &obj.Fidelity, oldVal, oldObj != nil)...) - } - - { // field ateapipb.ExternalSnapshot.ActorTemplateUid - fn := func( - fldPath *field.Path, - obj, oldObj *string, - oldValueCorrelated bool) (errs field.ErrorList) { - // don't revalidate unchanged data - if oldValueCorrelated && op.Type == operation.Update { - if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { - return nil - } - } - // call field-attached validations - earlyReturn := false - if e := validate.OptionalValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { - earlyReturn = true - } - if earlyReturn { - return // do not proceed - } - if e := validate.UUID(ctx, op, fldPath, obj, oldObj); len(e) != 0 { - errs = append(errs, e...) - } - return - } - oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.ExternalSnapshot) *string { - return &oldObj.ActorTemplateUid - }) - errs = append(errs, fn(fldPath.Child("actor_template_uid"), &obj.ActorTemplateUid, oldVal, oldObj != nil)...) - } - - return errs -} - // Validate_ExternalVolume validates an instance of ExternalVolume according // to declarative validation rules in the API schema. func Validate_ExternalVolume( @@ -5609,7 +5466,8 @@ func Validate_LocalSnapshot( } // call field-attached validations earlyReturn := false - if e := validate.OptionalValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) earlyReturn = true } if earlyReturn { @@ -5627,40 +5485,6 @@ func Validate_LocalSnapshot( errs = append(errs, fn(fldPath.Child("snapshot_name"), &obj.SnapshotName, oldVal, oldObj != nil)...) } - { // field ateapipb.LocalSnapshot.Fidelity - fn := func( - fldPath *field.Path, - obj, oldObj *ateapipb.SnapshotFidelity, - oldValueCorrelated bool) (errs field.ErrorList) { - // don't revalidate unchanged data - if oldValueCorrelated && op.Type == operation.Update { - if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { - return nil - } - } - // call field-attached validations - earlyReturn := false - if e := validate.OptionalValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { - earlyReturn = true - } - if earlyReturn { - return // do not proceed - } - if e := validate.Maximum(ctx, op, fldPath, obj, oldObj, 3); len(e) != 0 { - errs = append(errs, e...) - } - if e := validate.Minimum(ctx, op, fldPath, obj, oldObj, 1); len(e) != 0 { - errs = append(errs, e...) - } - return - } - oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.LocalSnapshot) *ateapipb.SnapshotFidelity { - return &oldObj.Fidelity - }) - errs = append(errs, fn(fldPath.Child("fidelity"), &obj.Fidelity, oldVal, oldObj != nil)...) - } - return errs } @@ -6077,6 +5901,47 @@ func Validate_ObjectRef( return errs } +// Validate_ObjectSnapshot validates an instance of ObjectSnapshot according +// to declarative validation rules in the API schema. +func Validate_ObjectSnapshot( + ctx context.Context, op operation.Operation, fldPath *field.Path, + obj, oldObj *ateapipb.ObjectSnapshot) (errs field.ErrorList) { + + { // field ateapipb.ObjectSnapshot.SnapshotUri + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.MaxLength(ctx, op, fldPath, obj, oldObj, 2048); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.ObjectSnapshot) *string { + return &oldObj.SnapshotUri + }) + errs = append(errs, fn(fldPath.Child("snapshot_uri"), &obj.SnapshotUri, oldVal, oldObj != nil)...) + } + + return errs +} + // Validate_PauseActorRequest validates an instance of PauseActorRequest according // to declarative validation rules in the API schema. func Validate_PauseActorRequest( @@ -7058,6 +6923,164 @@ func Validate_Selector( return errs } +// Validate_Snapshot validates an instance of Snapshot according +// to declarative validation rules in the API schema. +func Validate_Snapshot( + ctx context.Context, op operation.Operation, fldPath *field.Path, + obj, oldObj *ateapipb.Snapshot) (errs field.ErrorList) { + + { // field ateapipb.Snapshot.Generation + fn := func( + fldPath *field.Path, + obj, oldObj *int32, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.Minimum(ctx, op, fldPath, obj, oldObj, 1); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.Snapshot) *int32 { + return &oldObj.Generation + }) + errs = append(errs, fn(fldPath.Child("generation"), &obj.Generation, oldVal, oldObj != nil)...) + } + + { // field ateapipb.Snapshot.Owner + fn := func( + fldPath *field.Path, + obj, oldObj *ateapipb.SnapshotOwner, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.Maximum(ctx, op, fldPath, obj, oldObj, 2); len(e) != 0 { + errs = append(errs, e...) + } + if e := validate.Minimum(ctx, op, fldPath, obj, oldObj, 1); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.Snapshot) *ateapipb.SnapshotOwner { + return &oldObj.Owner + }) + errs = append(errs, fn(fldPath.Child("owner"), &obj.Owner, oldVal, oldObj != nil)...) + } + + { // field ateapipb.Snapshot.ActorTemplateUid + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.OptionalValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.UUID(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.Snapshot) *string { + return &oldObj.ActorTemplateUid + }) + errs = append(errs, fn(fldPath.Child("actor_template_uid"), &obj.ActorTemplateUid, oldVal, oldObj != nil)...) + } + + { // field ateapipb.Snapshot.Storage + fn := func( + fldPath *field.Path, + obj, oldObj []*ateapipb.SnapshotStorage, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.PtrSliceNoNils[ateapipb.SnapshotStorage](ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.MaxItems(ctx, op, fldPath, obj, oldObj, 2).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.RequiredSlice(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // lists with map semantics require unique keys + if e := validate.PtrSliceUnique(ctx, op, fldPath, obj, oldObj, + func(a *ateapipb.SnapshotStorage, b *ateapipb.SnapshotStorage) bool { + return a.Durability == b.Durability + }); len(e) != 0 { + errs = append(errs, e...) + } + // iterate the list and call the type's validation function + if e := validate.EachPtrSliceVal(ctx, op, fldPath, obj, oldObj, + func(a *ateapipb.SnapshotStorage, b *ateapipb.SnapshotStorage) bool { + return a.Durability == b.Durability + }, ateDeepEqual, Validate_SnapshotStorage); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.Snapshot) []*ateapipb.SnapshotStorage { + return oldObj.Storage + }) + errs = append(errs, fn(fldPath.Child("storage"), obj.Storage, oldVal, oldObj != nil)...) + } + + return errs +} + // Validate_SnapshotConfig validates an instance of SnapshotConfig according // to declarative validation rules in the API schema. func Validate_SnapshotConfig( @@ -7142,6 +7165,198 @@ func Validate_SnapshotConfig( return errs } +var unionMembershipFor_github_com_agent_substrate_substrate_pkg_proto_ateapipb_SnapshotStorage_ = validate.NewUnionMembership(validate.NewUnionMember("local"), validate.NewUnionMember("object")) + +// Validate_SnapshotStorage validates an instance of SnapshotStorage according +// to declarative validation rules in the API schema. +func Validate_SnapshotStorage( + ctx context.Context, op operation.Operation, fldPath *field.Path, + obj, oldObj *ateapipb.SnapshotStorage) (errs field.ErrorList) { + + if e := validate.Union(ctx, op, fldPath, obj, oldObj, unionMembershipFor_github_com_agent_substrate_substrate_pkg_proto_ateapipb_SnapshotStorage_, + func(obj *ateapipb.SnapshotStorage) bool { + if obj == nil { + return false + } + return obj.Local != nil + }, + func(obj *ateapipb.SnapshotStorage) bool { + if obj == nil { + return false + } + return obj.Object != nil + }); len(e) != 0 { + errs = append(errs, e...) + } + + { // field ateapipb.SnapshotStorage.Durability + fn := func( + fldPath *field.Path, + obj, oldObj *ateapipb.SnapshotDurability, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.Maximum(ctx, op, fldPath, obj, oldObj, 2); len(e) != 0 { + errs = append(errs, e...) + } + if e := validate.Minimum(ctx, op, fldPath, obj, oldObj, 1); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.SnapshotStorage) *ateapipb.SnapshotDurability { + return &oldObj.Durability + }) + errs = append(errs, fn(fldPath.Child("durability"), &obj.Durability, oldVal, oldObj != nil)...) + } + + { // field ateapipb.SnapshotStorage.Status + fn := func( + fldPath *field.Path, + obj, oldObj *ateapipb.SnapshotStorageStatus, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.Maximum(ctx, op, fldPath, obj, oldObj, 2); len(e) != 0 { + errs = append(errs, e...) + } + if e := validate.Minimum(ctx, op, fldPath, obj, oldObj, 1); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.SnapshotStorage) *ateapipb.SnapshotStorageStatus { + return &oldObj.Status + }) + errs = append(errs, fn(fldPath.Child("status"), &obj.Status, oldVal, oldObj != nil)...) + } + + { // field ateapipb.SnapshotStorage.Fidelity + fn := func( + fldPath *field.Path, + obj, oldObj *ateapipb.SnapshotFidelity, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.Maximum(ctx, op, fldPath, obj, oldObj, 3); len(e) != 0 { + errs = append(errs, e...) + } + if e := validate.Minimum(ctx, op, fldPath, obj, oldObj, 1); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.SnapshotStorage) *ateapipb.SnapshotFidelity { + return &oldObj.Fidelity + }) + errs = append(errs, fn(fldPath.Child("fidelity"), &obj.Fidelity, oldVal, oldObj != nil)...) + } + + { // field ateapipb.SnapshotStorage.Local + fn := func( + fldPath *field.Path, + obj, oldObj *ateapipb.LocalSnapshot, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.OptionalPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // call the type's validation function + errs = append(errs, Validate_LocalSnapshot(ctx, op, fldPath, obj, oldObj)...) + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.SnapshotStorage) *ateapipb.LocalSnapshot { + return oldObj.Local + }) + errs = append(errs, fn(fldPath.Child("local"), obj.Local, oldVal, oldObj != nil)...) + } + + { // field ateapipb.SnapshotStorage.Object + fn := func( + fldPath *field.Path, + obj, oldObj *ateapipb.ObjectSnapshot, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.OptionalPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // call the type's validation function + errs = append(errs, Validate_ObjectSnapshot(ctx, op, fldPath, obj, oldObj)...) + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.SnapshotStorage) *ateapipb.ObjectSnapshot { + return oldObj.Object + }) + errs = append(errs, fn(fldPath.Child("object"), obj.Object, oldVal, oldObj != nil)...) + } + + return errs +} + // Validate_SuspendActorRequest validates an instance of SuspendActorRequest according // to declarative validation rules in the API schema. func Validate_SuspendActorRequest( @@ -7601,7 +7816,7 @@ func Validate_TagStatus( { // field ateapipb.TagStatus.Snapshot fn := func( fldPath *field.Path, - obj, oldObj *ateapipb.ExternalSnapshot, + obj, oldObj *ateapipb.Snapshot, oldValueCorrelated bool) (errs field.ErrorList) { // don't revalidate unchanged data if oldValueCorrelated && op.Type == operation.Update { @@ -7611,18 +7826,19 @@ func Validate_TagStatus( } // call field-attached validations earlyReturn := false - if e := validate.OptionalPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + if e := validate.RequiredPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) earlyReturn = true } if earlyReturn { return // do not proceed } // call the type's validation function - errs = append(errs, Validate_ExternalSnapshot(ctx, op, fldPath, obj, oldObj)...) + errs = append(errs, Validate_Snapshot(ctx, op, fldPath, obj, oldObj)...) return } oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.TagStatus) *ateapipb.ExternalSnapshot { + func(oldObj *ateapipb.TagStatus) *ateapipb.Snapshot { return oldObj.Snapshot }) errs = append(errs, fn(fldPath.Child("snapshot"), obj.Snapshot, oldVal, oldObj != nil)...) @@ -7660,38 +7876,6 @@ func Validate_TagStatus( errs = append(errs, fn(fldPath.Child("actor_template_uid"), &obj.ActorTemplateUid, oldVal, oldObj != nil)...) } - { // field ateapipb.TagStatus.StorageLocation - fn := func( - fldPath *field.Path, - obj, oldObj *string, - oldValueCorrelated bool) (errs field.ErrorList) { - // don't revalidate unchanged data - if oldValueCorrelated && op.Type == operation.Update { - if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { - return nil - } - } - // call field-attached validations - earlyReturn := false - if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { - errs = append(errs, e...) - earlyReturn = true - } - if earlyReturn { - return // do not proceed - } - if e := validate.MaxLength(ctx, op, fldPath, obj, oldObj, 1024); len(e) != 0 { - errs = append(errs, e...) - } - return - } - oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.TagStatus) *string { - return &oldObj.StorageLocation - }) - errs = append(errs, fn(fldPath.Child("storage_location"), &obj.StorageLocation, oldVal, oldObj != nil)...) - } - return errs } diff --git a/cmd/ateapi/internal/controlapi/actor.go b/cmd/ateapi/internal/controlapi/actor.go index a3a75247ed..3c8c1aacdb 100644 --- a/cmd/ateapi/internal/controlapi/actor.go +++ b/cmd/ateapi/internal/controlapi/actor.go @@ -128,20 +128,19 @@ func (s *ServiceImpl) CreateActor(ctx context.Context, inActor *ateapipb.Actor) ActorVolumes: initVols, } if sourceTag != nil { - // The Actor starts out borrowing the tag's external snapshot rather than + // The Actor starts out borrowing the tag's snapshot rather than // copying it. The snapshot URI is under the tag's prefix, not the Actor's, which // is what keeps the Actor from collecting those objects. Its first // suspend writes a snapshot under its own prefix and takes over from // there. - outActor.Status.ExternalSnapshot = proto.CloneOf(sourceTag.GetStatus().GetSnapshot()) + snap := proto.CloneOf(sourceTag.GetStatus().GetSnapshot()) + snap.Generation = 1 + snap.Owner = ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG // The Actor is born with guest state, so stamp the template that state - // was built on now rather than at the first resume. The Tag records it - // beside its snapshot rather than on it, so the clone above does not - // carry it. Left empty, a repoint before that first resume reads as "no - // guest state" instead of "replaced template", and the resume restores - // the old template's memory and rootfs in full instead of the volume - // data alone. - outActor.Status.ExternalSnapshot.ActorTemplateUid = sourceTag.GetStatus().GetActorTemplateUid() + // was built on now rather than at the first resume. + snap.ActorTemplateUid = sourceTag.GetStatus().GetActorTemplateUid() + outActor.Status.Snapshots = []*ateapipb.Snapshot{snap} + outActor.Status.LastAssignedGeneration = 1 } if errs := apivalidation.ValidateActorUpdate(ctx, field.NewPath("actor"), outActor, inActor, true); len(errs) > 0 { return nil, toGRPCInternalError(errs) @@ -187,7 +186,7 @@ func (s *ServiceImpl) resolveTagSource(ctx context.Context, actorAtespace string return nil, apierror.FailedPrecondition("source Tag has an invalid scope") } // A tag might have an empty Snapshot URI if the tag creation failed or is ongoing. - if tag.GetStatus().GetSnapshot().GetSnapshotUri() == "" { + if tagDurableSnapshotURI(tag, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) == "" { return nil, apierror.FailedPrecondition("source Tag is still being created or failed creation") } // TODO: Permit compatible VOLUMES snapshots when runtimes can extract portable data. @@ -395,7 +394,8 @@ func validateTemplateVolumesUnchanged(oldTemplate, newTemplate *ateapipb.ActorTe // Deleting an actor collects everything under its external snapshot prefix. If // the location prefix ever changes, we risk leaking the snapshots under the old prefix. func validateSnapshotLocationUnchanged(actor *ateapipb.Actor, newTemplate *ateapipb.ActorTemplate) error { - currentSnapshotURI := actor.GetStatus().GetExternalSnapshot().GetSnapshotUri() + _, st := findLatestSnapshotStorage(actor.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + currentSnapshotURI := st.GetObject().GetSnapshotUri() if currentSnapshotURI == "" { return nil } diff --git a/cmd/ateapi/internal/controlapi/actor_template_test.go b/cmd/ateapi/internal/controlapi/actor_template_test.go index 8692040ed1..07937d0abe 100644 --- a/cmd/ateapi/internal/controlapi/actor_template_test.go +++ b/cmd/ateapi/internal/controlapi/actor_template_test.go @@ -230,10 +230,9 @@ func TestDeleteActorTemplate(t *testing.T) { actorURI := mustActorSnapshotURI(t, tmpl, actor, "snapshot") objects.PutSnapshot(t, actorURI, "manifest.json") actor = mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapipb.ActorStatus) { - s.ExternalSnapshot = &ateapipb.ExternalSnapshot{ - SnapshotUri: actorURI.String(), - Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, - ActorTemplateUid: tmpl.GetMetadata().GetUid(), + s.LastAssignedGeneration = 1 + s.Snapshots = []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, tmpl.GetMetadata().GetUid(), actorURI.String(), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), } }) var tag *ateapipb.Tag diff --git a/cmd/ateapi/internal/controlapi/actor_test.go b/cmd/ateapi/internal/controlapi/actor_test.go index 6a9eb5b627..de1e64f5f1 100644 --- a/cmd/ateapi/internal/controlapi/actor_test.go +++ b/cmd/ateapi/internal/controlapi/actor_test.go @@ -509,7 +509,10 @@ func TestUpdateActor_RepointTemplateStorageLocation(t *testing.T) { if tt.snapshot != nil { uri := tt.snapshot(t, actor) actor = mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapipb.ActorStatus) { - s.ExternalSnapshot = &ateapipb.ExternalSnapshot{SnapshotUri: uri} + s.LastAssignedGeneration = 1 + s.Snapshots = []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", uri, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + } }) } @@ -814,7 +817,7 @@ func TestCreateActor_GoldenTagDefault(t *testing.T) { Scope: ateapipb.TagScope_TAG_SCOPE_PUBLISHED, Status: &ateapipb.TagStatus{ ActorTemplateUid: tmpl.GetMetadata().GetUid(), - Snapshot: &ateapipb.ExternalSnapshot{SnapshotUri: "gs://bucket/atespaces/ate-golden/tags/" + someActorUID, Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY}, + Snapshot: newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, tmpl.GetMetadata().GetUid(), "gs://bucket/atespaces/ate-golden/tags/"+someActorUID, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), }, } wantCode := codes.OK @@ -828,7 +831,7 @@ func TestCreateActor_GoldenTagDefault(t *testing.T) { tag.Status.ActorTemplateUid = "other" wantCode = codes.FailedPrecondition case "data scope": - tag.Status.Snapshot.Fidelity = ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES + tag.Status.Snapshot.Storage[0].Fidelity = ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES wantCode = codes.FailedPrecondition } if scenario != "missing" { @@ -845,7 +848,7 @@ func TestCreateActor_GoldenTagDefault(t *testing.T) { actor := &ateapipb.Actor{Metadata: &ateapipb.ResourceMetadata{Atespace: "team-a", Name: "actor"}, ActorTemplate: resources.ActorTemplateRefFromActorTemplate(tmpl).ToObjectRef()} if scenario == "explicit tag" { tag.Metadata.Name = "explicit" - tag.Status.Snapshot.SnapshotUri = "gs://bucket/atespaces/ate-golden/tags/explicit" + tag.Status.Snapshot = newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, tmpl.GetMetadata().GetUid(), "gs://bucket/atespaces/ate-golden/tags/explicit", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) if _, err := persistence.CreateTag(ctx, tag); err != nil { t.Fatal(err) } @@ -859,8 +862,9 @@ func TestCreateActor_GoldenTagDefault(t *testing.T) { if err != nil { return } - if got := created.GetStatus(); got.GetExternalSnapshot().GetSnapshotUri() != tag.GetStatus().GetSnapshot().GetSnapshotUri() || got.GetExternalSnapshot().GetActorTemplateUid() != tmpl.GetMetadata().GetUid() { - t.Fatalf("incorrect initial status: %v", got) + gotSnap, gotSt := findLatestSnapshotStorage(created.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + if gotSt.GetObject().GetSnapshotUri() != tagDurableSnapshotURI(tag, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) || gotSnap.GetActorTemplateUid() != tmpl.GetMetadata().GetUid() { + t.Fatalf("incorrect initial status: %v", created.GetStatus()) } if scenario == "own snapshot" { uri, err := resources.NewActorSnapshotURI(tmpl.GetSnapshotConfig().GetStorageLocation(), "team-a", created.GetMetadata().GetUid(), "snapshot") @@ -868,7 +872,10 @@ func TestCreateActor_GoldenTagDefault(t *testing.T) { t.Fatal(err) } if _, err := persistence.UpdateActor(ctx, resources.ActorRefFromActor(created), store.PreconditionFrom(created), func(db *ateapipb.Actor) error { - db.Status.ExternalSnapshot.SnapshotUri = uri.String() + db.Status.LastAssignedGeneration = 1 + db.Status.Snapshots = []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, tmpl.GetMetadata().GetUid(), uri.String(), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + } return nil }); err != nil { t.Fatal(err) diff --git a/cmd/ateapi/internal/controlapi/crash.go b/cmd/ateapi/internal/controlapi/crash.go index e32b5f8353..d25745e6d1 100644 --- a/cmd/ateapi/internal/controlapi/crash.go +++ b/cmd/ateapi/internal/controlapi/crash.go @@ -140,11 +140,10 @@ func crashActor(ctx context.Context, st crashActorStore, actorRef resources.Acto toUpdate.Status.Crash = newActorCrash(opName, message) } - // InProgressSnapshotUri and InProgressLocalSnapshotName are kept so a - // later DeleteActor or RevertActor can delete what they name: each is - // the only pointer to it, so clearing them here would leak the objects - // for good; failed workflow steps must never promote either of them to an - // ExternalSnapshot or to LocalSnapshot. + // In-progress snapshot entries in Status.Snapshots are kept so a later + // DeleteActor or RevertActor can delete what they name: each is the only + // pointer to it, so clearing them here would leak the objects for good; + // failed workflow steps must never promote them to COMPLETED. toUpdate.Status.WorkerAssignment = nil return nil }) diff --git a/cmd/ateapi/internal/controlapi/crash_test.go b/cmd/ateapi/internal/controlapi/crash_test.go index 54b34af65f..7cf02bcece 100644 --- a/cmd/ateapi/internal/controlapi/crash_test.go +++ b/cmd/ateapi/internal/controlapi/crash_test.go @@ -63,7 +63,10 @@ func seedActor(t *testing.T, ctx context.Context, st store.Interface, actorRef r WorkerPodUid: "uid", WorkerPodIps: []string{"1.2.3.4"}, }, - InProgressSnapshotUri: "gs://bucket/atespaces/as/actors/uid/snapshots/reserved-snapshot", + LastAssignedGeneration: 1, + Snapshots: []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "gs://bucket/atespaces/as/actors/uid/snapshots/reserved-snapshot", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS), + }, }, }) } @@ -106,8 +109,11 @@ func seedUnboundActor(t *testing.T, ctx context.Context, st store.Interface, act storetest.MustCreateActor(t, ctx, st, &ateapipb.Actor{ Metadata: &ateapipb.ResourceMetadata{Name: actorRef.Name, Atespace: actorRef.Atespace}, Status: &ateapipb.ActorStatus{ - State: ateapipb.ActorState_ACTOR_STATE_RUNNING, - InProgressSnapshotUri: "gs://bucket/atespaces/as/actors/uid/snapshots/reserved-snapshot", + State: ateapipb.ActorState_ACTOR_STATE_RUNNING, + LastAssignedGeneration: 1, + Snapshots: []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "gs://bucket/atespaces/as/actors/uid/snapshots/reserved-snapshot", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS), + }, }, }) } @@ -124,8 +130,8 @@ func assertCrashed(t *testing.T, ctx context.Context, st store.Interface, actorR t.Errorf("status = %v, want %v", got.GetStatus().GetState(), ateapipb.ActorState_ACTOR_STATE_CRASHED) } // Keep the snapshot uri for debugging. - if got.GetStatus().GetInProgressSnapshotUri() == "" { - t.Error(`InProgressSnapshotUri = "", want preserved`) + if _, st := findLatestSnapshotStorage(got.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS); st == nil { + t.Error(`in-progress durable snapshot URI = "", want preserved`) } if got.GetStatus().GetWorkerAssignment() != nil { t.Errorf("WorkerAssignment = %v, want cleared", got.GetStatus().GetWorkerAssignment()) diff --git a/cmd/ateapi/internal/controlapi/functionaltest/actor_template_test.go b/cmd/ateapi/internal/controlapi/functionaltest/actor_template_test.go index f4c65b273c..ee694af879 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/actor_template_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/actor_template_test.go @@ -174,7 +174,7 @@ func TestGoldenTagLifecycle(t *testing.T) { if err != nil { t.Fatal(err) } - uri := golden.GetStatus().GetSnapshot().GetSnapshotUri() + uri := tagSnapshotURI(golden) parsed, err := resources.ParseSnapshotURI(uri) if err != nil { t.Fatal(err) @@ -193,7 +193,7 @@ func TestGoldenTagLifecycle(t *testing.T) { if err != nil { t.Fatal(err) } - if late.GetStatus().GetExternalSnapshot().GetSnapshotUri() != uri || late.GetStatus().GetExternalSnapshot().GetActorTemplateUid() != tmpl.GetMetadata().GetUid() { + if durableSnapshotURI(late.GetStatus()) != uri || durableSnapshot(late.GetStatus()).GetActorTemplateUid() != tmpl.GetMetadata().GetUid() { t.Fatal("actor did not inherit golden tag snapshot and template UID") } waitForWorkerAvailable(t, tc, workerName) diff --git a/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go b/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go index 6eba54a915..6b9e95e187 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go @@ -78,8 +78,11 @@ func TestCreateActor_Success(t *testing.T) { Metadata: &ateapipb.ResourceMetadata{Name: "id1", Atespace: testAtespace, Version: 1}, ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"}, Status: &ateapipb.ActorStatus{ - State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, - ExternalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: goldenSnapshotURI(t), Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, ActorTemplateUid: tmpl.GetMetadata().GetUid()}, + State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, + LastAssignedGeneration: 1, + Snapshots: []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, tmpl.GetMetadata().GetUid(), goldenSnapshotURI(t)), + }, }, WorkerSelector: &ateapipb.Selector{MatchLabels: map[string]string{"tier": "free"}}, } @@ -294,7 +297,7 @@ func TestCreateActor_RejectsDifferentTemplateForDataSnapshot(t *testing.T) { createTemplateWithSelector(t, tc, "tmpl2", nil) seedTag(t, tc, "data-source", "data-snapshot", func(tag *ateapipb.Tag) { - tag.Status.Snapshot.Fidelity = ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES + tag.Status.Snapshot.Storage[0].Fidelity = ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES tag.Status.ActorTemplateUid = tmpl.GetMetadata().GetUid() }) @@ -364,11 +367,10 @@ func TestCreateActor_PendingTag(t *testing.T) { ctx := context.Background() tmpl := createTemplate(t, tc, ns) - // Simulates a tag creation that failed in while writing the snapshot to + // Simulates a tag creation that failed while writing the snapshot to // external storage. pending := seedTag(t, tc, "pending-source", "pending", func(tag *ateapipb.Tag) { - tag.Status.StorageLocation = testStorageLocation - tag.Status.Snapshot = nil + tag.Status.Snapshot.Storage[0].Status = ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS tag.Status.ActorTemplateUid = tmpl.GetMetadata().GetUid() }) tagRef := &ateapipb.ObjectRef{Atespace: testAtespace, Name: "pending"} @@ -382,17 +384,11 @@ func TestCreateActor_PendingTag(t *testing.T) { assertGrpcError(t, err, codes.FailedPrecondition, "source Tag is still being created or failed creation") // Finishing the tag creation, so now the tag is qualified to be a tag source. - snapshotURI, err := resources.NewTagSnapshotURI(pending.GetStatus().GetStorageLocation(), pending.GetMetadata().GetAtespace(), pending.GetMetadata().GetUid()) - if err != nil { - t.Fatalf("NewTagSnapshotURI: %v", err) - } + snapshotURI := pending.GetStatus().GetSnapshot().GetStorage()[0].GetObject().GetSnapshotUri() if _, err := tc.persistence.UpdateTag(ctx, resources.TagRefFromTag(pending), store.PreconditionFrom(pending), func(toUpdate *ateapipb.Tag) error { - toUpdate.Status.Snapshot = &ateapipb.ExternalSnapshot{ - SnapshotUri: snapshotURI.String(), - Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, - } + toUpdate.Status.Snapshot.Storage[0].Status = ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED return nil }); err != nil { t.Fatalf("finalizing the tag: %v", err) @@ -409,7 +405,7 @@ func TestCreateActor_PendingTag(t *testing.T) { } // The clone points at the tag's snapshot, under the tag's own prefix: the // tag still owns those objects. - if got := clone.GetStatus().GetExternalSnapshot().GetSnapshotUri(); got != snapshotURI.String() { + if got := durableSnapshotURI(clone.GetStatus()); got != snapshotURI { t.Errorf("clone external snapshot = %q, want the tag's %q", got, snapshotURI) } } @@ -699,8 +695,11 @@ func TestUpdateActor_Success(t *testing.T) { Metadata: &ateapipb.ResourceMetadata{Name: "id1", Atespace: testAtespace, Version: 2}, ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"}, Status: &ateapipb.ActorStatus{ - State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, - ExternalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: goldenSnapshotURI(t), Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, ActorTemplateUid: tmpl.GetMetadata().GetUid()}, + State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, + LastAssignedGeneration: 1, + Snapshots: []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, tmpl.GetMetadata().GetUid(), goldenSnapshotURI(t)), + }, }, WorkerSelector: &ateapipb.Selector{ MatchLabels: map[string]string{"tier": "paid"}, @@ -845,8 +844,11 @@ func TestUpdateActor(t *testing.T) { Metadata: &ateapipb.ResourceMetadata{Name: "id1", Atespace: testAtespace, Version: 2}, ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"}, Status: &ateapipb.ActorStatus{ - State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, - ExternalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: goldenSnapshotURI(t), Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, ActorTemplateUid: tmpl.GetMetadata().GetUid()}, + State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, + LastAssignedGeneration: 1, + Snapshots: []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, tmpl.GetMetadata().GetUid(), goldenSnapshotURI(t)), + }, }, WorkerSelector: &ateapipb.Selector{ MatchLabels: map[string]string{"tier": "paid"}, @@ -1117,16 +1119,13 @@ func TestDeleteActor_MissingSnapshotBucket(t *testing.T) { } if _, err := tc.persistence.UpdateActor(ctx, resources.ActorRefFromActor(actor), store.PreconditionFrom(actor), func(toUpdate *ateapipb.Actor) error { toUpdate.Status.State = state - toUpdate.Status.ExternalSnapshot = nil - if state == ateapipb.ActorState_ACTOR_STATE_SUSPENDED { - toUpdate.Status.ExternalSnapshot = &ateapipb.ExternalSnapshot{ - SnapshotUri: uri.String(), Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, - } - } else { + snap := newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", uri.String()) + if state != ateapipb.ActorState_ACTOR_STATE_SUSPENDED { // A failed upload leaves only the in-progress URI, even // when a prior deletion already marked the actor DELETING. - toUpdate.Status.InProgressSnapshotUri = uri.String() + snap.Storage[0].Status = ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS } + toUpdate.Status.Snapshots = []*ateapipb.Snapshot{snap} return nil }); err != nil { t.Fatalf("UpdateActor: %v", err) @@ -2908,9 +2907,12 @@ func TestResumeActor(t *testing.T) { Metadata: &ateapipb.ResourceMetadata{Name: name, Atespace: testAtespace}, ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"}, Status: &ateapipb.ActorStatus{ - State: ateapipb.ActorState_ACTOR_STATE_RUNNING, - AssignedNode: "node1", - ExternalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: goldenSnapshotURI(t), Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, ActorTemplateUid: tmpl.GetMetadata().GetUid()}, + State: ateapipb.ActorState_ACTOR_STATE_RUNNING, + AssignedNode: "node1", + LastAssignedGeneration: 1, + Snapshots: []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, tmpl.GetMetadata().GetUid(), goldenSnapshotURI(t)), + }, WorkerAssignment: &ateapipb.WorkerAssignment{ Worker: &ateapipb.ObjectRef{Name: podUID}, WorkerNamespace: ns, @@ -3066,8 +3068,7 @@ func createDataFidelityTemplate(t *testing.T, tc *testContext, ns string) *ateap SourceActor: &ateapipb.ObjectRef{Atespace: resources.GoldenActorAtespace, Name: created.GetMetadata().GetUid()}, Scope: ateapipb.TagScope_TAG_SCOPE_PUBLISHED, Status: &ateapipb.TagStatus{ - Snapshot: &ateapipb.ExternalSnapshot{SnapshotUri: goldenSnapshotURI(t), Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY}, - StorageLocation: testStorageLocation, + Snapshot: newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, created.GetMetadata().GetUid(), goldenSnapshotURI(t)), ActorTemplateUid: created.GetMetadata().GetUid(), }, }) @@ -3120,7 +3121,7 @@ func TestResumeActor_DataSnapshotIgnoresGolden(t *testing.T) { t.Fatalf("SuspendActor failed: %v", err) } waitForWorkerAvailable(t, tc, workerName) - actorSnapshotURI := suspended.GetActor().GetStatus().GetExternalSnapshot().GetSnapshotUri() + actorSnapshotURI := durableSnapshotURI(suspended.GetActor().GetStatus()) if actorSnapshotURI == "" { t.Fatal("SuspendActor recorded no external snapshot") } @@ -3173,7 +3174,7 @@ func TestSuspendActor_ReplacedSnapshotReleaseFailure(t *testing.T) { t.Fatalf("SuspendActor (first) failed: %v", err) } waitForWorkerAvailable(t, tc, workerName) - replacedURI := first.GetActor().GetStatus().GetExternalSnapshot().GetSnapshotUri() + replacedURI := durableSnapshotURI(first.GetActor().GetStatus()) assertSnapshotOwnedByActor(t, first.GetActor(), replacedURI) if _, err := tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{Actor: actorRef}); err != nil { @@ -3191,7 +3192,7 @@ func TestSuspendActor_ReplacedSnapshotReleaseFailure(t *testing.T) { if got := second.GetActor().GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_SUSPENDED { t.Errorf("state after the second suspend = %v, want SUSPENDED", got) } - freshURI := second.GetActor().GetStatus().GetExternalSnapshot().GetSnapshotUri() + freshURI := durableSnapshotURI(second.GetActor().GetStatus()) if freshURI == "" || freshURI == replacedURI { t.Errorf("external snapshot after the second suspend = %q, want a new one replacing %q", freshURI, replacedURI) } @@ -3213,7 +3214,7 @@ func TestSuspendActor_ReplacedSnapshotReleaseFailure(t *testing.T) { t.Fatalf("SuspendActor (third) failed: %v", err) } waitForWorkerAvailable(t, tc, workerName) - lastURI := third.GetActor().GetStatus().GetExternalSnapshot().GetSnapshotUri() + lastURI := durableSnapshotURI(third.GetActor().GetStatus()) assertSnapshotCollected(t, tc, freshURI) assertSnapshotPresent(t, tc, replacedURI) @@ -3608,7 +3609,7 @@ func TestSuspendActor(t *testing.T) { } sourceActor := suspended.GetActor() - snapshotURI := sourceActor.GetStatus().GetExternalSnapshot().GetSnapshotUri() + snapshotURI := durableSnapshotURI(sourceActor.GetStatus()) if snapshotURI == "" { t.Fatalf("SuspendActor wrote no external snapshot: %v", suspended) } @@ -3632,21 +3633,20 @@ func TestSuspendActor(t *testing.T) { // The tag owns a copy of its own, so the Actor's later suspends and its // deletion cannot collect the tag's snapshot copy. - tagSnapshotURI := tagged.GetStatus().GetSnapshot().GetSnapshotUri() - if tagSnapshotURI == snapshotURI || tagSnapshotURI == "" { - t.Fatalf("tag snapshot uri = %q, want an external snapshot of its own", tagSnapshotURI) + taggedSnapshotURI := tagSnapshotURI(tagged) + if taggedSnapshotURI == snapshotURI || taggedSnapshotURI == "" { + t.Fatalf("tag snapshot uri = %q, want an external snapshot of its own", taggedSnapshotURI) } assertSnapshotPresent(t, tc, snapshotURI) - assertSnapshotPresent(t, tc, tagSnapshotURI) + assertSnapshotPresent(t, tc, taggedSnapshotURI) wantTag := &ateapipb.Tag{ Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: tagName}, Scope: ateapipb.TagScope_TAG_SCOPE_ATESPACE, SourceActor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: name}, Status: &ateapipb.TagStatus{ - Snapshot: &ateapipb.ExternalSnapshot{SnapshotUri: tagSnapshotURI, Fidelity: sourceActor.GetStatus().GetExternalSnapshot().GetFidelity()}, + Snapshot: newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG, durableSnapshotStorage(sourceActor.GetStatus()).GetFidelity(), "", taggedSnapshotURI), ActorTemplateUid: tmpl.GetMetadata().GetUid(), - StorageLocation: tmpl.GetSnapshotConfig().GetStorageLocation(), }, } stored, err := tc.client.GetTag(context.Background(), &ateapipb.GetTagRequest{Tag: tagRef}) @@ -3681,8 +3681,8 @@ func TestSuspendActor(t *testing.T) { if err != nil || updated.GetScope() != ateapipb.TagScope_TAG_SCOPE_PUBLISHED { t.Fatalf("UpdateTag = (%v, %v), want published", updated, err) } - if updated.GetStatus().GetSnapshot().GetSnapshotUri() != tagSnapshotURI { - t.Errorf("tag snapshot uri after publication = %q, want %q", updated.GetStatus().GetSnapshot().GetSnapshotUri(), tagSnapshotURI) + if tagSnapshotURI(updated) != taggedSnapshotURI { + t.Errorf("tag snapshot uri after publication = %q, want %q", tagSnapshotURI(updated), taggedSnapshotURI) } if _, err := tc.client.CreateActor(context.Background(), crossAtespaceClone); err != nil { t.Fatalf("CreateActor from published tag failed: %v", err) @@ -3701,11 +3701,11 @@ func TestSuspendActor(t *testing.T) { if err != nil { t.Fatalf("CreateActor from tag failed: %v", err) } - if got := clone.GetStatus().GetExternalSnapshot().GetSnapshotUri(); got != tagSnapshotURI { - t.Errorf("clone snapshot uri = %q, want the tag's %q", got, tagSnapshotURI) + if got := durableSnapshotURI(clone.GetStatus()); got != taggedSnapshotURI { + t.Errorf("clone snapshot uri = %q, want the tag's %q", got, taggedSnapshotURI) } - if snapshotOwnedByActor(t, clone, tagSnapshotURI) { - t.Errorf("clone snapshot %s sits under the clone's own prefix, want it left under the tag's", tagSnapshotURI) + if snapshotOwnedByActor(t, clone, taggedSnapshotURI) { + t.Errorf("clone snapshot %s sits under the clone's own prefix, want it left under the tag's", taggedSnapshotURI) } if !proto.Equal(clone.GetSourceTag(), tagRef) { t.Errorf("clone source tag = %v, want %v", clone.GetSourceTag(), tagRef) @@ -3723,13 +3723,13 @@ func TestSuspendActor(t *testing.T) { if err != nil { t.Fatalf("SuspendActor clone failed: %v", err) } - cloneSnapshotURI := cloneSuspended.GetActor().GetStatus().GetExternalSnapshot().GetSnapshotUri() - if cloneSnapshotURI == tagSnapshotURI || cloneSnapshotURI == "" { + cloneSnapshotURI := durableSnapshotURI(cloneSuspended.GetActor().GetStatus()) + if cloneSnapshotURI == taggedSnapshotURI || cloneSnapshotURI == "" { t.Errorf("clone snapshot uri after suspension = %q, want an external snapshot of its own", cloneSnapshotURI) } assertSnapshotOwnedByActor(t, cloneSuspended.GetActor(), cloneSnapshotURI) // It stopped borrowing without releasing what it had borrowed. - assertSnapshotPresent(t, tc, tagSnapshotURI) + assertSnapshotPresent(t, tc, taggedSnapshotURI) assertSnapshotPresent(t, tc, cloneSnapshotURI) // The untagged suspend created no tag. listed, err = tc.client.ListTags(context.Background(), &ateapipb.ListTagsRequest{Atespace: testAtespace}) @@ -3747,11 +3747,10 @@ func TestSuspendActor(t *testing.T) { Metadata: &ateapipb.ResourceMetadata{Name: name, Atespace: testAtespace}, ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"}, Status: &ateapipb.ActorStatus{ - State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, - ExternalSnapshot: &ateapipb.ExternalSnapshot{ - SnapshotUri: snapshotURI, - Fidelity: sourceActor.GetStatus().GetExternalSnapshot().GetFidelity(), - ActorTemplateUid: tmpl.GetMetadata().GetUid(), + State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, + LastAssignedGeneration: 2, + Snapshots: []*ateapipb.Snapshot{ + newDurableSnapshot(2, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, durableSnapshotStorage(sourceActor.GetStatus()).GetFidelity(), tmpl.GetMetadata().GetUid(), snapshotURI), }, }, } @@ -3774,7 +3773,7 @@ func TestSuspendActor(t *testing.T) { } // The Actor took only what it owned with it. assertSnapshotCollected(t, tc, snapshotURI) - assertSnapshotPresent(t, tc, tagSnapshotURI) + assertSnapshotPresent(t, tc, taggedSnapshotURI) if deleted, err := tc.client.DeleteTag(context.Background(), &ateapipb.DeleteTagRequest{Tag: tagRef}); err != nil || deleted.GetMetadata().GetName() != tagRef.GetName() { t.Fatalf("DeleteTag = (%v, %v)", deleted, err) @@ -3784,7 +3783,7 @@ func TestSuspendActor(t *testing.T) { } // Deleting the tag is what ends its snapshot's life. The clone that // borrowed it has one of its own by now and is unaffected. - assertSnapshotCollected(t, tc, tagSnapshotURI) + assertSnapshotCollected(t, tc, taggedSnapshotURI) assertSnapshotPresent(t, tc, cloneSnapshotURI) } @@ -3902,8 +3901,8 @@ func TestResumeActor_RepointTemplateBeforeResume(t *testing.T) { } // Either way the restore reads the snapshot the clone borrowed // from the tag, not the template's golden image. - if got := restoreReq.GetExternalConfig().GetSnapshotUri(); got != cloneActor.GetStatus().GetExternalSnapshot().GetSnapshotUri() { - t.Errorf("restore request to atelet had snapshot uri = %q, want the clone's borrowed %q", got, cloneActor.GetStatus().GetExternalSnapshot().GetSnapshotUri()) + if got := restoreReq.GetExternalConfig().GetSnapshotUri(); got != durableSnapshotURI(cloneActor.GetStatus()) { + t.Errorf("restore request to atelet had snapshot uri = %q, want the clone's borrowed %q", got, durableSnapshotURI(cloneActor.GetStatus())) } }) } @@ -4039,12 +4038,13 @@ func TestPauseActor(t *testing.T) { Metadata: &ateapipb.ResourceMetadata{Name: name, Atespace: testAtespace}, ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"}, Status: &ateapipb.ActorStatus{ - State: ateapipb.ActorState_ACTOR_STATE_PAUSED, - AssignedNode: "node1", - LocalSnapshot: &ateapipb.LocalSnapshot{ - Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + State: ateapipb.ActorState_ACTOR_STATE_PAUSED, + AssignedNode: "node1", + LastAssignedGeneration: 2, + Snapshots: []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, tmpl.GetMetadata().GetUid(), goldenSnapshotURI(t)), + newLocalSnapshot(2, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, tmpl.GetMetadata().GetUid(), "ignored"), }, - ExternalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: goldenSnapshotURI(t), Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, ActorTemplateUid: tmpl.GetMetadata().GetUid()}, }, } @@ -4057,7 +4057,7 @@ func TestPauseActor(t *testing.T) { ); diff != "" { t.Errorf("GetActor response mismatch (-want +got):\n%s", diff) } - if getResp.GetStatus().GetLocalSnapshot().GetSnapshotName() == "" { + if _, ls := localSnapshot(getResp.GetStatus()); ls.GetSnapshotName() == "" { t.Error("LocalSnapshot.SnapshotName is empty, want the name the pause checkpointed under") } } @@ -4106,7 +4106,7 @@ func TestResumeActor_PausedLocalSnapshotMissing_Crashes(t *testing.T) { if getResp.GetStatus().GetState() != ateapipb.ActorState_ACTOR_STATE_PAUSED { t.Fatalf("actor state = %v, want ACTOR_STATE_PAUSED", getResp.GetStatus().GetState()) } - if getResp.GetStatus().GetLocalSnapshot() == nil { + if _, ls := localSnapshot(getResp.GetStatus()); ls == nil { t.Fatal("expected LocalSnapshot to be present on paused actor") } waitForWorkerAvailable(t, tc, workerName) @@ -4731,7 +4731,8 @@ func TestSuspendActor_FromPaused(t *testing.T) { t.Error("atelet Checkpoint called for a paused actor; there is no workload to checkpoint") } upload := tc.fakeAtelet.UploadRequest - if got, want := upload.GetLocalSnapshotName(), paused.GetStatus().GetLocalSnapshot().GetSnapshotName(); got != want { + _, pausedLS := localSnapshot(paused.GetStatus()) + if got, want := upload.GetLocalSnapshotName(), pausedLS.GetSnapshotName(); got != want { t.Errorf("upload local_snapshot_name = %q, want the pause snapshot %q", got, want) } if got, want := upload.GetAtespace(), testAtespace; got != want { @@ -4745,13 +4746,13 @@ func TestSuspendActor_FromPaused(t *testing.T) { if actor.GetStatus().GetState() != ateapipb.ActorState_ACTOR_STATE_SUSPENDED { t.Errorf("state = %v, want SUSPENDED", actor.GetStatus().GetState()) } - if actor.GetStatus().GetLocalSnapshot() != nil { - t.Errorf("LocalSnapshot = %v, want cleared (node pinning must not survive suspend)", actor.GetStatus().GetLocalSnapshot()) + if _, ls := localSnapshot(actor.GetStatus()); ls != nil { + t.Errorf("LocalSnapshot = %v, want cleared (node pinning must not survive suspend)", ls) } - if got, want := actor.GetStatus().GetExternalSnapshot().GetSnapshotUri(), upload.GetDestinationSnapshotUri(); got != want { + if got, want := durableSnapshotURI(actor.GetStatus()), upload.GetDestinationSnapshotUri(); got != want { t.Errorf("snapshot URI = %q, want the upload destination %q", got, want) } - if got := actor.GetStatus().GetExternalSnapshot().GetFidelity(); got != ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY { + if got := durableSnapshotStorage(actor.GetStatus()).GetFidelity(); got != ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY { t.Errorf("snapshot Fidelity = %v, want FULL", got) } } @@ -4984,8 +4985,8 @@ func TestResumeActor_RelocatesAfterSuspendFromPaused(t *testing.T) { if got := suspended.GetActor().GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_SUSPENDED { t.Fatalf("state after suspend = %v, want SUSPENDED", got) } - if got := suspended.GetActor().GetStatus().GetLocalSnapshot(); got != nil { - t.Fatalf("LocalSnapshot = %v, want cleared so the actor can be scheduled anywhere", got) + if _, ls := localSnapshot(suspended.GetActor().GetStatus()); ls != nil { + t.Fatalf("LocalSnapshot = %v, want cleared so the actor can be scheduled anywhere", ls) } // Resume should succeed now and the actor scheduled on node2. @@ -5295,7 +5296,7 @@ func TestRevertActor(t *testing.T) { t.Fatalf("SuspendActor failed: %v", err) } waitForWorkerAvailable(t, tc, workerName) - snapshotURI := suspended.GetActor().GetStatus().GetExternalSnapshot().GetSnapshotUri() + snapshotURI := durableSnapshotURI(suspended.GetActor().GetStatus()) if snapshotURI == "" { t.Fatalf("SuspendActor wrote no external snapshot: %v", suspended) } @@ -5318,7 +5319,7 @@ func TestRevertActor(t *testing.T) { if got.GetState() != ateapipb.ActorState_ACTOR_STATE_SUSPENDED { t.Errorf("state = %v, want SUSPENDED", got.GetState()) } - if uri := got.GetExternalSnapshot().GetSnapshotUri(); uri != snapshotURI { + if uri := durableSnapshotURI(got); uri != snapshotURI { t.Errorf("external snapshot = %q, want it untouched at %q", uri, snapshotURI) } assertSnapshotPresent(t, tc, snapshotURI) @@ -5363,7 +5364,7 @@ func TestRevertActor_FromPaused(t *testing.T) { t.Fatalf("SuspendActor failed: %v", err) } waitForWorkerAvailable(t, tc, workerName) - snapshotURI := suspended.GetActor().GetStatus().GetExternalSnapshot().GetSnapshotUri() + snapshotURI := durableSnapshotURI(suspended.GetActor().GetStatus()) if snapshotURI == "" { t.Fatalf("SuspendActor wrote no external snapshot: %v", suspended) } @@ -5389,12 +5390,12 @@ func TestRevertActor_FromPaused(t *testing.T) { if got.GetState() != ateapipb.ActorState_ACTOR_STATE_SUSPENDED { t.Errorf("state = %v, want SUSPENDED", got.GetState()) } - if uri := got.GetExternalSnapshot().GetSnapshotUri(); uri != snapshotURI { + if uri := durableSnapshotURI(got); uri != snapshotURI { t.Errorf("external snapshot = %q, want it untouched at %q", uri, snapshotURI) } assertSnapshotPresent(t, tc, snapshotURI) - if got.GetLocalSnapshot() != nil { - t.Errorf("local snapshot info = %v, want nil", got.GetLocalSnapshot()) + if _, ls := localSnapshot(got); ls != nil { + t.Errorf("local snapshot info = %v, want nil", ls) } if got.GetWorkerAssignment() != nil { t.Errorf("worker assignment = %v, want nil", got.GetWorkerAssignment()) @@ -5441,7 +5442,7 @@ func TestRevertActor_FromCrashed(t *testing.T) { t.Fatalf("SuspendActor failed: %v", err) } waitForWorkerAvailable(t, tc, workerName) - snapshotURI := suspended.GetActor().GetStatus().GetExternalSnapshot().GetSnapshotUri() + snapshotURI := durableSnapshotURI(suspended.GetActor().GetStatus()) if snapshotURI == "" { t.Fatalf("SuspendActor wrote no external snapshot: %v", suspended) } @@ -5473,7 +5474,7 @@ func TestRevertActor_FromCrashed(t *testing.T) { if got.GetState() != ateapipb.ActorState_ACTOR_STATE_SUSPENDED { t.Errorf("state = %v, want SUSPENDED", got.GetState()) } - if uri := got.GetExternalSnapshot().GetSnapshotUri(); uri != snapshotURI { + if uri := durableSnapshotURI(got); uri != snapshotURI { t.Errorf("external snapshot = %q, want it untouched at %q", uri, snapshotURI) } assertSnapshotPresent(t, tc, snapshotURI) diff --git a/cmd/ateapi/internal/controlapi/functionaltest/common_test.go b/cmd/ateapi/internal/controlapi/functionaltest/common_test.go index 8748f36261..398e9a05f6 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/common_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/common_test.go @@ -459,8 +459,7 @@ func createTemplateWithContainersAndVolumes(t *testing.T, tc *testContext, ns st SourceActor: &ateapipb.ObjectRef{Atespace: resources.GoldenActorAtespace, Name: created.GetMetadata().GetUid()}, Scope: ateapipb.TagScope_TAG_SCOPE_PUBLISHED, Status: &ateapipb.TagStatus{ - Snapshot: &ateapipb.ExternalSnapshot{SnapshotUri: goldenSnapshotURI(t), Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY}, - StorageLocation: testStorageLocation, + Snapshot: newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, created.GetMetadata().GetUid(), goldenSnapshotURI(t)), ActorTemplateUid: created.GetMetadata().GetUid(), }, }) @@ -487,6 +486,86 @@ func createTemplateWithContainersAndVolumes(t *testing.T, tc *testContext, ns st return updated } +func newDurableSnapshot(gen int32, owner ateapipb.SnapshotOwner, fidelity ateapipb.SnapshotFidelity, templateUID, uri string) *ateapipb.Snapshot { + return &ateapipb.Snapshot{ + Generation: gen, + Owner: owner, + ActorTemplateUid: templateUID, + Storage: []*ateapipb.SnapshotStorage{{ + Durability: ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, + Status: ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED, + Fidelity: fidelity, + Object: &ateapipb.ObjectSnapshot{SnapshotUri: uri}, + }}, + } +} + +func newLocalSnapshot(gen int32, fidelity ateapipb.SnapshotFidelity, templateUID, name string) *ateapipb.Snapshot { + return &ateapipb.Snapshot{ + Generation: gen, + Owner: ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, + ActorTemplateUid: templateUID, + Storage: []*ateapipb.SnapshotStorage{{ + Durability: ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL, + Status: ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED, + Fidelity: fidelity, + Local: &ateapipb.LocalSnapshot{SnapshotName: name}, + }}, + } +} + +func durableSnapshot(status *ateapipb.ActorStatus) *ateapipb.Snapshot { + var best *ateapipb.Snapshot + for _, snap := range status.GetSnapshots() { + for _, st := range snap.GetStorage() { + if st.GetDurability() == ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE && + st.GetStatus() == ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED && + st.GetObject().GetSnapshotUri() != "" { + if best == nil || snap.GetGeneration() > best.GetGeneration() { + best = snap + } + } + } + } + return best +} + +func durableSnapshotStorage(status *ateapipb.ActorStatus) *ateapipb.SnapshotStorage { + snap := durableSnapshot(status) + for _, st := range snap.GetStorage() { + if st.GetDurability() == ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE { + return st + } + } + return nil +} + +func durableSnapshotURI(status *ateapipb.ActorStatus) string { + return durableSnapshotStorage(status).GetObject().GetSnapshotUri() +} + +func localSnapshot(status *ateapipb.ActorStatus) (*ateapipb.Snapshot, *ateapipb.LocalSnapshot) { + for _, snap := range status.GetSnapshots() { + for _, st := range snap.GetStorage() { + if st.GetDurability() == ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL && + st.GetStatus() == ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED && + st.GetLocal().GetSnapshotName() != "" { + return snap, st.GetLocal() + } + } + } + return nil, nil +} + +func tagSnapshotURI(tag *ateapipb.Tag) string { + for _, st := range tag.GetStatus().GetSnapshot().GetStorage() { + if st.GetDurability() == ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE { + return st.GetObject().GetSnapshotUri() + } + } + return "" +} + // testPauseImage is the pause image the default test SandboxConfig carries; // it is what a resolved WorkloadSpec's sandbox assets should name. const testPauseImage = "pause@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" diff --git a/cmd/ateapi/internal/controlapi/functionaltest/tag_test.go b/cmd/ateapi/internal/controlapi/functionaltest/tag_test.go index 14ae066bf1..205a022cb0 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/tag_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/tag_test.go @@ -47,43 +47,40 @@ func seedTag(t *testing.T, tc *testContext, actorName, tagName string, opts ...f t.Fatalf("NewActorSnapshotURI: %v", err) } actor, err = tc.persistence.UpdateActor(ctx, resources.ActorRefFromActor(actor), store.PreconditionFrom(actor), func(toUpdate *ateapipb.Actor) error { - toUpdate.Status.ExternalSnapshot = &ateapipb.ExternalSnapshot{SnapshotUri: actorSnapshotURI.String(), Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY} + toUpdate.Status.LastAssignedGeneration = 1 + toUpdate.Status.Snapshots = []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", actorSnapshotURI.String()), + } return nil }) if err != nil { t.Fatalf("recording the actor's external snapshot: %v", err) } + uri, err := resources.NewTagSnapshotURI(testStorageLocation, testAtespace, resources.NewSnapshotName()) + if err != nil { + t.Fatalf("NewTagSnapshotURI: %v", err) + } tag := &ateapipb.Tag{ Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: tagName}, Scope: ateapipb.TagScope_TAG_SCOPE_ATESPACE, SourceActor: resources.ActorRefFromActor(actor).ToObjectRef(), Status: &ateapipb.TagStatus{ - Snapshot: &ateapipb.ExternalSnapshot{Fidelity: actor.GetStatus().GetExternalSnapshot().GetFidelity()}, - StorageLocation: testStorageLocation, + Snapshot: &ateapipb.Snapshot{ + Generation: 1, + Owner: ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG, + Storage: []*ateapipb.SnapshotStorage{{ + Durability: ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, + Status: ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED, + Fidelity: durableSnapshotStorage(actor.GetStatus()).GetFidelity(), + Object: &ateapipb.ObjectSnapshot{SnapshotUri: uri.String()}, + }}, + }, }, } for _, opt := range opts { opt(tag) } - snapshot := tag.Status.Snapshot - tag.Status.Snapshot = nil - tag = storetest.MustCreateTag(t, ctx, tc.persistence, tag) - if snapshot == nil { - return tag - } - uri, err := resources.NewTagSnapshotURI(testStorageLocation, testAtespace, tag.GetMetadata().GetUid()) - if err != nil { - t.Fatalf("NewTagSnapshotURI: %v", err) - } - snapshot.SnapshotUri = uri.String() - tag, err = tc.persistence.UpdateTag(ctx, resources.TagRefFromTag(tag), store.PreconditionFrom(tag), func(toUpdate *ateapipb.Tag) error { - toUpdate.Status.Snapshot = snapshot - return nil - }) - if err != nil { - t.Fatalf("finalizing tag: %v", err) - } - return tag + return storetest.MustCreateTag(t, ctx, tc.persistence, tag) } // TestCreateTag_ReusedTagName tests that a tag does not move @@ -111,7 +108,7 @@ func TestCreateTag_ReusedTagName(t *testing.T) { if err != nil { t.Fatalf("CreateTag(actor-a) failed: %v", err) } - tagSnapshotURI := first.GetStatus().GetSnapshot().GetSnapshotUri() + tagSnapshotURI := tagSnapshotURI(first) // Tag owns a snapshot. So it must not be empty. wantObjects := snapshotObjectNames(t, tc, tagSnapshotURI) if len(wantObjects) == 0 { @@ -189,7 +186,7 @@ func suspendActorForTest(t *testing.T, tc *testContext, workerName, name string) if err != nil { t.Fatalf("SuspendActor(%s) failed: %v", name, err) } - uri := suspended.GetActor().GetStatus().GetExternalSnapshot().GetSnapshotUri() + uri := durableSnapshotURI(suspended.GetActor().GetStatus()) if uri == "" { t.Fatalf("SuspendActor(%s) wrote no external snapshot: %v", name, suspended) } diff --git a/cmd/ateapi/internal/controlapi/snapshot_helpers.go b/cmd/ateapi/internal/controlapi/snapshot_helpers.go new file mode 100644 index 0000000000..e493914b49 --- /dev/null +++ b/cmd/ateapi/internal/controlapi/snapshot_helpers.go @@ -0,0 +1,186 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlapi + +import ( + "math" + + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" +) + +// findSnapshotByGeneration returns the Snapshot in snapshots with the given +// generation, or nil if none exists. +func findSnapshotByGeneration(snapshots []*ateapipb.Snapshot, gen int32) *ateapipb.Snapshot { + for _, snap := range snapshots { + if snap.GetGeneration() == gen { + return snap + } + } + return nil +} + +// snapshotAtLatestGeneration returns the Snapshot at status.last_assigned_generation, or +// nil if none exists. +func snapshotAtLatestGeneration(status *ateapipb.ActorStatus) *ateapipb.Snapshot { + if status == nil { + return nil + } + return findSnapshotByGeneration(status.GetSnapshots(), status.GetLastAssignedGeneration()) +} + +// findSnapshotStorage returns the SnapshotStorage entry on snap with the given +// durability, or nil if none exists. +func findSnapshotStorage(snap *ateapipb.Snapshot, durability ateapipb.SnapshotDurability) *ateapipb.SnapshotStorage { + for _, st := range snap.GetStorage() { + if st.GetDurability() == durability { + return st + } + } + return nil +} + +// setSnapshotStorage replaces an existing SnapshotStorage entry on snap with +// the same durability, or appends entry if none exists. +func setSnapshotStorage(snap *ateapipb.Snapshot, entry *ateapipb.SnapshotStorage) { + for i, st := range snap.Storage { + if st.GetDurability() == entry.GetDurability() { + snap.Storage[i] = entry + return + } + } + snap.Storage = append(snap.Storage, entry) +} + +// removeSnapshotStorage removes any SnapshotStorage entry on snap with the +// given durability. +func removeSnapshotStorage(snap *ateapipb.Snapshot, durability ateapipb.SnapshotDurability) { + if snap == nil { + return + } + filtered := snap.Storage[:0] + for _, st := range snap.Storage { + if st.GetDurability() != durability { + filtered = append(filtered, st) + } + } + snap.Storage = filtered +} + +// findLatestSnapshotStorage returns the highest-generation Snapshot on status +// that holds a SnapshotStorage entry matching durability and storageStatus, +// along with that SnapshotStorage entry, or (nil, nil) if none exists. +func findLatestSnapshotStorage(status *ateapipb.ActorStatus, durability ateapipb.SnapshotDurability, storageStatus ateapipb.SnapshotStorageStatus) (*ateapipb.Snapshot, *ateapipb.SnapshotStorage) { + var bestSnap *ateapipb.Snapshot + var bestStorage *ateapipb.SnapshotStorage + for _, snap := range status.GetSnapshots() { + st := findSnapshotStorage(snap, durability) + if st.GetStatus() != storageStatus { + continue + } + if bestSnap == nil || snap.GetGeneration() >= bestSnap.GetGeneration() { + bestSnap = snap + bestStorage = st + } + } + return bestSnap, bestStorage +} + +// removeSnapshotStorageEntries removes SnapshotStorage entries matching +// durability (and *storageStatus, if non-nil) from status.Snapshots, dropping +// any Snapshot left with no storage entries. +// +// Called from ensureWorkerReleased during DeleteActor, ensureRevertedFinalized +// during revert, and ensureSuspendedFinalized during suspend. +func removeSnapshotStorageEntries(status *ateapipb.ActorStatus, durability ateapipb.SnapshotDurability, storageStatus *ateapipb.SnapshotStorageStatus) { + removeOlderSnapshotStorageEntries(status, durability, storageStatus, math.MaxInt32) +} + +// removeOlderSnapshotStorageEntries removes SnapshotStorage entries matching +// durability (and *storageStatus, if non-nil) from Snapshots in +// status.Snapshots with generation older than keepGen, dropping any Snapshot +// left with no storage entries. +// +// status.LastAssignedGeneration is not decreased so that we can keep track of +// stale snapshots that need to be cleaned up and discarded generation numbers +// are never reused. +// +// Called from ensurePausedFinalized on pause success and +// ensureSuspendedFinalized during suspend. +func removeOlderSnapshotStorageEntries(status *ateapipb.ActorStatus, durability ateapipb.SnapshotDurability, storageStatus *ateapipb.SnapshotStorageStatus, keepGen int32) { + if status == nil { + return + } + kept := status.Snapshots[:0] + for _, snap := range status.Snapshots { + if snap.GetGeneration() < keepGen { + storage := snap.Storage[:0] + for _, st := range snap.Storage { + if st.GetDurability() == durability && + (storageStatus == nil || st.GetStatus() == *storageStatus) { + continue + } + storage = append(storage, st) + } + snap.Storage = storage + } + if len(snap.GetStorage()) > 0 { + kept = append(kept, snap) + } + } + status.Snapshots = kept +} + +// newDurableSnapshot constructs a Snapshot with a single DURABLE ObjectSnapshot +// storage entry. +func newDurableSnapshot(gen int32, owner ateapipb.SnapshotOwner, fidelity ateapipb.SnapshotFidelity, templateUID, uri string, storageStatus ateapipb.SnapshotStorageStatus) *ateapipb.Snapshot { + return &ateapipb.Snapshot{ + Generation: gen, + Owner: owner, + ActorTemplateUid: templateUID, + Storage: []*ateapipb.SnapshotStorage{{ + Durability: ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, + Status: storageStatus, + Fidelity: fidelity, + Object: &ateapipb.ObjectSnapshot{SnapshotUri: uri}, + }}, + } +} + +// newLocalSnapshot constructs an actor Snapshot with a single LOCAL +// LocalSnapshot storage entry. +func newLocalSnapshot(gen int32, fidelity ateapipb.SnapshotFidelity, templateUID, snapshotName string, storageStatus ateapipb.SnapshotStorageStatus) *ateapipb.Snapshot { + return &ateapipb.Snapshot{ + Generation: gen, + Owner: ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, + ActorTemplateUid: templateUID, + Storage: []*ateapipb.SnapshotStorage{{ + Durability: ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL, + Status: storageStatus, + Fidelity: fidelity, + Local: &ateapipb.LocalSnapshot{SnapshotName: snapshotName}, + }}, + } +} + +// tagDurableSnapshotURI returns the durable snapshot URI on tag matching +// storageStatus (or any status if storageStatus is UNSPECIFIED), or "" if none +// exists. +func tagDurableSnapshotURI(tag *ateapipb.Tag, storageStatus ateapipb.SnapshotStorageStatus) string { + st := findSnapshotStorage(tag.GetStatus().GetSnapshot(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE) + if storageStatus != ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_UNSPECIFIED && st.GetStatus() != storageStatus { + return "" + } + return st.GetObject().GetSnapshotUri() +} diff --git a/cmd/ateapi/internal/controlapi/snapshot_helpers_test.go b/cmd/ateapi/internal/controlapi/snapshot_helpers_test.go new file mode 100644 index 0000000000..0c00dc49ca --- /dev/null +++ b/cmd/ateapi/internal/controlapi/snapshot_helpers_test.go @@ -0,0 +1,230 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlapi + +import ( + "testing" + + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "github.com/google/go-cmp/cmp" + "google.golang.org/protobuf/testing/protocmp" +) + +func TestFindSnapshotByGenerationAndSnapshotAtLatestGeneration(t *testing.T) { + if got := snapshotAtLatestGeneration(nil); got != nil { + t.Errorf("snapshotAtLatestGeneration(nil) = %v, want nil", got) + } + if got := findSnapshotByGeneration(nil, 1); got != nil { + t.Errorf("findSnapshotByGeneration(nil, 1) = %v, want nil", got) + } + + s1 := newLocalSnapshot(1, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "local-1", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + s2 := newDurableSnapshot(2, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "gs://b/snap-2", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + status := &ateapipb.ActorStatus{ + LastAssignedGeneration: 2, + Snapshots: []*ateapipb.Snapshot{s1, s2}, + } + + if got := findSnapshotByGeneration(status.GetSnapshots(), 1); got != s1 { + t.Errorf("findSnapshotByGeneration(1) = %v, want %v", got, s1) + } + if got := findSnapshotByGeneration(status.GetSnapshots(), 99); got != nil { + t.Errorf("findSnapshotByGeneration(99) = %v, want nil", got) + } + if got := snapshotAtLatestGeneration(status); got != s2 { + t.Errorf("snapshotAtLatestGeneration() = %v, want %v", got, s2) + } +} + +func TestSnapshotStorageAccessors(t *testing.T) { + if got := findSnapshotStorage(nil, ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL); got != nil { + t.Errorf("findSnapshotStorage(nil) = %v, want nil", got) + } + // Should not panic on nil snapshot. + removeSnapshotStorage(nil, ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL) + + snap := newLocalSnapshot(1, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "local-in-progress", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS) + + // Replace existing LOCAL entry. + completedLocal := &ateapipb.SnapshotStorage{ + Durability: ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL, + Status: ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED, + Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + Local: &ateapipb.LocalSnapshot{SnapshotName: "local-completed"}, + } + setSnapshotStorage(snap, completedLocal) + if len(snap.GetStorage()) != 1 { + t.Fatalf("len(snap.Storage) = %d, want 1", len(snap.GetStorage())) + } + if got := findSnapshotStorage(snap, ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL); got != completedLocal { + t.Errorf("findSnapshotStorage(LOCAL) = %v, want %v", got, completedLocal) + } + + // Append DURABLE entry alongside LOCAL. + durableEntry := &ateapipb.SnapshotStorage{ + Durability: ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, + Status: ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED, + Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + Object: &ateapipb.ObjectSnapshot{SnapshotUri: "gs://b/snap-1"}, + } + setSnapshotStorage(snap, durableEntry) + if len(snap.GetStorage()) != 2 { + t.Fatalf("len(snap.Storage) = %d, want 2", len(snap.GetStorage())) + } + if got := findSnapshotStorage(snap, ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE); got != durableEntry { + t.Errorf("findSnapshotStorage(DURABLE) = %v, want %v", got, durableEntry) + } + + // Remove LOCAL and verify DURABLE remains. + removeSnapshotStorage(snap, ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL) + if got := findSnapshotStorage(snap, ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL); got != nil { + t.Errorf("findSnapshotStorage(LOCAL) after remove = %v, want nil", got) + } + if got := findSnapshotStorage(snap, ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE); got != durableEntry { + t.Errorf("findSnapshotStorage(DURABLE) after removing LOCAL = %v, want %v", got, durableEntry) + } +} + +func TestFindLatestSnapshotStorage(t *testing.T) { + if snap, st := findLatestSnapshotStorage(nil, ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED); snap != nil || st != nil { + t.Errorf("findLatestSnapshotStorage(nil) = (%v, %v), want (nil, nil)", snap, st) + } + + s1 := newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "gs://b/snap-1", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + s2 := newDurableSnapshot(2, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "gs://b/snap-2", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + s3 := newDurableSnapshot(3, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "gs://b/snap-3", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS) + s4 := newLocalSnapshot(4, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "local-4", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + + // Place s2 before s1 to verify generation comparison rather than slice order. + status := &ateapipb.ActorStatus{ + Snapshots: []*ateapipb.Snapshot{s2, s1, s3, s4}, + } + + gotSnap, gotSt := findLatestSnapshotStorage(status, ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + if gotSnap != s2 || gotSt.GetObject().GetSnapshotUri() != "gs://b/snap-2" { + t.Errorf("findLatestSnapshotStorage(DURABLE, COMPLETED) = (gen %d, %v), want (gen 2, gs://b/snap-2)", gotSnap.GetGeneration(), gotSt) + } + + gotSnap, gotSt = findLatestSnapshotStorage(status, ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS) + if gotSnap != s3 || gotSt.GetObject().GetSnapshotUri() != "gs://b/snap-3" { + t.Errorf("findLatestSnapshotStorage(DURABLE, IN_PROGRESS) = (gen %d, %v), want (gen 3, gs://b/snap-3)", gotSnap.GetGeneration(), gotSt) + } + + gotSnap, gotSt = findLatestSnapshotStorage(status, ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + if gotSnap != s4 || gotSt.GetLocal().GetSnapshotName() != "local-4" { + t.Errorf("findLatestSnapshotStorage(LOCAL, COMPLETED) = (gen %d, %v), want (gen 4, local-4)", gotSnap.GetGeneration(), gotSt) + } + + gotSnap, gotSt = findLatestSnapshotStorage(status, ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS) + if gotSnap != nil || gotSt != nil { + t.Errorf("findLatestSnapshotStorage(LOCAL, IN_PROGRESS) = (%v, %v), want (nil, nil)", gotSnap, gotSt) + } +} + +func TestRemoveSnapshotStorageEntries(t *testing.T) { + // Should not panic on nil status. + removeSnapshotStorageEntries(nil, ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL, nil) + removeOlderSnapshotStorageEntries(nil, ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL, nil, 3) + + t.Run("removes older local snapshots while preserving keepGen, newer generations, and durable storage", func(t *testing.T) { + bothGen1 := newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "gs://b/snap-1", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + setSnapshotStorage(bothGen1, &ateapipb.SnapshotStorage{ + Durability: ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL, + Status: ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED, + Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + Local: &ateapipb.LocalSnapshot{SnapshotName: "local-1"}, + }) + localGen2 := newLocalSnapshot(2, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "local-2", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + localGen3 := newLocalSnapshot(3, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "local-3", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + localGen4 := newLocalSnapshot(4, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "local-4", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS) + + status := &ateapipb.ActorStatus{ + LastAssignedGeneration: 4, + Snapshots: []*ateapipb.Snapshot{bothGen1, localGen2, localGen3, localGen4}, + } + + removeOlderSnapshotStorageEntries(status, ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL, nil, 3) + + want := []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "gs://b/snap-1", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + newLocalSnapshot(3, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "local-3", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + newLocalSnapshot(4, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "local-4", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS), + } + if diff := cmp.Diff(want, status.GetSnapshots(), protocmp.Transform()); diff != "" { + t.Errorf("status.Snapshots mismatch (-want +got):\n%s", diff) + } + if status.GetLastAssignedGeneration() != 4 { + t.Errorf("LastAssignedGeneration = %d, want 4", status.GetLastAssignedGeneration()) + } + }) + + t.Run("removes all matching snapshot storage entries across all generations", func(t *testing.T) { + status := &ateapipb.ActorStatus{ + Snapshots: []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "gs://b/snap-1", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + newLocalSnapshot(2, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "local-2", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + newLocalSnapshot(3, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "local-3", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS), + }, + } + + removeSnapshotStorageEntries(status, ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL, nil) + + want := []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "gs://b/snap-1", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + } + if diff := cmp.Diff(want, status.GetSnapshots(), protocmp.Transform()); diff != "" { + t.Errorf("status.Snapshots mismatch (-want +got):\n%s", diff) + } + }) + + t.Run("filters by specific storageStatus", func(t *testing.T) { + status := &ateapipb.ActorStatus{ + Snapshots: []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "gs://b/snap-1", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS), + newDurableSnapshot(2, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "gs://b/snap-2", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + }, + } + + removeSnapshotStorageEntries(status, ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, new(ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS)) + + want := []*ateapipb.Snapshot{ + newDurableSnapshot(2, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "gs://b/snap-2", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + } + if diff := cmp.Diff(want, status.GetSnapshots(), protocmp.Transform()); diff != "" { + t.Errorf("status.Snapshots mismatch (-want +got):\n%s", diff) + } + }) +} + +func TestTagDurableSnapshotURI(t *testing.T) { + if got := tagDurableSnapshotURI(nil, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_UNSPECIFIED); got != "" { + t.Errorf("tagDurableSnapshotURI(nil) = %q, want empty", got) + } + + inProgressTag := &ateapipb.Tag{ + Status: &ateapipb.TagStatus{ + Snapshot: newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "gs://b/tag-1", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS), + }, + } + if got := tagDurableSnapshotURI(inProgressTag, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED); got != "" { + t.Errorf("tagDurableSnapshotURI(inProgressTag, COMPLETED) = %q, want empty", got) + } + if got := tagDurableSnapshotURI(inProgressTag, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS); got != "gs://b/tag-1" { + t.Errorf("tagDurableSnapshotURI(inProgressTag, IN_PROGRESS) = %q, want gs://b/tag-1", got) + } + if got := tagDurableSnapshotURI(inProgressTag, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_UNSPECIFIED); got != "gs://b/tag-1" { + t.Errorf("tagDurableSnapshotURI(inProgressTag, UNSPECIFIED) = %q, want gs://b/tag-1", got) + } +} diff --git a/cmd/ateapi/internal/controlapi/tag.go b/cmd/ateapi/internal/controlapi/tag.go index 7a3b3d6b7e..6c8789e7b8 100644 --- a/cmd/ateapi/internal/controlapi/tag.go +++ b/cmd/ateapi/internal/controlapi/tag.go @@ -123,7 +123,7 @@ func (s *RPCService) UpdateTag(ctx context.Context, req *ateapipb.UpdateTagReque // A tag whose create never finished names a partial copy. Publishing it // — or changing its scope at all — would hand out content that is still // being written, or may never be. - if toUpdate.GetStatus().GetSnapshot().GetSnapshotUri() == "" { + if tagDurableSnapshotURI(toUpdate, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) == "" { return errTagPending } // Metadata and status are server-owned fields. diff --git a/cmd/ateapi/internal/controlapi/tag_test.go b/cmd/ateapi/internal/controlapi/tag_test.go index 9c7b31d232..90b3779578 100644 --- a/cmd/ateapi/internal/controlapi/tag_test.go +++ b/cmd/ateapi/internal/controlapi/tag_test.go @@ -135,7 +135,7 @@ func TestUpdateTag(t *testing.T) { req: &ateapipb.Tag{ Scope: ateapipb.TagScope_TAG_SCOPE_PUBLISHED, Status: &ateapipb.TagStatus{ - Snapshot: &ateapipb.ExternalSnapshot{SnapshotUri: "gs://attacker/elsewhere", Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES}, + Snapshot: newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, "", "gs://attacker/elsewhere", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), ActorTemplateUid: "other-template-uid", }, }, @@ -249,7 +249,10 @@ func newTestSuspendedActor(t *testing.T, ctx context.Context, st store.Interface t.Fatalf("NewActorSnapshotURI: %v", err) } return mustUpdateActorStatus(t, ctx, st, actor, func(status *ateapipb.ActorStatus) { - status.ExternalSnapshot = &ateapipb.ExternalSnapshot{SnapshotUri: uri.String(), Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY} + status.LastAssignedGeneration = 1 + status.Snapshots = []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", uri.String(), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + } }) } @@ -263,27 +266,24 @@ func newTestTag(t *testing.T, name string, actor *ateapipb.Actor) *ateapipb.Tag if err != nil { t.Fatalf("NewTagSnapshotURI: %v", err) } + actorSnap, actorSt := findLatestSnapshotStorage(actor.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) return &ateapipb.Tag{ Metadata: &ateapipb.ResourceMetadata{Atespace: atespace, Name: name}, Scope: ateapipb.TagScope_TAG_SCOPE_ATESPACE, SourceActor: resources.ActorRefFromActor(actor).ToObjectRef(), Status: &ateapipb.TagStatus{ - Snapshot: &ateapipb.ExternalSnapshot{ - SnapshotUri: uri.String(), - Fidelity: actor.GetStatus().GetExternalSnapshot().GetFidelity(), - }, + Snapshot: newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG, actorSt.GetFidelity(), actorSnap.GetActorTemplateUid(), uri.String(), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), }, } } // newPendingTestTag builds the tag a create leaves behind when it dies between // reserving the name and finishing the copy: the row names the prefix it was -// writing into, and nothing else. +// writing into as IN_PROGRESS. func newPendingTestTag(t *testing.T, name string, actor *ateapipb.Actor) *ateapipb.Tag { t.Helper() tag := newTestTag(t, name, actor) - tag.Status.StorageLocation = testStorageLocation - tag.Status.Snapshot = nil + tag.Status.Snapshot.Storage[0].Status = ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS return tag } diff --git a/cmd/ateapi/internal/controlapi/template_reconciler.go b/cmd/ateapi/internal/controlapi/template_reconciler.go index 131bc80e96..03ef425127 100644 --- a/cmd/ateapi/internal/controlapi/template_reconciler.go +++ b/cmd/ateapi/internal/controlapi/template_reconciler.go @@ -224,7 +224,7 @@ func (r *ActorTemplateReconciler) reconcileOne(ctx context.Context, ref resource if tag.GetStatus().GetActorTemplateUid() != tmpl.GetMetadata().GetUid() || resources.ActorRefFromObjectRef(tag.GetSourceActor()) != resources.ActorRefFromObjectRef(goldenActorRef) { return 0, r.fail(ctx, tmpl, reasonGoldenTagConflict, "golden tag belongs to another actor or template") } - if tag.GetStatus().GetSnapshot().GetSnapshotUri() != "" { + if tagDurableSnapshotURI(tag, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) != "" { return 0, r.saveGoldenTag(ctx, tmpl, goldenActorRef) } // CreateTag cannot resume an incomplete copy. Delete it before retrying. @@ -284,7 +284,7 @@ func (r *ActorTemplateReconciler) reconcileOne(ctx context.Context, ref resource ateapipb.ActorState_ACTOR_STATE_SUSPENDED: // The golden actor was never resumed, or a previous resume didn't // finish; ResumeActor is reentrant from both. - if actor.GetStatus().GetExternalSnapshot().GetSnapshotUri() != "" { + if _, st := findLatestSnapshotStorage(actor.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED); st != nil { // Golden actors never start from a source snapshot, so an // existing snapshot means an earlier suspend completed // without being recorded. @@ -319,8 +319,7 @@ func (r *ActorTemplateReconciler) suspendActor(ctx context.Context, goldenRef *a // A crash during suspend is observed as CRASHED on the retry. return fmt.Errorf("while suspending golden actor: %w", err) } - suspended := resp.GetActor().GetStatus().GetExternalSnapshot() - if suspended.GetSnapshotUri() == "" { + if _, st := findLatestSnapshotStorage(resp.GetActor().GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED); st == nil { return fmt.Errorf("suspending golden actor produced no external snapshot") } return nil diff --git a/cmd/ateapi/internal/controlapi/template_reconciler_test.go b/cmd/ateapi/internal/controlapi/template_reconciler_test.go index 1a2d51bc35..7effb0c55b 100644 --- a/cmd/ateapi/internal/controlapi/template_reconciler_test.go +++ b/cmd/ateapi/internal/controlapi/template_reconciler_test.go @@ -182,6 +182,17 @@ func (c *fakeGoldenControl) CreateAtespace(_ context.Context, req *ateapipb.Crea return req.GetAtespace(), nil } +func (c *fakeGoldenControl) goldenActorStatus() *ateapipb.ActorStatus { + st := &ateapipb.ActorStatus{State: c.goldenState} + if c.goldenSnapshot != "" { + st.LastAssignedGeneration = 1 + st.Snapshots = []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, testTemplateUID, c.goldenSnapshot, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + } + } + return st +} + func (c *fakeGoldenControl) CreateActor(_ context.Context, req *ateapipb.CreateActorRequest) (*ateapipb.Actor, error) { c.mu.Lock() defer c.mu.Unlock() @@ -198,7 +209,7 @@ func (c *fakeGoldenControl) CreateActor(_ context.Context, req *ateapipb.CreateA // a status observing the initial SUSPENDED state. return &ateapipb.Actor{ Metadata: req.GetActor().GetMetadata(), - Status: &ateapipb.ActorStatus{State: c.goldenState, ExternalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: c.goldenSnapshot}}, + Status: c.goldenActorStatus(), }, nil } @@ -213,7 +224,7 @@ func (c *fakeGoldenControl) GetActor(_ context.Context, req *ateapipb.GetActorRe } return &ateapipb.Actor{ Metadata: &ateapipb.ResourceMetadata{Atespace: req.GetActor().GetAtespace(), Name: req.GetActor().GetName()}, - Status: &ateapipb.ActorStatus{State: c.goldenState, ExternalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: c.goldenSnapshot}}, + Status: c.goldenActorStatus(), }, nil } @@ -240,7 +251,7 @@ func (c *fakeGoldenControl) SuspendActor(_ context.Context, req *ateapipb.Suspen c.goldenSnapshot = c.snapshot } return &ateapipb.SuspendActorResponse{ - Actor: &ateapipb.Actor{Status: &ateapipb.ActorStatus{ExternalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: c.goldenSnapshot}}}, + Actor: &ateapipb.Actor{Status: c.goldenActorStatus()}, }, nil } @@ -261,7 +272,10 @@ func (c *fakeGoldenControl) CreateTag(_ context.Context, req *ateapipb.CreateTag return nil, c.tagErr } c.tag = proto.CloneOf(req.GetTag()) - c.tag.Status = &ateapipb.TagStatus{ActorTemplateUid: testTemplateUID, Snapshot: &ateapipb.ExternalSnapshot{SnapshotUri: c.goldenSnapshot}} + c.tag.Status = &ateapipb.TagStatus{ + ActorTemplateUid: testTemplateUID, + Snapshot: newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, testTemplateUID, c.goldenSnapshot, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + } return proto.CloneOf(c.tag), nil } @@ -856,7 +870,10 @@ func TestReconcileOne_GoldenTagRecovery(t *testing.T) { Metadata: &ateapipb.ResourceMetadata{Atespace: ref.Atespace, Name: ref.Name}, SourceActor: ref, Scope: ateapipb.TagScope_TAG_SCOPE_PUBLISHED, - Status: &ateapipb.TagStatus{ActorTemplateUid: testTemplateUID, Snapshot: &ateapipb.ExternalSnapshot{SnapshotUri: "gs://bucket/tag-snapshot"}}, + Status: &ateapipb.TagStatus{ + ActorTemplateUid: testTemplateUID, + Snapshot: newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, testTemplateUID, "gs://bucket/tag-snapshot", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + }, } incomplete := proto.CloneOf(completed) incomplete.Status.Snapshot = nil @@ -913,7 +930,7 @@ func TestReconcileOne_GoldenTagRecovery(t *testing.T) { if !proto.Equal(st.storedStatus(t, testTemplateRef).GetGoldenSnapshotStatus().GetGoldenTag(), ref) { t.Fatal("golden tag not recorded") } - if control.tag.GetStatus().GetSnapshot().GetSnapshotUri() == "" { + if tagDurableSnapshotURI(control.tag, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) == "" { t.Fatal("golden tag has no snapshot") } if len(control.createReqs) != 0 || len(control.resumeReqs) != 0 || len(control.suspendReqs) != 0 { diff --git a/cmd/ateapi/internal/controlapi/workflow_delete.go b/cmd/ateapi/internal/controlapi/workflow_delete.go index ef0a671375..5f98de04f9 100644 --- a/cmd/ateapi/internal/controlapi/workflow_delete.go +++ b/cmd/ateapi/internal/controlapi/workflow_delete.go @@ -289,7 +289,7 @@ func (w *ActorWorkflow) ensureWorkerReleased(ctx context.Context, actorRef resou updatedActor, err := w.store.UpdateActor(ctx, actorRef, store.PreconditionFrom(latestActor), func(dbActor *ateapipb.Actor) error { if dbActor.Status != nil { - dbActor.Status.LocalSnapshot = nil + removeSnapshotStorageEntries(dbActor.Status, ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL, nil) dbActor.Status.AssignedNode = "" dbActor.Status.WorkerAssignment = nil } @@ -397,7 +397,8 @@ func (w *ActorWorkflow) ensureExternalSnapshotsReleased(ctx context.Context, act // wrote anything. func actorSnapshotStoragePrefix(actor *ateapipb.Actor) (resources.StoragePrefix, error) { actorOwner := actorSnapshotOwner(actor) - if snapshotURI := actor.GetStatus().GetExternalSnapshot().GetSnapshotUri(); snapshotURI != "" { + if _, st := findLatestSnapshotStorage(actor.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED); st != nil { + snapshotURI := st.GetObject().GetSnapshotUri() uri, err := resources.ParseSnapshotURI(snapshotURI) if err != nil { return resources.StoragePrefix{}, fmt.Errorf("while parsing the external snapshot %q: %w", snapshotURI, err) @@ -412,7 +413,8 @@ func actorSnapshotStoragePrefix(actor *ateapipb.Actor) (resources.StoragePrefix, // Nothing of the actor's own is recorded. Unless a suspend died partway, // nothing was ever written under its prefix: the in-progress URI is // recorded before atelet uploads the first object. - inProgress := actor.GetStatus().GetInProgressSnapshotUri() + _, inProgressSt := findLatestSnapshotStorage(actor.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS) + inProgress := inProgressSt.GetObject().GetSnapshotUri() if inProgress == "" { return resources.StoragePrefix{}, nil } diff --git a/cmd/ateapi/internal/controlapi/workflow_delete_test.go b/cmd/ateapi/internal/controlapi/workflow_delete_test.go index f391973870..214f0b3965 100644 --- a/cmd/ateapi/internal/controlapi/workflow_delete_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_delete_test.go @@ -263,8 +263,10 @@ func TestEnsureExternalSnapshotsReleased(t *testing.T) { // The actor's prefix is keyed on the UID the store just assigned, so // its snapshots can only be placed now. current := mustActorSnapshotURI(t, template, actor, "current") + currentType := ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR if tt.tagOwnedSnapshot { current = mustTagSnapshotURI(t, template, "team-a", "v1-snapshot") + currentType = ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG } objects.PutSnapshot(t, current, "manifest.json") inFlight := mustActorSnapshotURI(t, template, actor, inFlightSnapshotName) @@ -272,8 +274,11 @@ func TestEnsureExternalSnapshotsReleased(t *testing.T) { objects.PutSnapshot(t, inFlight, "manifest.json") } actor = mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapipb.ActorStatus) { - s.ExternalSnapshot = &ateapipb.ExternalSnapshot{SnapshotUri: current.String()} - s.InProgressSnapshotUri = inFlight.String() + s.LastAssignedGeneration = 2 + s.Snapshots = []*ateapipb.Snapshot{ + newDurableSnapshot(1, currentType, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", current.String(), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + newDurableSnapshot(2, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", inFlight.String(), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS), + } }) if err := w.ensureExternalSnapshotsReleased(ctx, actor); err != nil { @@ -318,7 +323,10 @@ func TestEnsureExternalSnapshotsReleased_CollectsStrandedSnapshots(t *testing.T) objects.PutSnapshot(t, uri, "manifest.json") } actor = mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapipb.ActorStatus) { - s.ExternalSnapshot = &ateapipb.ExternalSnapshot{SnapshotUri: currentSnapshot.String()} + s.LastAssignedGeneration = 1 + s.Snapshots = []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", currentSnapshot.String(), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + } }) if err := w.ensureExternalSnapshotsReleased(ctx, actor); err != nil { @@ -355,7 +363,10 @@ func TestDeleteActor_CollectsInFlightSnapshotWithoutTemplate(t *testing.T) { }, actor, "abandoned") objects.PutSnapshot(t, inFlight, "manifest.json") mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapipb.ActorStatus) { - s.InProgressSnapshotUri = inFlight.String() + s.LastAssignedGeneration = 1 + s.Snapshots = []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", inFlight.String(), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS), + } }) if _, err := w.DeleteActor(ctx, actorRef, true, store.DeletePreconditions{}); err != nil { @@ -385,7 +396,10 @@ func TestEnsureExternalSnapshotsReleased_DeletePrefixFailure(t *testing.T) { current := mustActorSnapshotURI(t, template, actor, "current") objects.PutSnapshot(t, current, "manifest.json", "memory.zst") actor = mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapipb.ActorStatus) { - s.ExternalSnapshot = &ateapipb.ExternalSnapshot{SnapshotUri: current.String()} + s.LastAssignedGeneration = 1 + s.Snapshots = []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", current.String(), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + } }) errTransient := errors.New("simulated transient delete failure") @@ -416,8 +430,8 @@ func TestEnsureExternalSnapshotsReleased_DeletePrefixFailure(t *testing.T) { // TestDeleteActor_CollectsSnapshotsAfterWorkerDelete verifies that // deleting an actor whose suspend a worker delete crashed mid-finalize deletes // every object that suspend wrote. When an actor crashes mid-suspend, only -// DeleteActor or RevertActor can delete the in-progress snapshot -// (in_progress_snapshot_uri): whatever they cannot name is leaked for good. +// DeleteActor or RevertActor can delete the in-progress snapshot: +// whatever they cannot name is leaked for good. func TestDeleteActor_CollectsSnapshotsAfterWorkerDelete(t *testing.T) { tests := []struct { name string @@ -479,7 +493,10 @@ func TestDeleteActor_CollectsSnapshotsAfterWorkerDelete(t *testing.T) { previous := mustActorSnapshotURI(t, template, actor, "old") objects.PutSnapshot(t, previous, "manifest.json") actor = mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapipb.ActorStatus) { - s.ExternalSnapshot = &ateapipb.ExternalSnapshot{SnapshotUri: previous.String()} + s.LastAssignedGeneration = 1 + s.Snapshots = []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", previous.String(), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + } }) } @@ -490,7 +507,8 @@ func TestDeleteActor_CollectsSnapshotsAfterWorkerDelete(t *testing.T) { if err != nil { t.Fatalf("ensureMarkedSuspending: %v", err) } - fresh := mustParseSnapshotURI(t, actor.GetStatus().GetInProgressSnapshotUri()) + _, freshSt := findLatestSnapshotStorage(actor.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS) + fresh := mustParseSnapshotURI(t, freshSt.GetObject().GetSnapshotUri()) objects.PutSnapshot(t, fresh, "manifest.json") // The worker's pod goes away with the commit still outstanding, so diff --git a/cmd/ateapi/internal/controlapi/workflow_pause.go b/cmd/ateapi/internal/controlapi/workflow_pause.go index 0c0f125e0f..b00ae7a8cb 100644 --- a/cmd/ateapi/internal/controlapi/workflow_pause.go +++ b/cmd/ateapi/internal/controlapi/workflow_pause.go @@ -68,7 +68,7 @@ func (w *ActorWorkflow) PauseActor(ctx context.Context, actorRef resources.Actor return actor, nil } var marked *ateapipb.Actor - if marked, err = w.ensureMarkedPausing(leaseCtx, actorRef, actor); err != nil { + if marked, err = w.ensureMarkedPausing(leaseCtx, actorRef, actor, actorTemplate); err != nil { return nil, err } actor = marked @@ -85,7 +85,7 @@ func (w *ActorWorkflow) PauseActor(ctx context.Context, actorRef resources.Actor // them here, as crash.go does for the crash counter. finalAttrs = lifecycleOpAttrs(actor, actorTemplate, "", wireFidelity) var finalized *ateapipb.Actor - if finalized, err = w.ensurePausedFinalized(leaseCtx, actorRef, actorTemplate); err != nil { + if finalized, err = w.ensurePausedFinalized(leaseCtx, actorRef); err != nil { return nil, err } actor = finalized @@ -112,7 +112,7 @@ func (w *ActorWorkflow) loadActorForPause(ctx context.Context, actorRef resource // local snapshot name. Skips when a previous attempt already marked the // actor; the persisted name then stays authoritative for the rest of the // workflow. -func (w *ActorWorkflow) ensureMarkedPausing(ctx context.Context, actorRef resources.ActorRef, actor *ateapipb.Actor) (_ *ateapipb.Actor, err error) { +func (w *ActorWorkflow) ensureMarkedPausing(ctx context.Context, actorRef resources.ActorRef, actor *ateapipb.Actor, actorTemplate *ateapipb.ActorTemplate) (_ *ateapipb.Actor, err error) { ctx, done := stepSpan(ctx, "MarkPausing") defer func() { err = done(err) }() @@ -133,7 +133,17 @@ func (w *ActorWorkflow) ensureMarkedPausing(ctx context.Context, actorRef resour snapshotName := resources.NewSnapshotName() storedActor, err := w.store.UpdateActor(ctx, actorRef, store.PreconditionFrom(actor), func(toUpdate *ateapipb.Actor) error { toUpdate.Status.State = ateapipb.ActorState_ACTOR_STATE_PAUSING - toUpdate.Status.InProgressLocalSnapshotName = snapshotName + // Increment last_assigned_generation for the new Pause request. + gen := toUpdate.Status.LastAssignedGeneration + 1 + toUpdate.Status.LastAssignedGeneration = gen + toUpdate.Status.Snapshots = append(toUpdate.Status.Snapshots, + newLocalSnapshot( + gen, + actorTemplate.GetSnapshotConfig().GetPreferredFidelity(), + actorTemplate.GetMetadata().GetUid(), + snapshotName, + ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS, + )) return nil }) if err != nil { @@ -179,6 +189,7 @@ func (w *ActorWorkflow) ensureAteletPaused(ctx context.Context, actorRef resourc // Checkpoint does not carry the sandbox config: atelet uses the version the // actor is currently running (recorded on-node at Run/Restore) and pins it // into the snapshot manifest. + _, inProgressLocal := findLatestSnapshotStorage(actor.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS) req := &ateletpb.CheckpointRequest{ TargetAteomUid: assignment.GetWorkerPodUid(), Atespace: actor.GetMetadata().GetAtespace(), @@ -189,7 +200,7 @@ func (w *ActorWorkflow) ensureAteletPaused(ctx context.Context, actorRef resourc Type: ateletpb.CheckpointType_CHECKPOINT_TYPE_LOCAL, Config: &ateletpb.CheckpointRequest_LocalConfig{ LocalConfig: &ateletpb.LocalCheckpointConfiguration{ - SnapshotName: actor.GetStatus().GetInProgressLocalSnapshotName(), + SnapshotName: inProgressLocal.GetLocal().GetSnapshotName(), }, }, Fidelity: fidelityToAtelet(actorTemplate.GetSnapshotConfig().GetPreferredFidelity()), @@ -210,7 +221,7 @@ func (w *ActorWorkflow) ensureAteletPaused(ctx context.Context, actorRef resourc // never be resumed. It re-reads the actor first so an out-of-band transition // (e.g. the syncer crashing the actor after its worker died) is not // overwritten: with no assignment left there is nothing to finalize. -func (w *ActorWorkflow) ensurePausedFinalized(ctx context.Context, actorRef resources.ActorRef, actorTemplate *ateapipb.ActorTemplate) (_ *ateapipb.Actor, err error) { +func (w *ActorWorkflow) ensurePausedFinalized(ctx context.Context, actorRef resources.ActorRef) (_ *ateapipb.Actor, err error) { ctx, done := stepSpan(ctx, "FinalizePaused") defer func() { err = done(err) }() @@ -257,7 +268,6 @@ func (w *ActorWorkflow) ensurePausedFinalized(ctx context.Context, actorRef reso newState = ateapipb.ActorState_ACTOR_STATE_CRASHED crashStatus = newActorCrash(ateattr.OperationPause, crashMessageLocalSnapshotNodeUnknown) } - fidelity := actorTemplate.GetSnapshotConfig().GetPreferredFidelity() sandboxClass := "" if worker != nil { sandboxClass = worker.GetSandboxClass() @@ -271,14 +281,16 @@ func (w *ActorWorkflow) ensurePausedFinalized(ctx context.Context, actorRef reso if newState == ateapipb.ActorState_ACTOR_STATE_CRASHED && !wasAlreadyCrashed { toUpdate.Status.Crash = crashStatus } - // TODO(dberkov) - what if InProgressLocalSnapshotName is empty? That shouldn't be possible. - if toUpdate.GetStatus().GetInProgressLocalSnapshotName() != "" { - localSnapshot := &ateapipb.LocalSnapshot{ - SnapshotName: toUpdate.GetStatus().GetInProgressLocalSnapshotName(), - Fidelity: fidelity, + if newState != ateapipb.ActorState_ACTOR_STATE_CRASHED { + // Finalize the snapshot and move it from IN_PROGRESS to COMPLETED. + snap := snapshotAtLatestGeneration(toUpdate.Status) + if snap == nil { + return fmt.Errorf("actor %s has no latest snapshot", actorRef) + } + if inProgressSt := findSnapshotStorage(snap, ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL); inProgressSt.GetLocal().GetSnapshotName() != "" { + inProgressSt.Status = ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + removeOlderSnapshotStorageEntries(toUpdate.Status, ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL, nil, snap.GetGeneration()) } - toUpdate.Status.LocalSnapshot = localSnapshot - toUpdate.Status.InProgressLocalSnapshotName = "" } toUpdate.Status.WorkerAssignment = nil return nil diff --git a/cmd/ateapi/internal/controlapi/workflow_pause_test.go b/cmd/ateapi/internal/controlapi/workflow_pause_test.go index 9f8ee00448..d8725d34d7 100644 --- a/cmd/ateapi/internal/controlapi/workflow_pause_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_pause_test.go @@ -50,14 +50,17 @@ func TestEnsurePausedFinalized_WorkerGone(t *testing.T) { WorkerPool: "pool1", WorkerPod: "worker-pod-1", }, - InProgressLocalSnapshotName: "local-snap-1", + LastAssignedGeneration: 1, + Snapshots: []*ateapipb.Snapshot{ + newLocalSnapshot(1, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "local-snap-1", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS), + }, }, } storetest.MustCreateActor(t, ctx, st, actor) // Intentionally NOT creating the worker in store, simulates worker already gone. w := &ActorWorkflow{store: st} - finalized, err := w.ensurePausedFinalized(ctx, actorRef, &ateapipb.ActorTemplate{}) + finalized, err := w.ensurePausedFinalized(ctx, actorRef) if err != nil { t.Fatalf("ensurePausedFinalized: %v", err) } @@ -76,8 +79,8 @@ func TestEnsurePausedFinalized_WorkerGone(t *testing.T) { if got.GetStatus().GetAssignedNode() != "" { t.Errorf("AssignedNode = %q, want empty", got.GetStatus().GetAssignedNode()) } - if gotSnap := got.GetStatus().GetInProgressLocalSnapshotName(); gotSnap != "" { - t.Errorf("InProgressLocalSnapshotName = %q, want cleared on crash", gotSnap) + if _, gotSt := findLatestSnapshotStorage(got.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS); gotSt.GetLocal().GetSnapshotName() != "local-snap-1" { + t.Errorf("in-progress local snapshot name = %q, want %q preserved on crash", gotSt.GetLocal().GetSnapshotName(), "local-snap-1") } if finalized.GetStatus().GetWorkerAssignment() != nil { @@ -99,8 +102,8 @@ func TestEnsurePausedFinalized_WorkerGone(t *testing.T) { // TestEnsurePausedFinalized_AlreadyCrashed verifies that if the actor was // already crashed out-of-band when ensurePausedFinalized runs with no -// AssignedNode, its existing Crash status is preserved and no duplicate crash -// log record is emitted. +// AssignedNode, its existing Crash status and snapshots are preserved and no +// duplicate crash log record is emitted. func TestEnsurePausedFinalized_AlreadyCrashed(t *testing.T) { st, cleanup := storetest.SetupTestStore(t) defer cleanup() @@ -120,13 +123,16 @@ func TestEnsurePausedFinalized_AlreadyCrashed(t *testing.T) { WorkerPool: "pool1", WorkerPod: "worker-pod-1", }, - InProgressLocalSnapshotName: "local-snap-1", + LastAssignedGeneration: 1, + Snapshots: []*ateapipb.Snapshot{ + newLocalSnapshot(1, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "local-snap-1", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS), + }, }, } storetest.MustCreateActor(t, ctx, st, actor) w := &ActorWorkflow{store: st} - finalized, err := w.ensurePausedFinalized(ctx, actorRef, &ateapipb.ActorTemplate{}) + finalized, err := w.ensurePausedFinalized(ctx, actorRef) if err != nil { t.Fatalf("ensurePausedFinalized: %v", err) } @@ -136,6 +142,9 @@ func TestEnsurePausedFinalized_AlreadyCrashed(t *testing.T) { if got, want := finalized.GetStatus().GetCrash().GetMessage(), originalCrash.GetMessage(); got != want { t.Errorf("crash message = %q, want original %q preserved", got, want) } + if _, gotSt := findLatestSnapshotStorage(finalized.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS); gotSt.GetLocal().GetSnapshotName() != "local-snap-1" { + t.Errorf("in-progress local snapshot name = %q, want %q preserved", gotSt.GetLocal().GetSnapshotName(), "local-snap-1") + } if finalized.GetStatus().GetWorkerAssignment() != nil { t.Errorf("WorkerAssignment = %v, want nil", finalized.GetStatus().GetWorkerAssignment()) } @@ -144,10 +153,10 @@ func TestEnsurePausedFinalized_AlreadyCrashed(t *testing.T) { } } -// TestEnsurePausedFinalized_RecordsFidelity verifies pause finalization -// records the scope the pause checkpoint captured (the template's preferredFidelity) -// in LocalSnapshot, so a later suspend or resume of the PAUSED actor knows -// what the local snapshot contains. +// TestEnsurePausedFinalized_RecordsFidelity verifies pause marks the +// scope the pause checkpoint captures (the template's preferredFidelity) in +// SnapshotStorage and finalization promotes it to COMPLETED, so a later suspend +// or resume of the PAUSED actor knows what the local snapshot contains. func TestEnsurePausedFinalized_RecordsFidelity(t *testing.T) { tests := []struct { name string @@ -168,7 +177,7 @@ func TestEnsurePausedFinalized_RecordsFidelity(t *testing.T) { created := storetest.MustCreateActor(t, ctx, st, &ateapipb.Actor{ Metadata: &ateapipb.ResourceMetadata{Atespace: actorRef.Atespace, Name: actorRef.Name}, Status: &ateapipb.ActorStatus{ - State: ateapipb.ActorState_ACTOR_STATE_PAUSING, + State: ateapipb.ActorState_ACTOR_STATE_RUNNING, AssignedNode: "node1", WorkerAssignment: &ateapipb.WorkerAssignment{ Worker: &ateapipb.ObjectRef{Name: workerName}, @@ -177,7 +186,6 @@ func TestEnsurePausedFinalized_RecordsFidelity(t *testing.T) { WorkerPod: "worker-pod-1", WorkerPodUid: workerName, }, - InProgressLocalSnapshotName: "snap-prefix", }, }) if _, err := st.CreateWorker(ctx, &ateapipb.Worker{ @@ -200,7 +208,10 @@ func TestEnsurePausedFinalized_RecordsFidelity(t *testing.T) { tmpl := &ateapipb.ActorTemplate{ SnapshotConfig: &ateapipb.SnapshotConfig{PreferredFidelity: tc.fidelity}, } - got, err := w.ensurePausedFinalized(ctx, actorRef, tmpl) + if _, err := w.ensureMarkedPausing(ctx, actorRef, created, tmpl); err != nil { + t.Fatalf("ensureMarkedPausing: %v", err) + } + got, err := w.ensurePausedFinalized(ctx, actorRef) if err != nil { t.Fatalf("ensurePausedFinalized: %v", err) } @@ -208,8 +219,9 @@ func TestEnsurePausedFinalized_RecordsFidelity(t *testing.T) { if got.GetStatus().GetState() != ateapipb.ActorState_ACTOR_STATE_PAUSED { t.Fatalf("state = %v, want PAUSED", got.GetStatus().GetState()) } - if scope := got.GetStatus().GetLocalSnapshot().GetFidelity(); scope != tc.want { - t.Errorf("LocalSnapshot.Fidelity = %v, want %v", scope, tc.want) + _, localSt := findLatestSnapshotStorage(got.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + if scope := localSt.GetFidelity(); scope != tc.want { + t.Errorf("local SnapshotStorage.Fidelity = %v, want %v", scope, tc.want) } if got.GetStatus().GetAssignedNode() != "node1" { t.Errorf("AssignedNode = %q, want %q", got.GetStatus().GetAssignedNode(), "node1") @@ -303,7 +315,7 @@ func TestEnsureMarkedPausing_StateMatrix(t *testing.T) { Status: &ateapipb.ActorStatus{State: seedState}, }) - marked, err := w.ensureMarkedPausing(ctx, actorRef, actor) + marked, err := w.ensureMarkedPausing(ctx, actorRef, actor, &ateapipb.ActorTemplate{}) assertPrerequisiteResult(t, seedState, err, allowed[seedState]) if err == nil && marked.GetStatus().GetState() != ateapipb.ActorState_ACTOR_STATE_PAUSING { t.Errorf("state %v: ensureMarkedPausing returned actor in %v, want PAUSING", seedState, marked.GetStatus().GetState()) @@ -331,6 +343,12 @@ func TestEnsureAteletPaused_DialFailureLeavesActorRetryable(t *testing.T) { ctx := context.Background() persistence := newTestPersistence(t) + var snapshots []*ateapipb.Snapshot + if tt.prevSnapshot != "" { + snapshots = append(snapshots, newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", tt.prevSnapshot, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED)) + } + snapshots = append(snapshots, newLocalSnapshot(2, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "actor-1-never-written", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS)) + actor := &ateapipb.Actor{ Metadata: &ateapipb.ResourceMetadata{Atespace: "team-a", Name: "actor-1"}, Status: &ateapipb.ActorStatus{ @@ -341,8 +359,8 @@ func TestEnsureAteletPaused_DialFailureLeavesActorRetryable(t *testing.T) { WorkerPod: "pod-gone", NodeName: "node-gone", }, - InProgressLocalSnapshotName: "actor-1-never-written", - ExternalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: tt.prevSnapshot}, + LastAssignedGeneration: 2, + Snapshots: snapshots, }, } created := storetest.MustCreateActor(t, ctx, persistence, actor) @@ -359,10 +377,11 @@ func TestEnsureAteletPaused_DialFailureLeavesActorRetryable(t *testing.T) { if stored.GetStatus().GetState() != ateapipb.ActorState_ACTOR_STATE_PAUSING { t.Errorf("state = %v, want unchanged PAUSING", stored.GetStatus().GetState()) } - if got := stored.GetStatus().GetInProgressLocalSnapshotName(); got != "actor-1-never-written" { - t.Errorf("InProgressLocalSnapshotName = %q, want preserved for debugging", got) + if _, gotSt := findLatestSnapshotStorage(stored.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS); gotSt.GetLocal().GetSnapshotName() != "actor-1-never-written" { + t.Errorf("in-progress local snapshot name = %q, want preserved for debugging", gotSt.GetLocal().GetSnapshotName()) } - if got := stored.GetStatus().GetExternalSnapshot().GetSnapshotUri(); got != tt.prevSnapshot { + _, gotSt := findLatestSnapshotStorage(stored.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + if got := gotSt.GetObject().GetSnapshotUri(); got != tt.prevSnapshot { t.Errorf("SnapshotUri = %q, want %q", got, tt.prevSnapshot) } }) @@ -407,8 +426,9 @@ func TestEnsureMarkedPausing_GoldenAtespaceRejected(t *testing.T) { _, err := w.ensureMarkedPausing(context.Background(), resources.ActorRef{Atespace: resources.GoldenActorAtespace, Name: "golden-1"}, - &ateapipb.Actor{Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_RUNNING}}) + &ateapipb.Actor{Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_RUNNING}}, + &ateapipb.ActorTemplate{}) if got := apierror.Code(err); got != codes.FailedPrecondition { - t.Fatalf("status.Code = %v (err %v), want FailedPrecondition", got, err) + t.Fatalf("apierror.Code = %v (err %v), want FailedPrecondition", got, err) } } diff --git a/cmd/ateapi/internal/controlapi/workflow_reconcile_assignments_test.go b/cmd/ateapi/internal/controlapi/workflow_reconcile_assignments_test.go index 9adf4f12db..5a59ab01e0 100644 --- a/cmd/ateapi/internal/controlapi/workflow_reconcile_assignments_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_reconcile_assignments_test.go @@ -177,7 +177,10 @@ func TestReconcileAssignments_CrashesEarlierActors(t *testing.T) { svc, persistence := newWorkerAPIService(t) seedEpochWorker(t, ctx, persistence, 1, 1) actor := seedAPIActor(t, ctx, persistence, ateapipb.ActorState_ACTOR_STATE_RUNNING, func(a *ateapipb.Actor) { - a.Status.InProgressLocalSnapshotName = "partial-local-snapshot" + a.Status.LastAssignedGeneration = 1 + a.Status.Snapshots = []*ateapipb.Snapshot{ + newLocalSnapshot(1, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "partial-local-snapshot", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS), + } }) assignAPIWorker(t, ctx, persistence, apiWorkerName, actor.GetMetadata().GetUid()) mustRaiseEpoch(t, ctx, svc, persistence, 2) @@ -204,8 +207,8 @@ func TestReconcileAssignments_CrashesEarlierActors(t *testing.T) { if got.GetStatus().GetWorkerAssignment() != nil { t.Errorf("actor worker assignment = %v, want it cleared", got.GetStatus().GetWorkerAssignment()) } - if got.GetStatus().GetInProgressLocalSnapshotName() != "" { - t.Errorf("in-progress local checkpoint not cleared: %v", got.GetStatus()) + if _, gotSt := findLatestSnapshotStorage(got.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS); gotSt.GetLocal().GetSnapshotName() != "partial-local-snapshot" { + t.Errorf("in-progress local checkpoint = %q, want %q preserved", gotSt.GetLocal().GetSnapshotName(), "partial-local-snapshot") } if a := firstAssignment(t, persistence, apiWorkerName); a != nil { t.Errorf("worker still hosts %v, want the assignment released", a) diff --git a/cmd/ateapi/internal/controlapi/workflow_resume.go b/cmd/ateapi/internal/controlapi/workflow_resume.go index 85223ec7d1..54b483b1ee 100644 --- a/cmd/ateapi/internal/controlapi/workflow_resume.go +++ b/cmd/ateapi/internal/controlapi/workflow_resume.go @@ -137,11 +137,12 @@ func (w *ActorWorkflow) ResumeActor(ctx context.Context, actorRef resources.Acto // commit MEMORY (preferredFidelity), so this only trips on a golden snapshot // whose record drifted — surface a clear error instead of shipping a restore // request atelet would reject (or that would boot an empty guest). -func validateGoldenSnapshotFidelity(snapshot *ateapipb.ExternalSnapshot) error { - if fidelity := snapshot.GetFidelity(); fidelity != ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY { +func validateGoldenSnapshotFidelity(snapshot *ateapipb.Snapshot) error { + st := findSnapshotStorage(snapshot, ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE) + if fidelity := st.GetFidelity(); fidelity != ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY { return apierror.FailedPrecondition( "ActorTemplate golden snapshot %q was taken with fidelity %s, not MEMORY; regenerate the golden snapshot", - snapshot.GetSnapshotUri(), fidelity) + st.GetObject().GetSnapshotUri(), fidelity) } return nil } @@ -172,12 +173,12 @@ func (w *ActorWorkflow) loadActorForResume(ctx context.Context, actorRef resourc if err != nil { return nil, nil, src, err } - if uri := actor.GetStatus().GetExternalSnapshot().GetSnapshotUri(); uri != "" { - if src.SnapshotURI, err = resources.ParseSnapshotURI(uri); err != nil { + if snap, st := findLatestSnapshotStorage(actor.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED); st != nil { + if src.SnapshotURI, err = resources.ParseSnapshotURI(st.GetObject().GetSnapshotUri()); err != nil { return nil, nil, src, apierror.DataLoss("Actor %s external snapshot: %v", actorRef, err) } - src.Fidelity = actor.GetStatus().GetExternalSnapshot().GetFidelity() - capturedUnder := actor.GetStatus().GetExternalSnapshot().GetActorTemplateUid() + src.Fidelity = st.GetFidelity() + capturedUnder := snap.GetActorTemplateUid() src.TemplateReplaced = capturedUnder != "" && capturedUnder != actorTemplate.GetMetadata().GetUid() } @@ -694,7 +695,7 @@ func (w *ActorWorkflow) ensureAteletRestored(ctx context.Context, actorRef resou return tele, fmt.Errorf("while resolving sandbox assets: %w", err) } - if local := actor.GetStatus().GetLocalSnapshot(); local != nil { + if _, localSt := findLatestSnapshotStorage(actor.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED); localSt != nil { slog.InfoContext(ctx, "Actor has snapshot; Restoring from snapshot") tele.SnapshotKind = ateattr.SnapshotKindLocal @@ -713,9 +714,9 @@ func (w *ActorWorkflow) ensureAteletRestored(ctx context.Context, actorRef resou } req.Type = ateletpb.CheckpointType_CHECKPOINT_TYPE_LOCAL req.Config = &ateletpb.RestoreRequest_LocalConfig{ - LocalConfig: &ateletpb.LocalCheckpointConfiguration{SnapshotName: local.GetSnapshotName()}, + LocalConfig: &ateletpb.LocalCheckpointConfiguration{SnapshotName: localSt.GetLocal().GetSnapshotName()}, } - req.Fidelity = fidelityToAtelet(local.GetFidelity()) + req.Fidelity = fidelityToAtelet(localSt.GetFidelity()) tele.WireFidelity = ateattr.SnapshotFidelityValue(req.Fidelity) if _, err = client.Restore(ctx, req); err != nil { diff --git a/cmd/ateapi/internal/controlapi/workflow_resume_test.go b/cmd/ateapi/internal/controlapi/workflow_resume_test.go index 4faa4d0752..9e14dad80c 100644 --- a/cmd/ateapi/internal/controlapi/workflow_resume_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_resume_test.go @@ -969,10 +969,9 @@ func TestLoadActorForResume_TemplateReplaced(t *testing.T) { var seedOpts []func(*ateapipb.Actor) if !tt.noSnapshot { seedOpts = append(seedOpts, func(a *ateapipb.Actor) { - a.Status.ExternalSnapshot = &ateapipb.ExternalSnapshot{ - SnapshotUri: someActorSnapshotURI(t, testStorageLocation, actorRef.Atespace, "snap-1"), - Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, - ActorTemplateUid: resolve(tt.snapshotUID), + a.Status.LastAssignedGeneration = 1 + a.Status.Snapshots = []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, resolve(tt.snapshotUID), someActorSnapshotURI(t, testStorageLocation, actorRef.Atespace, "snap-1"), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), } }) } @@ -1142,13 +1141,20 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { dataScope := ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES unspecScope := ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED + extSnap := func(uri string, fidelity ateapipb.SnapshotFidelity) *ateapipb.Snapshot { + return newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, fidelity, "", uri, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + } + localSnap := func(name string, fidelity ateapipb.SnapshotFidelity) *ateapipb.Snapshot { + return newLocalSnapshot(1, fidelity, "", name, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + } + // actorSeed is the actor status a row persists before resuming. type actorSeed struct { - // localSnapshot seeds Status.LocalSnapshot (the pause checkpoint); + // localSnapshot seeds a completed local snapshot (the pause checkpoint); // a non-nil value also parks the actor PAUSED instead of SUSPENDED. - localSnapshot *ateapipb.LocalSnapshot - // externalSnapshot seeds Status.ExternalSnapshot (the durable snapshot). - externalSnapshot *ateapipb.ExternalSnapshot + localSnapshot *ateapipb.Snapshot + // externalSnapshot seeds a completed durable snapshot. + externalSnapshot *ateapipb.Snapshot // tmplUID seeds the template UID the snapshot's guest state was built // on, stamped onto externalSnapshot: "current" stands for the created // template's store-assigned UID (unknown until runtime), "" leaves the @@ -1158,7 +1164,7 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { // templateSeed is the ActorTemplate configuration a row persists. type templateSeed struct { // golden seeds the template's golden tag snapshot. - golden *ateapipb.ExternalSnapshot + golden *ateapipb.Snapshot // configName is the SandboxConfig the template names; "" means the // "gvisor" config the workflow's lister serves. configName string @@ -1191,8 +1197,8 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { }, { name: "02 inherited golden snapshot restores in Full", - actor: actorSeed{externalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: goldenURI, Fidelity: fullScope}, tmplUID: "current"}, - tmpl: templateSeed{golden: &ateapipb.ExternalSnapshot{SnapshotUri: goldenURI, Fidelity: fullScope}}, + actor: actorSeed{externalSnapshot: extSnap(goldenURI, fullScope), tmplUID: "current"}, + tmpl: templateSeed{golden: extSnap(goldenURI, fullScope)}, want: restoreWant{ checkpointType: ateletpb.CheckpointType_CHECKPOINT_TYPE_EXTERNAL, snapshotURI: goldenURI, @@ -1201,24 +1207,24 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { }, { name: "03 late non-Full golden does not change a cold boot", - tmpl: templateSeed{golden: &ateapipb.ExternalSnapshot{SnapshotUri: goldenURI, Fidelity: dataScope}}, + tmpl: templateSeed{golden: extSnap(goldenURI, dataScope)}, want: restoreWant{run: true}, }, { name: "04 inherited golden snapshot rejects a malformed URI", - actor: actorSeed{externalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: malformedURI, Fidelity: fullScope}, tmplUID: "current"}, - tmpl: templateSeed{golden: &ateapipb.ExternalSnapshot{SnapshotUri: malformedURI, Fidelity: fullScope}}, + actor: actorSeed{externalSnapshot: extSnap(malformedURI, fullScope), tmplUID: "current"}, + tmpl: templateSeed{golden: extSnap(malformedURI, fullScope)}, want: restoreWant{code: codes.DataLoss}, }, { name: "05 template repoint with a late golden still cold-boots", actor: actorSeed{tmplUID: "old-template-uid"}, - tmpl: templateSeed{golden: &ateapipb.ExternalSnapshot{SnapshotUri: goldenURI, Fidelity: fullScope}}, + tmpl: templateSeed{golden: extSnap(goldenURI, fullScope)}, want: restoreWant{run: true}, }, { name: "06 Full durable snapshot restores itself in Full", - actor: actorSeed{externalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: actorURI, Fidelity: fullScope}}, + actor: actorSeed{externalSnapshot: extSnap(actorURI, fullScope)}, want: restoreWant{ checkpointType: ateletpb.CheckpointType_CHECKPOINT_TYPE_EXTERNAL, snapshotURI: actorURI, @@ -1228,7 +1234,7 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { { name: "07 durable snapshot built on the current template stays Full", actor: actorSeed{ - externalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: actorURI, Fidelity: fullScope}, + externalSnapshot: extSnap(actorURI, fullScope), tmplUID: "current", }, want: restoreWant{ @@ -1243,7 +1249,7 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { // boots fresh and only the volume data carries over. name: "08 repointed actor's Full durable snapshot drops to Data", actor: actorSeed{ - externalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: actorURI, Fidelity: fullScope}, + externalSnapshot: extSnap(actorURI, fullScope), tmplUID: "mismatch", }, want: restoreWant{ @@ -1254,7 +1260,7 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { }, { name: "09 Data durable snapshot restores as Data", - actor: actorSeed{externalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: actorURI, Fidelity: dataScope}}, + actor: actorSeed{externalSnapshot: extSnap(actorURI, dataScope)}, want: restoreWant{ checkpointType: ateletpb.CheckpointType_CHECKPOINT_TYPE_EXTERNAL, snapshotURI: actorURI, @@ -1265,8 +1271,8 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { // The template's golden snapshot never supplies guest state for a // Data snapshot: it restores as plain Data. name: "10 Data durable snapshot ignores the template's golden", - actor: actorSeed{externalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: actorURI, Fidelity: dataScope}}, - tmpl: templateSeed{golden: &ateapipb.ExternalSnapshot{SnapshotUri: goldenURI, Fidelity: fullScope}}, + actor: actorSeed{externalSnapshot: extSnap(actorURI, dataScope)}, + tmpl: templateSeed{golden: extSnap(goldenURI, fullScope)}, want: restoreWant{ checkpointType: ateletpb.CheckpointType_CHECKPOINT_TYPE_EXTERNAL, snapshotURI: actorURI, @@ -1274,13 +1280,11 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { }, }, { - // Snapshots recorded before fidelity existed carry - // UNSPECIFIED; the conversion sends them out as Full. // A stored snapshot always records its fidelity; one without is a // drifted record, and ateapi forwards it unchanged for atelet's // validation to reject rather than guessing MEMORY. name: "11 unspecified durable fidelity goes out unspecified", - actor: actorSeed{externalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: actorURI, Fidelity: unspecScope}}, + actor: actorSeed{externalSnapshot: extSnap(actorURI, unspecScope)}, want: restoreWant{ checkpointType: ateletpb.CheckpointType_CHECKPOINT_TYPE_EXTERNAL, snapshotURI: actorURI, @@ -1289,13 +1293,13 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { }, { name: "12 malformed durable snapshot URI fails with DataLoss", - actor: actorSeed{externalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: malformedURI, Fidelity: fullScope}}, + actor: actorSeed{externalSnapshot: extSnap(malformedURI, fullScope)}, want: restoreWant{code: codes.DataLoss}, }, { name: "13 Full pause snapshot restores locally as Full", actor: actorSeed{ - localSnapshot: &ateapipb.LocalSnapshot{SnapshotName: localSnapshotName, Fidelity: fullScope}, + localSnapshot: localSnap(localSnapshotName, fullScope), }, want: restoreWant{ checkpointType: ateletpb.CheckpointType_CHECKPOINT_TYPE_LOCAL, @@ -1309,7 +1313,7 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { // resume-source resolution is being reworked. name: "14 local snapshot built on the current template stays Full", actor: actorSeed{ - localSnapshot: &ateapipb.LocalSnapshot{SnapshotName: localSnapshotName, Fidelity: fullScope}, + localSnapshot: localSnap(localSnapshotName, fullScope), tmplUID: "current", }, want: restoreWant{ @@ -1321,7 +1325,7 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { { name: "15 Data pause snapshot restores locally as Data", actor: actorSeed{ - localSnapshot: &ateapipb.LocalSnapshot{SnapshotName: localSnapshotName, Fidelity: dataScope}, + localSnapshot: localSnap(localSnapshotName, dataScope), }, want: restoreWant{ checkpointType: ateletpb.CheckpointType_CHECKPOINT_TYPE_LOCAL, @@ -1332,10 +1336,10 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { { name: "16 Data pause snapshot ignores the template's golden", actor: actorSeed{ - localSnapshot: &ateapipb.LocalSnapshot{SnapshotName: localSnapshotName, Fidelity: dataScope}, + localSnapshot: localSnap(localSnapshotName, dataScope), }, tmpl: templateSeed{ - golden: &ateapipb.ExternalSnapshot{SnapshotUri: goldenURI, Fidelity: fullScope}, + golden: extSnap(goldenURI, fullScope), }, want: restoreWant{ checkpointType: ateletpb.CheckpointType_CHECKPOINT_TYPE_LOCAL, @@ -1348,8 +1352,8 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { // scope wins over the durable snapshot's. name: "17 local snapshot wins over a Full durable snapshot", actor: actorSeed{ - localSnapshot: &ateapipb.LocalSnapshot{SnapshotName: localSnapshotName, Fidelity: dataScope}, - externalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: actorURI, Fidelity: fullScope}, + localSnapshot: localSnap(localSnapshotName, dataScope), + externalSnapshot: extSnap(actorURI, fullScope), }, want: restoreWant{ checkpointType: ateletpb.CheckpointType_CHECKPOINT_TYPE_LOCAL, @@ -1364,8 +1368,8 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { // template. name: "18 local snapshot ignores an older external snapshot's template mismatch", actor: actorSeed{ - localSnapshot: &ateapipb.LocalSnapshot{SnapshotName: localSnapshotName, Fidelity: fullScope}, - externalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: actorURI, Fidelity: fullScope}, + localSnapshot: localSnap(localSnapshotName, fullScope), + externalSnapshot: extSnap(actorURI, fullScope), tmplUID: "mismatch", }, want: restoreWant{ @@ -1378,13 +1382,13 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { // Restores take their sandbox from the template, like cold boots, // so an unresolvable SandboxConfig stops them before atelet. name: "19 durable snapshot restore with a missing SandboxConfig is rejected", - actor: actorSeed{externalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: actorURI, Fidelity: fullScope}}, + actor: actorSeed{externalSnapshot: extSnap(actorURI, fullScope)}, tmpl: templateSeed{configName: "missing"}, want: restoreWant{code: codes.FailedPrecondition}, }, { name: "20 local snapshot restore with a missing SandboxConfig is rejected", - actor: actorSeed{localSnapshot: &ateapipb.LocalSnapshot{SnapshotName: localSnapshotName}}, + actor: actorSeed{localSnapshot: localSnap(localSnapshotName, unspecScope)}, tmpl: templateSeed{configName: "missing"}, want: restoreWant{code: codes.FailedPrecondition}, }, @@ -1450,21 +1454,31 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { actorRef := resources.ActorRef{Atespace: "team-a", Name: "id1"} seedWorkflowActor(t, ctx, persistence, actorRef, "ns", "tmpl1", actorState, func(a *ateapipb.Actor) { a.Status.WorkerAssignment = wireTestAssignment() - a.Status.LocalSnapshot = tt.actor.localSnapshot - if tt.actor.localSnapshot != nil { - a.Status.AssignedNode = "node-1" - } uid := tt.actor.tmplUID if uid == "current" { uid = createdTmpl.GetMetadata().GetUid() } + var gen int32 if tt.actor.externalSnapshot != nil { + gen++ ext := proto.CloneOf(tt.actor.externalSnapshot) + ext.Generation = gen if ext.ActorTemplateUid == "" { ext.ActorTemplateUid = uid } - a.Status.ExternalSnapshot = ext + a.Status.Snapshots = append(a.Status.Snapshots, ext) + } + if tt.actor.localSnapshot != nil { + gen++ + a.Status.AssignedNode = "node-1" + loc := proto.CloneOf(tt.actor.localSnapshot) + loc.Generation = gen + if loc.ActorTemplateUid == "" { + loc.ActorTemplateUid = uid + } + a.Status.Snapshots = append(a.Status.Snapshots, loc) } + a.Status.LastAssignedGeneration = gen }) actor, loadedTmpl, src, err := w.loadActorForResume(ctx, actorRef) diff --git a/cmd/ateapi/internal/controlapi/workflow_revert.go b/cmd/ateapi/internal/controlapi/workflow_revert.go index 26695ee975..2f4b3c6e2f 100644 --- a/cmd/ateapi/internal/controlapi/workflow_revert.go +++ b/cmd/ateapi/internal/controlapi/workflow_revert.go @@ -199,7 +199,8 @@ func (w *ActorWorkflow) ensureInProgressSnapshotDiscarded(ctx context.Context, a ctx, done := stepSpan(ctx, "DiscardInProgressSnapshot") defer func() { err = done(err) }() - inProgress := actor.GetStatus().GetInProgressSnapshotUri() + _, inProgressSt := findLatestSnapshotStorage(actor.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS) + inProgress := inProgressSt.GetObject().GetSnapshotUri() switch { case w.snapshotPlugin == nil: markSkipped(ctx, "no object store configured") @@ -245,9 +246,13 @@ func (w *ActorWorkflow) ensureRevertedFinalized(ctx context.Context, actorRef re toUpdate.Status.State = ateapipb.ActorState_ACTOR_STATE_SUSPENDED toUpdate.Status.WorkerAssignment = nil toUpdate.Status.AssignedNode = "" - toUpdate.Status.InProgressSnapshotUri = "" - toUpdate.Status.InProgressLocalSnapshotName = "" - toUpdate.Status.LocalSnapshot = nil + // We revert back to the latest durable snapshot, so all local snapshots become invalid. + // We already cleaned up the node before this in ensureWorkerDiscarded. + // Even during revert, we do not decrease LastAssignedGeneration so that we + // can keep track of stale snapshots that need to be cleaned up. + removeSnapshotStorageEntries(toUpdate.Status, ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL, nil) + // We already cleaned up the inprogress durable snapshot in ensureInProgressSnapshotDiscarded + removeSnapshotStorageEntries(toUpdate.Status, ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, new(ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS)) toUpdate.Status.Crash = nil return nil }) diff --git a/cmd/ateapi/internal/controlapi/workflow_revert_test.go b/cmd/ateapi/internal/controlapi/workflow_revert_test.go index 15dcd49373..acec0029e5 100644 --- a/cmd/ateapi/internal/controlapi/workflow_revert_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_revert_test.go @@ -97,9 +97,12 @@ func TestRevertActor_ReturnsActorToItsSnapshot(t *testing.T) { // pause left behind, which it must not. const keptURI = "gs://snapshots/team-a/actors/keep/snapshot" mustUpdateActorStatus(t, ctx, st, actor, func(s *ateapipb.ActorStatus) { - s.ExternalSnapshot = &ateapipb.ExternalSnapshot{SnapshotUri: keptURI} - s.LocalSnapshot = &ateapipb.LocalSnapshot{SnapshotName: "local-1"} - s.InProgressLocalSnapshotName = "local-in-progress" + s.LastAssignedGeneration = 3 + s.Snapshots = []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", keptURI, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + newLocalSnapshot(2, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "local-1", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + newLocalSnapshot(3, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "local-in-progress", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS), + } }) reverted, err := w.RevertActor(ctx, actorRef) @@ -111,20 +114,30 @@ func TestRevertActor_ReturnsActorToItsSnapshot(t *testing.T) { if got := gotStatus.GetState(); got != ateapipb.ActorState_ACTOR_STATE_SUSPENDED { t.Errorf("state = %v, want SUSPENDED", got) } - if got := gotStatus.GetExternalSnapshot().GetSnapshotUri(); got != keptURI { + // LastAssignedGeneration is not decreased on revert so we can keep track + // of stale snapshots that need to be cleaned up and future snapshots + // never reuse generations 2 or 3 after revert discards them. + if got := gotStatus.GetLastAssignedGeneration(); got != 3 { + t.Errorf("last_assigned_generation = %d, want 3", got) + } + gotSnap, gotSt := findLatestSnapshotStorage(gotStatus, ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + if got := gotSnap.GetGeneration(); got != 1 { + t.Errorf("retained snapshot generation = %d, want 1", got) + } + if got := gotSt.GetObject().GetSnapshotUri(); got != keptURI { t.Errorf("external snapshot = %q, want it untouched at %q", got, keptURI) } // Resume prefers a local snapshot over the external one, so a // surviving LocalSnapshot would restore the execution this // revert just discarded. - if got := gotStatus.GetLocalSnapshot(); got != nil { + if _, got := findLatestSnapshotStorage(gotStatus, ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED); got != nil { t.Errorf("local snapshot info = %v, want nil", got) } - if got := gotStatus.GetInProgressSnapshotUri(); got != "" { - t.Errorf("in-progress snapshot uri = %q, want empty", got) + if _, got := findLatestSnapshotStorage(gotStatus, ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS); got != nil { + t.Errorf("in-progress durable snapshot = %v, want nil", got) } - if got := gotStatus.GetInProgressLocalSnapshotName(); got != "" { - t.Errorf("in-progress local snapshot name = %q, want empty", got) + if _, got := findLatestSnapshotStorage(gotStatus, ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS); got != nil { + t.Errorf("in-progress local snapshot = %v, want nil", got) } if got := gotStatus.GetAssignedNode(); got != "" { t.Errorf("assigned node = %q, want empty", got) @@ -132,6 +145,25 @@ func TestRevertActor_ReturnsActorToItsSnapshot(t *testing.T) { if got := gotStatus.GetWorkerAssignment(); got != nil { t.Errorf("worker assignment = %v, want nil", got) } + + // If the actor runs again and takes another snapshot, it must + // allocate generation 4 rather than reusing 2 or 3. + running := mustUpdateActorStatus(t, ctx, st, reverted, func(s *ateapipb.ActorStatus) { + s.State = ateapipb.ActorState_ACTOR_STATE_RUNNING + }) + tmpl := &ateapipb.ActorTemplate{ + SnapshotConfig: &ateapipb.SnapshotConfig{StorageLocation: "gs://snapshots/"}, + } + suspending, err := w.ensureMarkedSuspending(ctx, actorRef, running, tmpl) + if err != nil { + t.Fatalf("ensureMarkedSuspending after revert: %v", err) + } + if got := suspending.GetStatus().GetLastAssignedGeneration(); got != 4 { + t.Errorf("last_assigned_generation after post-revert suspend = %d, want 4", got) + } + if got := snapshotAtLatestGeneration(suspending.GetStatus()).GetGeneration(); got != 4 { + t.Errorf("new snapshot generation after post-revert suspend = %d, want 4", got) + } }) } } @@ -219,7 +251,8 @@ func TestRevertActor_NoSnapshotToRevertTo(t *testing.T) { if got := reverted.GetStatus().GetCrash(); got != nil { t.Errorf("Crash = %v, want cleared by the revert", got) } - if got := reverted.GetStatus().GetExternalSnapshot().GetSnapshotUri(); got != "" { + _, gotSt := findLatestSnapshotStorage(reverted.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + if got := gotSt.GetObject().GetSnapshotUri(); got != "" { t.Errorf("external snapshot = %q, want none", got) } } @@ -283,8 +316,10 @@ func TestEnsureInProgressSnapshotDiscarded(t *testing.T) { // The actor's prefix is keyed on the UID the store just assigned, // so its snapshots can only be placed now. current := mustActorSnapshotURI(t, template, actor, "current") + currentType := ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR if tt.tagOwnedSnapshot { current = mustTagSnapshotURI(t, template, "team-a", "v1-snapshot") + currentType = ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG } objects.PutSnapshot(t, current, "manifest.json") inFlight := mustActorSnapshotURI(t, template, actor, inFlightSnapshotName) @@ -295,9 +330,13 @@ func TestEnsureInProgressSnapshotDiscarded(t *testing.T) { objects.PutSnapshot(t, inFlight, "manifest.json") } actor = mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapipb.ActorStatus) { - s.ExternalSnapshot = &ateapipb.ExternalSnapshot{SnapshotUri: current.String()} + s.LastAssignedGeneration = 1 + s.Snapshots = []*ateapipb.Snapshot{ + newDurableSnapshot(1, currentType, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", current.String(), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + } if tt.inFlight != "" { - s.InProgressSnapshotUri = inFlight.String() + s.LastAssignedGeneration = 2 + s.Snapshots = append(s.Snapshots, newDurableSnapshot(2, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", inFlight.String(), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS)) } }) @@ -342,9 +381,12 @@ func TestEnsureRevertedFinalized_NoObjectStore(t *testing.T) { Metadata: &ateapipb.ResourceMetadata{Atespace: "team-a", Name: "actor-1"}, ActorTemplate: &ateapipb.ObjectRef{Atespace: "team-a", Name: "sub-tmpl"}, Status: &ateapipb.ActorStatus{ - State: ateapipb.ActorState_ACTOR_STATE_REVERTING, - AssignedNode: "node-1", - InProgressSnapshotUri: someActorSnapshotURI(t, testStorageLocation, "team-a", "abandoned"), + State: ateapipb.ActorState_ACTOR_STATE_REVERTING, + AssignedNode: "node-1", + LastAssignedGeneration: 1, + Snapshots: []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", someActorSnapshotURI(t, testStorageLocation, "team-a", "abandoned"), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS), + }, }, }) @@ -363,7 +405,7 @@ func TestEnsureRevertedFinalized_NoObjectStore(t *testing.T) { if got := finalized.GetStatus().GetAssignedNode(); got != "" { t.Errorf("assigned node = %q, want empty", got) } - if got := finalized.GetStatus().GetInProgressSnapshotUri(); got != "" { - t.Errorf("in-progress snapshot uri = %q, want empty", got) + if _, got := findLatestSnapshotStorage(finalized.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS); got != nil { + t.Errorf("in-progress durable snapshot = %v, want nil", got) } } diff --git a/cmd/ateapi/internal/controlapi/workflow_suspend.go b/cmd/ateapi/internal/controlapi/workflow_suspend.go index 1caa7d8270..e6d462ddbb 100644 --- a/cmd/ateapi/internal/controlapi/workflow_suspend.go +++ b/cmd/ateapi/internal/controlapi/workflow_suspend.go @@ -28,7 +28,6 @@ import ( "github.com/agent-substrate/substrate/internal/resources" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "go.opentelemetry.io/otel/attribute" - "google.golang.org/protobuf/proto" ) // SuspendActor executes the workflow to suspend a running or paused actor: @@ -93,7 +92,7 @@ func (w *ActorWorkflow) SuspendActor(ctx context.Context, actorRef resources.Act // them here, as crash.go does for the crash counter. finalAttrs = lifecycleOpAttrs(actor, actorTemplate, "", wireFidelity) var finalized *ateapipb.Actor - if finalized, err = w.ensureSuspendedFinalized(leaseCtx, actorRef, actorTemplate); err != nil { + if finalized, err = w.ensureSuspendedFinalized(leaseCtx, actorRef); err != nil { return nil, err } actor = finalized @@ -132,6 +131,16 @@ func (w *ActorWorkflow) ensureMarkedSuspending(ctx context.Context, actorRef res if got := actor.GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_RUNNING && got != ateapipb.ActorState_ACTOR_STATE_PAUSED { return nil, apierror.FailedPrecondition("MarkSuspending prerequisite not met for Actor: %s (got: %v, want %s or %s)", actorRef, got, ateapipb.ActorState_ACTOR_STATE_RUNNING, ateapipb.ActorState_ACTOR_STATE_PAUSED) } + if actor.GetStatus().GetState() == ateapipb.ActorState_ACTOR_STATE_PAUSED { + localSnap, _ := findLatestSnapshotStorage(actor.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + if localSnap == nil { + if err := crashActor(ctx, w.store, actorRef, ateattr.OperationSuspend, crashMessageLocalSnapshotNodeUnknown); err != nil { + slog.ErrorContext(ctx, "Failed to crash actor", slog.String("err", err.Error())) + } + return nil, fmt.Errorf("actor is CRASHED because it was in PAUSED state with no completed local snapshot") + } + } + // Fail here rather than at checkpoint time if the template's location // cannot produce a usable URI: nothing has been written yet. uri, err := newInProgressSnapshotURI(actorTemplate, actor) @@ -139,8 +148,37 @@ func (w *ActorWorkflow) ensureMarkedSuspending(ctx context.Context, actorRef res return nil, err } storedActor, err := w.store.UpdateActor(ctx, actorRef, store.PreconditionFrom(actor), func(toUpdate *ateapipb.Actor) error { + wasPaused := toUpdate.Status.State == ateapipb.ActorState_ACTOR_STATE_PAUSED toUpdate.Status.State = ateapipb.ActorState_ACTOR_STATE_SUSPENDING - toUpdate.Status.InProgressSnapshotUri = uri.String() + if wasPaused { + // Add a new in progress durable snapshot to the current generation + snap, _ := findLatestSnapshotStorage(toUpdate.Status, ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + if snap == nil { + return fmt.Errorf("actor %s is PAUSED but has no completed local snapshot", actorRef) + } + snap.ActorTemplateUid = actorTemplate.GetMetadata().GetUid() + durableInProgress := &ateapipb.SnapshotStorage{ + Durability: ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, + Status: ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS, + Fidelity: preferredFidelity(actorRef.Atespace, actorTemplate), + Object: &ateapipb.ObjectSnapshot{SnapshotUri: uri.String()}, + } + setSnapshotStorage(snap, durableInProgress) + return nil + } + // Increment last_assigned_generation for the new Suspend request and create + // an in-progress snapshot for that generation. + gen := toUpdate.Status.LastAssignedGeneration + 1 + toUpdate.Status.LastAssignedGeneration = gen + toUpdate.Status.Snapshots = append(toUpdate.Status.Snapshots, + newDurableSnapshot( + gen, + ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, + preferredFidelity(actorRef.Atespace, actorTemplate), + actorTemplate.GetMetadata().GetUid(), + uri.String(), + ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS, + )) return nil }) if err != nil { @@ -171,10 +209,15 @@ func preferredFidelity(atespace string, tmpl *ateapipb.ActorTemplate) ateapipb.S // nil worker assignment disambiguates — running-origin suspends keep their // assignment until finalize, paused actors never have one. func isPausedOriginSuspend(actor *ateapipb.Actor) bool { - return actor.GetStatus().GetState() == ateapipb.ActorState_ACTOR_STATE_PAUSED || - (actor.GetStatus().GetState() == ateapipb.ActorState_ACTOR_STATE_SUSPENDING && - actor.GetStatus().GetWorkerAssignment() == nil && - actor.GetStatus().GetLocalSnapshot() != nil) + if actor.GetStatus().GetState() == ateapipb.ActorState_ACTOR_STATE_PAUSED { + return true + } + if actor.GetStatus().GetState() == ateapipb.ActorState_ACTOR_STATE_SUSPENDING && + actor.GetStatus().GetWorkerAssignment() == nil { + _, localSt := findLatestSnapshotStorage(actor.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + return localSt != nil + } + return false } // ensureAteletSuspended checkpoints the workload to the actor's persisted @@ -210,6 +253,7 @@ func (w *ActorWorkflow) ensureAteletSuspended(ctx context.Context, actorRef reso // Checkpoint does not carry the sandbox config: atelet uses the version the // actor is currently running (recorded on-node at Run/Restore) and pins it // into the snapshot manifest. + _, inProgressDurable := findLatestSnapshotStorage(actor.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS) req := &ateletpb.CheckpointRequest{ TargetAteomUid: assignment.GetWorkerPodUid(), Atespace: actor.GetMetadata().GetAtespace(), @@ -220,7 +264,7 @@ func (w *ActorWorkflow) ensureAteletSuspended(ctx context.Context, actorRef reso Type: ateletpb.CheckpointType_CHECKPOINT_TYPE_EXTERNAL, Config: &ateletpb.CheckpointRequest_ExternalConfig{ ExternalConfig: &ateletpb.ExternalCheckpointConfiguration{ - SnapshotUri: actor.GetStatus().GetInProgressSnapshotUri(), + SnapshotUri: inProgressDurable.GetObject().GetSnapshotUri(), }, }, Fidelity: fidelityToAtelet(preferredFidelity(actor.GetMetadata().GetAtespace(), actorTemplate)), @@ -244,9 +288,9 @@ func (w *ActorWorkflow) ensurePausedSnapshotUploaded(ctx context.Context, actorR ctx, done := stepSpan(ctx, "UploadPausedCheckpoint") defer func() { err = done(err) }() - local := actor.GetStatus().GetLocalSnapshot() + _, localSt := findLatestSnapshotStorage(actor.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) nodeName := actor.GetStatus().GetAssignedNode() - if nodeName == "" { + if localSt == nil || nodeName == "" { // Without the node the snapshot can never be found (mirrors // FinalizePaused, which crashes rather than record an unknown node). if err := crashActor(ctx, w.store, actorRef, ateattr.OperationSuspend, crashMessageLocalSnapshotNodeUnknown); err != nil { @@ -264,14 +308,15 @@ func (w *ActorWorkflow) ensurePausedSnapshotUploaded(ctx context.Context, actorR } client := ateletpb.NewAteomHerderClient(ateletConn) + _, inProgressDurable := findLatestSnapshotStorage(actor.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS) req := &ateletpb.UploadPausedCheckpointRequest{ Atespace: actor.GetMetadata().GetAtespace(), ActorName: actor.GetMetadata().GetName(), ActorUid: actor.GetMetadata().GetUid(), ActorTemplateAtespace: actor.GetActorTemplate().GetAtespace(), ActorTemplateName: actor.GetActorTemplate().GetName(), - LocalSnapshotName: local.GetSnapshotName(), - DestinationSnapshotUri: actor.GetStatus().GetInProgressSnapshotUri(), + LocalSnapshotName: localSt.GetLocal().GetSnapshotName(), + DestinationSnapshotUri: inProgressDurable.GetObject().GetSnapshotUri(), // The commit scope, like a running-origin suspend; atelet converts // from the captured scope in the snapshot's manifest where possible. DesiredFidelity: fidelityToAtelet(preferredFidelity(actor.GetMetadata().GetAtespace(), actorTemplate)), @@ -309,12 +354,12 @@ func (w *ActorWorkflow) ensureVolumesDetached(ctx context.Context, actor *ateapi // ensureSuspendedFinalized releases the actor's worker (only when it is still // owned by this actor), records the in-progress snapshot as the actor's -// external snapshot, and commits SUSPENDED with the assignment cleared in a -// single update, then releases the external snapshot that update replaced. +// durable snapshot, and commits SUSPENDED with the assignment cleared in a +// single update, then releases the durable snapshot that update replaced. // It re-reads the actor first so an out-of-band transition (e.g. the syncer // crashing the actor after its worker died) is not overwritten: with no // assignment left there is nothing to finalize. -func (w *ActorWorkflow) ensureSuspendedFinalized(ctx context.Context, actorRef resources.ActorRef, actorTemplate *ateapipb.ActorTemplate) (_ *ateapipb.Actor, err error) { +func (w *ActorWorkflow) ensureSuspendedFinalized(ctx context.Context, actorRef resources.ActorRef) (_ *ateapipb.Actor, err error) { ctx, done := stepSpan(ctx, "FinalizeSuspended") defer func() { err = done(err) }() @@ -360,32 +405,29 @@ func (w *ActorWorkflow) ensureSuspendedFinalized(ctx context.Context, actorRef r } } - // 2. Finalize the actor: record its new external snapshot and mark it SUSPENDED. This + // 2. Finalize the actor: record its new durable snapshot and mark it SUSPENDED. This // must run even with no worker assignment (nothing to free), or the actor // would be left SUSPENDING forever with the workflow reporting success. - inProgressSnapshotURI := latestActor.GetStatus().GetInProgressSnapshotUri() - externalSnapshot := latestActor.GetStatus().GetExternalSnapshot() - if inProgressSnapshotURI != "" { - externalSnapshot = &ateapipb.ExternalSnapshot{ - SnapshotUri: inProgressSnapshotURI, - Fidelity: preferredFidelity(actorRef.Atespace, actorTemplate), - ActorTemplateUid: actorTemplate.GetMetadata().GetUid(), - } - } + _, prevSt := findLatestSnapshotStorage(latestActor.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) // 3. Commit the actor. t = time.Now() storedActor, err := w.store.UpdateActor(ctx, actorRef, store.PreconditionFrom(latestActor), func(toUpdate *ateapipb.Actor) error { toUpdate.Status.State = ateapipb.ActorState_ACTOR_STATE_SUSPENDED - if inProgressSnapshotURI != "" { - // The recorded URI is under the actor's own prefix, so the actor now - // owns its external snapshot rather than borrowing the tag's it may - // have been created from. - toUpdate.Status.ExternalSnapshot = proto.CloneOf(externalSnapshot) - toUpdate.Status.InProgressSnapshotUri = "" + snap := snapshotAtLatestGeneration(toUpdate.Status) + if snap == nil { + return fmt.Errorf("actor %s has no latest snapshot", actorRef) + } + if inProgressSt := findSnapshotStorage(snap, ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE); inProgressSt.GetObject().GetSnapshotUri() != "" { + inProgressSt.Status = ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED } + // Remove all local snapshots since they are superseded by the new durable snapshot. + // TODO: This will change when we merge pause and suspend and allow for both local + // and durable snapshots to be kept. + removeSnapshotStorageEntries(toUpdate.Status, ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL, nil) + // Remove all durable snapshots older than the new durable snapshot. + removeOlderSnapshotStorageEntries(toUpdate.Status, ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, new(ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), snap.GetGeneration()) toUpdate.Status.WorkerAssignment = nil - toUpdate.Status.LocalSnapshot = nil toUpdate.Status.AssignedNode = "" return nil }) @@ -404,12 +446,13 @@ func (w *ActorWorkflow) ensureSuspendedFinalized(ctx context.Context, actorRef r // snapshot's objects in storage until the actor is deleted, which removes // its whole prefix. t = time.Now() - releaseErr := w.releaseReplacedSnapshot(ctx, latestActor, externalSnapshot) + _, nextSt := findLatestSnapshotStorage(storedActor.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + releaseErr := w.releaseReplacedSnapshot(ctx, latestActor, nextSt) dReleaseSnapshot = time.Since(t) if releaseErr != nil { slog.WarnContext(ctx, "Failed to release the external snapshot a suspend replaced; its objects are left in storage until the actor is deleted", slog.Any("actor", actorRef), - slog.String("snapshot_uri", latestActor.GetStatus().GetExternalSnapshot().GetSnapshotUri()), + slog.String("snapshot_uri", prevSt.GetObject().GetSnapshotUri()), slog.String("err", releaseErr.Error())) } return storedActor, nil @@ -419,11 +462,12 @@ func (w *ActorWorkflow) ensureSuspendedFinalized(ctx context.Context, actorRef r // this suspend. actor is the record as it was before the suspend committed. // An actor that borrowed its current snapshot from a tag releases nothing — // the snapshot lives under the tag's prefix, and the tag outlives the actor. -func (w *ActorWorkflow) releaseReplacedSnapshot(ctx context.Context, actor *ateapipb.Actor, nextSnapshot *ateapipb.ExternalSnapshot) (err error) { +func (w *ActorWorkflow) releaseReplacedSnapshot(ctx context.Context, actor *ateapipb.Actor, nextSnapshot *ateapipb.SnapshotStorage) (err error) { ctx, done := stepSpan(ctx, "ReleaseReplacedSnapshot") defer func() { err = done(err) }() - previous := actor.GetStatus().GetExternalSnapshot().GetSnapshotUri() + _, prevSt := findLatestSnapshotStorage(actor.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + previous := prevSt.GetObject().GetSnapshotUri() switch { case w.snapshotPlugin == nil: markSkipped(ctx, "no object store configured") @@ -431,7 +475,7 @@ func (w *ActorWorkflow) releaseReplacedSnapshot(ctx context.Context, actor *atea case previous == "": markSkipped(ctx, "the actor held no external snapshot") return nil - case previous == nextSnapshot.GetSnapshotUri(): + case previous == nextSnapshot.GetObject().GetSnapshotUri(): markSkipped(ctx, "the actor's external snapshot is unchanged") return nil } diff --git a/cmd/ateapi/internal/controlapi/workflow_suspend_test.go b/cmd/ateapi/internal/controlapi/workflow_suspend_test.go index 2c651628c1..845283878b 100644 --- a/cmd/ateapi/internal/controlapi/workflow_suspend_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_suspend_test.go @@ -48,7 +48,8 @@ func TestEnsureMarkedSuspending_SnapshotURI(t *testing.T) { t.Fatalf("ensureMarkedSuspending: %v", err) } - uri := mustParseSnapshotURI(t, marked.GetStatus().GetInProgressSnapshotUri()) + _, inProgressSt := findLatestSnapshotStorage(marked.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS) + uri := mustParseSnapshotURI(t, inProgressSt.GetObject().GetSnapshotUri()) if !resources.IsValidResourceName(uri.Name()) { t.Errorf("in-progress snapshot name = %q, want a valid resource name", uri.Name()) } @@ -69,8 +70,11 @@ func TestEnsureMarkedSuspending_ReentryKeepsPersistedSnapshotLocation(t *testing actor := storetest.MustCreateActor(t, ctx, persistence, &ateapipb.Actor{ Metadata: &ateapipb.ResourceMetadata{Atespace: "team-a", Name: "actor-1"}, Status: &ateapipb.ActorStatus{ - State: ateapipb.ActorState_ACTOR_STATE_SUSPENDING, - InProgressSnapshotUri: firstAttempt, + State: ateapipb.ActorState_ACTOR_STATE_SUSPENDING, + LastAssignedGeneration: 1, + Snapshots: []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", firstAttempt, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS), + }, }, }) w := &ActorWorkflow{store: persistence} @@ -78,8 +82,8 @@ func TestEnsureMarkedSuspending_ReentryKeepsPersistedSnapshotLocation(t *testing if err != nil { t.Fatalf("ensureMarkedSuspending: %v", err) } - if got := marked.GetStatus().GetInProgressSnapshotUri(); got != firstAttempt { - t.Errorf("InProgressSnapshotUri = %q, want the first attempt's location", got) + if _, gotSt := findLatestSnapshotStorage(marked.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS); gotSt.GetObject().GetSnapshotUri() != firstAttempt { + t.Errorf("in-progress durable snapshot URI = %q, want the first attempt's location", gotSt.GetObject().GetSnapshotUri()) } } @@ -157,9 +161,16 @@ func TestEnsureMarkedSuspending_StateMatrix(t *testing.T) { w := &ActorWorkflow{store: persistence} actorRef := resources.ActorRef{Atespace: "team-a", Name: "id1"} + actorStatus := &ateapipb.ActorStatus{State: seedState} + if seedState == ateapipb.ActorState_ACTOR_STATE_PAUSED { + actorStatus.LastAssignedGeneration = 1 + actorStatus.Snapshots = []*ateapipb.Snapshot{ + newLocalSnapshot(1, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "snap-1", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + } + } actor := storetest.MustCreateActor(t, ctx, persistence, &ateapipb.Actor{ Metadata: &ateapipb.ResourceMetadata{Atespace: actorRef.Atespace, Name: actorRef.Name}, - Status: &ateapipb.ActorStatus{State: seedState}, + Status: actorStatus, }) tmpl := &ateapipb.ActorTemplate{ @@ -237,6 +248,12 @@ func TestEnsureAteletSuspended_DialFailureLeavesActorRetryable(t *testing.T) { t.Run(tt.name, func(t *testing.T) { ctx := context.Background() persistence := newTestPersistence(t) + var snapshots []*ateapipb.Snapshot + if tt.prevSnapshot != "" { + snapshots = append(snapshots, newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", tt.prevSnapshot, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED)) + } + snapshots = append(snapshots, newDurableSnapshot(2, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", neverWritten, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS)) + actor := &ateapipb.Actor{ Metadata: &ateapipb.ResourceMetadata{Atespace: "team-a", Name: "actor-1"}, Status: &ateapipb.ActorStatus{ @@ -247,8 +264,8 @@ func TestEnsureAteletSuspended_DialFailureLeavesActorRetryable(t *testing.T) { WorkerPod: "pod-gone", NodeName: "node-gone", }, - InProgressSnapshotUri: neverWritten, - ExternalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: tt.prevSnapshot}, + LastAssignedGeneration: 2, + Snapshots: snapshots, }, } created := storetest.MustCreateActor(t, ctx, persistence, actor) @@ -268,10 +285,11 @@ func TestEnsureAteletSuspended_DialFailureLeavesActorRetryable(t *testing.T) { if stored.GetStatus().GetState() != ateapipb.ActorState_ACTOR_STATE_SUSPENDING { t.Errorf("state = %v, want unchanged SUSPENDING", stored.GetStatus().GetState()) } - if got := stored.GetStatus().GetInProgressSnapshotUri(); got != neverWritten { - t.Errorf("InProgressSnapshotUri = %q, want preserved for debugging", got) + if _, gotSt := findLatestSnapshotStorage(stored.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS); gotSt.GetObject().GetSnapshotUri() != neverWritten { + t.Errorf("in-progress durable snapshot URI = %q, want preserved for debugging", gotSt.GetObject().GetSnapshotUri()) } - if got := stored.GetStatus().GetExternalSnapshot().GetSnapshotUri(); got != tt.prevSnapshot { + _, gotSt := findLatestSnapshotStorage(stored.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + if got := gotSt.GetObject().GetSnapshotUri(); got != tt.prevSnapshot { t.Errorf("SnapshotUri = %q, want %q", got, tt.prevSnapshot) } }) @@ -289,25 +307,26 @@ func TestEnsureSuspendedFinalized_NoAssignment(t *testing.T) { persistence := newTestPersistence(t) snapshotURI := someActorSnapshotURI(t, testStorageLocation, "team-a", "2026-01-01t00-00-00z-abc") + snap := newLocalSnapshot(1, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-uid-1", "actor-1-pause-snapshot", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + snap.Storage = append(snap.Storage, &ateapipb.SnapshotStorage{ + Durability: ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, + Status: ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS, + Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + Object: &ateapipb.ObjectSnapshot{SnapshotUri: snapshotURI}, + }) actor := &ateapipb.Actor{ Metadata: &ateapipb.ResourceMetadata{Atespace: "team-a", Name: "actor-1"}, Status: &ateapipb.ActorStatus{ - State: ateapipb.ActorState_ACTOR_STATE_SUSPENDING, - AssignedNode: "node1", - InProgressSnapshotUri: snapshotURI, - LocalSnapshot: &ateapipb.LocalSnapshot{ - SnapshotName: "actor-1-pause-snapshot", - }, + State: ateapipb.ActorState_ACTOR_STATE_SUSPENDING, + AssignedNode: "node1", + LastAssignedGeneration: 1, + Snapshots: []*ateapipb.Snapshot{snap}, }, } storetest.MustCreateActor(t, ctx, persistence, actor) w := &ActorWorkflow{store: persistence} - tmpl := &ateapipb.ActorTemplate{ - Metadata: &ateapipb.ResourceMetadata{Atespace: "team-a", Name: "tmpl", Uid: "tmpl-uid-1"}, - SnapshotConfig: &ateapipb.SnapshotConfig{StorageLocation: testStorageLocation}, - } - stored, err := w.ensureSuspendedFinalized(ctx, resources.ActorRef{Atespace: "team-a", Name: "actor-1"}, tmpl) + stored, err := w.ensureSuspendedFinalized(ctx, resources.ActorRef{Atespace: "team-a", Name: "actor-1"}) if err != nil { t.Fatalf("ensureSuspendedFinalized: %v", err) } @@ -315,19 +334,20 @@ func TestEnsureSuspendedFinalized_NoAssignment(t *testing.T) { if stored.GetStatus().GetState() != ateapipb.ActorState_ACTOR_STATE_SUSPENDED { t.Errorf("state = %v, want SUSPENDED", stored.GetStatus().GetState()) } - if got := stored.GetStatus().GetExternalSnapshot().GetSnapshotUri(); got != snapshotURI { + gotSnap, gotSt := findLatestSnapshotStorage(stored.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + if got := gotSt.GetObject().GetSnapshotUri(); got != snapshotURI { t.Errorf("SnapshotUri = %q, want %q", got, snapshotURI) } // The snapshot carries the template it was captured under, so a later // repoint can tell that its guest state no longer matches. - if got := stored.GetStatus().GetExternalSnapshot().GetActorTemplateUid(); got != "tmpl-uid-1" { - t.Errorf("ExternalSnapshot.ActorTemplateUid = %q, want %q", got, "tmpl-uid-1") + if got := gotSnap.GetActorTemplateUid(); got != "tmpl-uid-1" { + t.Errorf("Snapshot.ActorTemplateUid = %q, want %q", got, "tmpl-uid-1") } - if got := stored.GetStatus().GetInProgressSnapshotUri(); got != "" { - t.Errorf("InProgressSnapshotUri = %q, want cleared", got) + if _, got := findLatestSnapshotStorage(stored.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS); got != nil { + t.Errorf("in-progress durable snapshot = %v, want cleared", got) } - if stored.GetStatus().GetLocalSnapshot() != nil { - t.Errorf("LocalSnapshot = %v, want cleared", stored.GetStatus().GetLocalSnapshot()) + if _, local := findLatestSnapshotStorage(stored.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED); local != nil { + t.Errorf("LocalSnapshot = %v, want cleared", local) } if got := stored.GetStatus().GetAssignedNode(); got != "" { t.Errorf("AssignedNode = %q, want cleared", got) @@ -377,25 +397,31 @@ func TestEnsureSuspendedFinalized_ReleasesReplacedSnapshot(t *testing.T) { }) previous := mustActorSnapshotURI(t, template, actor, "old") + prevType := ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR if tt.tagOwnedSnapshot { previous = mustTagSnapshotURI(t, template, "team-a", "v1-snapshot") + prevType = ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG } fresh := mustActorSnapshotURI(t, template, actor, snapshotName) objects.PutSnapshot(t, previous, "manifest.json") objects.PutSnapshot(t, fresh, "manifest.json") mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapipb.ActorStatus) { - s.InProgressSnapshotUri = fresh.String() - s.ExternalSnapshot = &ateapipb.ExternalSnapshot{SnapshotUri: previous.String()} + s.LastAssignedGeneration = 2 + s.Snapshots = []*ateapipb.Snapshot{ + newDurableSnapshot(1, prevType, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", previous.String(), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + newDurableSnapshot(2, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", fresh.String(), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS), + } }) - stored, err := w.ensureSuspendedFinalized(ctx, actorRef, template) + stored, err := w.ensureSuspendedFinalized(ctx, actorRef) if err != nil { t.Fatalf("ensureSuspendedFinalized: %v", err) } // Whichever way the previous snapshot went, the actor now owns the // one it just wrote and is no longer borrowing. - if got := stored.GetStatus().GetExternalSnapshot().GetSnapshotUri(); got != fresh.String() { + _, gotSt := findLatestSnapshotStorage(stored.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + if got := gotSt.GetObject().GetSnapshotUri(); got != fresh.String() { t.Errorf("external snapshot = %q, want the one this suspend wrote, %q", got, fresh) } if released := len(objects.Snapshot(t, previous)) == 0; released != tt.wantReleased { @@ -443,23 +469,27 @@ func TestEnsureSuspendedFinalized_CommitsDespiteObjectStoreFailure(t *testing.T) objects.PutSnapshot(t, fresh, "manifest.json") mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapipb.ActorStatus) { - s.InProgressSnapshotUri = fresh.String() - s.ExternalSnapshot = &ateapipb.ExternalSnapshot{SnapshotUri: previous.String()} + s.LastAssignedGeneration = 2 + s.Snapshots = []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", previous.String(), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + newDurableSnapshot(2, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", fresh.String(), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS), + } }) objects.OnDelete = func(string, string) error { return errObjectStore } - stored, err := w.ensureSuspendedFinalized(ctx, actorRef, template) + stored, err := w.ensureSuspendedFinalized(ctx, actorRef) if err != nil { t.Fatalf("ensureSuspendedFinalized: %v", err) } if got := stored.GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_SUSPENDED { t.Errorf("state = %v, want SUSPENDED", got) } - if got := stored.GetStatus().GetExternalSnapshot().GetSnapshotUri(); got != fresh.String() { + _, gotSt := findLatestSnapshotStorage(stored.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + if got := gotSt.GetObject().GetSnapshotUri(); got != fresh.String() { t.Errorf("actor snapshot uri = %q, want the one this suspend wrote, %q", got, fresh) } - if got := stored.GetStatus().GetInProgressSnapshotUri(); got != "" { - t.Errorf("InProgressSnapshotUri = %q, want cleared", got) + if _, got := findLatestSnapshotStorage(stored.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS); got != nil { + t.Errorf("in-progress durable snapshot = %v, want cleared", got) } if len(objects.Snapshot(t, previous)) == 0 { t.Error("replaced external snapshot was collected despite every delete failing") @@ -491,12 +521,15 @@ func TestEnsureSuspendedFinalized_KeepsReplacedSnapshotOnConflict(t *testing.T) objects.PutSnapshot(t, previous, "manifest.json") objects.PutSnapshot(t, fresh, "manifest.json") mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapipb.ActorStatus) { - s.InProgressSnapshotUri = fresh.String() - s.ExternalSnapshot = &ateapipb.ExternalSnapshot{SnapshotUri: previous.String()} + s.LastAssignedGeneration = 2 + s.Snapshots = []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", previous.String(), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + newDurableSnapshot(2, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", fresh.String(), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS), + } }) w := &ActorWorkflow{store: &conflictingUpdateStore{Interface: persistence}, snapshotPlugin: objectstoreplugintest.ControlClient(objects)} - if _, err := w.ensureSuspendedFinalized(ctx, actorRef, template); apierror.Code(err) != codes.Aborted { + if _, err := w.ensureSuspendedFinalized(ctx, actorRef); apierror.Code(err) != codes.Aborted { t.Fatalf("ensureSuspendedFinalized = %v, want code Aborted", err) } if len(objects.Snapshot(t, previous)) == 0 { @@ -554,7 +587,10 @@ func TestEnsureSuspendedFinalized_ReleasesOnlyOwnWorker(t *testing.T) { WorkerPod: "pod-1", WorkerPodUid: testWorkerUID("pod-1"), }, - InProgressSnapshotUri: someActorSnapshotURI(t, testStorageLocation, "team-a", "snapshot-1"), + LastAssignedGeneration: 1, + Snapshots: []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", someActorSnapshotURI(t, testStorageLocation, "team-a", "snapshot-1"), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS), + }, }, } created := storetest.MustCreateActor(t, ctx, persistence, actor) @@ -580,8 +616,7 @@ func TestEnsureSuspendedFinalized_ReleasesOnlyOwnWorker(t *testing.T) { }) w := &ActorWorkflow{store: persistence} - tmpl := &ateapipb.ActorTemplate{SnapshotConfig: &ateapipb.SnapshotConfig{StorageLocation: "gs://bucket/root"}} - if _, err := w.ensureSuspendedFinalized(ctx, resources.ActorRef{Atespace: "team-a", Name: "shared"}, tmpl); err != nil { + if _, err := w.ensureSuspendedFinalized(ctx, resources.ActorRef{Atespace: "team-a", Name: "shared"}); err != nil { t.Fatalf("ensureSuspendedFinalized: %v", err) } @@ -630,16 +665,18 @@ func TestPreferredFidelity(t *testing.T) { // no worker assignment, means the suspend uploads a local snapshot. func TestIsPausedOriginSuspend(t *testing.T) { assignment := &ateapipb.WorkerAssignment{WorkerNamespace: "ns", WorkerPool: "pool", WorkerPod: "pod-1"} - localInfo := &ateapipb.LocalSnapshot{SnapshotName: "snap"} + localSnaps := []*ateapipb.Snapshot{ + newLocalSnapshot(1, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "snap", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + } tests := []struct { name string actor *ateapipb.Actor want bool }{ - {"paused actor", &ateapipb.Actor{Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_PAUSED, AssignedNode: "node1", LocalSnapshot: localInfo}}, true}, - {"suspending retry of a paused-origin suspend", &ateapipb.Actor{Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_SUSPENDING, AssignedNode: "node1", LocalSnapshot: localInfo}}, true}, - {"running actor with stale local snapshot info", &ateapipb.Actor{Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_RUNNING, LocalSnapshot: localInfo, WorkerAssignment: assignment}}, false}, - {"suspending retry of a running-origin suspend with stale local snapshot info", &ateapipb.Actor{Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_SUSPENDING, LocalSnapshot: localInfo, WorkerAssignment: assignment}}, false}, + {"paused actor", &ateapipb.Actor{Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_PAUSED, AssignedNode: "node1", Snapshots: localSnaps}}, true}, + {"suspending retry of a paused-origin suspend", &ateapipb.Actor{Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_SUSPENDING, AssignedNode: "node1", Snapshots: localSnaps}}, true}, + {"running actor with stale local snapshot info", &ateapipb.Actor{Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_RUNNING, Snapshots: localSnaps, WorkerAssignment: assignment}}, false}, + {"suspending retry of a running-origin suspend with stale local snapshot info", &ateapipb.Actor{Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_SUSPENDING, Snapshots: localSnaps, WorkerAssignment: assignment}}, false}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { @@ -663,8 +700,11 @@ func TestEnsurePausedSnapshotUploaded_Preconditions(t *testing.T) { created := storetest.MustCreateActor(t, ctx, persistence, &ateapipb.Actor{ Metadata: &ateapipb.ResourceMetadata{Atespace: "team-a", Name: "actor-1"}, Status: &ateapipb.ActorStatus{ - State: ateapipb.ActorState_ACTOR_STATE_SUSPENDING, - LocalSnapshot: &ateapipb.LocalSnapshot{SnapshotName: "snap"}, + State: ateapipb.ActorState_ACTOR_STATE_SUSPENDING, + LastAssignedGeneration: 1, + Snapshots: []*ateapipb.Snapshot{ + newLocalSnapshot(1, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "snap", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + }, }, }) @@ -686,13 +726,20 @@ func TestEnsurePausedSnapshotUploaded_Preconditions(t *testing.T) { persistence := newTestPersistence(t) w := &ActorWorkflow{store: persistence, dialer: newDanglingDialer()} + snap := newLocalSnapshot(1, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "snap", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + snap.Storage = append(snap.Storage, &ateapipb.SnapshotStorage{ + Durability: ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, + Status: ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS, + Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + Object: &ateapipb.ObjectSnapshot{SnapshotUri: someActorSnapshotURI(t, testStorageLocation, "team-a", "snap-dest")}, + }) created := storetest.MustCreateActor(t, ctx, persistence, &ateapipb.Actor{ Metadata: &ateapipb.ResourceMetadata{Atespace: "team-a", Name: "actor-1"}, Status: &ateapipb.ActorStatus{ - State: ateapipb.ActorState_ACTOR_STATE_SUSPENDING, - AssignedNode: "node1", - InProgressSnapshotUri: someActorSnapshotURI(t, testStorageLocation, "team-a", "snap-dest"), - LocalSnapshot: &ateapipb.LocalSnapshot{SnapshotName: "snap"}, + State: ateapipb.ActorState_ACTOR_STATE_SUSPENDING, + AssignedNode: "node1", + LastAssignedGeneration: 1, + Snapshots: []*ateapipb.Snapshot{snap}, }, }) diff --git a/cmd/ateapi/internal/controlapi/workflow_tag.go b/cmd/ateapi/internal/controlapi/workflow_tag.go index c6a0ce6453..634f781ce9 100644 --- a/cmd/ateapi/internal/controlapi/workflow_tag.go +++ b/cmd/ateapi/internal/controlapi/workflow_tag.go @@ -67,16 +67,12 @@ func (w *ActorWorkflow) TagActorSnapshot(ctx context.Context, tag *ateapipb.Tag) if err != nil { return nil, err } - snapshot := actor.GetStatus().GetExternalSnapshot() + snapshot, _ := findLatestSnapshotStorage(actor.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) - reserved, err := w.ensureTagReserved(leaseCtx, tagRef, actor, actorTemplate, tag) + reserved, dst, err := w.ensureTagReserved(leaseCtx, tagRef, actor, actorTemplate, tag) if err != nil { return nil, err } - dst, err := resources.NewTagSnapshotURI(reserved.GetStatus().GetStorageLocation(), tagRef.Atespace, reserved.GetMetadata().GetUid()) - if err != nil { - return nil, fmt.Errorf("while building the snapshot URI for tag %s: %w", tagRef, err) - } if err := w.ensureTagSnapshotCopied(leaseCtx, reserved, snapshot, dst); err != nil { return nil, err } @@ -159,7 +155,7 @@ func (w *ActorWorkflow) ensureTagSnapshotReleased(ctx context.Context, tag *atea return nil } tagRef := resources.TagRefFromTag(tag) - uri, err := resources.NewTagSnapshotURI(tag.GetStatus().GetStorageLocation(), tagRef.Atespace, tag.GetMetadata().GetUid()) + uri, err := resources.ParseSnapshotURI(tagDurableSnapshotURI(tag, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_UNSPECIFIED)) if err != nil { return fmt.Errorf("while resolving the external snapshot of tag %s: %w", tagRef, err) } @@ -205,8 +201,8 @@ func (w *ActorWorkflow) loadActorForTag(ctx context.Context, actorRef resources. if got := actor.GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_SUSPENDED { return nil, nil, apierror.FailedPrecondition("Actor %s must be %s to be tagged (got: %v)", actorRef, ateapipb.ActorState_ACTOR_STATE_SUSPENDED, got) } - snapshotURI := actor.GetStatus().GetExternalSnapshot().GetSnapshotUri() - if snapshotURI == "" { + snap, st := findLatestSnapshotStorage(actor.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + if st == nil { return nil, nil, apierror.FailedPrecondition("Actor %s holds no external snapshot to tag", actorRef) } // Every way an Actor comes to hold an external snapshot records the @@ -214,7 +210,7 @@ func (w *ActorWorkflow) loadActorForTag(ctx context.Context, actorRef resources. // ensureSuspendedFinalized, a create from a tag through the tag's own UID. // A snapshot without one is a broken row, and tagging it would mint a tag // that names no template. - if actor.GetStatus().GetExternalSnapshot().GetActorTemplateUid() == "" { + if snap.GetActorTemplateUid() == "" { return nil, nil, apierror.Internal("Actor %s holds an external snapshot but records no template it was built under", actorRef) } actorTemplate, err := resolveActorTemplate(ctx, w.store, actor) @@ -224,21 +220,26 @@ func (w *ActorWorkflow) loadActorForTag(ctx context.Context, actorRef resources. return actor, actorTemplate, nil } -// ensureTagReserved takes the tag's name and records the storage location. +// ensureTagReserved takes the tag's name and records the in-progress snapshot. // // A name already taken is AlreadyExists, whether the tag holding it is finished // or was left pending by a create that died. Resuming a pending row would mean // deciding whether the objects under it still belong to the snapshot being // tagged, and the row may not even be this actor's; deleting the tag collects // them and frees the name, so a retry is a delete followed by a create. -func (w *ActorWorkflow) ensureTagReserved(ctx context.Context, tagRef resources.TagRef, actor *ateapipb.Actor, actorTemplate *ateapipb.ActorTemplate, tag *ateapipb.Tag) (_ *ateapipb.Tag, err error) { +func (w *ActorWorkflow) ensureTagReserved(ctx context.Context, tagRef resources.TagRef, actor *ateapipb.Actor, actorTemplate *ateapipb.ActorTemplate, tag *ateapipb.Tag) (_ *ateapipb.Tag, _ resources.SnapshotURI, err error) { ctx, done := stepSpan(ctx, "ReserveTag") defer func() { err = done(err) }() location := actorTemplate.GetSnapshotConfig().GetStorageLocation() if err := resources.ValidateSnapshotLocation(location); err != nil { - return nil, fmt.Errorf("invalid storage location for tag %s: %w", tagRef, err) + return nil, resources.SnapshotURI{}, fmt.Errorf("invalid storage location for tag %s: %w", tagRef, err) } + dst, err := resources.NewTagSnapshotURI(location, tagRef.Atespace, resources.NewSnapshotName()) + if err != nil { + return nil, resources.SnapshotURI{}, fmt.Errorf("while building the snapshot URI for tag %s: %w", tagRef, err) + } + snap, st := findLatestSnapshotStorage(actor.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) tagToCreate := &ateapipb.Tag{ Metadata: &ateapipb.ResourceMetadata{Atespace: tagRef.Atespace, Name: tagRef.Name}, Scope: tag.GetScope(), @@ -249,27 +250,34 @@ func (w *ActorWorkflow) ensureTagReserved(ctx context.Context, tagRef resources. // and a tag that claimed the new template would hand clones the old template's // memory under the new one's identity, past the data-only downgrade a resume of // the actor itself would take. - ActorTemplateUid: actor.GetStatus().GetExternalSnapshot().GetActorTemplateUid(), - StorageLocation: location, + ActorTemplateUid: snap.GetActorTemplateUid(), + Snapshot: newDurableSnapshot( + 1, + ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG, + st.GetFidelity(), + "", + dst.String(), + ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS, + ), }, } stored, err := w.store.CreateTag(ctx, tagToCreate) switch { case err == nil: - return stored, nil + return stored, dst, nil case errors.Is(err, store.ErrFailedPrecondition): - return nil, apierror.FailedPrecondition("Atespace %s not found", tagRef.Atespace) + return nil, resources.SnapshotURI{}, apierror.FailedPrecondition("Atespace %s not found", tagRef.Atespace) case errors.Is(err, store.ErrAlreadyExists): - return nil, apierror.AlreadyExists("Tag %s already exists; delete it and create it again to retry", tagRef) + return nil, resources.SnapshotURI{}, apierror.AlreadyExists("Tag %s already exists; delete it and create it again to retry", tagRef) } - return nil, fmt.Errorf("while reserving tag %s: %w", tagRef, err) + return nil, resources.SnapshotURI{}, fmt.Errorf("while reserving tag %s: %w", tagRef, err) } // ensureTagSnapshotCopied copies the actor's external snapshot to the tag's own -// prefix, derived from the reserved row's freshly minted UID. The prefix is -// empty by construction, so the copy never blends with another attempt's objects. -func (w *ActorWorkflow) ensureTagSnapshotCopied(ctx context.Context, tag *ateapipb.Tag, snapshot *ateapipb.ExternalSnapshot, dst resources.SnapshotURI) (err error) { +// prefix. The prefix is empty by construction, so the copy never blends with +// another attempt's objects. +func (w *ActorWorkflow) ensureTagSnapshotCopied(ctx context.Context, tag *ateapipb.Tag, snapshot *ateapipb.Snapshot, dst resources.SnapshotURI) (err error) { ctx, done := stepSpan(ctx, "CopyTagSnapshot") defer func() { err = done(err) }() @@ -278,9 +286,10 @@ func (w *ActorWorkflow) ensureTagSnapshotCopied(ctx context.Context, tag *ateapi return nil } tagRef := resources.TagRefFromTag(tag) - src, err := resources.ParseSnapshotURI(snapshot.GetSnapshotUri()) + srcURI := findSnapshotStorage(snapshot, ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE).GetObject().GetSnapshotUri() + src, err := resources.ParseSnapshotURI(srcURI) if err != nil { - return fmt.Errorf("while parsing the external snapshot %q of the source actor: %w", snapshot.GetSnapshotUri(), err) + return fmt.Errorf("while parsing the external snapshot %q of the source actor: %w", srcURI, err) } if err := w.copySnapshot(ctx, src.Prefix(), dst.Prefix()); err != nil { return fmt.Errorf("while copying the external snapshot for tag %s: %w", tagRef, err) @@ -288,18 +297,24 @@ func (w *ActorWorkflow) ensureTagSnapshotCopied(ctx context.Context, tag *ateapi return nil } -// ensureTagFinalized publishes the copy by setting status.snapshot. Until this -// lands the tag is pending and unusable; deleting it collects any partial copy. -func (w *ActorWorkflow) ensureTagFinalized(ctx context.Context, tag *ateapipb.Tag, snapshot *ateapipb.ExternalSnapshot, dst resources.SnapshotURI) (_ *ateapipb.Tag, err error) { +// ensureTagFinalized publishes the copy by marking status.snapshot COMPLETED. +// Until this lands the tag is pending and unusable; deleting it collects any +// partial copy. +func (w *ActorWorkflow) ensureTagFinalized(ctx context.Context, tag *ateapipb.Tag, snapshot *ateapipb.Snapshot, dst resources.SnapshotURI) (_ *ateapipb.Tag, err error) { ctx, done := stepSpan(ctx, "FinalizeTag") defer func() { err = done(err) }() tagRef := resources.TagRefFromTag(tag) // The copy is byte-identical to the source, so it carries the same content. - finalSnapshot := &ateapipb.ExternalSnapshot{ - SnapshotUri: dst.String(), - Fidelity: snapshot.GetFidelity(), - } + srcSt := findSnapshotStorage(snapshot, ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE) + finalSnapshot := newDurableSnapshot( + 1, + ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG, + srcSt.GetFidelity(), + "", + dst.String(), + ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED, + ) stored, err := w.store.UpdateTag(ctx, tagRef, store.PreconditionFrom(tag), func(toUpdate *ateapipb.Tag) error { toUpdate.Status.Snapshot = finalSnapshot return nil diff --git a/cmd/ateapi/internal/controlapi/workflow_tag_test.go b/cmd/ateapi/internal/controlapi/workflow_tag_test.go index e8536fb8fa..1e7ee697b3 100644 --- a/cmd/ateapi/internal/controlapi/workflow_tag_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_tag_test.go @@ -44,10 +44,9 @@ func seedTagSource(t *testing.T, ctx context.Context, persistence store.Interfac uri := mustActorSnapshotURI(t, template, actor, name+"-snapshot") objects.PutSnapshot(t, uri, objectNames...) actor = mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapipb.ActorStatus) { - s.ExternalSnapshot = &ateapipb.ExternalSnapshot{ - SnapshotUri: uri.String(), - Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, - ActorTemplateUid: template.GetMetadata().GetUid(), + s.LastAssignedGeneration = 1 + s.Snapshots = []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, template.GetMetadata().GetUid(), uri.String(), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), } }) return actor, uri @@ -90,7 +89,7 @@ func TestTagActorSnapshot(t *testing.T) { t.Fatalf("TagActorSnapshot: %v", err) } - tagSnapshot := tag.GetStatus().GetSnapshot().GetSnapshotUri() + tagSnapshot := tagDurableSnapshotURI(tag, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) if tagSnapshot == "" || tagSnapshot == actorSnapshot.String() { t.Fatalf("tag snapshot uri = %q, want a copy of its own rather than the actor's %q", tagSnapshot, actorSnapshot) } @@ -100,8 +99,8 @@ func TestTagActorSnapshot(t *testing.T) { if got, want := tag.GetStatus().GetActorTemplateUid(), template.GetMetadata().GetUid(); got != want { t.Errorf("actor template uid = %q, want %q", got, want) } - if got, want := tag.GetStatus().GetSnapshot().GetFidelity(), ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY; got != want { - t.Errorf("content scope = %v, want the source's %v", got, want) + if got, want := findSnapshotStorage(tag.GetStatus().GetSnapshot(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE).GetFidelity(), ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY; got != want { + t.Errorf("fidelity = %v, want the source's %v", got, want) } // Both prefixes hold the same objects: the tag copied rather than moved. @@ -198,9 +197,13 @@ func TestTagActorSnapshot_Preconditions(t *testing.T) { uri := mustActorSnapshotURI(t, template, actor, "actor-1-snapshot") objects.PutSnapshot(t, uri, "manifest.json") mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapipb.ActorStatus) { - s.ExternalSnapshot = &ateapipb.ExternalSnapshot{SnapshotUri: uri.String()} + var tmplUID string if tt.builtOnTemplate { - s.ExternalSnapshot.ActorTemplateUid = template.GetMetadata().GetUid() + tmplUID = template.GetMetadata().GetUid() + } + s.LastAssignedGeneration = 1 + s.Snapshots = []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, tmplUID, uri.String(), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), } }) } @@ -248,7 +251,7 @@ func TestTagActorSnapshot_RecreateAfterCopyFailure(t *testing.T) { if err != nil { t.Fatalf("GetTag after the external store failure: %v", err) } - if got := pending.GetStatus().GetSnapshot().GetSnapshotUri(); got != "" { + if got := tagDurableSnapshotURI(pending, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED); got != "" { t.Errorf("snapshot uri after the failure = %q, want unset: the copy never finished", got) } strandedURI := mustReservedTagSnapshotURI(t, pending) @@ -287,7 +290,7 @@ func TestTagActorSnapshot_RecreateAfterCopyFailure(t *testing.T) { if err != nil { t.Fatalf("TagActorSnapshot after the delete: %v", err) } - recreated := tag.GetStatus().GetSnapshot().GetSnapshotUri() + recreated := tagDurableSnapshotURI(tag, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) if recreated == "" || recreated == stranded { t.Fatalf("snapshot uri after the delete = %q, want a fresh prefix rather than the stranded %q", recreated, stranded) } @@ -350,7 +353,7 @@ func TestTagActorSnapshot_RacesDelete(t *testing.T) { // Nothing was collected out from under the copy, and the row in the store it // is still there. - if got := tag.GetStatus().GetSnapshot().GetSnapshotUri(); got != pendingURI { + if got := tagDurableSnapshotURI(tag, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED); got != pendingURI { t.Errorf("snapshot uri = %q, want the prefix the create reserved, %q", got, pendingURI) } if diff := cmp.Diff([]string{"manifest.json", "memory.zst"}, objects.Snapshot(t, mustParseSnapshotURI(t, pendingURI))); diff != "" { @@ -516,9 +519,9 @@ func TestDeleteTag_NotFound(t *testing.T) { func mustReservedTagSnapshotURI(t *testing.T, tag *ateapipb.Tag) resources.SnapshotURI { t.Helper() - uri, err := resources.NewTagSnapshotURI(tag.GetStatus().GetStorageLocation(), tag.GetMetadata().GetAtespace(), tag.GetMetadata().GetUid()) + uri, err := resources.ParseSnapshotURI(tagDurableSnapshotURI(tag, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_UNSPECIFIED)) if err != nil { - t.Fatalf("NewTagSnapshotURI: %v", err) + t.Fatalf("ParseSnapshotURI: %v", err) } return uri } diff --git a/cmd/ateapi/internal/controlapi/workflow_test.go b/cmd/ateapi/internal/controlapi/workflow_test.go index cb3b48ea96..c89f43f9aa 100644 --- a/cmd/ateapi/internal/controlapi/workflow_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_test.go @@ -59,8 +59,8 @@ func TestActorStateChangeRecords(t *testing.T) { { name: "pause marks the actor pausing", seedState: ateapipb.ActorState_ACTOR_STATE_RUNNING, - transition: func(t *testing.T, w *ActorWorkflow, ref resources.ActorRef, actor *ateapipb.Actor, _ *ateapipb.ActorTemplate) { - if _, err := w.ensureMarkedPausing(context.Background(), ref, actor); err != nil { + transition: func(t *testing.T, w *ActorWorkflow, ref resources.ActorRef, actor *ateapipb.Actor, tmpl *ateapipb.ActorTemplate) { + if _, err := w.ensureMarkedPausing(context.Background(), ref, actor, tmpl); err != nil { t.Fatalf("ensureMarkedPausing: %v", err) } }, @@ -258,7 +258,10 @@ func TestActorStateChangeRecordSkippedOnConflict(t *testing.T) { // Bump the stored version so the workflow's precondition is stale. if _, err := persistence.UpdateActor(ctx, actorRef, store.PreconditionFrom(stale), func(toUpdate *ateapipb.Actor) error { - toUpdate.Status.InProgressSnapshotUri = someActorSnapshotURI(t, testStorageLocation, "team-a", "someone-else") + toUpdate.Status.LastAssignedGeneration = 1 + toUpdate.Status.Snapshots = []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", someActorSnapshotURI(t, testStorageLocation, "team-a", "someone-else"), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS), + } return nil }); err != nil { t.Fatalf("bump version: %v", err) diff --git a/cmd/ateapi/internal/controlapi/workflow_worker_delete.go b/cmd/ateapi/internal/controlapi/workflow_worker_delete.go index 5d8847376e..742f7179ce 100644 --- a/cmd/ateapi/internal/controlapi/workflow_worker_delete.go +++ b/cmd/ateapi/internal/controlapi/workflow_worker_delete.go @@ -227,10 +227,6 @@ func (w *WorkerWorkflow) crashBoundActor(ctx context.Context, worker *ateapipb.W toUpdate.Status.Crash = newActorCrash(opName, crashMsg) } toUpdate.Status.WorkerAssignment = nil - // Local in-progress checkpoint dies with the sandbox that was writing - // it. The external in-progress checkpoint is kept so delete or revert - // can delete it. - toUpdate.Status.InProgressLocalSnapshotName = "" return nil }) switch { diff --git a/cmd/ateapi/internal/controlapi/workflow_worker_delete_test.go b/cmd/ateapi/internal/controlapi/workflow_worker_delete_test.go index bbfe22b461..15ebdc91ba 100644 --- a/cmd/ateapi/internal/controlapi/workflow_worker_delete_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_worker_delete_test.go @@ -109,9 +109,12 @@ func TestDeleteWorkerWorkflow_ReleasesBoundActor(t *testing.T) { actor := seedAPIActor(t, ctx, persistence, ateapipb.ActorState_ACTOR_STATE_RUNNING, func(a *ateapipb.Actor) { // Both in-progress checkpoints are set so the assertion covers the // shared crash path, which cannot know which workflow was in flight. - a.Status.InProgressSnapshotUri = someActorSnapshotURI(t, testStorageLocation, apiActorRef.Atespace, "partial-snapshot") - a.Status.InProgressLocalSnapshotName = "partial-local-snapshot" - a.Status.ExternalSnapshot = &ateapipb.ExternalSnapshot{SnapshotUri: someActorSnapshotURI(t, testStorageLocation, apiActorRef.Atespace, "last")} + a.Status.LastAssignedGeneration = 3 + a.Status.Snapshots = []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", someActorSnapshotURI(t, testStorageLocation, apiActorRef.Atespace, "last"), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + newDurableSnapshot(2, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", someActorSnapshotURI(t, testStorageLocation, apiActorRef.Atespace, "partial-snapshot"), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS), + newLocalSnapshot(3, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "partial-local-snapshot", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS), + } }) assignAPIWorker(t, ctx, persistence, apiWorkerName, actor.GetMetadata().GetUid()) @@ -132,19 +135,21 @@ func TestDeleteWorkerWorkflow_ReleasesBoundActor(t *testing.T) { if got.GetStatus().GetWorkerAssignment() != nil { t.Errorf("actor worker assignment = %v, want it cleared", got.GetStatus().GetWorkerAssignment()) } - // The local checkpoint lived on the node that went away, so it dies with - // the worker. - if got.GetStatus().GetInProgressLocalSnapshotName() != "" { - t.Errorf("in-progress local checkpoint not cleared: %v", got.GetStatus()) + // In-progress checkpoints are kept so delete or revert can clean them up + // while assigned_node or the object prefix is still recorded. + if _, gotSt := findLatestSnapshotStorage(got.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS); gotSt.GetLocal().GetSnapshotName() != "partial-local-snapshot" { + t.Errorf("in-progress local checkpoint = %q, want %q preserved", gotSt.GetLocal().GetSnapshotName(), "partial-local-snapshot") } // The durable one is kept: it names the prefix whatever atelet already // uploaded lives under, which delete or revert needs to collect it. - if want := someActorSnapshotURI(t, testStorageLocation, apiActorRef.Atespace, "partial-snapshot"); got.GetStatus().GetInProgressSnapshotUri() != want { + _, gotInProgressSt := findLatestSnapshotStorage(got.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS) + if want := someActorSnapshotURI(t, testStorageLocation, apiActorRef.Atespace, "partial-snapshot"); gotInProgressSt.GetObject().GetSnapshotUri() != want { t.Errorf("in-progress external checkpoint not preserved: %v", got.GetStatus()) } // The last completed snapshot is what makes the actor resumable, so it stays. - if want := someActorSnapshotURI(t, testStorageLocation, apiActorRef.Atespace, "last"); got.GetStatus().GetExternalSnapshot().GetSnapshotUri() != want { - t.Errorf("external snapshot = %q, want it preserved as %q", got.GetStatus().GetExternalSnapshot().GetSnapshotUri(), want) + _, gotCompletedSt := findLatestSnapshotStorage(got.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + if want := someActorSnapshotURI(t, testStorageLocation, apiActorRef.Atespace, "last"); gotCompletedSt.GetObject().GetSnapshotUri() != want { + t.Errorf("external snapshot = %q, want it preserved as %q", gotCompletedSt.GetObject().GetSnapshotUri(), want) } } diff --git a/cmd/ateapi/internal/store/atepg/tag.go b/cmd/ateapi/internal/store/atepg/tag.go index c902a2bdd5..543e1fa15a 100644 --- a/cmd/ateapi/internal/store/atepg/tag.go +++ b/cmd/ateapi/internal/store/atepg/tag.go @@ -185,6 +185,22 @@ func (p *Persistence) CreateTag(ctx context.Context, tag *ateapipb.Tag) (*ateapi return dbTag, nil } +func isAllowedTagSnapshotTransition(stored, mutated *ateapipb.Snapshot) bool { + if proto.Equal(stored, mutated) || stored == nil { + return true + } + if mutated == nil || len(stored.GetStorage()) != 1 || len(mutated.GetStorage()) != 1 { + return false + } + if stored.GetStorage()[0].GetStatus() != ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS || + mutated.GetStorage()[0].GetStatus() != ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED { + return false + } + expected := proto.CloneOf(stored) + expected.Storage[0].Status = ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + return proto.Equal(expected, mutated) +} + func validateUpdateTagMutation(storedTag, mutatedTag *ateapipb.Tag) error { if stored, mutated := storedTag.GetMetadata().GetAtespace(), mutatedTag.GetMetadata().GetAtespace(); stored != mutated { return fmt.Errorf("metadata.atespace is immutable: mutation changed it from %q to %q", stored, mutated) @@ -192,12 +208,9 @@ func validateUpdateTagMutation(storedTag, mutatedTag *ateapipb.Tag) error { if stored, mutated := storedTag.GetMetadata().GetName(), mutatedTag.GetMetadata().GetName(); stored != mutated { return fmt.Errorf("metadata.name is immutable: mutation changed it from %q to %q", stored, mutated) } - if stored, mutated := storedTag.GetStatus().GetSnapshot(), mutatedTag.GetStatus().GetSnapshot(); stored != nil && !proto.Equal(stored, mutated) { + if stored, mutated := storedTag.GetStatus().GetSnapshot(), mutatedTag.GetStatus().GetSnapshot(); !isAllowedTagSnapshotTransition(stored, mutated) { return fmt.Errorf("status.snapshot is immutable once set: mutation changed it from %s to %s", stored, mutated) } - if stored, mutated := storedTag.GetStatus().GetStorageLocation(), mutatedTag.GetStatus().GetStorageLocation(); stored != mutated { - return fmt.Errorf("status.storage_location is immutable: mutation changed it from %q to %q", stored, mutated) - } if stored, mutated := storedTag.GetStatus().GetActorTemplateUid(), mutatedTag.GetStatus().GetActorTemplateUid(); stored != mutated { return fmt.Errorf("status.actor_template_uid is immutable: mutation changed it from %q to %q", stored, mutated) } diff --git a/cmd/ateapi/internal/store/atepg/tag_test.go b/cmd/ateapi/internal/store/atepg/tag_test.go index 50ae7b4826..3d51f7a6eb 100644 --- a/cmd/ateapi/internal/store/atepg/tag_test.go +++ b/cmd/ateapi/internal/store/atepg/tag_test.go @@ -35,7 +35,16 @@ func createTestTag(t *testing.T, s *Persistence, tagAtespace, tagName string) *a Metadata: &ateapipb.ResourceMetadata{Atespace: tagAtespace, Name: tagName}, Scope: ateapipb.TagScope_TAG_SCOPE_ATESPACE, Status: &ateapipb.TagStatus{ - Snapshot: &ateapipb.ExternalSnapshot{SnapshotUri: "gs://bucket/atespaces/" + tagAtespace + "/tags/" + tagName}, + Snapshot: &ateapipb.Snapshot{ + Generation: 1, + Owner: ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG, + Storage: []*ateapipb.SnapshotStorage{{ + Durability: ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, + Status: ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED, + Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + Object: &ateapipb.ObjectSnapshot{SnapshotUri: "gs://bucket/atespaces/" + tagAtespace + "/tags/" + tagName}, + }}, + }, }, }) if err != nil { @@ -84,9 +93,7 @@ func TestCreateTag_TagForeignKeyErrors(t *testing.T) { // A tag in an atespace that does not exist trips the tag's atespace FK. _, err := s.CreateTag(ctx, &ateapipb.Tag{ Metadata: &ateapipb.ResourceMetadata{Atespace: "gone", Name: "latest"}, - Status: &ateapipb.TagStatus{ - StorageLocation: "gs://bucket", - }, + Status: &ateapipb.TagStatus{}, }) if !errors.Is(err, store.ErrFailedPrecondition) { t.Errorf("missing tag atespace error = %v, want ErrFailedPrecondition", err) diff --git a/cmd/ateapi/internal/store/storecontract/contract.go b/cmd/ateapi/internal/store/storecontract/contract.go index c597100393..3589e6849c 100644 --- a/cmd/ateapi/internal/store/storecontract/contract.go +++ b/cmd/ateapi/internal/store/storecontract/contract.go @@ -865,16 +865,36 @@ func runActorContractTests(t *testing.T, setup func(t *testing.T) store.Interfac Metadata: &ateapipb.ResourceMetadata{Name: "id1", Atespace: testAtespace}, ActorTemplate: &ateapipb.ObjectRef{Atespace: "ns1", Name: "tmpl1"}, Status: &ateapipb.ActorStatus{ - State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, - ExternalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: testActorSnapshotURI("gs://bucket", testAtespace, "snapshot-1")}, + State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, + LastAssignedGeneration: 1, + Snapshots: []*ateapipb.Snapshot{{ + Generation: 1, + Owner: ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, + Storage: []*ateapipb.SnapshotStorage{{ + Durability: ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, + Status: ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED, + Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + Object: &ateapipb.ObjectSnapshot{SnapshotUri: testActorSnapshotURI("gs://bucket", testAtespace, "snapshot-1")}, + }}, + }}, }, } actor2 := &ateapipb.Actor{ Metadata: &ateapipb.ResourceMetadata{Name: "id2", Atespace: testAtespace}, ActorTemplate: &ateapipb.ObjectRef{Atespace: "ns1", Name: "tmpl1"}, Status: &ateapipb.ActorStatus{ - State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, - ExternalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: testActorSnapshotURI("gs://bucket", testAtespace, "snapshot-2")}, + State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, + LastAssignedGeneration: 1, + Snapshots: []*ateapipb.Snapshot{{ + Generation: 1, + Owner: ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, + Storage: []*ateapipb.SnapshotStorage{{ + Durability: ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, + Status: ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED, + Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + Object: &ateapipb.ObjectSnapshot{SnapshotUri: testActorSnapshotURI("gs://bucket", testAtespace, "snapshot-2")}, + }}, + }}, }, } if _, err := s.CreateActor(ctx, actor1); err != nil { @@ -1137,25 +1157,32 @@ func runActorTemplateContractTests(t *testing.T, setup func(t *testing.T) store. }) } -// newTestSuspendedActor builds an actor that already holds an external +// newTestSuspendedActor builds an actor that already holds a durable // snapshot, the state CreateTag tags from. func newTestSuspendedActor(atespace, name string) *ateapipb.Actor { return &ateapipb.Actor{ Metadata: &ateapipb.ResourceMetadata{Atespace: atespace, Name: name}, ActorTemplate: &ateapipb.ObjectRef{Atespace: "ns1", Name: "tmpl1"}, Status: &ateapipb.ActorStatus{ - State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, - ExternalSnapshot: &ateapipb.ExternalSnapshot{ - SnapshotUri: testActorSnapshotURI("gs://private", atespace, name), - Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, - }, + State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, + LastAssignedGeneration: 1, + Snapshots: []*ateapipb.Snapshot{{ + Generation: 1, + Owner: ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, + Storage: []*ateapipb.SnapshotStorage{{ + Durability: ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, + Status: ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED, + Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + Object: &ateapipb.ObjectSnapshot{SnapshotUri: testActorSnapshotURI("gs://private", atespace, name)}, + }}, + }}, }, } } // newTestInProgressTag builds the row CreateTag reserves for a -// tag of actor: ATESPACE-scoped, with no snapshot yet and the destination of -// the copy still to come derived from status.storage_location and its UID. +// tag of actor: ATESPACE-scoped, with an IN_PROGRESS snapshot naming where the +// copy is going. func newTestInProgressTag(name string, actor *ateapipb.Actor) *ateapipb.Tag { atespace := actor.GetMetadata().GetAtespace() return &ateapipb.Tag{ @@ -1163,7 +1190,18 @@ func newTestInProgressTag(name string, actor *ateapipb.Actor) *ateapipb.Tag { Scope: ateapipb.TagScope_TAG_SCOPE_ATESPACE, Status: &ateapipb.TagStatus{ ActorTemplateUid: "template-uid", - StorageLocation: "gs://private", + Snapshot: &ateapipb.Snapshot{ + Generation: 1, + Owner: ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG, + Storage: []*ateapipb.SnapshotStorage{{ + Durability: ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, + Status: ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS, + Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + Object: &ateapipb.ObjectSnapshot{ + SnapshotUri: testTagSnapshotURI("gs://private", atespace, name), + }, + }}, + }, }, } } @@ -1171,7 +1209,7 @@ func newTestInProgressTag(name string, actor *ateapipb.Actor) *ateapipb.Tag { func runTagContractTests(t *testing.T, setup func(t *testing.T) store.Interface) { t.Helper() - // seedSuspendedActor seeds an atespace and an actor holding an external + // seedSuspendedActor seeds an atespace and an actor holding a durable // snapshot, the precondition every tag test starts from. seedSuspendedActor := func(t *testing.T, s store.Interface, atespace, name string) *ateapipb.Actor { t.Helper() @@ -1204,10 +1242,10 @@ func runTagContractTests(t *testing.T, setup func(t *testing.T) store.Interface) if err != nil { t.Fatalf("GetTag failed: %v", err) } - // A reserved tag names where its copy is going and nothing else: it is - // not usable until the copy lands and finalize names it. - if tag.GetStatus().GetSnapshot() != nil { - t.Errorf("reserved tag snapshot = %v, want unset", tag.GetStatus().GetSnapshot()) + // A reserved tag has an IN_PROGRESS snapshot storage entry and is not + // usable until the copy lands and finalize marks it COMPLETED. + if got := tag.GetStatus().GetSnapshot().GetStorage()[0].GetStatus(); got != ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS { + t.Errorf("reserved tag snapshot storage status = %v, want IN_PROGRESS", got) } if diff := cmp.Diff(inProgressTag, tag, protocmp.Transform(), ignoreUID, ignoreVersion, ignoreTimestamps); diff != "" { t.Errorf("stored tag mismatch (-want +got):\n%s", diff) @@ -1223,12 +1261,16 @@ func runTagContractTests(t *testing.T, setup func(t *testing.T) store.Interface) if err != nil { t.Fatalf("finalizing tag failed: %v", err) } - wantURI := testTagSnapshotURI(tag.GetStatus().GetStorageLocation(), "team-a", tag.GetMetadata().GetUid()) - if got := ready.GetStatus().GetSnapshot().GetSnapshotUri(); got != wantURI { - t.Errorf("finalized tag snapshot uri = %q, want %q", got, wantURI) + wantURI := testTagSnapshotURI("gs://private", "team-a", "production") + var gotURI string + if len(ready.GetStatus().GetSnapshot().GetStorage()) > 0 { + gotURI = ready.GetStatus().GetSnapshot().GetStorage()[0].GetObject().GetSnapshotUri() + } + if gotURI != wantURI { + t.Errorf("finalized tag snapshot uri = %q, want %q", gotURI, wantURI) } - if got := ready.GetStatus().GetStorageLocation(); got != tag.GetStatus().GetStorageLocation() { - t.Errorf("finalized tag storage location = %q, want unchanged", got) + if got := ready.GetStatus().GetSnapshot().GetStorage()[0].GetStatus(); got != ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED { + t.Errorf("finalized tag snapshot storage status = %v, want COMPLETED", got) } updated, err := s.UpdateTag(ctx, resources.TagRef{Atespace: "team-a", Name: "production"}, store.PreconditionFrom(ready), func(toUpdate *ateapipb.Tag) error { @@ -1332,13 +1374,13 @@ func runTagContractTests(t *testing.T, setup func(t *testing.T) store.Interface) { name: "snapshot uri", mutate: func(toUpdate *ateapipb.Tag) { - toUpdate.Status.Snapshot.SnapshotUri = "gs://private/elsewhere" + toUpdate.Status.Snapshot.Storage[0].Object.SnapshotUri = "gs://private/elsewhere" }, }, { name: "snapshot content scope", mutate: func(toUpdate *ateapipb.Tag) { - toUpdate.Status.Snapshot.Fidelity = ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES + toUpdate.Status.Snapshot.Storage[0].Fidelity = ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES }, }, { @@ -1347,16 +1389,6 @@ func runTagContractTests(t *testing.T, setup func(t *testing.T) store.Interface) toUpdate.Status.Snapshot = nil }, }, - { - name: "storage location", - mutate: func(toUpdate *ateapipb.Tag) { - toUpdate.Status.StorageLocation = "gs://elsewhere" - }, - }, - { - name: "clearing the storage location", - mutate: func(toUpdate *ateapipb.Tag) { toUpdate.Status.StorageLocation = "" }, - }, { name: "actor template uid", mutate: func(toUpdate *ateapipb.Tag) { toUpdate.Status.ActorTemplateUid = "other-template-uid" }, @@ -1494,12 +1526,9 @@ func runTagContractTests(t *testing.T, setup func(t *testing.T) store.Interface) } // finalizeTag mutates a reserved tag the way the tag workflow's second -// transaction does: it names the copy that landed under the tag UID. +// transaction does: it marks the in-progress snapshot storage COMPLETED. func finalizeTag(toUpdate *ateapipb.Tag) error { - toUpdate.Status.Snapshot = &ateapipb.ExternalSnapshot{ - SnapshotUri: testTagSnapshotURI(toUpdate.GetStatus().GetStorageLocation(), toUpdate.GetMetadata().GetAtespace(), toUpdate.GetMetadata().GetUid()), - Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, - } + toUpdate.Status.Snapshot.Storage[0].Status = ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED return nil } diff --git a/cmd/kubectl-ate/internal/cmd/tag_test.go b/cmd/kubectl-ate/internal/cmd/tag_test.go index b225a65c7a..1f9b769433 100644 --- a/cmd/kubectl-ate/internal/cmd/tag_test.go +++ b/cmd/kubectl-ate/internal/cmd/tag_test.go @@ -60,7 +60,14 @@ func TestUpdateTagScope(t *testing.T) { }, Scope: ateapipb.TagScope_TAG_SCOPE_ATESPACE, Status: &ateapipb.TagStatus{ - Snapshot: &ateapipb.ExternalSnapshot{SnapshotUri: "gs://private/atespaces/space-1/tags/tag-1"}, + Snapshot: &ateapipb.Snapshot{ + Storage: []*ateapipb.SnapshotStorage{{ + Durability: ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, + Status: ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED, + Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + Object: &ateapipb.ObjectSnapshot{SnapshotUri: "gs://private/atespaces/space-1/tags/tag-1"}, + }}, + }, }, } diff --git a/cmd/kubectl-ate/internal/printer/printer.go b/cmd/kubectl-ate/internal/printer/printer.go index 6d32492468..59c3e309fb 100644 --- a/cmd/kubectl-ate/internal/printer/printer.go +++ b/cmd/kubectl-ate/internal/printer/printer.go @@ -338,11 +338,11 @@ func PrintTagsTo(out io.Writer, tags []*ateapipb.Tag, format string) error { fmt.Fprintln(w, "ATESPACE\tNAME\tSCOPE\tSTATE\tSNAPSHOT\tFIDELITY\tAGE") for _, tag := range tags { // A pending tag has no snapshot yet, so neither its URI nor its - // content scope says anything. + // fidelity says anything. snapshotURI, fidelity := "", "" - if snapshot := tag.GetStatus().GetSnapshot(); snapshot.GetSnapshotUri() != "" { - snapshotURI = snapshot.GetSnapshotUri() - fidelity = snapshot.GetFidelity().String() + if st := durableSnapshotStorage(tag.GetStatus().GetSnapshot()); st != nil { + snapshotURI = st.GetObject().GetSnapshotUri() + fidelity = st.GetFidelity().String() } fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%s\n", tag.GetMetadata().GetAtespace(), tag.GetMetadata().GetName(), tag.GetScope(), @@ -355,11 +355,22 @@ func PrintTagsTo(out io.Writer, tags []*ateapipb.Tag, format string) error { } } +func durableSnapshotStorage(snapshot *ateapipb.Snapshot) *ateapipb.SnapshotStorage { + for _, st := range snapshot.GetStorage() { + if st.GetDurability() == ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE && + st.GetStatus() == ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED && + st.GetObject().GetSnapshotUri() != "" { + return st + } + } + return nil +} + // tagState reports whether a tag is usable. A tag is Pending until // the copy of its own snapshot lands; until then it names nothing an Actor can // be created from, and deleting it collects whatever the create stranded. func tagState(tag *ateapipb.Tag) string { - if tag.GetStatus().GetSnapshot().GetSnapshotUri() == "" { + if durableSnapshotStorage(tag.GetStatus().GetSnapshot()) == nil { return "Pending" } return "Ready" diff --git a/cmd/kubectl-ate/internal/printer/printer_test.go b/cmd/kubectl-ate/internal/printer/printer_test.go index 292cceb02f..70612bdeaa 100644 --- a/cmd/kubectl-ate/internal/printer/printer_test.go +++ b/cmd/kubectl-ate/internal/printer/printer_test.go @@ -599,7 +599,14 @@ func TestPrintTagsTo_Table(t *testing.T) { }, Scope: ateapipb.TagScope_TAG_SCOPE_PUBLISHED, Status: &ateapipb.TagStatus{ - Snapshot: &ateapipb.ExternalSnapshot{SnapshotUri: "gs://private/atespaces/team-a/tags/v2", Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY}, + Snapshot: &ateapipb.Snapshot{ + Storage: []*ateapipb.SnapshotStorage{{ + Durability: ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, + Status: ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED, + Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + Object: &ateapipb.ObjectSnapshot{SnapshotUri: "gs://private/atespaces/team-a/tags/v2"}, + }}, + }, }, }, { @@ -610,7 +617,14 @@ func TestPrintTagsTo_Table(t *testing.T) { }, Scope: ateapipb.TagScope_TAG_SCOPE_ATESPACE, Status: &ateapipb.TagStatus{ - Snapshot: &ateapipb.ExternalSnapshot{SnapshotUri: "gs://private/atespaces/team-a/tags/v1", Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY}, + Snapshot: &ateapipb.Snapshot{ + Storage: []*ateapipb.SnapshotStorage{{ + Durability: ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, + Status: ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED, + Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + Object: &ateapipb.ObjectSnapshot{SnapshotUri: "gs://private/atespaces/team-a/tags/v1"}, + }}, + }, }, }, { @@ -623,7 +637,14 @@ func TestPrintTagsTo_Table(t *testing.T) { }, Scope: ateapipb.TagScope_TAG_SCOPE_ATESPACE, Status: &ateapipb.TagStatus{ - StorageLocation: "gs://private", + Snapshot: &ateapipb.Snapshot{ + Storage: []*ateapipb.SnapshotStorage{{ + Durability: ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, + Status: ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS, + Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + Object: &ateapipb.ObjectSnapshot{SnapshotUri: "gs://private/atespaces/team-a/tags/v3"}, + }}, + }, }, }, } diff --git a/docs/api-guide.md b/docs/api-guide.md index 58dfab4b69..73d7602157 100644 --- a/docs/api-guide.md +++ b/docs/api-guide.md @@ -375,7 +375,7 @@ An actor takes a series of snapshots over its life, so it gets a prefix of its o An owner is collected by deleting everything under its prefix, and it can delete nothing else. That is what makes a borrowed snapshot safe: an actor created from a tag points at a URI under `tags/`, which its own prefix does not cover. See [Snapshot lifetime](#snapshot-lifetime). -An `Actor` reports its current snapshot in the server-managed `status.externalSnapshot` and a `Tag` in `status.snapshot`, each an `ExternalSnapshot` carrying `snapshotUri`, `fidelity`, and `actorTemplateUid`. The URI is recorded when the snapshot is written. `actorTemplateUid` records the `ActorTemplate` whose sandbox the guest state was captured from, which is not always the template the actor points at now: an actor may be repointed while `SUSPENDED`, and the snapshot on disk still came from the old one. A resume that finds the two disagree restores the durable data only and boots the guest fresh, because memory captured under one sandbox image cannot be resumed under another. An `ActorTemplate` references its golden tag with the `ObjectRef` in `status.goldenSnapshotStatus.goldenTag`. These status fields are server-owned and ignored on input. Parse a URI only against the scheme above. +An `Actor` reports its snapshots in the server-managed `status.snapshots` (and `status.lastAssignedGeneration`) and a `Tag` in `status.snapshot`, each a `Snapshot` carrying `generation`, `owner`, `actorTemplateUid`, and `storage`. The URI and `fidelity` are recorded in a `DURABLE` `SnapshotStorage` entry (`object.snapshotUri` and `fidelity`) when the snapshot is written. `actorTemplateUid` records the `ActorTemplate` whose sandbox the guest state was captured from, which is not always the template the actor points at now: an actor may be repointed while `SUSPENDED`, and the snapshot on disk still came from the old one. A resume that finds the two disagree restores the durable data only and boots the guest fresh, because memory captured under one sandbox image cannot be resumed under another. An `ActorTemplate` references its golden tag with the `ObjectRef` in `status.goldenSnapshotStatus.goldenTag`. These status fields are server-owned and ignored on input. Parse a URI only against the scheme above. An `ActorTemplate` belongs to one atespace, but one `storageLocation` still holds snapshots for many atespaces: the golden actor lives in the reserved `ate-golden` atespace, and a `PUBLISHED` snapshot may be cloned from other atespaces. The `` level exists so that access can be granted per tenant: an object-storage policy can only condition on an **object-name prefix**, and cannot read the identity recorded inside a snapshot's manifest. Binding a per-atespace grant on GCS looks like: @@ -488,7 +488,7 @@ The Substrate Control Plane (`ate-api-server`) exposes a gRPC interface for mana Registers a new logical actor in the system. * **Request:** `CreateActorRequest` * `actor`: `Actor` — the actor to create. Its `metadata` carries the atespace and name (name must be a DNS-1123 label); the `actor_template` ref (atespace + name) selects the `ActorTemplate`. - * `actor.source_tag`: (Optional) `ObjectRef` of a `Tag` to seed the actor from. The tag must be taken under the same `ActorTemplate`, and either in the actor's own atespace or `PUBLISHED`. Nothing is copied: the new actor's `status.externalSnapshot` points at the tag's snapshot, under the tag's prefix, until its own first suspend. + * `actor.source_tag`: (Optional) `ObjectRef` of a `Tag` to seed the actor from. The tag must be taken under the same `ActorTemplate`, and either in the actor's own atespace or `PUBLISHED`. Nothing is copied: the new actor's `status.snapshots` points at the tag's snapshot, under the tag's prefix, until its own first suspend. * **Response:** the initialized `Actor`. #### `UpdateActor` @@ -511,7 +511,7 @@ Activates a suspended actor by restoring it onto a physical worker. Hibernate a running actor, capturing its current RAM and disk state into a snapshot. * **Request:** `SuspendActorRequest` * `actor`: `ObjectRef` of the actor to suspend. -* **Response:** `SuspendActorResponse` containing the `Actor` object in `ACTOR_STATE_SUSPENDED`, with its snapshot in `status.externalSnapshot`. +* **Response:** `SuspendActorResponse` containing the `Actor` object in `ACTOR_STATE_SUSPENDED`, with its snapshot in `status.snapshots`. * A successful suspend releases the actor's previous external snapshot: an actor keeps one, and only tags outlive it. To keep the snapshot a suspend just wrote, tag it with `CreateTag` while the actor is still suspended. #### Snapshot lifetime @@ -520,21 +520,21 @@ Every external snapshot has exactly one owner, and the control plane deletes it | Owner | Released when | | :--- | :--- | -| The actor that took it (`status.externalSnapshot`) | The actor's next successful suspend replaces it, or the actor is deleted. | +| The actor that took it (`status.snapshots`) | The actor's next successful suspend replaces it, or the actor is deleted. | | The tag that copied it (`status.snapshot`) | The tag is deleted. | An actor created from a tag borrows the tag's copy instead of taking one of its own. The borrowed URI sits under the tag's prefix, which the actor's own prefix does not cover, so neither suspending nor deleting the actor can reach it; its first own suspend writes a snapshot under the actor's prefix, and it owns its snapshots from then on. Deletion always runs before the database reference is dropped, and a failure fails the whole RPC. Clients are expected to retry with the same arguments: destinations are deterministic and every phase tolerates a partly-completed predecessor, so a retry resumes rather than duplicating work. The cost of that ordering is that a crash between the two can leave an external snapshot no row names; the reverse order would instead lose the handle needed to ever delete it. -> **Do not delete a tag while actors created from it exist.** A clone borrows the tag's snapshot rather than copying it, and only stops borrowing at its own first suspend (its `status.externalSnapshot.snapshotUri` still names the tag's prefix while it is). Deleting the tag leaves such a clone unable to resume. This is not prevented today. +> **Do not delete a tag while actors created from it exist.** A clone borrows the tag's snapshot rather than copying it, and only stops borrowing at its own first suspend (its `status.snapshots` entry still names the tag's prefix while it is). Deleting the tag leaves such a clone unable to resume. This is not prevented today. #### `RevertActor` -Discards an actor's live or crashed execution and transitions it to `ACTOR_STATE_SUSPENDED` at its last completed external snapshot (`status.externalSnapshot`). +Discards an actor's live or crashed execution and transitions it to `ACTOR_STATE_SUSPENDED` at its last completed external snapshot (`status.snapshots`). * **Request:** `RevertActorRequest` * `actor`: `ObjectRef` of the actor to revert. Accepted from `ACTOR_STATE_RUNNING`, `ACTOR_STATE_PAUSED`, and `ACTOR_STATE_CRASHED` (plus `ACTOR_STATE_REVERTING` for idempotent retries). Calling `RevertActor` on an already `ACTOR_STATE_SUSPENDED` actor returns `FAILED_PRECONDITION`. * **Response:** `RevertActorResponse` containing the reverted `Actor` in `ACTOR_STATE_SUSPENDED`. -* Reverting terminates any bound worker sandbox, clears node-local pause checkpoints (`localSnapshot`), and garbage-collects any partial external snapshot left by an interrupted suspend while preserving the last committed `externalSnapshot`. +* Reverting terminates any bound worker sandbox, clears node-local pause checkpoints, and garbage-collects any partial external snapshot left by an interrupted suspend while preserving the last committed durable snapshot in `status.snapshots`. * External volumes are not reverted. Their contents are never part of a snapshot, so a reverted actor comes back with its memory and root filesystem rewound but its volumes exactly as the discarded execution left them. #### `DeleteActor` diff --git a/docs/architecture.md b/docs/architecture.md index 1034ad5808..a09707ade9 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -474,7 +474,7 @@ Triggered by an explicit `SuspendActor` call. borrowed from a tag is left alone, since the tag owns it. 5. **State**: State transitions back to `ACTOR_STATE_SUSPENDED`, and the Actor's - `status.externalSnapshot` names the external snapshot it resumes from. + `status.snapshots` records the external snapshot it resumes from. Snapshots may be given tags owned and addressed by an Atespace. The same tag name may exist in different Atespaces. A tag is an immutable alias and retention pin: diff --git a/internal/e2e/suites/demo/demo_test.go b/internal/e2e/suites/demo/demo_test.go index 894d1bca48..9fdfe062a8 100644 --- a/internal/e2e/suites/demo/demo_test.go +++ b/internal/e2e/suites/demo/demo_test.go @@ -140,7 +140,7 @@ func TestActorSnapshotLifecycle(t *testing.T) { if err != nil { t.Fatalf("failed to suspend source Actor: %v", err) } - snapshotURI := suspended.GetActor().GetStatus().GetExternalSnapshot().GetSnapshotUri() + snapshotURI := durableSnapshotURI(suspended.GetActor().GetStatus()) if snapshotURI == "" { t.Fatal("suspended Actor has no external snapshot") } @@ -156,15 +156,11 @@ func TestActorSnapshotLifecycle(t *testing.T) { if err != nil { t.Fatalf("failed to create the Tag: %v", err) } - if got := tagToUpdate.GetStatus().GetSnapshot().GetSnapshotUri(); got == "" || got == snapshotURI { + if got := tagSnapshotURI(tagToUpdate); got == "" || got == snapshotURI { t.Fatalf("Tag %s snapshot uri = %q, want a copy of its own", tagRef.GetName(), got) } - wantTagURI, err := resources.NewTagSnapshotURI(tagToUpdate.GetStatus().GetStorageLocation(), tagToUpdate.GetMetadata().GetAtespace(), tagToUpdate.GetMetadata().GetUid()) - if err != nil { - t.Fatalf("NewTagSnapshotURI: %v", err) - } - if got := tagToUpdate.GetStatus().GetSnapshot().GetSnapshotUri(); got != wantTagURI.String() { - t.Errorf("tag snapshot URI = %q, want UID-based URI %q", got, wantTagURI) + if _, err := resources.ParseSnapshotURI(tagSnapshotURI(tagToUpdate)); err != nil { + t.Errorf("ParseSnapshotURI(%q): %v", tagSnapshotURI(tagToUpdate), err) } listed, err := clients.SubstrateAPI.ListTags(ctx, &ateapipb.ListTagsRequest{Atespace: demoAtespace}) if err != nil { @@ -648,8 +644,8 @@ func runActorLifecycleTestCase(t *testing.T, prefix string, createTemplate func( if err != nil { t.Fatalf("failed to get suspended Actor: %v", err) } - if suspendedActor.GetStatus().GetLocalSnapshot() != nil { - t.Errorf("suspended Actor still carries LocalSnapshot: %v", suspendedActor.GetStatus().GetLocalSnapshot()) + if hasLocalSnapshot(suspendedActor.GetStatus()) { + t.Errorf("suspended Actor still carries LocalSnapshot: %v", suspendedActor.GetStatus().GetSnapshots()) } } @@ -686,12 +682,62 @@ func validateSnapshotFidelity(ctx context.Context, t *testing.T, clients *e2e.Cl if err != nil { t.Fatalf("failed to get suspended Actor: %v", err) } - if actor.GetStatus().GetExternalSnapshot().GetSnapshotUri() == "" { + if durableSnapshotURI(actor.GetStatus()) == "" { t.Fatal("suspended Actor has no external snapshot") } - if got := actor.GetStatus().GetExternalSnapshot().GetFidelity(); got != want { - t.Errorf("snapshot %q content scope = %v, want %v", actor.GetStatus().GetExternalSnapshot().GetSnapshotUri(), got, want) + if got := durableSnapshotStorage(actor.GetStatus()).GetFidelity(); got != want { + t.Errorf("snapshot %q content scope = %v, want %v", durableSnapshotURI(actor.GetStatus()), got, want) + } +} + +func durableSnapshot(status *ateapipb.ActorStatus) *ateapipb.Snapshot { + var best *ateapipb.Snapshot + for _, snap := range status.GetSnapshots() { + for _, st := range snap.GetStorage() { + if st.GetDurability() == ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE && + st.GetStatus() == ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED && + st.GetObject().GetSnapshotUri() != "" { + if best == nil || snap.GetGeneration() > best.GetGeneration() { + best = snap + } + } + } + } + return best +} + +func durableSnapshotStorage(status *ateapipb.ActorStatus) *ateapipb.SnapshotStorage { + snap := durableSnapshot(status) + for _, st := range snap.GetStorage() { + if st.GetDurability() == ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE { + return st + } + } + return nil +} + +func durableSnapshotURI(status *ateapipb.ActorStatus) string { + return durableSnapshotStorage(status).GetObject().GetSnapshotUri() +} + +func hasLocalSnapshot(status *ateapipb.ActorStatus) bool { + for _, snap := range status.GetSnapshots() { + for _, st := range snap.GetStorage() { + if st.GetDurability() == ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL { + return true + } + } + } + return false +} + +func tagSnapshotURI(tag *ateapipb.Tag) string { + for _, st := range tag.GetStatus().GetSnapshot().GetStorage() { + if st.GetDurability() == ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE { + return st.GetObject().GetSnapshotUri() + } } + return "" } // validateSecondFileCounter checks the counter the workload keeps in its second @@ -958,7 +1004,7 @@ func revertActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj if err != nil { t.Fatalf("failed to suspend Actor: %v", err) } - snapshotURI := suspended.GetActor().GetStatus().GetExternalSnapshot().GetSnapshotUri() + snapshotURI := durableSnapshotURI(suspended.GetActor().GetStatus()) if snapshotURI == "" { t.Fatal("suspend wrote no external snapshot") } @@ -983,7 +1029,7 @@ func revertActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj if got := reverted.GetActor().GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_SUSPENDED { t.Fatalf("state after revert = %v, want SUSPENDED", got) } - if got := reverted.GetActor().GetStatus().GetExternalSnapshot().GetSnapshotUri(); got != snapshotURI { + if got := durableSnapshotURI(reverted.GetActor().GetStatus()); got != snapshotURI { t.Fatalf("snapshot URI after revert = %q, want %q", got, snapshotURI) } if reverted.GetActor().GetStatus().GetWorkerAssignment() != nil { @@ -1018,10 +1064,10 @@ func revertActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj if got := reverted.GetActor().GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_SUSPENDED { t.Fatalf("state after revert from PAUSED = %v, want SUSPENDED", got) } - if reverted.GetActor().GetStatus().GetLocalSnapshot() != nil { + if hasLocalSnapshot(reverted.GetActor().GetStatus()) { t.Fatal("local snapshot pointer survived revert from PAUSED") } - if got := reverted.GetActor().GetStatus().GetExternalSnapshot().GetSnapshotUri(); got != snapshotURI { + if got := durableSnapshotURI(reverted.GetActor().GetStatus()); got != snapshotURI { t.Fatalf("snapshot URI after revert from PAUSED = %q, want %q", got, snapshotURI) } @@ -1614,7 +1660,7 @@ func TestRevertCrashedActor(t *testing.T) { if err != nil { t.Fatalf("failed to suspend Actor: %v", err) } - snapshotURI := suspended.GetActor().GetStatus().GetExternalSnapshot().GetSnapshotUri() + snapshotURI := durableSnapshotURI(suspended.GetActor().GetStatus()) if snapshotURI == "" { t.Fatal("suspend wrote no external snapshot") } @@ -1664,7 +1710,7 @@ func TestRevertCrashedActor(t *testing.T) { if got := reverted.GetActor().GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_SUSPENDED { t.Fatalf("state after revert = %v, want SUSPENDED", got) } - if got := reverted.GetActor().GetStatus().GetExternalSnapshot().GetSnapshotUri(); got != snapshotURI { + if got := durableSnapshotURI(reverted.GetActor().GetStatus()); got != snapshotURI { t.Fatalf("snapshot URI after revert = %q, want %q", got, snapshotURI) } if reverted.GetActor().GetStatus().GetWorkerAssignment() != nil { diff --git a/internal/e2e/suites/demo/updatetemplate_test.go b/internal/e2e/suites/demo/updatetemplate_test.go index dd507e9296..b689189043 100644 --- a/internal/e2e/suites/demo/updatetemplate_test.go +++ b/internal/e2e/suites/demo/updatetemplate_test.go @@ -138,10 +138,10 @@ func runUpdateTemplateTestCase(t *testing.T, fidelity ateapipb.SnapshotFidelity) if err != nil { t.Fatalf("failed to get suspended Actor: %v", err) } - if got, want := suspended.GetStatus().GetExternalSnapshot().GetActorTemplateUid(), createdA.GetMetadata().GetUid(); got != want { + if got, want := durableSnapshot(suspended.GetStatus()).GetActorTemplateUid(), createdA.GetMetadata().GetUid(); got != want { t.Errorf("suspended Actor external_snapshot.actor_template_uid = %q, want template A's %q", got, want) } - if suspended.GetStatus().GetExternalSnapshot().GetSnapshotUri() == "" { + if durableSnapshotURI(suspended.GetStatus()) == "" { t.Error("suspended Actor has no external snapshot") } if wa := suspended.GetStatus().GetWorkerAssignment(); wa != nil { @@ -188,7 +188,7 @@ func runUpdateTemplateTestCase(t *testing.T, fidelity ateapipb.SnapshotFidelity) t.Errorf("[after template update] expected %q (template B validating the preserved file), got response: %s", want, resp) } - // Pause under template B while status.external_snapshot still holds the last + // Pause under template B while status.snapshots still holds the last // committed snapshot from template A. Resuming from PAUSED restores the // local checkpoint (which was captured under template B, since templates // can only be updated while SUSPENDED) and preserves the in-memory counter @@ -207,7 +207,7 @@ func runUpdateTemplateTestCase(t *testing.T, fidelity ateapipb.SnapshotFidelity) if err != nil { t.Fatalf("failed to get paused Actor: %v", err) } - if got, want := paused.GetStatus().GetExternalSnapshot().GetActorTemplateUid(), createdA.GetMetadata().GetUid(); got != want { + if got, want := durableSnapshot(paused.GetStatus()).GetActorTemplateUid(), createdA.GetMetadata().GetUid(); got != want { t.Errorf("paused Actor external_snapshot.actor_template_uid = %q, want template A's %q", got, want) } @@ -248,15 +248,15 @@ func runUpdateTemplateTestCase(t *testing.T, fidelity ateapipb.SnapshotFidelity) } // Resume prefers a local checkpoint over the external snapshot, so one // surviving here would hide the repoint the next resume has to detect. - if got := revertedStatus.GetLocalSnapshot(); got != nil { - t.Errorf("reverted Actor local_snapshot = %v, want cleared", got) + if hasLocalSnapshot(revertedStatus) { + t.Errorf("reverted Actor local_snapshot present in %v, want cleared", revertedStatus.GetSnapshots()) } // The two halves of the mismatch the next resume has to spot: the spec // names B, the snapshot it would restore was captured under A. if got := reverted.GetActor().GetActorTemplate().GetName(); got != nameB { t.Errorf("reverted Actor actor_template = %q, want %q", got, nameB) } - if got, want := revertedStatus.GetExternalSnapshot().GetActorTemplateUid(), createdA.GetMetadata().GetUid(); got != want { + if got, want := durableSnapshot(revertedStatus).GetActorTemplateUid(), createdA.GetMetadata().GetUid(); got != want { t.Errorf("reverted Actor external_snapshot.actor_template_uid = %q, want template A's %q", got, want) } @@ -298,7 +298,7 @@ func runUpdateTemplateTestCase(t *testing.T, fidelity ateapipb.SnapshotFidelity) if err != nil { t.Fatalf("failed to get re-suspended Actor: %v", err) } - if got, want := suspended.GetStatus().GetExternalSnapshot().GetActorTemplateUid(), createdB.GetMetadata().GetUid(); got != want { + if got, want := durableSnapshot(suspended.GetStatus()).GetActorTemplateUid(), createdB.GetMetadata().GetUid(); got != want { t.Errorf("re-suspended Actor external_snapshot.actor_template_uid = %q, want template B's %q", got, want) } } diff --git a/internal/e2e/suites/security/crossatespace_test.go b/internal/e2e/suites/security/crossatespace_test.go index f40fa865fd..355229d310 100644 --- a/internal/e2e/suites/security/crossatespace_test.go +++ b/internal/e2e/suites/security/crossatespace_test.go @@ -225,7 +225,7 @@ func testTagScopeBoundary(t *testing.T, ctx context.Context, clients *e2e.Client if err != nil { t.Fatalf("SuspendActor %s/%s: %v", ns.a, source, err) } - if suspended.GetActor().GetStatus().GetExternalSnapshot().GetSnapshotUri() == "" { + if durableSnapshotURI(suspended.GetActor().GetStatus()) == "" { t.Fatalf("suspended actor %s/%s has no external snapshot, so there is nothing for a tag to capture", ns.a, source) } @@ -246,7 +246,7 @@ func testTagScopeBoundary(t *testing.T, ctx context.Context, clients *e2e.Client if err != nil { t.Fatalf("CreateTag %s/%s: %v", ns.a, tagRef.GetName(), err) } - if tag.GetStatus().GetSnapshot().GetSnapshotUri() == "" { + if snapshotURI(tag.GetStatus().GetSnapshot()) == "" { t.Fatalf("tag %s/%s has no snapshot uri, so it could seed an Actor in NO atespace and the negative case below would be vacuous", ns.a, tagRef.GetName()) } @@ -451,3 +451,25 @@ func probeFileOp(t *testing.T, ctx context.Context, rc *e2e.RouterClient, actor } return out } + +func durableSnapshotURI(status *ateapipb.ActorStatus) string { + var best *ateapipb.Snapshot + for _, snap := range status.GetSnapshots() { + if uri := snapshotURI(snap); uri != "" { + if best == nil || snap.GetGeneration() > best.GetGeneration() { + best = snap + } + } + } + return snapshotURI(best) +} + +func snapshotURI(snap *ateapipb.Snapshot) string { + for _, st := range snap.GetStorage() { + if st.GetDurability() == ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE && + st.GetStatus() == ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED { + return st.GetObject().GetSnapshotUri() + } + } + return "" +} diff --git a/pkg/proto/ateapipb/ateapi.pb.go b/pkg/proto/ateapipb/ateapi.pb.go index 10bd750fae..b87d5d48d5 100644 --- a/pkg/proto/ateapipb/ateapi.pb.go +++ b/pkg/proto/ateapipb/ateapi.pb.go @@ -36,6 +36,165 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) +// SnapshotOwner indicates the owner of a snapshot. +type SnapshotOwner int32 + +const ( + SnapshotOwner_SNAPSHOT_OWNER_UNSPECIFIED SnapshotOwner = 0 + // Snapshot captured directly from this Actor. + SnapshotOwner_SNAPSHOT_OWNER_ACTOR SnapshotOwner = 1 + // Snapshot owned by a Tag (also includes Golden Snapshot) + SnapshotOwner_SNAPSHOT_OWNER_TAG SnapshotOwner = 2 // Keep this in sync with Snapshot.owner's maximum. +) + +// Enum value maps for SnapshotOwner. +var ( + SnapshotOwner_name = map[int32]string{ + 0: "SNAPSHOT_OWNER_UNSPECIFIED", + 1: "SNAPSHOT_OWNER_ACTOR", + 2: "SNAPSHOT_OWNER_TAG", + } + SnapshotOwner_value = map[string]int32{ + "SNAPSHOT_OWNER_UNSPECIFIED": 0, + "SNAPSHOT_OWNER_ACTOR": 1, + "SNAPSHOT_OWNER_TAG": 2, + } +) + +func (x SnapshotOwner) Enum() *SnapshotOwner { + p := new(SnapshotOwner) + *p = x + return p +} + +func (x SnapshotOwner) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (SnapshotOwner) Descriptor() protoreflect.EnumDescriptor { + return file_ateapi_proto_enumTypes[0].Descriptor() +} + +func (SnapshotOwner) Type() protoreflect.EnumType { + return &file_ateapi_proto_enumTypes[0] +} + +func (x SnapshotOwner) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use SnapshotOwner.Descriptor instead. +func (SnapshotOwner) EnumDescriptor() ([]byte, []int) { + return file_ateapi_proto_rawDescGZIP(), []int{0} +} + +// SnapshotDurability indicates the durability level of the snapshot. +// TODO: potentially rename this to survivability level. Is it at the actor or snapshot level? +type SnapshotDurability int32 + +const ( + SnapshotDurability_SNAPSHOT_DURABILITY_UNSPECIFIED SnapshotDurability = 0 + // Node-local checkpoint on the worker node + // Survives node reboot. + SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL SnapshotDurability = 1 + // Durable snapshot in external storage. + // Survives node crash and deletion. + SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE SnapshotDurability = 2 // Keep this in sync with SnapshotStorage.durability's maximum. +) + +// Enum value maps for SnapshotDurability. +var ( + SnapshotDurability_name = map[int32]string{ + 0: "SNAPSHOT_DURABILITY_UNSPECIFIED", + 1: "SNAPSHOT_DURABILITY_LOCAL", + 2: "SNAPSHOT_DURABILITY_DURABLE", + } + SnapshotDurability_value = map[string]int32{ + "SNAPSHOT_DURABILITY_UNSPECIFIED": 0, + "SNAPSHOT_DURABILITY_LOCAL": 1, + "SNAPSHOT_DURABILITY_DURABLE": 2, + } +) + +func (x SnapshotDurability) Enum() *SnapshotDurability { + p := new(SnapshotDurability) + *p = x + return p +} + +func (x SnapshotDurability) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (SnapshotDurability) Descriptor() protoreflect.EnumDescriptor { + return file_ateapi_proto_enumTypes[1].Descriptor() +} + +func (SnapshotDurability) Type() protoreflect.EnumType { + return &file_ateapi_proto_enumTypes[1] +} + +func (x SnapshotDurability) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use SnapshotDurability.Descriptor instead. +func (SnapshotDurability) EnumDescriptor() ([]byte, []int) { + return file_ateapi_proto_rawDescGZIP(), []int{1} +} + +// SnapshotStorageStatus indicates the lifecycle stage of a snapshot storage entry. +type SnapshotStorageStatus int32 + +const ( + SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_UNSPECIFIED SnapshotStorageStatus = 0 + // Snapshot creation or promotion is in progress. + SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS SnapshotStorageStatus = 1 + // Snapshot is complete and ready for use. + SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED SnapshotStorageStatus = 2 // Keep this in sync with SnapshotStorage.status's maximum. +) + +// Enum value maps for SnapshotStorageStatus. +var ( + SnapshotStorageStatus_name = map[int32]string{ + 0: "SNAPSHOT_STORAGE_STATUS_UNSPECIFIED", + 1: "SNAPSHOT_STORAGE_STATUS_IN_PROGRESS", + 2: "SNAPSHOT_STORAGE_STATUS_COMPLETED", + } + SnapshotStorageStatus_value = map[string]int32{ + "SNAPSHOT_STORAGE_STATUS_UNSPECIFIED": 0, + "SNAPSHOT_STORAGE_STATUS_IN_PROGRESS": 1, + "SNAPSHOT_STORAGE_STATUS_COMPLETED": 2, + } +) + +func (x SnapshotStorageStatus) Enum() *SnapshotStorageStatus { + p := new(SnapshotStorageStatus) + *p = x + return p +} + +func (x SnapshotStorageStatus) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (SnapshotStorageStatus) Descriptor() protoreflect.EnumDescriptor { + return file_ateapi_proto_enumTypes[2].Descriptor() +} + +func (SnapshotStorageStatus) Type() protoreflect.EnumType { + return &file_ateapi_proto_enumTypes[2] +} + +func (x SnapshotStorageStatus) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use SnapshotStorageStatus.Descriptor instead. +func (SnapshotStorageStatus) EnumDescriptor() ([]byte, []int) { + return file_ateapi_proto_rawDescGZIP(), []int{2} +} + // SnapshotFidelity orders the state layers a snapshot can hold, lowest to // highest. Each level includes the ones below it. type SnapshotFidelity int32 @@ -80,11 +239,11 @@ func (x SnapshotFidelity) String() string { } func (SnapshotFidelity) Descriptor() protoreflect.EnumDescriptor { - return file_ateapi_proto_enumTypes[0].Descriptor() + return file_ateapi_proto_enumTypes[3].Descriptor() } func (SnapshotFidelity) Type() protoreflect.EnumType { - return &file_ateapi_proto_enumTypes[0] + return &file_ateapi_proto_enumTypes[3] } func (x SnapshotFidelity) Number() protoreflect.EnumNumber { @@ -93,7 +252,7 @@ func (x SnapshotFidelity) Number() protoreflect.EnumNumber { // Deprecated: Use SnapshotFidelity.Descriptor instead. func (SnapshotFidelity) EnumDescriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{0} + return file_ateapi_proto_rawDescGZIP(), []int{3} } type TagScope int32 @@ -133,11 +292,11 @@ func (x TagScope) String() string { } func (TagScope) Descriptor() protoreflect.EnumDescriptor { - return file_ateapi_proto_enumTypes[1].Descriptor() + return file_ateapi_proto_enumTypes[4].Descriptor() } func (TagScope) Type() protoreflect.EnumType { - return &file_ateapi_proto_enumTypes[1] + return &file_ateapi_proto_enumTypes[4] } func (x TagScope) Number() protoreflect.EnumNumber { @@ -146,7 +305,7 @@ func (x TagScope) Number() protoreflect.EnumNumber { // Deprecated: Use TagScope.Descriptor instead. func (TagScope) EnumDescriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{1} + return file_ateapi_proto_rawDescGZIP(), []int{4} } type ActorState int32 @@ -203,11 +362,11 @@ func (x ActorState) String() string { } func (ActorState) Descriptor() protoreflect.EnumDescriptor { - return file_ateapi_proto_enumTypes[2].Descriptor() + return file_ateapi_proto_enumTypes[5].Descriptor() } func (ActorState) Type() protoreflect.EnumType { - return &file_ateapi_proto_enumTypes[2] + return &file_ateapi_proto_enumTypes[5] } func (x ActorState) Number() protoreflect.EnumNumber { @@ -216,7 +375,7 @@ func (x ActorState) Number() protoreflect.EnumNumber { // Deprecated: Use ActorState.Descriptor instead. func (ActorState) EnumDescriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{2} + return file_ateapi_proto_rawDescGZIP(), []int{5} } // SandboxClass selects the sandbox runtime family. Snapshots are not portable @@ -254,11 +413,11 @@ func (x SandboxClass) String() string { } func (SandboxClass) Descriptor() protoreflect.EnumDescriptor { - return file_ateapi_proto_enumTypes[3].Descriptor() + return file_ateapi_proto_enumTypes[6].Descriptor() } func (SandboxClass) Type() protoreflect.EnumType { - return &file_ateapi_proto_enumTypes[3] + return &file_ateapi_proto_enumTypes[6] } func (x SandboxClass) Number() protoreflect.EnumNumber { @@ -267,7 +426,7 @@ func (x SandboxClass) Number() protoreflect.EnumNumber { // Deprecated: Use SandboxClass.Descriptor instead. func (SandboxClass) EnumDescriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{3} + return file_ateapi_proto_rawDescGZIP(), []int{6} } // ActorMetadataField selects one identity field of the actor. @@ -307,11 +466,11 @@ func (x ActorMetadataField) String() string { } func (ActorMetadataField) Descriptor() protoreflect.EnumDescriptor { - return file_ateapi_proto_enumTypes[4].Descriptor() + return file_ateapi_proto_enumTypes[7].Descriptor() } func (ActorMetadataField) Type() protoreflect.EnumType { - return &file_ateapi_proto_enumTypes[4] + return &file_ateapi_proto_enumTypes[7] } func (x ActorMetadataField) Number() protoreflect.EnumNumber { @@ -320,7 +479,7 @@ func (x ActorMetadataField) Number() protoreflect.EnumNumber { // Deprecated: Use ActorMetadataField.Descriptor instead. func (ActorMetadataField) EnumDescriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{4} + return file_ateapi_proto_rawDescGZIP(), []int{7} } type WorkerState int32 @@ -358,11 +517,11 @@ func (x WorkerState) String() string { } func (WorkerState) Descriptor() protoreflect.EnumDescriptor { - return file_ateapi_proto_enumTypes[5].Descriptor() + return file_ateapi_proto_enumTypes[8].Descriptor() } func (WorkerState) Type() protoreflect.EnumType { - return &file_ateapi_proto_enumTypes[5] + return &file_ateapi_proto_enumTypes[8] } func (x WorkerState) Number() protoreflect.EnumNumber { @@ -371,7 +530,7 @@ func (x WorkerState) Number() protoreflect.EnumNumber { // Deprecated: Use WorkerState.Descriptor instead. func (WorkerState) EnumDescriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{5} + return file_ateapi_proto_rawDescGZIP(), []int{8} } type ExternalVolume_Status int32 @@ -413,11 +572,11 @@ func (x ExternalVolume_Status) String() string { } func (ExternalVolume_Status) Descriptor() protoreflect.EnumDescriptor { - return file_ateapi_proto_enumTypes[6].Descriptor() + return file_ateapi_proto_enumTypes[9].Descriptor() } func (ExternalVolume_Status) Type() protoreflect.EnumType { - return &file_ateapi_proto_enumTypes[6] + return &file_ateapi_proto_enumTypes[9] } func (x ExternalVolume_Status) Number() protoreflect.EnumNumber { @@ -426,47 +585,35 @@ func (x ExternalVolume_Status) Number() protoreflect.EnumNumber { // Deprecated: Use ExternalVolume_Status.Descriptor instead. func (ExternalVolume_Status) EnumDescriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{4, 0} + return file_ateapi_proto_rawDescGZIP(), []int{6, 0} } -// ExternalSnapshot addresses a snapshot held in object storage. -type ExternalSnapshot struct { +// ObjectSnapshot addresses a snapshot held in object storage. +type ObjectSnapshot struct { state protoimpl.MessageState `protogen:"open.v1"` // snapshot_uri addresses the snapshot's prefix in object storage. // // +k8s:required // +k8s:maxLength=2048 # the template's storage_location bound plus the owner prefix and snapshot name - SnapshotUri string `protobuf:"bytes,1,opt,name=snapshot_uri,json=snapshotUri,proto3" json:"snapshot_uri,omitempty"` - // fidelity is what the snapshot holds. - // - // +k8s:optional - // +k8s:minimum=1 - // +k8s:maximum=3 # keep this in sync with the SnapshotFidelity enum - Fidelity SnapshotFidelity `protobuf:"varint,2,opt,name=fidelity,proto3,enum=ateapi.SnapshotFidelity" json:"fidelity,omitempty"` - // UID of the ActorTemplate whose sandbox this snapshot's guest state was - // captured from. - // - // +k8s:optional - // +k8s:format=k8s-uuid - ActorTemplateUid string `protobuf:"bytes,3,opt,name=actor_template_uid,json=actorTemplateUid,proto3" json:"actor_template_uid,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + SnapshotUri string `protobuf:"bytes,1,opt,name=snapshot_uri,json=snapshotUri,proto3" json:"snapshot_uri,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } -func (x *ExternalSnapshot) Reset() { - *x = ExternalSnapshot{} +func (x *ObjectSnapshot) Reset() { + *x = ObjectSnapshot{} mi := &file_ateapi_proto_msgTypes[0] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } -func (x *ExternalSnapshot) String() string { +func (x *ObjectSnapshot) String() string { return protoimpl.X.MessageStringOf(x) } -func (*ExternalSnapshot) ProtoMessage() {} +func (*ObjectSnapshot) ProtoMessage() {} -func (x *ExternalSnapshot) ProtoReflect() protoreflect.Message { +func (x *ObjectSnapshot) ProtoReflect() protoreflect.Message { mi := &file_ateapi_proto_msgTypes[0] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -478,48 +625,29 @@ func (x *ExternalSnapshot) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use ExternalSnapshot.ProtoReflect.Descriptor instead. -func (*ExternalSnapshot) Descriptor() ([]byte, []int) { +// Deprecated: Use ObjectSnapshot.ProtoReflect.Descriptor instead. +func (*ObjectSnapshot) Descriptor() ([]byte, []int) { return file_ateapi_proto_rawDescGZIP(), []int{0} } -func (x *ExternalSnapshot) GetSnapshotUri() string { +func (x *ObjectSnapshot) GetSnapshotUri() string { if x != nil { return x.SnapshotUri } return "" } -func (x *ExternalSnapshot) GetFidelity() SnapshotFidelity { - if x != nil { - return x.Fidelity - } - return SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED -} - -func (x *ExternalSnapshot) GetActorTemplateUid() string { - if x != nil { - return x.ActorTemplateUid - } - return "" -} - // LocalSnapshot records information about a node-local snapshot. type LocalSnapshot struct { state protoimpl.MessageState `protogen:"open.v1"` // The name of the local checkpoint on the node. Checkpoint names are // server-generated UUIDs, but any resource name is valid here. // - // +k8s:optional - // +k8s:format=k8s-short-name - SnapshotName string `protobuf:"bytes,1,opt,name=snapshot_name,json=snapshotName,proto3" json:"snapshot_name,omitempty"` - // Fidelity the pause checkpoint captured: the template's preferredFidelity - // at pause time. + // TODO: consider removing this since it's the same as the snapshot_id // - // +k8s:optional - // +k8s:minimum=1 - // +k8s:maximum=3 # keep this in sync with the SnapshotFidelity enum - Fidelity SnapshotFidelity `protobuf:"varint,2,opt,name=fidelity,proto3,enum=ateapi.SnapshotFidelity" json:"fidelity,omitempty"` + // +k8s:required + // +k8s:format=k8s-short-name + SnapshotName string `protobuf:"bytes,1,opt,name=snapshot_name,json=snapshotName,proto3" json:"snapshot_name,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -561,13 +689,200 @@ func (x *LocalSnapshot) GetSnapshotName() string { return "" } -func (x *LocalSnapshot) GetFidelity() SnapshotFidelity { +// SnapshotStorage records the physical storage representation of a Snapshot. +type SnapshotStorage struct { + state protoimpl.MessageState `protogen:"open.v1"` + // durability is the durability tier of this storage entry. Only + // SNAPSHOT_DURABILITY_LOCAL and SNAPSHOT_DURABILITY_DURABLE are allowed. + // + // +k8s:required + // +k8s:minimum=1 + // +k8s:maximum=2 # keep this in sync with the SnapshotDurability enum + Durability SnapshotDurability `protobuf:"varint,1,opt,name=durability,proto3,enum=ateapi.SnapshotDurability" json:"durability,omitempty"` + // status is the lifecycle status of this storage representation. + // + // +k8s:required + // +k8s:minimum=1 + // +k8s:maximum=2 # keep this in sync with the SnapshotStorageStatus enum + Status SnapshotStorageStatus `protobuf:"varint,2,opt,name=status,proto3,enum=ateapi.SnapshotStorageStatus" json:"status,omitempty"` + // fidelity is what the snapshot holds. + // + // +k8s:required + // +k8s:minimum=1 + // +k8s:maximum=3 # keep this in sync with the SnapshotFidelity enum + Fidelity SnapshotFidelity `protobuf:"varint,3,opt,name=fidelity,proto3,enum=ateapi.SnapshotFidelity" json:"fidelity,omitempty"` + // Exactly one of local or object must be set. + // + // local snapshot is stored on the node where the actor was suspended. + // + // +k8s:optional + // +k8s:unionMember + Local *LocalSnapshot `protobuf:"bytes,4,opt,name=local,proto3" json:"local,omitempty"` + // object snapshot is stored in object storage. + // + // +k8s:optional + // +k8s:unionMember + Object *ObjectSnapshot `protobuf:"bytes,5,opt,name=object,proto3" json:"object,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SnapshotStorage) Reset() { + *x = SnapshotStorage{} + mi := &file_ateapi_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SnapshotStorage) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SnapshotStorage) ProtoMessage() {} + +func (x *SnapshotStorage) ProtoReflect() protoreflect.Message { + mi := &file_ateapi_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SnapshotStorage.ProtoReflect.Descriptor instead. +func (*SnapshotStorage) Descriptor() ([]byte, []int) { + return file_ateapi_proto_rawDescGZIP(), []int{2} +} + +func (x *SnapshotStorage) GetDurability() SnapshotDurability { + if x != nil { + return x.Durability + } + return SnapshotDurability_SNAPSHOT_DURABILITY_UNSPECIFIED +} + +func (x *SnapshotStorage) GetStatus() SnapshotStorageStatus { + if x != nil { + return x.Status + } + return SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_UNSPECIFIED +} + +func (x *SnapshotStorage) GetFidelity() SnapshotFidelity { if x != nil { return x.Fidelity } return SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED } +func (x *SnapshotStorage) GetLocal() *LocalSnapshot { + if x != nil { + return x.Local + } + return nil +} + +func (x *SnapshotStorage) GetObject() *ObjectSnapshot { + if x != nil { + return x.Object + } + return nil +} + +// Snapshot represents one generation of an Actor's or Tag's snapshot state. +type Snapshot struct { + state protoimpl.MessageState `protogen:"open.v1"` + // generation is the monotonically increasing generation number of this + // snapshot, starting at 1. + // + // +k8s:required + // +k8s:minimum=1 + Generation int32 `protobuf:"varint,1,opt,name=generation,proto3" json:"generation,omitempty"` + // owner identifies the origin and ownership of this snapshot. + // + // +k8s:required + // +k8s:minimum=1 + // +k8s:maximum=2 # keep this in sync with the SnapshotOwner enum + Owner SnapshotOwner `protobuf:"varint,2,opt,name=owner,proto3,enum=ateapi.SnapshotOwner" json:"owner,omitempty"` + // UID of the ActorTemplate whose sandbox this snapshot's guest state was + // captured from. + // + // +k8s:optional + // +k8s:format=k8s-uuid + ActorTemplateUid string `protobuf:"bytes,3,opt,name=actor_template_uid,json=actorTemplateUid,proto3" json:"actor_template_uid,omitempty"` + // storage lists the storage representations of this snapshot by durability. + // Only SNAPSHOT_DURABILITY_LOCAL and SNAPSHOT_DURABILITY_DURABLE are allowed. + // + // +k8s:required + // +k8s:maxItems=2 + // +k8s:listType=map + // +k8s:listMapKey=durability + Storage []*SnapshotStorage `protobuf:"bytes,4,rep,name=storage,proto3" json:"storage,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Snapshot) Reset() { + *x = Snapshot{} + mi := &file_ateapi_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Snapshot) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Snapshot) ProtoMessage() {} + +func (x *Snapshot) ProtoReflect() protoreflect.Message { + mi := &file_ateapi_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Snapshot.ProtoReflect.Descriptor instead. +func (*Snapshot) Descriptor() ([]byte, []int) { + return file_ateapi_proto_rawDescGZIP(), []int{3} +} + +func (x *Snapshot) GetGeneration() int32 { + if x != nil { + return x.Generation + } + return 0 +} + +func (x *Snapshot) GetOwner() SnapshotOwner { + if x != nil { + return x.Owner + } + return SnapshotOwner_SNAPSHOT_OWNER_UNSPECIFIED +} + +func (x *Snapshot) GetActorTemplateUid() string { + if x != nil { + return x.ActorTemplateUid + } + return "" +} + +func (x *Snapshot) GetStorage() []*SnapshotStorage { + if x != nil { + return x.Storage + } + return nil +} + // Selector matches worker pools by label. // Only equality-based matching is supported. type Selector struct { @@ -587,7 +902,7 @@ type Selector struct { func (x *Selector) Reset() { *x = Selector{} - mi := &file_ateapi_proto_msgTypes[2] + mi := &file_ateapi_proto_msgTypes[4] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -599,7 +914,7 @@ func (x *Selector) String() string { func (*Selector) ProtoMessage() {} func (x *Selector) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[2] + mi := &file_ateapi_proto_msgTypes[4] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -612,7 +927,7 @@ func (x *Selector) ProtoReflect() protoreflect.Message { // Deprecated: Use Selector.ProtoReflect.Descriptor instead. func (*Selector) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{2} + return file_ateapi_proto_rawDescGZIP(), []int{4} } func (x *Selector) GetMatchLabels() map[string]string { @@ -683,7 +998,7 @@ type ResourceMetadata struct { func (x *ResourceMetadata) Reset() { *x = ResourceMetadata{} - mi := &file_ateapi_proto_msgTypes[3] + mi := &file_ateapi_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -695,7 +1010,7 @@ func (x *ResourceMetadata) String() string { func (*ResourceMetadata) ProtoMessage() {} func (x *ResourceMetadata) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[3] + mi := &file_ateapi_proto_msgTypes[5] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -708,7 +1023,7 @@ func (x *ResourceMetadata) ProtoReflect() protoreflect.Message { // Deprecated: Use ResourceMetadata.ProtoReflect.Descriptor instead. func (*ResourceMetadata) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{3} + return file_ateapi_proto_rawDescGZIP(), []int{5} } func (x *ResourceMetadata) GetAtespace() string { @@ -807,7 +1122,7 @@ type ExternalVolume struct { func (x *ExternalVolume) Reset() { *x = ExternalVolume{} - mi := &file_ateapi_proto_msgTypes[4] + mi := &file_ateapi_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -819,7 +1134,7 @@ func (x *ExternalVolume) String() string { func (*ExternalVolume) ProtoMessage() {} func (x *ExternalVolume) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[4] + mi := &file_ateapi_proto_msgTypes[6] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -832,7 +1147,7 @@ func (x *ExternalVolume) ProtoReflect() protoreflect.Message { // Deprecated: Use ExternalVolume.ProtoReflect.Descriptor instead. func (*ExternalVolume) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{4} + return file_ateapi_proto_rawDescGZIP(), []int{6} } func (x *ExternalVolume) GetVolumeName() string { @@ -908,7 +1223,7 @@ type Actor struct { func (x *Actor) Reset() { *x = Actor{} - mi := &file_ateapi_proto_msgTypes[5] + mi := &file_ateapi_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -920,7 +1235,7 @@ func (x *Actor) String() string { func (*Actor) ProtoMessage() {} func (x *Actor) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[5] + mi := &file_ateapi_proto_msgTypes[7] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -933,7 +1248,7 @@ func (x *Actor) ProtoReflect() protoreflect.Message { // Deprecated: Use Actor.ProtoReflect.Descriptor instead. func (*Actor) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{5} + return file_ateapi_proto_rawDescGZIP(), []int{7} } func (x *Actor) GetMetadata() *ResourceMetadata { @@ -1011,7 +1326,7 @@ type EgressPolicy struct { func (x *EgressPolicy) Reset() { *x = EgressPolicy{} - mi := &file_ateapi_proto_msgTypes[6] + mi := &file_ateapi_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1023,7 +1338,7 @@ func (x *EgressPolicy) String() string { func (*EgressPolicy) ProtoMessage() {} func (x *EgressPolicy) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[6] + mi := &file_ateapi_proto_msgTypes[8] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1036,7 +1351,7 @@ func (x *EgressPolicy) ProtoReflect() protoreflect.Message { // Deprecated: Use EgressPolicy.ProtoReflect.Descriptor instead. func (*EgressPolicy) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{6} + return file_ateapi_proto_rawDescGZIP(), []int{8} } func (x *EgressPolicy) GetMetadata() *ResourceMetadata { @@ -1086,7 +1401,7 @@ type EgressRule struct { func (x *EgressRule) Reset() { *x = EgressRule{} - mi := &file_ateapi_proto_msgTypes[7] + mi := &file_ateapi_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1098,7 +1413,7 @@ func (x *EgressRule) String() string { func (*EgressRule) ProtoMessage() {} func (x *EgressRule) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[7] + mi := &file_ateapi_proto_msgTypes[9] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1111,7 +1426,7 @@ func (x *EgressRule) ProtoReflect() protoreflect.Message { // Deprecated: Use EgressRule.ProtoReflect.Descriptor instead. func (*EgressRule) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{7} + return file_ateapi_proto_rawDescGZIP(), []int{9} } func (x *EgressRule) GetHttp() *HTTPRule { @@ -1179,7 +1494,7 @@ type HTTPRule struct { func (x *HTTPRule) Reset() { *x = HTTPRule{} - mi := &file_ateapi_proto_msgTypes[8] + mi := &file_ateapi_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1191,7 +1506,7 @@ func (x *HTTPRule) String() string { func (*HTTPRule) ProtoMessage() {} func (x *HTTPRule) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[8] + mi := &file_ateapi_proto_msgTypes[10] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1204,7 +1519,7 @@ func (x *HTTPRule) ProtoReflect() protoreflect.Message { // Deprecated: Use HTTPRule.ProtoReflect.Descriptor instead. func (*HTTPRule) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{8} + return file_ateapi_proto_rawDescGZIP(), []int{10} } func (x *HTTPRule) GetHostnames() []string { @@ -1264,7 +1579,7 @@ type HTTPSRule struct { func (x *HTTPSRule) Reset() { *x = HTTPSRule{} - mi := &file_ateapi_proto_msgTypes[9] + mi := &file_ateapi_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1276,7 +1591,7 @@ func (x *HTTPSRule) String() string { func (*HTTPSRule) ProtoMessage() {} func (x *HTTPSRule) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[9] + mi := &file_ateapi_proto_msgTypes[11] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1289,7 +1604,7 @@ func (x *HTTPSRule) ProtoReflect() protoreflect.Message { // Deprecated: Use HTTPSRule.ProtoReflect.Descriptor instead. func (*HTTPSRule) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{9} + return file_ateapi_proto_rawDescGZIP(), []int{11} } func (x *HTTPSRule) GetHostnames() []string { @@ -1340,7 +1655,7 @@ type TLSPassthroughRule struct { func (x *TLSPassthroughRule) Reset() { *x = TLSPassthroughRule{} - mi := &file_ateapi_proto_msgTypes[10] + mi := &file_ateapi_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1352,7 +1667,7 @@ func (x *TLSPassthroughRule) String() string { func (*TLSPassthroughRule) ProtoMessage() {} func (x *TLSPassthroughRule) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[10] + mi := &file_ateapi_proto_msgTypes[12] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1365,7 +1680,7 @@ func (x *TLSPassthroughRule) ProtoReflect() protoreflect.Message { // Deprecated: Use TLSPassthroughRule.ProtoReflect.Descriptor instead. func (*TLSPassthroughRule) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{10} + return file_ateapi_proto_rawDescGZIP(), []int{12} } func (x *TLSPassthroughRule) GetHostnames() []string { @@ -1406,7 +1721,7 @@ type Ports struct { func (x *Ports) Reset() { *x = Ports{} - mi := &file_ateapi_proto_msgTypes[11] + mi := &file_ateapi_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1418,7 +1733,7 @@ func (x *Ports) String() string { func (*Ports) ProtoMessage() {} func (x *Ports) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[11] + mi := &file_ateapi_proto_msgTypes[13] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1431,7 +1746,7 @@ func (x *Ports) ProtoReflect() protoreflect.Message { // Deprecated: Use Ports.ProtoReflect.Descriptor instead. func (*Ports) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{11} + return file_ateapi_proto_rawDescGZIP(), []int{13} } func (x *Ports) GetAll() *AllPorts { @@ -1457,7 +1772,7 @@ type AllPorts struct { func (x *AllPorts) Reset() { *x = AllPorts{} - mi := &file_ateapi_proto_msgTypes[12] + mi := &file_ateapi_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1469,7 +1784,7 @@ func (x *AllPorts) String() string { func (*AllPorts) ProtoMessage() {} func (x *AllPorts) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[12] + mi := &file_ateapi_proto_msgTypes[14] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1482,7 +1797,7 @@ func (x *AllPorts) ProtoReflect() protoreflect.Message { // Deprecated: Use AllPorts.ProtoReflect.Descriptor instead. func (*AllPorts) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{12} + return file_ateapi_proto_rawDescGZIP(), []int{14} } // HttpRuleEffects contains effects applied by a matching HTTP or HTTPS rule. @@ -1516,7 +1831,7 @@ type HttpRuleEffects struct { func (x *HttpRuleEffects) Reset() { *x = HttpRuleEffects{} - mi := &file_ateapi_proto_msgTypes[13] + mi := &file_ateapi_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1528,7 +1843,7 @@ func (x *HttpRuleEffects) String() string { func (*HttpRuleEffects) ProtoMessage() {} func (x *HttpRuleEffects) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[13] + mi := &file_ateapi_proto_msgTypes[15] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1541,7 +1856,7 @@ func (x *HttpRuleEffects) ProtoReflect() protoreflect.Message { // Deprecated: Use HttpRuleEffects.ProtoReflect.Descriptor instead. func (*HttpRuleEffects) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{13} + return file_ateapi_proto_rawDescGZIP(), []int{15} } func (x *HttpRuleEffects) GetReplaceHeaders() []*CredentialHeader { @@ -1591,7 +1906,7 @@ type CredentialHeader struct { func (x *CredentialHeader) Reset() { *x = CredentialHeader{} - mi := &file_ateapi_proto_msgTypes[14] + mi := &file_ateapi_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1603,7 +1918,7 @@ func (x *CredentialHeader) String() string { func (*CredentialHeader) ProtoMessage() {} func (x *CredentialHeader) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[14] + mi := &file_ateapi_proto_msgTypes[16] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1616,7 +1931,7 @@ func (x *CredentialHeader) ProtoReflect() protoreflect.Message { // Deprecated: Use CredentialHeader.ProtoReflect.Descriptor instead. func (*CredentialHeader) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{14} + return file_ateapi_proto_rawDescGZIP(), []int{16} } func (x *CredentialHeader) GetHeader() string { @@ -1673,7 +1988,7 @@ type ActorJWTSource struct { func (x *ActorJWTSource) Reset() { *x = ActorJWTSource{} - mi := &file_ateapi_proto_msgTypes[15] + mi := &file_ateapi_proto_msgTypes[17] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1685,7 +2000,7 @@ func (x *ActorJWTSource) String() string { func (*ActorJWTSource) ProtoMessage() {} func (x *ActorJWTSource) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[15] + mi := &file_ateapi_proto_msgTypes[17] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1698,7 +2013,7 @@ func (x *ActorJWTSource) ProtoReflect() protoreflect.Message { // Deprecated: Use ActorJWTSource.ProtoReflect.Descriptor instead. func (*ActorJWTSource) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{15} + return file_ateapi_proto_rawDescGZIP(), []int{17} } func (x *ActorJWTSource) GetAudiences() []string { @@ -1729,22 +2044,6 @@ type ActorStatus struct { // +k8s:optional // +k8s:update=NoModify # can be set and cleared, but not changed in place WorkerAssignment *WorkerAssignment `protobuf:"bytes,2,opt,name=worker_assignment,json=workerAssignment,proto3" json:"worker_assignment,omitempty"` - // in_progress_snapshot_uri is the URI in object storage of the durable - // snapshot the Actor is currently taking. - // - // +k8s:optional - // +k8s:maxLength=2048 # the template's storage_location bound plus the owner prefix and snapshot name - InProgressSnapshotUri string `protobuf:"bytes,3,opt,name=in_progress_snapshot_uri,json=inProgressSnapshotUri,proto3" json:"in_progress_snapshot_uri,omitempty"` - // external_snapshot is the Actor's current external snapshot. - // If the Actor was created from a Tag this is the tag's snapshot, borrowed - // until the Actor's first suspend writes one of its own. Otherwise it is - // unset until the Actor is first suspended. - // +k8s:optional - ExternalSnapshot *ExternalSnapshot `protobuf:"bytes,4,opt,name=external_snapshot,json=externalSnapshot,proto3" json:"external_snapshot,omitempty"` - // Node-local state used only while the Actor is paused. - // - // +k8s:optional - LocalSnapshot *LocalSnapshot `protobuf:"bytes,5,opt,name=local_snapshot,json=localSnapshot,proto3" json:"local_snapshot,omitempty"` // Volumes attached to the actor. These volumes only live as long as the actor. // They are deleted when the actor is deleted. Each template volume // appears at most once. @@ -1754,12 +2053,6 @@ type ActorStatus struct { // +k8s:listType=map // +k8s:listMapKey=volume_name ActorVolumes []*ExternalVolume `protobuf:"bytes,7,rep,name=actor_volumes,json=actorVolumes,proto3" json:"actor_volumes,omitempty"` - // The name of the in-progress node-local checkpoint. Like durable snapshot - // names, these are server-generated UUIDs. - // - // +k8s:optional - // +k8s:format=k8s-short-name - InProgressLocalSnapshotName string `protobuf:"bytes,8,opt,name=in_progress_local_snapshot_name,json=inProgressLocalSnapshotName,proto3" json:"in_progress_local_snapshot_name,omitempty"` // crash records why and when the Actor entered CRASHED. It is set with the // CRASHED state and cleared when a revert returns the Actor to SUSPENDED. // @@ -1771,14 +2064,32 @@ type ActorStatus struct { // // +k8s:optional // +k8s:format=k8s-long-name - AssignedNode string `protobuf:"bytes,10,opt,name=assigned_node,json=assignedNode,proto3" json:"assigned_node,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + AssignedNode string `protobuf:"bytes,10,opt,name=assigned_node,json=assignedNode,proto3" json:"assigned_node,omitempty"` + // snapshots holds the Actor's active and in-progress snapshot generations. + // + // +k8s:optional + // +k8s:maxItems=16 + // +k8s:listType=atomic + // +k8s:unique=map + // +k8s:listMapKey=generation + Snapshots []*Snapshot `protobuf:"bytes,11,rep,name=snapshots,proto3" json:"snapshots,omitempty"` + // last_assigned_generation tracks the last snapshot generation assigned for + // this Actor during a Suspend or Pause request. + // Even during Revert, the generation is not decreased to keep track of snapshots + // that have become stale and need to be cleaned up. + // + // +k8s:optional + // +k8s:alpha(since: "0.0")=+k8s:monotonic # TODO: get rid of alpha prefix + // +k8s:minimum=0 + // +k8s:update=NoUnset + LastAssignedGeneration int32 `protobuf:"varint,12,opt,name=last_assigned_generation,json=lastAssignedGeneration,proto3" json:"last_assigned_generation,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *ActorStatus) Reset() { *x = ActorStatus{} - mi := &file_ateapi_proto_msgTypes[16] + mi := &file_ateapi_proto_msgTypes[18] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1790,7 +2101,7 @@ func (x *ActorStatus) String() string { func (*ActorStatus) ProtoMessage() {} func (x *ActorStatus) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[16] + mi := &file_ateapi_proto_msgTypes[18] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1803,7 +2114,7 @@ func (x *ActorStatus) ProtoReflect() protoreflect.Message { // Deprecated: Use ActorStatus.ProtoReflect.Descriptor instead. func (*ActorStatus) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{16} + return file_ateapi_proto_rawDescGZIP(), []int{18} } func (x *ActorStatus) GetState() ActorState { @@ -1820,53 +2131,39 @@ func (x *ActorStatus) GetWorkerAssignment() *WorkerAssignment { return nil } -func (x *ActorStatus) GetInProgressSnapshotUri() string { - if x != nil { - return x.InProgressSnapshotUri - } - return "" -} - -func (x *ActorStatus) GetExternalSnapshot() *ExternalSnapshot { - if x != nil { - return x.ExternalSnapshot - } - return nil -} - -func (x *ActorStatus) GetLocalSnapshot() *LocalSnapshot { +func (x *ActorStatus) GetActorVolumes() []*ExternalVolume { if x != nil { - return x.LocalSnapshot + return x.ActorVolumes } return nil } -func (x *ActorStatus) GetActorVolumes() []*ExternalVolume { +func (x *ActorStatus) GetCrash() *ActorCrash { if x != nil { - return x.ActorVolumes + return x.Crash } return nil } -func (x *ActorStatus) GetInProgressLocalSnapshotName() string { +func (x *ActorStatus) GetAssignedNode() string { if x != nil { - return x.InProgressLocalSnapshotName + return x.AssignedNode } return "" } -func (x *ActorStatus) GetCrash() *ActorCrash { +func (x *ActorStatus) GetSnapshots() []*Snapshot { if x != nil { - return x.Crash + return x.Snapshots } return nil } -func (x *ActorStatus) GetAssignedNode() string { +func (x *ActorStatus) GetLastAssignedGeneration() int32 { if x != nil { - return x.AssignedNode + return x.LastAssignedGeneration } - return "" + return 0 } // ActorCrash describes the failure that moved an Actor to CRASHED. @@ -1887,7 +2184,7 @@ type ActorCrash struct { func (x *ActorCrash) Reset() { *x = ActorCrash{} - mi := &file_ateapi_proto_msgTypes[17] + mi := &file_ateapi_proto_msgTypes[19] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1899,7 +2196,7 @@ func (x *ActorCrash) String() string { func (*ActorCrash) ProtoMessage() {} func (x *ActorCrash) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[17] + mi := &file_ateapi_proto_msgTypes[19] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1912,7 +2209,7 @@ func (x *ActorCrash) ProtoReflect() protoreflect.Message { // Deprecated: Use ActorCrash.ProtoReflect.Descriptor instead. func (*ActorCrash) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{17} + return file_ateapi_proto_rawDescGZIP(), []int{19} } func (x *ActorCrash) GetMessage() string { @@ -1993,7 +2290,7 @@ type WorkerAssignment struct { func (x *WorkerAssignment) Reset() { *x = WorkerAssignment{} - mi := &file_ateapi_proto_msgTypes[18] + mi := &file_ateapi_proto_msgTypes[20] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2005,7 +2302,7 @@ func (x *WorkerAssignment) String() string { func (*WorkerAssignment) ProtoMessage() {} func (x *WorkerAssignment) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[18] + mi := &file_ateapi_proto_msgTypes[20] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2018,7 +2315,7 @@ func (x *WorkerAssignment) ProtoReflect() protoreflect.Message { // Deprecated: Use WorkerAssignment.ProtoReflect.Descriptor instead. func (*WorkerAssignment) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{18} + return file_ateapi_proto_rawDescGZIP(), []int{20} } func (x *WorkerAssignment) GetWorker() *ObjectRef { @@ -2082,33 +2379,26 @@ func (x *WorkerAssignment) GetWorkerEpoch() int64 { // the rest is immutable provenance. type TagStatus struct { state protoimpl.MessageState `protogen:"open.v1"` - // snapshot is the external snapshot this tag points at. Unset while the tag - // is still being created, and immutable once set: a tag never moves between - // snapshots. A tag is usable only once this is set. + // snapshot is the external snapshot this tag points at. Its storage entry is + // IN_PROGRESS while the tag is being created and transitions to COMPLETED + // once the copy finishes: a tag never moves between snapshots and is usable + // only once COMPLETED. // - // +k8s:optional - Snapshot *ExternalSnapshot `protobuf:"bytes,1,opt,name=snapshot,proto3" json:"snapshot,omitempty"` + // +k8s:required + Snapshot *Snapshot `protobuf:"bytes,1,opt,name=snapshot,proto3" json:"snapshot,omitempty"` // UID of the ActorTemplate the snapshot was taken under. Actors can only be // seeded from a tag under the same template. // // +k8s:required // +k8s:format=k8s-uuid ActorTemplateUid string `protobuf:"bytes,2,opt,name=actor_template_uid,json=actorTemplateUid,proto3" json:"actor_template_uid,omitempty"` - // storage_location is the base object-storage URI for this tag's snapshot. - // Set by the server from the source actor's template when the tag is created - // and immutable thereafter. The full snapshot URI is available in - // snapshot.snapshot_uri once tag creation completes. - // - // +k8s:required - // +k8s:maxLength=1024 - StorageLocation string `protobuf:"bytes,3,opt,name=storage_location,json=storageLocation,proto3" json:"storage_location,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *TagStatus) Reset() { *x = TagStatus{} - mi := &file_ateapi_proto_msgTypes[19] + mi := &file_ateapi_proto_msgTypes[21] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2120,7 +2410,7 @@ func (x *TagStatus) String() string { func (*TagStatus) ProtoMessage() {} func (x *TagStatus) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[19] + mi := &file_ateapi_proto_msgTypes[21] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2133,10 +2423,10 @@ func (x *TagStatus) ProtoReflect() protoreflect.Message { // Deprecated: Use TagStatus.ProtoReflect.Descriptor instead. func (*TagStatus) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{19} + return file_ateapi_proto_rawDescGZIP(), []int{21} } -func (x *TagStatus) GetSnapshot() *ExternalSnapshot { +func (x *TagStatus) GetSnapshot() *Snapshot { if x != nil { return x.Snapshot } @@ -2150,13 +2440,6 @@ func (x *TagStatus) GetActorTemplateUid() string { return "" } -func (x *TagStatus) GetStorageLocation() string { - if x != nil { - return x.StorageLocation - } - return "" -} - // Tag is an Atespace-owned, stable name for one external snapshot. The // external snapshot pointed by a tag is a copy of an actor snapshot. The // snapshot is copied when the tag is created, and deleted when the tag is. Its @@ -2192,7 +2475,7 @@ type Tag struct { func (x *Tag) Reset() { *x = Tag{} - mi := &file_ateapi_proto_msgTypes[20] + mi := &file_ateapi_proto_msgTypes[22] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2204,7 +2487,7 @@ func (x *Tag) String() string { func (*Tag) ProtoMessage() {} func (x *Tag) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[20] + mi := &file_ateapi_proto_msgTypes[22] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2217,7 +2500,7 @@ func (x *Tag) ProtoReflect() protoreflect.Message { // Deprecated: Use Tag.ProtoReflect.Descriptor instead. func (*Tag) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{20} + return file_ateapi_proto_rawDescGZIP(), []int{22} } func (x *Tag) GetMetadata() *ResourceMetadata { @@ -2263,7 +2546,7 @@ type Atespace struct { func (x *Atespace) Reset() { *x = Atespace{} - mi := &file_ateapi_proto_msgTypes[21] + mi := &file_ateapi_proto_msgTypes[23] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2275,7 +2558,7 @@ func (x *Atespace) String() string { func (*Atespace) ProtoMessage() {} func (x *Atespace) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[21] + mi := &file_ateapi_proto_msgTypes[23] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2288,7 +2571,7 @@ func (x *Atespace) ProtoReflect() protoreflect.Message { // Deprecated: Use Atespace.ProtoReflect.Descriptor instead. func (*Atespace) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{21} + return file_ateapi_proto_rawDescGZIP(), []int{23} } func (x *Atespace) GetMetadata() *ResourceMetadata { @@ -2321,7 +2604,7 @@ type ObjectRef struct { func (x *ObjectRef) Reset() { *x = ObjectRef{} - mi := &file_ateapi_proto_msgTypes[22] + mi := &file_ateapi_proto_msgTypes[24] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2333,7 +2616,7 @@ func (x *ObjectRef) String() string { func (*ObjectRef) ProtoMessage() {} func (x *ObjectRef) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[22] + mi := &file_ateapi_proto_msgTypes[24] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2346,7 +2629,7 @@ func (x *ObjectRef) ProtoReflect() protoreflect.Message { // Deprecated: Use ObjectRef.ProtoReflect.Descriptor instead. func (*ObjectRef) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{22} + return file_ateapi_proto_rawDescGZIP(), []int{24} } func (x *ObjectRef) GetAtespace() string { @@ -2403,7 +2686,7 @@ type ActorTemplate struct { func (x *ActorTemplate) Reset() { *x = ActorTemplate{} - mi := &file_ateapi_proto_msgTypes[23] + mi := &file_ateapi_proto_msgTypes[25] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2415,7 +2698,7 @@ func (x *ActorTemplate) String() string { func (*ActorTemplate) ProtoMessage() {} func (x *ActorTemplate) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[23] + mi := &file_ateapi_proto_msgTypes[25] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2428,7 +2711,7 @@ func (x *ActorTemplate) ProtoReflect() protoreflect.Message { // Deprecated: Use ActorTemplate.ProtoReflect.Descriptor instead. func (*ActorTemplate) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{23} + return file_ateapi_proto_rawDescGZIP(), []int{25} } func (x *ActorTemplate) GetMetadata() *ResourceMetadata { @@ -2504,7 +2787,7 @@ type Resources struct { func (x *Resources) Reset() { *x = Resources{} - mi := &file_ateapi_proto_msgTypes[24] + mi := &file_ateapi_proto_msgTypes[26] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2516,7 +2799,7 @@ func (x *Resources) String() string { func (*Resources) ProtoMessage() {} func (x *Resources) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[24] + mi := &file_ateapi_proto_msgTypes[26] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2529,7 +2812,7 @@ func (x *Resources) ProtoReflect() protoreflect.Message { // Deprecated: Use Resources.ProtoReflect.Descriptor instead. func (*Resources) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{24} + return file_ateapi_proto_rawDescGZIP(), []int{26} } func (x *Resources) GetLimits() []*Limits { @@ -2557,7 +2840,7 @@ type Limits struct { func (x *Limits) Reset() { *x = Limits{} - mi := &file_ateapi_proto_msgTypes[25] + mi := &file_ateapi_proto_msgTypes[27] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2569,7 +2852,7 @@ func (x *Limits) String() string { func (*Limits) ProtoMessage() {} func (x *Limits) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[25] + mi := &file_ateapi_proto_msgTypes[27] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2582,7 +2865,7 @@ func (x *Limits) ProtoReflect() protoreflect.Message { // Deprecated: Use Limits.ProtoReflect.Descriptor instead. func (*Limits) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{25} + return file_ateapi_proto_rawDescGZIP(), []int{27} } func (x *Limits) GetName() string { @@ -2623,7 +2906,7 @@ type GoldenSnapshotStatus struct { func (x *GoldenSnapshotStatus) Reset() { *x = GoldenSnapshotStatus{} - mi := &file_ateapi_proto_msgTypes[26] + mi := &file_ateapi_proto_msgTypes[28] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2635,7 +2918,7 @@ func (x *GoldenSnapshotStatus) String() string { func (*GoldenSnapshotStatus) ProtoMessage() {} func (x *GoldenSnapshotStatus) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[26] + mi := &file_ateapi_proto_msgTypes[28] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2648,7 +2931,7 @@ func (x *GoldenSnapshotStatus) ProtoReflect() protoreflect.Message { // Deprecated: Use GoldenSnapshotStatus.ProtoReflect.Descriptor instead. func (*GoldenSnapshotStatus) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{26} + return file_ateapi_proto_rawDescGZIP(), []int{28} } func (x *GoldenSnapshotStatus) GetGoldenTag() *ObjectRef { @@ -2682,7 +2965,7 @@ type ActorTemplateStatus struct { func (x *ActorTemplateStatus) Reset() { *x = ActorTemplateStatus{} - mi := &file_ateapi_proto_msgTypes[27] + mi := &file_ateapi_proto_msgTypes[29] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2694,7 +2977,7 @@ func (x *ActorTemplateStatus) String() string { func (*ActorTemplateStatus) ProtoMessage() {} func (x *ActorTemplateStatus) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[27] + mi := &file_ateapi_proto_msgTypes[29] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2707,7 +2990,7 @@ func (x *ActorTemplateStatus) ProtoReflect() protoreflect.Message { // Deprecated: Use ActorTemplateStatus.ProtoReflect.Descriptor instead. func (*ActorTemplateStatus) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{27} + return file_ateapi_proto_rawDescGZIP(), []int{29} } func (x *ActorTemplateStatus) GetGoldenSnapshotStatus() *GoldenSnapshotStatus { @@ -2738,7 +3021,7 @@ type SandboxConfig struct { func (x *SandboxConfig) Reset() { *x = SandboxConfig{} - mi := &file_ateapi_proto_msgTypes[28] + mi := &file_ateapi_proto_msgTypes[30] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2750,7 +3033,7 @@ func (x *SandboxConfig) String() string { func (*SandboxConfig) ProtoMessage() {} func (x *SandboxConfig) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[28] + mi := &file_ateapi_proto_msgTypes[30] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2763,7 +3046,7 @@ func (x *SandboxConfig) ProtoReflect() protoreflect.Message { // Deprecated: Use SandboxConfig.ProtoReflect.Descriptor instead. func (*SandboxConfig) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{28} + return file_ateapi_proto_rawDescGZIP(), []int{30} } func (x *SandboxConfig) GetSandboxClass() SandboxClass { @@ -2811,7 +3094,7 @@ type SnapshotConfig struct { func (x *SnapshotConfig) Reset() { *x = SnapshotConfig{} - mi := &file_ateapi_proto_msgTypes[29] + mi := &file_ateapi_proto_msgTypes[31] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2823,7 +3106,7 @@ func (x *SnapshotConfig) String() string { func (*SnapshotConfig) ProtoMessage() {} func (x *SnapshotConfig) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[29] + mi := &file_ateapi_proto_msgTypes[31] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2836,7 +3119,7 @@ func (x *SnapshotConfig) ProtoReflect() protoreflect.Message { // Deprecated: Use SnapshotConfig.ProtoReflect.Descriptor instead. func (*SnapshotConfig) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{29} + return file_ateapi_proto_rawDescGZIP(), []int{31} } func (x *SnapshotConfig) GetStorageLocation() string { @@ -2923,7 +3206,7 @@ type Container struct { func (x *Container) Reset() { *x = Container{} - mi := &file_ateapi_proto_msgTypes[30] + mi := &file_ateapi_proto_msgTypes[32] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2935,7 +3218,7 @@ func (x *Container) String() string { func (*Container) ProtoMessage() {} func (x *Container) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[30] + mi := &file_ateapi_proto_msgTypes[32] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2948,7 +3231,7 @@ func (x *Container) ProtoReflect() protoreflect.Message { // Deprecated: Use Container.ProtoReflect.Descriptor instead. func (*Container) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{30} + return file_ateapi_proto_rawDescGZIP(), []int{32} } func (x *Container) GetName() string { @@ -3028,7 +3311,7 @@ type SecurityContext struct { func (x *SecurityContext) Reset() { *x = SecurityContext{} - mi := &file_ateapi_proto_msgTypes[31] + mi := &file_ateapi_proto_msgTypes[33] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3040,7 +3323,7 @@ func (x *SecurityContext) String() string { func (*SecurityContext) ProtoMessage() {} func (x *SecurityContext) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[31] + mi := &file_ateapi_proto_msgTypes[33] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3053,7 +3336,7 @@ func (x *SecurityContext) ProtoReflect() protoreflect.Message { // Deprecated: Use SecurityContext.ProtoReflect.Descriptor instead. func (*SecurityContext) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{31} + return file_ateapi_proto_rawDescGZIP(), []int{33} } func (x *SecurityContext) GetCapabilities() *Capabilities { @@ -3091,7 +3374,7 @@ type Capabilities struct { func (x *Capabilities) Reset() { *x = Capabilities{} - mi := &file_ateapi_proto_msgTypes[32] + mi := &file_ateapi_proto_msgTypes[34] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3103,7 +3386,7 @@ func (x *Capabilities) String() string { func (*Capabilities) ProtoMessage() {} func (x *Capabilities) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[32] + mi := &file_ateapi_proto_msgTypes[34] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3116,7 +3399,7 @@ func (x *Capabilities) ProtoReflect() protoreflect.Message { // Deprecated: Use Capabilities.ProtoReflect.Descriptor instead. func (*Capabilities) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{32} + return file_ateapi_proto_rawDescGZIP(), []int{34} } func (x *Capabilities) GetAdd() []string { @@ -3154,7 +3437,7 @@ type EnvVar struct { func (x *EnvVar) Reset() { *x = EnvVar{} - mi := &file_ateapi_proto_msgTypes[33] + mi := &file_ateapi_proto_msgTypes[35] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3166,7 +3449,7 @@ func (x *EnvVar) String() string { func (*EnvVar) ProtoMessage() {} func (x *EnvVar) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[33] + mi := &file_ateapi_proto_msgTypes[35] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3179,7 +3462,7 @@ func (x *EnvVar) ProtoReflect() protoreflect.Message { // Deprecated: Use EnvVar.ProtoReflect.Descriptor instead. func (*EnvVar) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{33} + return file_ateapi_proto_rawDescGZIP(), []int{35} } func (x *EnvVar) GetName() string { @@ -3219,7 +3502,7 @@ type ContainerWakeupProbe struct { func (x *ContainerWakeupProbe) Reset() { *x = ContainerWakeupProbe{} - mi := &file_ateapi_proto_msgTypes[34] + mi := &file_ateapi_proto_msgTypes[36] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3231,7 +3514,7 @@ func (x *ContainerWakeupProbe) String() string { func (*ContainerWakeupProbe) ProtoMessage() {} func (x *ContainerWakeupProbe) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[34] + mi := &file_ateapi_proto_msgTypes[36] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3244,7 +3527,7 @@ func (x *ContainerWakeupProbe) ProtoReflect() protoreflect.Message { // Deprecated: Use ContainerWakeupProbe.ProtoReflect.Descriptor instead. func (*ContainerWakeupProbe) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{34} + return file_ateapi_proto_rawDescGZIP(), []int{36} } func (x *ContainerWakeupProbe) GetHttpGet() *HTTPGetAction { @@ -3284,7 +3567,7 @@ type HTTPGetAction struct { func (x *HTTPGetAction) Reset() { *x = HTTPGetAction{} - mi := &file_ateapi_proto_msgTypes[35] + mi := &file_ateapi_proto_msgTypes[37] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3296,7 +3579,7 @@ func (x *HTTPGetAction) String() string { func (*HTTPGetAction) ProtoMessage() {} func (x *HTTPGetAction) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[35] + mi := &file_ateapi_proto_msgTypes[37] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3309,7 +3592,7 @@ func (x *HTTPGetAction) ProtoReflect() protoreflect.Message { // Deprecated: Use HTTPGetAction.ProtoReflect.Descriptor instead. func (*HTTPGetAction) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{35} + return file_ateapi_proto_rawDescGZIP(), []int{37} } func (x *HTTPGetAction) GetPath() string { @@ -3356,7 +3639,7 @@ type Volume struct { func (x *Volume) Reset() { *x = Volume{} - mi := &file_ateapi_proto_msgTypes[36] + mi := &file_ateapi_proto_msgTypes[38] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3368,7 +3651,7 @@ func (x *Volume) String() string { func (*Volume) ProtoMessage() {} func (x *Volume) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[36] + mi := &file_ateapi_proto_msgTypes[38] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3381,7 +3664,7 @@ func (x *Volume) ProtoReflect() protoreflect.Message { // Deprecated: Use Volume.ProtoReflect.Descriptor instead. func (*Volume) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{36} + return file_ateapi_proto_rawDescGZIP(), []int{38} } func (x *Volume) GetName() string { @@ -3437,7 +3720,7 @@ type ImageVolumeSource struct { func (x *ImageVolumeSource) Reset() { *x = ImageVolumeSource{} - mi := &file_ateapi_proto_msgTypes[37] + mi := &file_ateapi_proto_msgTypes[39] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3449,7 +3732,7 @@ func (x *ImageVolumeSource) String() string { func (*ImageVolumeSource) ProtoMessage() {} func (x *ImageVolumeSource) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[37] + mi := &file_ateapi_proto_msgTypes[39] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3462,7 +3745,7 @@ func (x *ImageVolumeSource) ProtoReflect() protoreflect.Message { // Deprecated: Use ImageVolumeSource.ProtoReflect.Descriptor instead. func (*ImageVolumeSource) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{37} + return file_ateapi_proto_rawDescGZIP(), []int{39} } func (x *ImageVolumeSource) GetReference() string { @@ -3482,7 +3765,7 @@ type DurableDirVolumeSource struct { func (x *DurableDirVolumeSource) Reset() { *x = DurableDirVolumeSource{} - mi := &file_ateapi_proto_msgTypes[38] + mi := &file_ateapi_proto_msgTypes[40] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3494,7 +3777,7 @@ func (x *DurableDirVolumeSource) String() string { func (*DurableDirVolumeSource) ProtoMessage() {} func (x *DurableDirVolumeSource) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[38] + mi := &file_ateapi_proto_msgTypes[40] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3507,7 +3790,7 @@ func (x *DurableDirVolumeSource) ProtoReflect() protoreflect.Message { // Deprecated: Use DurableDirVolumeSource.ProtoReflect.Descriptor instead. func (*DurableDirVolumeSource) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{38} + return file_ateapi_proto_rawDescGZIP(), []int{40} } // ExternalVolumeTemplate provisions an external volume per actor; the volume @@ -3533,7 +3816,7 @@ type ExternalVolumeTemplate struct { func (x *ExternalVolumeTemplate) Reset() { *x = ExternalVolumeTemplate{} - mi := &file_ateapi_proto_msgTypes[39] + mi := &file_ateapi_proto_msgTypes[41] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3545,7 +3828,7 @@ func (x *ExternalVolumeTemplate) String() string { func (*ExternalVolumeTemplate) ProtoMessage() {} func (x *ExternalVolumeTemplate) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[39] + mi := &file_ateapi_proto_msgTypes[41] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3558,7 +3841,7 @@ func (x *ExternalVolumeTemplate) ProtoReflect() protoreflect.Message { // Deprecated: Use ExternalVolumeTemplate.ProtoReflect.Descriptor instead. func (*ExternalVolumeTemplate) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{39} + return file_ateapi_proto_rawDescGZIP(), []int{41} } func (x *ExternalVolumeTemplate) GetCapacity() string { @@ -3595,7 +3878,7 @@ type SystemInfoVolumeSource struct { func (x *SystemInfoVolumeSource) Reset() { *x = SystemInfoVolumeSource{} - mi := &file_ateapi_proto_msgTypes[40] + mi := &file_ateapi_proto_msgTypes[42] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3607,7 +3890,7 @@ func (x *SystemInfoVolumeSource) String() string { func (*SystemInfoVolumeSource) ProtoMessage() {} func (x *SystemInfoVolumeSource) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[40] + mi := &file_ateapi_proto_msgTypes[42] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3620,7 +3903,7 @@ func (x *SystemInfoVolumeSource) ProtoReflect() protoreflect.Message { // Deprecated: Use SystemInfoVolumeSource.ProtoReflect.Descriptor instead. func (*SystemInfoVolumeSource) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{40} + return file_ateapi_proto_rawDescGZIP(), []int{42} } func (x *SystemInfoVolumeSource) GetDataSources() []*SystemInfoDataSource { @@ -3646,7 +3929,7 @@ type SystemInfoDataSource struct { func (x *SystemInfoDataSource) Reset() { *x = SystemInfoDataSource{} - mi := &file_ateapi_proto_msgTypes[41] + mi := &file_ateapi_proto_msgTypes[43] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3658,7 +3941,7 @@ func (x *SystemInfoDataSource) String() string { func (*SystemInfoDataSource) ProtoMessage() {} func (x *SystemInfoDataSource) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[41] + mi := &file_ateapi_proto_msgTypes[43] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3671,7 +3954,7 @@ func (x *SystemInfoDataSource) ProtoReflect() protoreflect.Message { // Deprecated: Use SystemInfoDataSource.ProtoReflect.Descriptor instead. func (*SystemInfoDataSource) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{41} + return file_ateapi_proto_rawDescGZIP(), []int{43} } func (x *SystemInfoDataSource) GetActorMetadata() *ActorMetadataDataSource { @@ -3708,7 +3991,7 @@ type ActorMetadataDataSource struct { func (x *ActorMetadataDataSource) Reset() { *x = ActorMetadataDataSource{} - mi := &file_ateapi_proto_msgTypes[42] + mi := &file_ateapi_proto_msgTypes[44] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3720,7 +4003,7 @@ func (x *ActorMetadataDataSource) String() string { func (*ActorMetadataDataSource) ProtoMessage() {} func (x *ActorMetadataDataSource) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[42] + mi := &file_ateapi_proto_msgTypes[44] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3733,7 +4016,7 @@ func (x *ActorMetadataDataSource) ProtoReflect() protoreflect.Message { // Deprecated: Use ActorMetadataDataSource.ProtoReflect.Descriptor instead. func (*ActorMetadataDataSource) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{42} + return file_ateapi_proto_rawDescGZIP(), []int{44} } func (x *ActorMetadataDataSource) GetItems() []*ActorMetadataItem { @@ -3765,7 +4048,7 @@ type ActorMetadataItem struct { func (x *ActorMetadataItem) Reset() { *x = ActorMetadataItem{} - mi := &file_ateapi_proto_msgTypes[43] + mi := &file_ateapi_proto_msgTypes[45] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3777,7 +4060,7 @@ func (x *ActorMetadataItem) String() string { func (*ActorMetadataItem) ProtoMessage() {} func (x *ActorMetadataItem) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[43] + mi := &file_ateapi_proto_msgTypes[45] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3790,7 +4073,7 @@ func (x *ActorMetadataItem) ProtoReflect() protoreflect.Message { // Deprecated: Use ActorMetadataItem.ProtoReflect.Descriptor instead. func (*ActorMetadataItem) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{43} + return file_ateapi_proto_rawDescGZIP(), []int{45} } func (x *ActorMetadataItem) GetField() ActorMetadataField { @@ -3842,7 +4125,7 @@ type TrustBundleDataSource struct { func (x *TrustBundleDataSource) Reset() { *x = TrustBundleDataSource{} - mi := &file_ateapi_proto_msgTypes[44] + mi := &file_ateapi_proto_msgTypes[46] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3854,7 +4137,7 @@ func (x *TrustBundleDataSource) String() string { func (*TrustBundleDataSource) ProtoMessage() {} func (x *TrustBundleDataSource) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[44] + mi := &file_ateapi_proto_msgTypes[46] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3867,7 +4150,7 @@ func (x *TrustBundleDataSource) ProtoReflect() protoreflect.Message { // Deprecated: Use TrustBundleDataSource.ProtoReflect.Descriptor instead. func (*TrustBundleDataSource) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{44} + return file_ateapi_proto_rawDescGZIP(), []int{46} } func (x *TrustBundleDataSource) GetNames() []string { @@ -3906,7 +4189,7 @@ type VolumeMount struct { func (x *VolumeMount) Reset() { *x = VolumeMount{} - mi := &file_ateapi_proto_msgTypes[45] + mi := &file_ateapi_proto_msgTypes[47] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3918,7 +4201,7 @@ func (x *VolumeMount) String() string { func (*VolumeMount) ProtoMessage() {} func (x *VolumeMount) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[45] + mi := &file_ateapi_proto_msgTypes[47] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3931,7 +4214,7 @@ func (x *VolumeMount) ProtoReflect() protoreflect.Message { // Deprecated: Use VolumeMount.ProtoReflect.Descriptor instead. func (*VolumeMount) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{45} + return file_ateapi_proto_rawDescGZIP(), []int{47} } func (x *VolumeMount) GetName() string { @@ -3960,7 +4243,7 @@ type CreateAtespaceRequest struct { func (x *CreateAtespaceRequest) Reset() { *x = CreateAtespaceRequest{} - mi := &file_ateapi_proto_msgTypes[46] + mi := &file_ateapi_proto_msgTypes[48] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3972,7 +4255,7 @@ func (x *CreateAtespaceRequest) String() string { func (*CreateAtespaceRequest) ProtoMessage() {} func (x *CreateAtespaceRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[46] + mi := &file_ateapi_proto_msgTypes[48] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3985,7 +4268,7 @@ func (x *CreateAtespaceRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use CreateAtespaceRequest.ProtoReflect.Descriptor instead. func (*CreateAtespaceRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{46} + return file_ateapi_proto_rawDescGZIP(), []int{48} } func (x *CreateAtespaceRequest) GetAtespace() *Atespace { @@ -4006,7 +4289,7 @@ type GetAtespaceRequest struct { func (x *GetAtespaceRequest) Reset() { *x = GetAtespaceRequest{} - mi := &file_ateapi_proto_msgTypes[47] + mi := &file_ateapi_proto_msgTypes[49] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4018,7 +4301,7 @@ func (x *GetAtespaceRequest) String() string { func (*GetAtespaceRequest) ProtoMessage() {} func (x *GetAtespaceRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[47] + mi := &file_ateapi_proto_msgTypes[49] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4031,7 +4314,7 @@ func (x *GetAtespaceRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetAtespaceRequest.ProtoReflect.Descriptor instead. func (*GetAtespaceRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{47} + return file_ateapi_proto_rawDescGZIP(), []int{49} } func (x *GetAtespaceRequest) GetAtespace() *ObjectRef { @@ -4062,7 +4345,7 @@ type ListAtespacesRequest struct { func (x *ListAtespacesRequest) Reset() { *x = ListAtespacesRequest{} - mi := &file_ateapi_proto_msgTypes[48] + mi := &file_ateapi_proto_msgTypes[50] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4074,7 +4357,7 @@ func (x *ListAtespacesRequest) String() string { func (*ListAtespacesRequest) ProtoMessage() {} func (x *ListAtespacesRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[48] + mi := &file_ateapi_proto_msgTypes[50] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4087,7 +4370,7 @@ func (x *ListAtespacesRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ListAtespacesRequest.ProtoReflect.Descriptor instead. func (*ListAtespacesRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{48} + return file_ateapi_proto_rawDescGZIP(), []int{50} } func (x *ListAtespacesRequest) GetPageSize() int32 { @@ -4116,7 +4399,7 @@ type ListAtespacesResponse struct { func (x *ListAtespacesResponse) Reset() { *x = ListAtespacesResponse{} - mi := &file_ateapi_proto_msgTypes[49] + mi := &file_ateapi_proto_msgTypes[51] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4128,7 +4411,7 @@ func (x *ListAtespacesResponse) String() string { func (*ListAtespacesResponse) ProtoMessage() {} func (x *ListAtespacesResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[49] + mi := &file_ateapi_proto_msgTypes[51] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4141,7 +4424,7 @@ func (x *ListAtespacesResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ListAtespacesResponse.ProtoReflect.Descriptor instead. func (*ListAtespacesResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{49} + return file_ateapi_proto_rawDescGZIP(), []int{51} } func (x *ListAtespacesResponse) GetAtespaces() []*Atespace { @@ -4173,7 +4456,7 @@ type DeleteAtespaceRequest struct { func (x *DeleteAtespaceRequest) Reset() { *x = DeleteAtespaceRequest{} - mi := &file_ateapi_proto_msgTypes[50] + mi := &file_ateapi_proto_msgTypes[52] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4185,7 +4468,7 @@ func (x *DeleteAtespaceRequest) String() string { func (*DeleteAtespaceRequest) ProtoMessage() {} func (x *DeleteAtespaceRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[50] + mi := &file_ateapi_proto_msgTypes[52] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4198,7 +4481,7 @@ func (x *DeleteAtespaceRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use DeleteAtespaceRequest.ProtoReflect.Descriptor instead. func (*DeleteAtespaceRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{50} + return file_ateapi_proto_rawDescGZIP(), []int{52} } func (x *DeleteAtespaceRequest) GetAtespace() *ObjectRef { @@ -4229,7 +4512,7 @@ type CreateActorTemplateRequest struct { func (x *CreateActorTemplateRequest) Reset() { *x = CreateActorTemplateRequest{} - mi := &file_ateapi_proto_msgTypes[51] + mi := &file_ateapi_proto_msgTypes[53] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4241,7 +4524,7 @@ func (x *CreateActorTemplateRequest) String() string { func (*CreateActorTemplateRequest) ProtoMessage() {} func (x *CreateActorTemplateRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[51] + mi := &file_ateapi_proto_msgTypes[53] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4254,7 +4537,7 @@ func (x *CreateActorTemplateRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use CreateActorTemplateRequest.ProtoReflect.Descriptor instead. func (*CreateActorTemplateRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{51} + return file_ateapi_proto_rawDescGZIP(), []int{53} } func (x *CreateActorTemplateRequest) GetActorTemplate() *ActorTemplate { @@ -4275,7 +4558,7 @@ type GetActorTemplateRequest struct { func (x *GetActorTemplateRequest) Reset() { *x = GetActorTemplateRequest{} - mi := &file_ateapi_proto_msgTypes[52] + mi := &file_ateapi_proto_msgTypes[54] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4287,7 +4570,7 @@ func (x *GetActorTemplateRequest) String() string { func (*GetActorTemplateRequest) ProtoMessage() {} func (x *GetActorTemplateRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[52] + mi := &file_ateapi_proto_msgTypes[54] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4300,7 +4583,7 @@ func (x *GetActorTemplateRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetActorTemplateRequest.ProtoReflect.Descriptor instead. func (*GetActorTemplateRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{52} + return file_ateapi_proto_rawDescGZIP(), []int{54} } func (x *GetActorTemplateRequest) GetActorTemplate() *ObjectRef { @@ -4337,7 +4620,7 @@ type ListActorTemplatesRequest struct { func (x *ListActorTemplatesRequest) Reset() { *x = ListActorTemplatesRequest{} - mi := &file_ateapi_proto_msgTypes[53] + mi := &file_ateapi_proto_msgTypes[55] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4349,7 +4632,7 @@ func (x *ListActorTemplatesRequest) String() string { func (*ListActorTemplatesRequest) ProtoMessage() {} func (x *ListActorTemplatesRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[53] + mi := &file_ateapi_proto_msgTypes[55] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4362,7 +4645,7 @@ func (x *ListActorTemplatesRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ListActorTemplatesRequest.ProtoReflect.Descriptor instead. func (*ListActorTemplatesRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{53} + return file_ateapi_proto_rawDescGZIP(), []int{55} } func (x *ListActorTemplatesRequest) GetAtespace() string { @@ -4399,7 +4682,7 @@ type ListActorTemplatesResponse struct { func (x *ListActorTemplatesResponse) Reset() { *x = ListActorTemplatesResponse{} - mi := &file_ateapi_proto_msgTypes[54] + mi := &file_ateapi_proto_msgTypes[56] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4411,7 +4694,7 @@ func (x *ListActorTemplatesResponse) String() string { func (*ListActorTemplatesResponse) ProtoMessage() {} func (x *ListActorTemplatesResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[54] + mi := &file_ateapi_proto_msgTypes[56] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4424,7 +4707,7 @@ func (x *ListActorTemplatesResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ListActorTemplatesResponse.ProtoReflect.Descriptor instead. func (*ListActorTemplatesResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{54} + return file_ateapi_proto_rawDescGZIP(), []int{56} } func (x *ListActorTemplatesResponse) GetActorTemplates() []*ActorTemplate { @@ -4456,7 +4739,7 @@ type DeleteActorTemplateRequest struct { func (x *DeleteActorTemplateRequest) Reset() { *x = DeleteActorTemplateRequest{} - mi := &file_ateapi_proto_msgTypes[55] + mi := &file_ateapi_proto_msgTypes[57] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4468,7 +4751,7 @@ func (x *DeleteActorTemplateRequest) String() string { func (*DeleteActorTemplateRequest) ProtoMessage() {} func (x *DeleteActorTemplateRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[55] + mi := &file_ateapi_proto_msgTypes[57] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4481,7 +4764,7 @@ func (x *DeleteActorTemplateRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use DeleteActorTemplateRequest.ProtoReflect.Descriptor instead. func (*DeleteActorTemplateRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{55} + return file_ateapi_proto_rawDescGZIP(), []int{57} } func (x *DeleteActorTemplateRequest) GetActorTemplate() *ObjectRef { @@ -4509,7 +4792,7 @@ type GetActorRequest struct { func (x *GetActorRequest) Reset() { *x = GetActorRequest{} - mi := &file_ateapi_proto_msgTypes[56] + mi := &file_ateapi_proto_msgTypes[58] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4521,7 +4804,7 @@ func (x *GetActorRequest) String() string { func (*GetActorRequest) ProtoMessage() {} func (x *GetActorRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[56] + mi := &file_ateapi_proto_msgTypes[58] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4534,7 +4817,7 @@ func (x *GetActorRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetActorRequest.ProtoReflect.Descriptor instead. func (*GetActorRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{56} + return file_ateapi_proto_rawDescGZIP(), []int{58} } func (x *GetActorRequest) GetActor() *ObjectRef { @@ -4557,7 +4840,7 @@ type CreateActorRequest struct { func (x *CreateActorRequest) Reset() { *x = CreateActorRequest{} - mi := &file_ateapi_proto_msgTypes[57] + mi := &file_ateapi_proto_msgTypes[59] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4569,7 +4852,7 @@ func (x *CreateActorRequest) String() string { func (*CreateActorRequest) ProtoMessage() {} func (x *CreateActorRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[57] + mi := &file_ateapi_proto_msgTypes[59] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4582,7 +4865,7 @@ func (x *CreateActorRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use CreateActorRequest.ProtoReflect.Descriptor instead. func (*CreateActorRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{57} + return file_ateapi_proto_rawDescGZIP(), []int{59} } func (x *CreateActorRequest) GetActor() *Actor { @@ -4613,7 +4896,7 @@ type UpdateActorRequest struct { func (x *UpdateActorRequest) Reset() { *x = UpdateActorRequest{} - mi := &file_ateapi_proto_msgTypes[58] + mi := &file_ateapi_proto_msgTypes[60] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4625,7 +4908,7 @@ func (x *UpdateActorRequest) String() string { func (*UpdateActorRequest) ProtoMessage() {} func (x *UpdateActorRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[58] + mi := &file_ateapi_proto_msgTypes[60] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4638,7 +4921,7 @@ func (x *UpdateActorRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use UpdateActorRequest.ProtoReflect.Descriptor instead. func (*UpdateActorRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{58} + return file_ateapi_proto_rawDescGZIP(), []int{60} } func (x *UpdateActorRequest) GetActor() *Actor { @@ -4659,7 +4942,7 @@ type SuspendActorRequest struct { func (x *SuspendActorRequest) Reset() { *x = SuspendActorRequest{} - mi := &file_ateapi_proto_msgTypes[59] + mi := &file_ateapi_proto_msgTypes[61] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4671,7 +4954,7 @@ func (x *SuspendActorRequest) String() string { func (*SuspendActorRequest) ProtoMessage() {} func (x *SuspendActorRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[59] + mi := &file_ateapi_proto_msgTypes[61] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4684,7 +4967,7 @@ func (x *SuspendActorRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use SuspendActorRequest.ProtoReflect.Descriptor instead. func (*SuspendActorRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{59} + return file_ateapi_proto_rawDescGZIP(), []int{61} } func (x *SuspendActorRequest) GetActor() *ObjectRef { @@ -4703,7 +4986,7 @@ type SuspendActorResponse struct { func (x *SuspendActorResponse) Reset() { *x = SuspendActorResponse{} - mi := &file_ateapi_proto_msgTypes[60] + mi := &file_ateapi_proto_msgTypes[62] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4715,7 +4998,7 @@ func (x *SuspendActorResponse) String() string { func (*SuspendActorResponse) ProtoMessage() {} func (x *SuspendActorResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[60] + mi := &file_ateapi_proto_msgTypes[62] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4728,7 +5011,7 @@ func (x *SuspendActorResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use SuspendActorResponse.ProtoReflect.Descriptor instead. func (*SuspendActorResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{60} + return file_ateapi_proto_rawDescGZIP(), []int{62} } func (x *SuspendActorResponse) GetActor() *Actor { @@ -4749,7 +5032,7 @@ type PauseActorRequest struct { func (x *PauseActorRequest) Reset() { *x = PauseActorRequest{} - mi := &file_ateapi_proto_msgTypes[61] + mi := &file_ateapi_proto_msgTypes[63] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4761,7 +5044,7 @@ func (x *PauseActorRequest) String() string { func (*PauseActorRequest) ProtoMessage() {} func (x *PauseActorRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[61] + mi := &file_ateapi_proto_msgTypes[63] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4774,7 +5057,7 @@ func (x *PauseActorRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use PauseActorRequest.ProtoReflect.Descriptor instead. func (*PauseActorRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{61} + return file_ateapi_proto_rawDescGZIP(), []int{63} } func (x *PauseActorRequest) GetActor() *ObjectRef { @@ -4793,7 +5076,7 @@ type PauseActorResponse struct { func (x *PauseActorResponse) Reset() { *x = PauseActorResponse{} - mi := &file_ateapi_proto_msgTypes[62] + mi := &file_ateapi_proto_msgTypes[64] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4805,7 +5088,7 @@ func (x *PauseActorResponse) String() string { func (*PauseActorResponse) ProtoMessage() {} func (x *PauseActorResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[62] + mi := &file_ateapi_proto_msgTypes[64] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4818,7 +5101,7 @@ func (x *PauseActorResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use PauseActorResponse.ProtoReflect.Descriptor instead. func (*PauseActorResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{62} + return file_ateapi_proto_rawDescGZIP(), []int{64} } func (x *PauseActorResponse) GetActor() *Actor { @@ -4839,7 +5122,7 @@ type ResumeActorRequest struct { func (x *ResumeActorRequest) Reset() { *x = ResumeActorRequest{} - mi := &file_ateapi_proto_msgTypes[63] + mi := &file_ateapi_proto_msgTypes[65] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4851,7 +5134,7 @@ func (x *ResumeActorRequest) String() string { func (*ResumeActorRequest) ProtoMessage() {} func (x *ResumeActorRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[63] + mi := &file_ateapi_proto_msgTypes[65] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4864,7 +5147,7 @@ func (x *ResumeActorRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ResumeActorRequest.ProtoReflect.Descriptor instead. func (*ResumeActorRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{63} + return file_ateapi_proto_rawDescGZIP(), []int{65} } func (x *ResumeActorRequest) GetActor() *ObjectRef { @@ -4886,7 +5169,7 @@ type ResumeActorResponse struct { func (x *ResumeActorResponse) Reset() { *x = ResumeActorResponse{} - mi := &file_ateapi_proto_msgTypes[64] + mi := &file_ateapi_proto_msgTypes[66] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4898,7 +5181,7 @@ func (x *ResumeActorResponse) String() string { func (*ResumeActorResponse) ProtoMessage() {} func (x *ResumeActorResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[64] + mi := &file_ateapi_proto_msgTypes[66] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4911,7 +5194,7 @@ func (x *ResumeActorResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ResumeActorResponse.ProtoReflect.Descriptor instead. func (*ResumeActorResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{64} + return file_ateapi_proto_rawDescGZIP(), []int{66} } func (x *ResumeActorResponse) GetActor() *Actor { @@ -4941,7 +5224,7 @@ type RevertActorRequest struct { func (x *RevertActorRequest) Reset() { *x = RevertActorRequest{} - mi := &file_ateapi_proto_msgTypes[65] + mi := &file_ateapi_proto_msgTypes[67] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4953,7 +5236,7 @@ func (x *RevertActorRequest) String() string { func (*RevertActorRequest) ProtoMessage() {} func (x *RevertActorRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[65] + mi := &file_ateapi_proto_msgTypes[67] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4966,7 +5249,7 @@ func (x *RevertActorRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use RevertActorRequest.ProtoReflect.Descriptor instead. func (*RevertActorRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{65} + return file_ateapi_proto_rawDescGZIP(), []int{67} } func (x *RevertActorRequest) GetActor() *ObjectRef { @@ -4988,7 +5271,7 @@ type RevertActorResponse struct { func (x *RevertActorResponse) Reset() { *x = RevertActorResponse{} - mi := &file_ateapi_proto_msgTypes[66] + mi := &file_ateapi_proto_msgTypes[68] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5000,7 +5283,7 @@ func (x *RevertActorResponse) String() string { func (*RevertActorResponse) ProtoMessage() {} func (x *RevertActorResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[66] + mi := &file_ateapi_proto_msgTypes[68] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5013,7 +5296,7 @@ func (x *RevertActorResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RevertActorResponse.ProtoReflect.Descriptor instead. func (*RevertActorResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{66} + return file_ateapi_proto_rawDescGZIP(), []int{68} } func (x *RevertActorResponse) GetActor() *Actor { @@ -5042,7 +5325,7 @@ type DeleteActorRequest struct { func (x *DeleteActorRequest) Reset() { *x = DeleteActorRequest{} - mi := &file_ateapi_proto_msgTypes[67] + mi := &file_ateapi_proto_msgTypes[69] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5054,7 +5337,7 @@ func (x *DeleteActorRequest) String() string { func (*DeleteActorRequest) ProtoMessage() {} func (x *DeleteActorRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[67] + mi := &file_ateapi_proto_msgTypes[69] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5067,7 +5350,7 @@ func (x *DeleteActorRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use DeleteActorRequest.ProtoReflect.Descriptor instead. func (*DeleteActorRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{67} + return file_ateapi_proto_rawDescGZIP(), []int{69} } func (x *DeleteActorRequest) GetActor() *ObjectRef { @@ -5105,7 +5388,7 @@ type GetActorEgressPolicyRequest struct { func (x *GetActorEgressPolicyRequest) Reset() { *x = GetActorEgressPolicyRequest{} - mi := &file_ateapi_proto_msgTypes[68] + mi := &file_ateapi_proto_msgTypes[70] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5117,7 +5400,7 @@ func (x *GetActorEgressPolicyRequest) String() string { func (*GetActorEgressPolicyRequest) ProtoMessage() {} func (x *GetActorEgressPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[68] + mi := &file_ateapi_proto_msgTypes[70] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5130,7 +5413,7 @@ func (x *GetActorEgressPolicyRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetActorEgressPolicyRequest.ProtoReflect.Descriptor instead. func (*GetActorEgressPolicyRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{68} + return file_ateapi_proto_rawDescGZIP(), []int{70} } func (x *GetActorEgressPolicyRequest) GetActor() *ObjectRef { @@ -5161,7 +5444,7 @@ type CreateActorEgressPolicyRequest struct { func (x *CreateActorEgressPolicyRequest) Reset() { *x = CreateActorEgressPolicyRequest{} - mi := &file_ateapi_proto_msgTypes[69] + mi := &file_ateapi_proto_msgTypes[71] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5173,7 +5456,7 @@ func (x *CreateActorEgressPolicyRequest) String() string { func (*CreateActorEgressPolicyRequest) ProtoMessage() {} func (x *CreateActorEgressPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[69] + mi := &file_ateapi_proto_msgTypes[71] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5186,7 +5469,7 @@ func (x *CreateActorEgressPolicyRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use CreateActorEgressPolicyRequest.ProtoReflect.Descriptor instead. func (*CreateActorEgressPolicyRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{69} + return file_ateapi_proto_rawDescGZIP(), []int{71} } func (x *CreateActorEgressPolicyRequest) GetActor() *ObjectRef { @@ -5224,7 +5507,7 @@ type UpdateActorEgressPolicyRequest struct { func (x *UpdateActorEgressPolicyRequest) Reset() { *x = UpdateActorEgressPolicyRequest{} - mi := &file_ateapi_proto_msgTypes[70] + mi := &file_ateapi_proto_msgTypes[72] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5236,7 +5519,7 @@ func (x *UpdateActorEgressPolicyRequest) String() string { func (*UpdateActorEgressPolicyRequest) ProtoMessage() {} func (x *UpdateActorEgressPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[70] + mi := &file_ateapi_proto_msgTypes[72] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5249,7 +5532,7 @@ func (x *UpdateActorEgressPolicyRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use UpdateActorEgressPolicyRequest.ProtoReflect.Descriptor instead. func (*UpdateActorEgressPolicyRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{70} + return file_ateapi_proto_rawDescGZIP(), []int{72} } func (x *UpdateActorEgressPolicyRequest) GetActor() *ObjectRef { @@ -5284,7 +5567,7 @@ type DeleteActorEgressPolicyRequest struct { func (x *DeleteActorEgressPolicyRequest) Reset() { *x = DeleteActorEgressPolicyRequest{} - mi := &file_ateapi_proto_msgTypes[71] + mi := &file_ateapi_proto_msgTypes[73] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5296,7 +5579,7 @@ func (x *DeleteActorEgressPolicyRequest) String() string { func (*DeleteActorEgressPolicyRequest) ProtoMessage() {} func (x *DeleteActorEgressPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[71] + mi := &file_ateapi_proto_msgTypes[73] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5309,7 +5592,7 @@ func (x *DeleteActorEgressPolicyRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use DeleteActorEgressPolicyRequest.ProtoReflect.Descriptor instead. func (*DeleteActorEgressPolicyRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{71} + return file_ateapi_proto_rawDescGZIP(), []int{73} } func (x *DeleteActorEgressPolicyRequest) GetActor() *ObjectRef { @@ -5337,7 +5620,7 @@ type GetTagRequest struct { func (x *GetTagRequest) Reset() { *x = GetTagRequest{} - mi := &file_ateapi_proto_msgTypes[72] + mi := &file_ateapi_proto_msgTypes[74] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5349,7 +5632,7 @@ func (x *GetTagRequest) String() string { func (*GetTagRequest) ProtoMessage() {} func (x *GetTagRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[72] + mi := &file_ateapi_proto_msgTypes[74] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5362,7 +5645,7 @@ func (x *GetTagRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetTagRequest.ProtoReflect.Descriptor instead. func (*GetTagRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{72} + return file_ateapi_proto_rawDescGZIP(), []int{74} } func (x *GetTagRequest) GetTag() *ObjectRef { @@ -5401,7 +5684,7 @@ type MintActorJWTRequest struct { func (x *MintActorJWTRequest) Reset() { *x = MintActorJWTRequest{} - mi := &file_ateapi_proto_msgTypes[73] + mi := &file_ateapi_proto_msgTypes[75] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5413,7 +5696,7 @@ func (x *MintActorJWTRequest) String() string { func (*MintActorJWTRequest) ProtoMessage() {} func (x *MintActorJWTRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[73] + mi := &file_ateapi_proto_msgTypes[75] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5426,7 +5709,7 @@ func (x *MintActorJWTRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use MintActorJWTRequest.ProtoReflect.Descriptor instead. func (*MintActorJWTRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{73} + return file_ateapi_proto_rawDescGZIP(), []int{75} } func (x *MintActorJWTRequest) GetActor() *ObjectRef { @@ -5476,7 +5759,7 @@ type MintActorJWTResponse struct { func (x *MintActorJWTResponse) Reset() { *x = MintActorJWTResponse{} - mi := &file_ateapi_proto_msgTypes[74] + mi := &file_ateapi_proto_msgTypes[76] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5488,7 +5771,7 @@ func (x *MintActorJWTResponse) String() string { func (*MintActorJWTResponse) ProtoMessage() {} func (x *MintActorJWTResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[74] + mi := &file_ateapi_proto_msgTypes[76] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5501,7 +5784,7 @@ func (x *MintActorJWTResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use MintActorJWTResponse.ProtoReflect.Descriptor instead. func (*MintActorJWTResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{74} + return file_ateapi_proto_rawDescGZIP(), []int{76} } func (x *MintActorJWTResponse) GetActorJwt() string { @@ -5546,7 +5829,7 @@ type MintActorCertificateRequest struct { func (x *MintActorCertificateRequest) Reset() { *x = MintActorCertificateRequest{} - mi := &file_ateapi_proto_msgTypes[75] + mi := &file_ateapi_proto_msgTypes[77] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5558,7 +5841,7 @@ func (x *MintActorCertificateRequest) String() string { func (*MintActorCertificateRequest) ProtoMessage() {} func (x *MintActorCertificateRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[75] + mi := &file_ateapi_proto_msgTypes[77] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5571,7 +5854,7 @@ func (x *MintActorCertificateRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use MintActorCertificateRequest.ProtoReflect.Descriptor instead. func (*MintActorCertificateRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{75} + return file_ateapi_proto_rawDescGZIP(), []int{77} } func (x *MintActorCertificateRequest) GetActor() *ObjectRef { @@ -5608,7 +5891,7 @@ type MintActorCertificateResponse struct { func (x *MintActorCertificateResponse) Reset() { *x = MintActorCertificateResponse{} - mi := &file_ateapi_proto_msgTypes[76] + mi := &file_ateapi_proto_msgTypes[78] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5620,7 +5903,7 @@ func (x *MintActorCertificateResponse) String() string { func (*MintActorCertificateResponse) ProtoMessage() {} func (x *MintActorCertificateResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[76] + mi := &file_ateapi_proto_msgTypes[78] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5633,7 +5916,7 @@ func (x *MintActorCertificateResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use MintActorCertificateResponse.ProtoReflect.Descriptor instead. func (*MintActorCertificateResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{76} + return file_ateapi_proto_rawDescGZIP(), []int{78} } func (x *MintActorCertificateResponse) GetActorCertificates() [][]byte { @@ -5669,7 +5952,7 @@ type ListTagsRequest struct { func (x *ListTagsRequest) Reset() { *x = ListTagsRequest{} - mi := &file_ateapi_proto_msgTypes[77] + mi := &file_ateapi_proto_msgTypes[79] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5681,7 +5964,7 @@ func (x *ListTagsRequest) String() string { func (*ListTagsRequest) ProtoMessage() {} func (x *ListTagsRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[77] + mi := &file_ateapi_proto_msgTypes[79] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5694,7 +5977,7 @@ func (x *ListTagsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ListTagsRequest.ProtoReflect.Descriptor instead. func (*ListTagsRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{77} + return file_ateapi_proto_rawDescGZIP(), []int{79} } func (x *ListTagsRequest) GetAtespace() string { @@ -5728,7 +6011,7 @@ type ListTagsResponse struct { func (x *ListTagsResponse) Reset() { *x = ListTagsResponse{} - mi := &file_ateapi_proto_msgTypes[78] + mi := &file_ateapi_proto_msgTypes[80] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5740,7 +6023,7 @@ func (x *ListTagsResponse) String() string { func (*ListTagsResponse) ProtoMessage() {} func (x *ListTagsResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[78] + mi := &file_ateapi_proto_msgTypes[80] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5753,7 +6036,7 @@ func (x *ListTagsResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ListTagsResponse.ProtoReflect.Descriptor instead. func (*ListTagsResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{78} + return file_ateapi_proto_rawDescGZIP(), []int{80} } func (x *ListTagsResponse) GetTags() []*Tag { @@ -5798,7 +6081,7 @@ type CreateTagRequest struct { func (x *CreateTagRequest) Reset() { *x = CreateTagRequest{} - mi := &file_ateapi_proto_msgTypes[79] + mi := &file_ateapi_proto_msgTypes[81] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5810,7 +6093,7 @@ func (x *CreateTagRequest) String() string { func (*CreateTagRequest) ProtoMessage() {} func (x *CreateTagRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[79] + mi := &file_ateapi_proto_msgTypes[81] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5823,7 +6106,7 @@ func (x *CreateTagRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use CreateTagRequest.ProtoReflect.Descriptor instead. func (*CreateTagRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{79} + return file_ateapi_proto_rawDescGZIP(), []int{81} } func (x *CreateTagRequest) GetTag() *Tag { @@ -5853,7 +6136,7 @@ type UpdateTagRequest struct { func (x *UpdateTagRequest) Reset() { *x = UpdateTagRequest{} - mi := &file_ateapi_proto_msgTypes[80] + mi := &file_ateapi_proto_msgTypes[82] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5865,7 +6148,7 @@ func (x *UpdateTagRequest) String() string { func (*UpdateTagRequest) ProtoMessage() {} func (x *UpdateTagRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[80] + mi := &file_ateapi_proto_msgTypes[82] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5878,7 +6161,7 @@ func (x *UpdateTagRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use UpdateTagRequest.ProtoReflect.Descriptor instead. func (*UpdateTagRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{80} + return file_ateapi_proto_rawDescGZIP(), []int{82} } func (x *UpdateTagRequest) GetTag() *Tag { @@ -5903,7 +6186,7 @@ type DeleteTagRequest struct { func (x *DeleteTagRequest) Reset() { *x = DeleteTagRequest{} - mi := &file_ateapi_proto_msgTypes[81] + mi := &file_ateapi_proto_msgTypes[83] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5915,7 +6198,7 @@ func (x *DeleteTagRequest) String() string { func (*DeleteTagRequest) ProtoMessage() {} func (x *DeleteTagRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[81] + mi := &file_ateapi_proto_msgTypes[83] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5928,7 +6211,7 @@ func (x *DeleteTagRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use DeleteTagRequest.ProtoReflect.Descriptor instead. func (*DeleteTagRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{81} + return file_ateapi_proto_rawDescGZIP(), []int{83} } func (x *DeleteTagRequest) GetTag() *ObjectRef { @@ -5967,7 +6250,7 @@ type DeleteOptions struct { func (x *DeleteOptions) Reset() { *x = DeleteOptions{} - mi := &file_ateapi_proto_msgTypes[82] + mi := &file_ateapi_proto_msgTypes[84] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5979,7 +6262,7 @@ func (x *DeleteOptions) String() string { func (*DeleteOptions) ProtoMessage() {} func (x *DeleteOptions) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[82] + mi := &file_ateapi_proto_msgTypes[84] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5992,7 +6275,7 @@ func (x *DeleteOptions) ProtoReflect() protoreflect.Message { // Deprecated: Use DeleteOptions.ProtoReflect.Descriptor instead. func (*DeleteOptions) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{82} + return file_ateapi_proto_rawDescGZIP(), []int{84} } func (x *DeleteOptions) GetVersion() int64 { @@ -6036,7 +6319,7 @@ type ListWorkerActorAssignmentsRequest struct { func (x *ListWorkerActorAssignmentsRequest) Reset() { *x = ListWorkerActorAssignmentsRequest{} - mi := &file_ateapi_proto_msgTypes[83] + mi := &file_ateapi_proto_msgTypes[85] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6048,7 +6331,7 @@ func (x *ListWorkerActorAssignmentsRequest) String() string { func (*ListWorkerActorAssignmentsRequest) ProtoMessage() {} func (x *ListWorkerActorAssignmentsRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[83] + mi := &file_ateapi_proto_msgTypes[85] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6061,7 +6344,7 @@ func (x *ListWorkerActorAssignmentsRequest) ProtoReflect() protoreflect.Message // Deprecated: Use ListWorkerActorAssignmentsRequest.ProtoReflect.Descriptor instead. func (*ListWorkerActorAssignmentsRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{83} + return file_ateapi_proto_rawDescGZIP(), []int{85} } func (x *ListWorkerActorAssignmentsRequest) GetWorker() *ObjectRef { @@ -6098,7 +6381,7 @@ type ListWorkerActorAssignmentsResponse struct { func (x *ListWorkerActorAssignmentsResponse) Reset() { *x = ListWorkerActorAssignmentsResponse{} - mi := &file_ateapi_proto_msgTypes[84] + mi := &file_ateapi_proto_msgTypes[86] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6110,7 +6393,7 @@ func (x *ListWorkerActorAssignmentsResponse) String() string { func (*ListWorkerActorAssignmentsResponse) ProtoMessage() {} func (x *ListWorkerActorAssignmentsResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[84] + mi := &file_ateapi_proto_msgTypes[86] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6123,7 +6406,7 @@ func (x *ListWorkerActorAssignmentsResponse) ProtoReflect() protoreflect.Message // Deprecated: Use ListWorkerActorAssignmentsResponse.ProtoReflect.Descriptor instead. func (*ListWorkerActorAssignmentsResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{84} + return file_ateapi_proto_rawDescGZIP(), []int{86} } func (x *ListWorkerActorAssignmentsResponse) GetActorAssignments() []*ActorAssignment { @@ -6161,7 +6444,7 @@ type ListWorkersRequest struct { func (x *ListWorkersRequest) Reset() { *x = ListWorkersRequest{} - mi := &file_ateapi_proto_msgTypes[85] + mi := &file_ateapi_proto_msgTypes[87] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6173,7 +6456,7 @@ func (x *ListWorkersRequest) String() string { func (*ListWorkersRequest) ProtoMessage() {} func (x *ListWorkersRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[85] + mi := &file_ateapi_proto_msgTypes[87] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6186,7 +6469,7 @@ func (x *ListWorkersRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ListWorkersRequest.ProtoReflect.Descriptor instead. func (*ListWorkersRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{85} + return file_ateapi_proto_rawDescGZIP(), []int{87} } func (x *ListWorkersRequest) GetPageSize() int32 { @@ -6215,7 +6498,7 @@ type ListWorkersResponse struct { func (x *ListWorkersResponse) Reset() { *x = ListWorkersResponse{} - mi := &file_ateapi_proto_msgTypes[86] + mi := &file_ateapi_proto_msgTypes[88] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6227,7 +6510,7 @@ func (x *ListWorkersResponse) String() string { func (*ListWorkersResponse) ProtoMessage() {} func (x *ListWorkersResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[86] + mi := &file_ateapi_proto_msgTypes[88] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6240,7 +6523,7 @@ func (x *ListWorkersResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ListWorkersResponse.ProtoReflect.Descriptor instead. func (*ListWorkersResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{86} + return file_ateapi_proto_rawDescGZIP(), []int{88} } func (x *ListWorkersResponse) GetWorkers() []*Worker { @@ -6270,7 +6553,7 @@ type GetWorkerRequest struct { func (x *GetWorkerRequest) Reset() { *x = GetWorkerRequest{} - mi := &file_ateapi_proto_msgTypes[87] + mi := &file_ateapi_proto_msgTypes[89] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6282,7 +6565,7 @@ func (x *GetWorkerRequest) String() string { func (*GetWorkerRequest) ProtoMessage() {} func (x *GetWorkerRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[87] + mi := &file_ateapi_proto_msgTypes[89] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6295,7 +6578,7 @@ func (x *GetWorkerRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetWorkerRequest.ProtoReflect.Descriptor instead. func (*GetWorkerRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{87} + return file_ateapi_proto_rawDescGZIP(), []int{89} } func (x *GetWorkerRequest) GetWorker() *ObjectRef { @@ -6317,7 +6600,7 @@ type CreateWorkerRequest struct { func (x *CreateWorkerRequest) Reset() { *x = CreateWorkerRequest{} - mi := &file_ateapi_proto_msgTypes[88] + mi := &file_ateapi_proto_msgTypes[90] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6329,7 +6612,7 @@ func (x *CreateWorkerRequest) String() string { func (*CreateWorkerRequest) ProtoMessage() {} func (x *CreateWorkerRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[88] + mi := &file_ateapi_proto_msgTypes[90] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6342,7 +6625,7 @@ func (x *CreateWorkerRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use CreateWorkerRequest.ProtoReflect.Descriptor instead. func (*CreateWorkerRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{88} + return file_ateapi_proto_rawDescGZIP(), []int{90} } func (x *CreateWorkerRequest) GetWorker() *Worker { @@ -6377,7 +6660,7 @@ type UpdateWorkerRequest struct { func (x *UpdateWorkerRequest) Reset() { *x = UpdateWorkerRequest{} - mi := &file_ateapi_proto_msgTypes[89] + mi := &file_ateapi_proto_msgTypes[91] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6389,7 +6672,7 @@ func (x *UpdateWorkerRequest) String() string { func (*UpdateWorkerRequest) ProtoMessage() {} func (x *UpdateWorkerRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[89] + mi := &file_ateapi_proto_msgTypes[91] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6402,7 +6685,7 @@ func (x *UpdateWorkerRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use UpdateWorkerRequest.ProtoReflect.Descriptor instead. func (*UpdateWorkerRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{89} + return file_ateapi_proto_rawDescGZIP(), []int{91} } func (x *UpdateWorkerRequest) GetWorker() *Worker { @@ -6429,7 +6712,7 @@ type DeleteWorkerRequest struct { func (x *DeleteWorkerRequest) Reset() { *x = DeleteWorkerRequest{} - mi := &file_ateapi_proto_msgTypes[90] + mi := &file_ateapi_proto_msgTypes[92] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6441,7 +6724,7 @@ func (x *DeleteWorkerRequest) String() string { func (*DeleteWorkerRequest) ProtoMessage() {} func (x *DeleteWorkerRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[90] + mi := &file_ateapi_proto_msgTypes[92] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6454,7 +6737,7 @@ func (x *DeleteWorkerRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use DeleteWorkerRequest.ProtoReflect.Descriptor instead. func (*DeleteWorkerRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{90} + return file_ateapi_proto_rawDescGZIP(), []int{92} } func (x *DeleteWorkerRequest) GetWorker() *ObjectRef { @@ -6484,7 +6767,7 @@ type DrainWorkerRequest struct { func (x *DrainWorkerRequest) Reset() { *x = DrainWorkerRequest{} - mi := &file_ateapi_proto_msgTypes[91] + mi := &file_ateapi_proto_msgTypes[93] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6496,7 +6779,7 @@ func (x *DrainWorkerRequest) String() string { func (*DrainWorkerRequest) ProtoMessage() {} func (x *DrainWorkerRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[91] + mi := &file_ateapi_proto_msgTypes[93] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6509,7 +6792,7 @@ func (x *DrainWorkerRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use DrainWorkerRequest.ProtoReflect.Descriptor instead. func (*DrainWorkerRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{91} + return file_ateapi_proto_rawDescGZIP(), []int{93} } func (x *DrainWorkerRequest) GetWorker() *ObjectRef { @@ -6547,7 +6830,7 @@ type ListActorsRequest struct { func (x *ListActorsRequest) Reset() { *x = ListActorsRequest{} - mi := &file_ateapi_proto_msgTypes[92] + mi := &file_ateapi_proto_msgTypes[94] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6559,7 +6842,7 @@ func (x *ListActorsRequest) String() string { func (*ListActorsRequest) ProtoMessage() {} func (x *ListActorsRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[92] + mi := &file_ateapi_proto_msgTypes[94] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6572,7 +6855,7 @@ func (x *ListActorsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ListActorsRequest.ProtoReflect.Descriptor instead. func (*ListActorsRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{92} + return file_ateapi_proto_rawDescGZIP(), []int{94} } func (x *ListActorsRequest) GetAtespace() string { @@ -6608,7 +6891,7 @@ type ListActorsResponse struct { func (x *ListActorsResponse) Reset() { *x = ListActorsResponse{} - mi := &file_ateapi_proto_msgTypes[93] + mi := &file_ateapi_proto_msgTypes[95] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6620,7 +6903,7 @@ func (x *ListActorsResponse) String() string { func (*ListActorsResponse) ProtoMessage() {} func (x *ListActorsResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[93] + mi := &file_ateapi_proto_msgTypes[95] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6633,7 +6916,7 @@ func (x *ListActorsResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ListActorsResponse.ProtoReflect.Descriptor instead. func (*ListActorsResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{93} + return file_ateapi_proto_rawDescGZIP(), []int{95} } func (x *ListActorsResponse) GetActors() []*Actor { @@ -6740,7 +7023,7 @@ type Worker struct { func (x *Worker) Reset() { *x = Worker{} - mi := &file_ateapi_proto_msgTypes[94] + mi := &file_ateapi_proto_msgTypes[96] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6752,7 +7035,7 @@ func (x *Worker) String() string { func (*Worker) ProtoMessage() {} func (x *Worker) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[94] + mi := &file_ateapi_proto_msgTypes[96] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6765,7 +7048,7 @@ func (x *Worker) ProtoReflect() protoreflect.Message { // Deprecated: Use Worker.ProtoReflect.Descriptor instead. func (*Worker) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{94} + return file_ateapi_proto_rawDescGZIP(), []int{96} } func (x *Worker) GetMetadata() *ResourceMetadata { @@ -6885,7 +7168,7 @@ type WorkerStatus struct { func (x *WorkerStatus) Reset() { *x = WorkerStatus{} - mi := &file_ateapi_proto_msgTypes[95] + mi := &file_ateapi_proto_msgTypes[97] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6897,7 +7180,7 @@ func (x *WorkerStatus) String() string { func (*WorkerStatus) ProtoMessage() {} func (x *WorkerStatus) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[95] + mi := &file_ateapi_proto_msgTypes[97] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6910,7 +7193,7 @@ func (x *WorkerStatus) ProtoReflect() protoreflect.Message { // Deprecated: Use WorkerStatus.ProtoReflect.Descriptor instead. func (*WorkerStatus) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{95} + return file_ateapi_proto_rawDescGZIP(), []int{97} } func (x *WorkerStatus) GetState() WorkerState { @@ -6966,7 +7249,7 @@ type HardwareIdentity struct { func (x *HardwareIdentity) Reset() { *x = HardwareIdentity{} - mi := &file_ateapi_proto_msgTypes[96] + mi := &file_ateapi_proto_msgTypes[98] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6978,7 +7261,7 @@ func (x *HardwareIdentity) String() string { func (*HardwareIdentity) ProtoMessage() {} func (x *HardwareIdentity) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[96] + mi := &file_ateapi_proto_msgTypes[98] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6991,7 +7274,7 @@ func (x *HardwareIdentity) ProtoReflect() protoreflect.Message { // Deprecated: Use HardwareIdentity.ProtoReflect.Descriptor instead. func (*HardwareIdentity) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{96} + return file_ateapi_proto_rawDescGZIP(), []int{98} } func (x *HardwareIdentity) GetAttributes() map[string]string { @@ -7023,7 +7306,7 @@ type WorkerResources struct { func (x *WorkerResources) Reset() { *x = WorkerResources{} - mi := &file_ateapi_proto_msgTypes[97] + mi := &file_ateapi_proto_msgTypes[99] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7035,7 +7318,7 @@ func (x *WorkerResources) String() string { func (*WorkerResources) ProtoMessage() {} func (x *WorkerResources) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[97] + mi := &file_ateapi_proto_msgTypes[99] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7048,7 +7331,7 @@ func (x *WorkerResources) ProtoReflect() protoreflect.Message { // Deprecated: Use WorkerResources.ProtoReflect.Descriptor instead. func (*WorkerResources) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{97} + return file_ateapi_proto_rawDescGZIP(), []int{99} } func (x *WorkerResources) GetResources() *Resources { @@ -7101,7 +7384,7 @@ type ActorAssignment struct { func (x *ActorAssignment) Reset() { *x = ActorAssignment{} - mi := &file_ateapi_proto_msgTypes[98] + mi := &file_ateapi_proto_msgTypes[100] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7113,7 +7396,7 @@ func (x *ActorAssignment) String() string { func (*ActorAssignment) ProtoMessage() {} func (x *ActorAssignment) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[98] + mi := &file_ateapi_proto_msgTypes[100] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7126,7 +7409,7 @@ func (x *ActorAssignment) ProtoReflect() protoreflect.Message { // Deprecated: Use ActorAssignment.ProtoReflect.Descriptor instead. func (*ActorAssignment) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{98} + return file_ateapi_proto_rawDescGZIP(), []int{100} } func (x *ActorAssignment) GetMetadata() *ResourceMetadata { @@ -7189,7 +7472,7 @@ type RegisterWorkerRequest struct { func (x *RegisterWorkerRequest) Reset() { *x = RegisterWorkerRequest{} - mi := &file_ateapi_proto_msgTypes[99] + mi := &file_ateapi_proto_msgTypes[101] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7201,7 +7484,7 @@ func (x *RegisterWorkerRequest) String() string { func (*RegisterWorkerRequest) ProtoMessage() {} func (x *RegisterWorkerRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[99] + mi := &file_ateapi_proto_msgTypes[101] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7214,7 +7497,7 @@ func (x *RegisterWorkerRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use RegisterWorkerRequest.ProtoReflect.Descriptor instead. func (*RegisterWorkerRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{99} + return file_ateapi_proto_rawDescGZIP(), []int{101} } func (x *RegisterWorkerRequest) GetWorker() *ObjectRef { @@ -7248,7 +7531,7 @@ type RegisterWorkerResponse struct { func (x *RegisterWorkerResponse) Reset() { *x = RegisterWorkerResponse{} - mi := &file_ateapi_proto_msgTypes[100] + mi := &file_ateapi_proto_msgTypes[102] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7260,7 +7543,7 @@ func (x *RegisterWorkerResponse) String() string { func (*RegisterWorkerResponse) ProtoMessage() {} func (x *RegisterWorkerResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[100] + mi := &file_ateapi_proto_msgTypes[102] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7273,7 +7556,7 @@ func (x *RegisterWorkerResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RegisterWorkerResponse.ProtoReflect.Descriptor instead. func (*RegisterWorkerResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{100} + return file_ateapi_proto_rawDescGZIP(), []int{102} } func (x *RegisterWorkerResponse) GetWorker() *Worker { @@ -7312,7 +7595,7 @@ type MintAteomActorCertificateRequest struct { func (x *MintAteomActorCertificateRequest) Reset() { *x = MintAteomActorCertificateRequest{} - mi := &file_ateapi_proto_msgTypes[101] + mi := &file_ateapi_proto_msgTypes[103] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7324,7 +7607,7 @@ func (x *MintAteomActorCertificateRequest) String() string { func (*MintAteomActorCertificateRequest) ProtoMessage() {} func (x *MintAteomActorCertificateRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[101] + mi := &file_ateapi_proto_msgTypes[103] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7337,7 +7620,7 @@ func (x *MintAteomActorCertificateRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use MintAteomActorCertificateRequest.ProtoReflect.Descriptor instead. func (*MintAteomActorCertificateRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{101} + return file_ateapi_proto_rawDescGZIP(), []int{103} } func (x *MintAteomActorCertificateRequest) GetActor() *ObjectRef { @@ -7374,7 +7657,7 @@ type MintAteomActorCertificateResponse struct { func (x *MintAteomActorCertificateResponse) Reset() { *x = MintAteomActorCertificateResponse{} - mi := &file_ateapi_proto_msgTypes[102] + mi := &file_ateapi_proto_msgTypes[104] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7386,7 +7669,7 @@ func (x *MintAteomActorCertificateResponse) String() string { func (*MintAteomActorCertificateResponse) ProtoMessage() {} func (x *MintAteomActorCertificateResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[102] + mi := &file_ateapi_proto_msgTypes[104] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7399,7 +7682,7 @@ func (x *MintAteomActorCertificateResponse) ProtoReflect() protoreflect.Message // Deprecated: Use MintAteomActorCertificateResponse.ProtoReflect.Descriptor instead. func (*MintAteomActorCertificateResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{102} + return file_ateapi_proto_rawDescGZIP(), []int{104} } func (x *MintAteomActorCertificateResponse) GetActorCertificates() [][]byte { @@ -7437,7 +7720,7 @@ type RequestActorSuspendRequest struct { func (x *RequestActorSuspendRequest) Reset() { *x = RequestActorSuspendRequest{} - mi := &file_ateapi_proto_msgTypes[103] + mi := &file_ateapi_proto_msgTypes[105] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7449,7 +7732,7 @@ func (x *RequestActorSuspendRequest) String() string { func (*RequestActorSuspendRequest) ProtoMessage() {} func (x *RequestActorSuspendRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[103] + mi := &file_ateapi_proto_msgTypes[105] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7462,7 +7745,7 @@ func (x *RequestActorSuspendRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use RequestActorSuspendRequest.ProtoReflect.Descriptor instead. func (*RequestActorSuspendRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{103} + return file_ateapi_proto_rawDescGZIP(), []int{105} } func (x *RequestActorSuspendRequest) GetWorker() *ObjectRef { @@ -7496,7 +7779,7 @@ type RequestActorSuspendResponse struct { func (x *RequestActorSuspendResponse) Reset() { *x = RequestActorSuspendResponse{} - mi := &file_ateapi_proto_msgTypes[104] + mi := &file_ateapi_proto_msgTypes[106] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7508,7 +7791,7 @@ func (x *RequestActorSuspendResponse) String() string { func (*RequestActorSuspendResponse) ProtoMessage() {} func (x *RequestActorSuspendResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[104] + mi := &file_ateapi_proto_msgTypes[106] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7521,7 +7804,7 @@ func (x *RequestActorSuspendResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RequestActorSuspendResponse.ProtoReflect.Descriptor instead. func (*RequestActorSuspendResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{104} + return file_ateapi_proto_rawDescGZIP(), []int{106} } func (x *RequestActorSuspendResponse) GetActor() *Actor { @@ -7556,7 +7839,7 @@ type AccessPolicy struct { func (x *AccessPolicy) Reset() { *x = AccessPolicy{} - mi := &file_ateapi_proto_msgTypes[105] + mi := &file_ateapi_proto_msgTypes[107] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7568,7 +7851,7 @@ func (x *AccessPolicy) String() string { func (*AccessPolicy) ProtoMessage() {} func (x *AccessPolicy) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[105] + mi := &file_ateapi_proto_msgTypes[107] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7581,7 +7864,7 @@ func (x *AccessPolicy) ProtoReflect() protoreflect.Message { // Deprecated: Use AccessPolicy.ProtoReflect.Descriptor instead. func (*AccessPolicy) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{105} + return file_ateapi_proto_rawDescGZIP(), []int{107} } func (x *AccessPolicy) GetMetadata() *ResourceMetadata { @@ -7623,7 +7906,7 @@ type Binding struct { func (x *Binding) Reset() { *x = Binding{} - mi := &file_ateapi_proto_msgTypes[106] + mi := &file_ateapi_proto_msgTypes[108] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7635,7 +7918,7 @@ func (x *Binding) String() string { func (*Binding) ProtoMessage() {} func (x *Binding) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[106] + mi := &file_ateapi_proto_msgTypes[108] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7648,7 +7931,7 @@ func (x *Binding) ProtoReflect() protoreflect.Message { // Deprecated: Use Binding.ProtoReflect.Descriptor instead. func (*Binding) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{106} + return file_ateapi_proto_rawDescGZIP(), []int{108} } func (x *Binding) GetRole() string { @@ -7674,7 +7957,7 @@ type GetGlobalAccessPolicyRequest struct { func (x *GetGlobalAccessPolicyRequest) Reset() { *x = GetGlobalAccessPolicyRequest{} - mi := &file_ateapi_proto_msgTypes[107] + mi := &file_ateapi_proto_msgTypes[109] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7686,7 +7969,7 @@ func (x *GetGlobalAccessPolicyRequest) String() string { func (*GetGlobalAccessPolicyRequest) ProtoMessage() {} func (x *GetGlobalAccessPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[107] + mi := &file_ateapi_proto_msgTypes[109] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7699,7 +7982,7 @@ func (x *GetGlobalAccessPolicyRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetGlobalAccessPolicyRequest.ProtoReflect.Descriptor instead. func (*GetGlobalAccessPolicyRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{107} + return file_ateapi_proto_rawDescGZIP(), []int{109} } // CreateGlobalAccessPolicyRequest creates the deployment-wide global access policy singleton. @@ -7716,7 +7999,7 @@ type CreateGlobalAccessPolicyRequest struct { func (x *CreateGlobalAccessPolicyRequest) Reset() { *x = CreateGlobalAccessPolicyRequest{} - mi := &file_ateapi_proto_msgTypes[108] + mi := &file_ateapi_proto_msgTypes[110] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7728,7 +8011,7 @@ func (x *CreateGlobalAccessPolicyRequest) String() string { func (*CreateGlobalAccessPolicyRequest) ProtoMessage() {} func (x *CreateGlobalAccessPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[108] + mi := &file_ateapi_proto_msgTypes[110] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7741,7 +8024,7 @@ func (x *CreateGlobalAccessPolicyRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use CreateGlobalAccessPolicyRequest.ProtoReflect.Descriptor instead. func (*CreateGlobalAccessPolicyRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{108} + return file_ateapi_proto_rawDescGZIP(), []int{110} } func (x *CreateGlobalAccessPolicyRequest) GetAccessPolicy() *AccessPolicy { @@ -7765,7 +8048,7 @@ type UpdateGlobalAccessPolicyRequest struct { func (x *UpdateGlobalAccessPolicyRequest) Reset() { *x = UpdateGlobalAccessPolicyRequest{} - mi := &file_ateapi_proto_msgTypes[109] + mi := &file_ateapi_proto_msgTypes[111] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7777,7 +8060,7 @@ func (x *UpdateGlobalAccessPolicyRequest) String() string { func (*UpdateGlobalAccessPolicyRequest) ProtoMessage() {} func (x *UpdateGlobalAccessPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[109] + mi := &file_ateapi_proto_msgTypes[111] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7790,7 +8073,7 @@ func (x *UpdateGlobalAccessPolicyRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use UpdateGlobalAccessPolicyRequest.ProtoReflect.Descriptor instead. func (*UpdateGlobalAccessPolicyRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{109} + return file_ateapi_proto_rawDescGZIP(), []int{111} } func (x *UpdateGlobalAccessPolicyRequest) GetAccessPolicy() *AccessPolicy { @@ -7814,7 +8097,7 @@ type GetAtespaceAccessPolicyRequest struct { func (x *GetAtespaceAccessPolicyRequest) Reset() { *x = GetAtespaceAccessPolicyRequest{} - mi := &file_ateapi_proto_msgTypes[110] + mi := &file_ateapi_proto_msgTypes[112] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7826,7 +8109,7 @@ func (x *GetAtespaceAccessPolicyRequest) String() string { func (*GetAtespaceAccessPolicyRequest) ProtoMessage() {} func (x *GetAtespaceAccessPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[110] + mi := &file_ateapi_proto_msgTypes[112] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7839,7 +8122,7 @@ func (x *GetAtespaceAccessPolicyRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetAtespaceAccessPolicyRequest.ProtoReflect.Descriptor instead. func (*GetAtespaceAccessPolicyRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{110} + return file_ateapi_proto_rawDescGZIP(), []int{112} } func (x *GetAtespaceAccessPolicyRequest) GetAtespace() *ObjectRef { @@ -7868,7 +8151,7 @@ type CreateAtespaceAccessPolicyRequest struct { func (x *CreateAtespaceAccessPolicyRequest) Reset() { *x = CreateAtespaceAccessPolicyRequest{} - mi := &file_ateapi_proto_msgTypes[111] + mi := &file_ateapi_proto_msgTypes[113] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7880,7 +8163,7 @@ func (x *CreateAtespaceAccessPolicyRequest) String() string { func (*CreateAtespaceAccessPolicyRequest) ProtoMessage() {} func (x *CreateAtespaceAccessPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[111] + mi := &file_ateapi_proto_msgTypes[113] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7893,7 +8176,7 @@ func (x *CreateAtespaceAccessPolicyRequest) ProtoReflect() protoreflect.Message // Deprecated: Use CreateAtespaceAccessPolicyRequest.ProtoReflect.Descriptor instead. func (*CreateAtespaceAccessPolicyRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{111} + return file_ateapi_proto_rawDescGZIP(), []int{113} } func (x *CreateAtespaceAccessPolicyRequest) GetAtespace() *ObjectRef { @@ -7929,7 +8212,7 @@ type UpdateAtespaceAccessPolicyRequest struct { func (x *UpdateAtespaceAccessPolicyRequest) Reset() { *x = UpdateAtespaceAccessPolicyRequest{} - mi := &file_ateapi_proto_msgTypes[112] + mi := &file_ateapi_proto_msgTypes[114] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7941,7 +8224,7 @@ func (x *UpdateAtespaceAccessPolicyRequest) String() string { func (*UpdateAtespaceAccessPolicyRequest) ProtoMessage() {} func (x *UpdateAtespaceAccessPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[112] + mi := &file_ateapi_proto_msgTypes[114] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7954,7 +8237,7 @@ func (x *UpdateAtespaceAccessPolicyRequest) ProtoReflect() protoreflect.Message // Deprecated: Use UpdateAtespaceAccessPolicyRequest.ProtoReflect.Descriptor instead. func (*UpdateAtespaceAccessPolicyRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{112} + return file_ateapi_proto_rawDescGZIP(), []int{114} } func (x *UpdateAtespaceAccessPolicyRequest) GetAtespace() *ObjectRef { @@ -7989,7 +8272,7 @@ type DeleteAtespaceAccessPolicyRequest struct { func (x *DeleteAtespaceAccessPolicyRequest) Reset() { *x = DeleteAtespaceAccessPolicyRequest{} - mi := &file_ateapi_proto_msgTypes[113] + mi := &file_ateapi_proto_msgTypes[115] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8001,7 +8284,7 @@ func (x *DeleteAtespaceAccessPolicyRequest) String() string { func (*DeleteAtespaceAccessPolicyRequest) ProtoMessage() {} func (x *DeleteAtespaceAccessPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[113] + mi := &file_ateapi_proto_msgTypes[115] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8014,7 +8297,7 @@ func (x *DeleteAtespaceAccessPolicyRequest) ProtoReflect() protoreflect.Message // Deprecated: Use DeleteAtespaceAccessPolicyRequest.ProtoReflect.Descriptor instead. func (*DeleteAtespaceAccessPolicyRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{113} + return file_ateapi_proto_rawDescGZIP(), []int{115} } func (x *DeleteAtespaceAccessPolicyRequest) GetAtespace() *ObjectRef { @@ -8035,14 +8318,26 @@ var File_ateapi_proto protoreflect.FileDescriptor const file_ateapi_proto_rawDesc = "" + "\n" + - "\fateapi.proto\x12\x06ateapi\x1a\x1fgoogle/protobuf/timestamp.proto\"\x99\x01\n" + - "\x10ExternalSnapshot\x12!\n" + - "\fsnapshot_uri\x18\x01 \x01(\tR\vsnapshotUri\x124\n" + - "\bfidelity\x18\x02 \x01(\x0e2\x18.ateapi.SnapshotFidelityR\bfidelity\x12,\n" + - "\x12actor_template_uid\x18\x03 \x01(\tR\x10actorTemplateUid\"j\n" + + "\fateapi.proto\x12\x06ateapi\x1a\x1fgoogle/protobuf/timestamp.proto\"3\n" + + "\x0eObjectSnapshot\x12!\n" + + "\fsnapshot_uri\x18\x01 \x01(\tR\vsnapshotUri\"4\n" + "\rLocalSnapshot\x12#\n" + - "\rsnapshot_name\x18\x01 \x01(\tR\fsnapshotName\x124\n" + - "\bfidelity\x18\x02 \x01(\x0e2\x18.ateapi.SnapshotFidelityR\bfidelity\"\x90\x01\n" + + "\rsnapshot_name\x18\x01 \x01(\tR\fsnapshotName\"\x97\x02\n" + + "\x0fSnapshotStorage\x12:\n" + + "\n" + + "durability\x18\x01 \x01(\x0e2\x1a.ateapi.SnapshotDurabilityR\n" + + "durability\x125\n" + + "\x06status\x18\x02 \x01(\x0e2\x1d.ateapi.SnapshotStorageStatusR\x06status\x124\n" + + "\bfidelity\x18\x03 \x01(\x0e2\x18.ateapi.SnapshotFidelityR\bfidelity\x12+\n" + + "\x05local\x18\x04 \x01(\v2\x15.ateapi.LocalSnapshotR\x05local\x12.\n" + + "\x06object\x18\x05 \x01(\v2\x16.ateapi.ObjectSnapshotR\x06object\"\xb8\x01\n" + + "\bSnapshot\x12\x1e\n" + + "\n" + + "generation\x18\x01 \x01(\x05R\n" + + "generation\x12+\n" + + "\x05owner\x18\x02 \x01(\x0e2\x15.ateapi.SnapshotOwnerR\x05owner\x12,\n" + + "\x12actor_template_uid\x18\x03 \x01(\tR\x10actorTemplateUid\x121\n" + + "\astorage\x18\x04 \x03(\v2\x17.ateapi.SnapshotStorageR\astorage\"\x90\x01\n" + "\bSelector\x12D\n" + "\fmatch_labels\x18\x01 \x03(\v2!.ateapi.Selector.MatchLabelsEntryR\vmatchLabels\x1a>\n" + "\x10MatchLabelsEntry\x12\x10\n" + @@ -8113,18 +8408,16 @@ const file_ateapi_proto_rawDesc = "" + "\tactor_jwt\x18\x04 \x01(\v2\x16.ateapi.ActorJWTSourceR\bactorJwt\"]\n" + "\x0eActorJWTSource\x12\x1c\n" + "\taudiences\x18\x01 \x03(\tR\taudiences\x12-\n" + - "\x12expiration_seconds\x18\x02 \x01(\x03R\x11expirationSeconds\"\x8e\x04\n" + + "\x12expiration_seconds\x18\x02 \x01(\x03R\x11expirationSeconds\"\xf4\x02\n" + "\vActorStatus\x12(\n" + "\x05state\x18\x01 \x01(\x0e2\x12.ateapi.ActorStateR\x05state\x12E\n" + - "\x11worker_assignment\x18\x02 \x01(\v2\x18.ateapi.WorkerAssignmentR\x10workerAssignment\x127\n" + - "\x18in_progress_snapshot_uri\x18\x03 \x01(\tR\x15inProgressSnapshotUri\x12E\n" + - "\x11external_snapshot\x18\x04 \x01(\v2\x18.ateapi.ExternalSnapshotR\x10externalSnapshot\x12<\n" + - "\x0elocal_snapshot\x18\x05 \x01(\v2\x15.ateapi.LocalSnapshotR\rlocalSnapshot\x12;\n" + - "\ractor_volumes\x18\a \x03(\v2\x16.ateapi.ExternalVolumeR\factorVolumes\x12D\n" + - "\x1fin_progress_local_snapshot_name\x18\b \x01(\tR\x1binProgressLocalSnapshotName\x12(\n" + + "\x11worker_assignment\x18\x02 \x01(\v2\x18.ateapi.WorkerAssignmentR\x10workerAssignment\x12;\n" + + "\ractor_volumes\x18\a \x03(\v2\x16.ateapi.ExternalVolumeR\factorVolumes\x12(\n" + "\x05crash\x18\t \x01(\v2\x12.ateapi.ActorCrashR\x05crash\x12#\n" + "\rassigned_node\x18\n" + - " \x01(\tR\fassignedNode\"a\n" + + " \x01(\tR\fassignedNode\x12.\n" + + "\tsnapshots\x18\v \x03(\v2\x10.ateapi.SnapshotR\tsnapshots\x128\n" + + "\x18last_assigned_generation\x18\f \x01(\x05R\x16lastAssignedGeneration\"a\n" + "\n" + "ActorCrash\x12\x18\n" + "\amessage\x18\x01 \x01(\tR\amessage\x129\n" + @@ -8140,11 +8433,10 @@ const file_ateapi_proto_rawDesc = "" + "\x0eworker_pod_uid\x18\x04 \x01(\tR\fworkerPodUid\x12$\n" + "\x0eworker_pod_ips\x18\x05 \x03(\tR\fworkerPodIps\x12\x1b\n" + "\tnode_name\x18\a \x01(\tR\bnodeName\x12!\n" + - "\fworker_epoch\x18\b \x01(\x03R\vworkerEpoch\"\x9a\x01\n" + - "\tTagStatus\x124\n" + - "\bsnapshot\x18\x01 \x01(\v2\x18.ateapi.ExternalSnapshotR\bsnapshot\x12,\n" + - "\x12actor_template_uid\x18\x02 \x01(\tR\x10actorTemplateUid\x12)\n" + - "\x10storage_location\x18\x03 \x01(\tR\x0fstorageLocation\"\xc4\x01\n" + + "\fworker_epoch\x18\b \x01(\x03R\vworkerEpoch\"g\n" + + "\tTagStatus\x12,\n" + + "\bsnapshot\x18\x01 \x01(\v2\x10.ateapi.SnapshotR\bsnapshot\x12,\n" + + "\x12actor_template_uid\x18\x02 \x01(\tR\x10actorTemplateUid\"\xc4\x01\n" + "\x03Tag\x124\n" + "\bmetadata\x18\x01 \x01(\v2\x18.ateapi.ResourceMetadataR\bmetadata\x12)\n" + "\x06status\x18\x02 \x01(\v2\x11.ateapi.TagStatusR\x06status\x12&\n" + @@ -8453,7 +8745,19 @@ const file_ateapi_proto_rawDesc = "" + "\raccess_policy\x18\x02 \x01(\v2\x14.ateapi.AccessPolicyR\faccessPolicy\"\x83\x01\n" + "!DeleteAtespaceAccessPolicyRequest\x12-\n" + "\batespace\x18\x01 \x01(\v2\x11.ateapi.ObjectRefR\batespace\x12/\n" + - "\aoptions\x18\x02 \x01(\v2\x15.ateapi.DeleteOptionsR\aoptions*\x90\x01\n" + + "\aoptions\x18\x02 \x01(\v2\x15.ateapi.DeleteOptionsR\aoptions*a\n" + + "\rSnapshotOwner\x12\x1e\n" + + "\x1aSNAPSHOT_OWNER_UNSPECIFIED\x10\x00\x12\x18\n" + + "\x14SNAPSHOT_OWNER_ACTOR\x10\x01\x12\x16\n" + + "\x12SNAPSHOT_OWNER_TAG\x10\x02*y\n" + + "\x12SnapshotDurability\x12#\n" + + "\x1fSNAPSHOT_DURABILITY_UNSPECIFIED\x10\x00\x12\x1d\n" + + "\x19SNAPSHOT_DURABILITY_LOCAL\x10\x01\x12\x1f\n" + + "\x1bSNAPSHOT_DURABILITY_DURABLE\x10\x02*\x90\x01\n" + + "\x15SnapshotStorageStatus\x12'\n" + + "#SNAPSHOT_STORAGE_STATUS_UNSPECIFIED\x10\x00\x12'\n" + + "#SNAPSHOT_STORAGE_STATUS_IN_PROGRESS\x10\x01\x12%\n" + + "!SNAPSHOT_STORAGE_STATUS_COMPLETED\x10\x02*\x90\x01\n" + "\x10SnapshotFidelity\x12!\n" + "\x1dSNAPSHOT_FIDELITY_UNSPECIFIED\x10\x00\x12\x1d\n" + "\x19SNAPSHOT_FIDELITY_VOLUMES\x10\x01\x12\x1c\n" + @@ -8550,381 +8854,390 @@ func file_ateapi_proto_rawDescGZIP() []byte { return file_ateapi_proto_rawDescData } -var file_ateapi_proto_enumTypes = make([]protoimpl.EnumInfo, 7) -var file_ateapi_proto_msgTypes = make([]protoimpl.MessageInfo, 118) +var file_ateapi_proto_enumTypes = make([]protoimpl.EnumInfo, 10) +var file_ateapi_proto_msgTypes = make([]protoimpl.MessageInfo, 120) var file_ateapi_proto_goTypes = []any{ - (SnapshotFidelity)(0), // 0: ateapi.SnapshotFidelity - (TagScope)(0), // 1: ateapi.TagScope - (ActorState)(0), // 2: ateapi.ActorState - (SandboxClass)(0), // 3: ateapi.SandboxClass - (ActorMetadataField)(0), // 4: ateapi.ActorMetadataField - (WorkerState)(0), // 5: ateapi.WorkerState - (ExternalVolume_Status)(0), // 6: ateapi.ExternalVolume.Status - (*ExternalSnapshot)(nil), // 7: ateapi.ExternalSnapshot - (*LocalSnapshot)(nil), // 8: ateapi.LocalSnapshot - (*Selector)(nil), // 9: ateapi.Selector - (*ResourceMetadata)(nil), // 10: ateapi.ResourceMetadata - (*ExternalVolume)(nil), // 11: ateapi.ExternalVolume - (*Actor)(nil), // 12: ateapi.Actor - (*EgressPolicy)(nil), // 13: ateapi.EgressPolicy - (*EgressRule)(nil), // 14: ateapi.EgressRule - (*HTTPRule)(nil), // 15: ateapi.HTTPRule - (*HTTPSRule)(nil), // 16: ateapi.HTTPSRule - (*TLSPassthroughRule)(nil), // 17: ateapi.TLSPassthroughRule - (*Ports)(nil), // 18: ateapi.Ports - (*AllPorts)(nil), // 19: ateapi.AllPorts - (*HttpRuleEffects)(nil), // 20: ateapi.HttpRuleEffects - (*CredentialHeader)(nil), // 21: ateapi.CredentialHeader - (*ActorJWTSource)(nil), // 22: ateapi.ActorJWTSource - (*ActorStatus)(nil), // 23: ateapi.ActorStatus - (*ActorCrash)(nil), // 24: ateapi.ActorCrash - (*WorkerAssignment)(nil), // 25: ateapi.WorkerAssignment - (*TagStatus)(nil), // 26: ateapi.TagStatus - (*Tag)(nil), // 27: ateapi.Tag - (*Atespace)(nil), // 28: ateapi.Atespace - (*ObjectRef)(nil), // 29: ateapi.ObjectRef - (*ActorTemplate)(nil), // 30: ateapi.ActorTemplate - (*Resources)(nil), // 31: ateapi.Resources - (*Limits)(nil), // 32: ateapi.Limits - (*GoldenSnapshotStatus)(nil), // 33: ateapi.GoldenSnapshotStatus - (*ActorTemplateStatus)(nil), // 34: ateapi.ActorTemplateStatus - (*SandboxConfig)(nil), // 35: ateapi.SandboxConfig - (*SnapshotConfig)(nil), // 36: ateapi.SnapshotConfig - (*Container)(nil), // 37: ateapi.Container - (*SecurityContext)(nil), // 38: ateapi.SecurityContext - (*Capabilities)(nil), // 39: ateapi.Capabilities - (*EnvVar)(nil), // 40: ateapi.EnvVar - (*ContainerWakeupProbe)(nil), // 41: ateapi.ContainerWakeupProbe - (*HTTPGetAction)(nil), // 42: ateapi.HTTPGetAction - (*Volume)(nil), // 43: ateapi.Volume - (*ImageVolumeSource)(nil), // 44: ateapi.ImageVolumeSource - (*DurableDirVolumeSource)(nil), // 45: ateapi.DurableDirVolumeSource - (*ExternalVolumeTemplate)(nil), // 46: ateapi.ExternalVolumeTemplate - (*SystemInfoVolumeSource)(nil), // 47: ateapi.SystemInfoVolumeSource - (*SystemInfoDataSource)(nil), // 48: ateapi.SystemInfoDataSource - (*ActorMetadataDataSource)(nil), // 49: ateapi.ActorMetadataDataSource - (*ActorMetadataItem)(nil), // 50: ateapi.ActorMetadataItem - (*TrustBundleDataSource)(nil), // 51: ateapi.TrustBundleDataSource - (*VolumeMount)(nil), // 52: ateapi.VolumeMount - (*CreateAtespaceRequest)(nil), // 53: ateapi.CreateAtespaceRequest - (*GetAtespaceRequest)(nil), // 54: ateapi.GetAtespaceRequest - (*ListAtespacesRequest)(nil), // 55: ateapi.ListAtespacesRequest - (*ListAtespacesResponse)(nil), // 56: ateapi.ListAtespacesResponse - (*DeleteAtespaceRequest)(nil), // 57: ateapi.DeleteAtespaceRequest - (*CreateActorTemplateRequest)(nil), // 58: ateapi.CreateActorTemplateRequest - (*GetActorTemplateRequest)(nil), // 59: ateapi.GetActorTemplateRequest - (*ListActorTemplatesRequest)(nil), // 60: ateapi.ListActorTemplatesRequest - (*ListActorTemplatesResponse)(nil), // 61: ateapi.ListActorTemplatesResponse - (*DeleteActorTemplateRequest)(nil), // 62: ateapi.DeleteActorTemplateRequest - (*GetActorRequest)(nil), // 63: ateapi.GetActorRequest - (*CreateActorRequest)(nil), // 64: ateapi.CreateActorRequest - (*UpdateActorRequest)(nil), // 65: ateapi.UpdateActorRequest - (*SuspendActorRequest)(nil), // 66: ateapi.SuspendActorRequest - (*SuspendActorResponse)(nil), // 67: ateapi.SuspendActorResponse - (*PauseActorRequest)(nil), // 68: ateapi.PauseActorRequest - (*PauseActorResponse)(nil), // 69: ateapi.PauseActorResponse - (*ResumeActorRequest)(nil), // 70: ateapi.ResumeActorRequest - (*ResumeActorResponse)(nil), // 71: ateapi.ResumeActorResponse - (*RevertActorRequest)(nil), // 72: ateapi.RevertActorRequest - (*RevertActorResponse)(nil), // 73: ateapi.RevertActorResponse - (*DeleteActorRequest)(nil), // 74: ateapi.DeleteActorRequest - (*GetActorEgressPolicyRequest)(nil), // 75: ateapi.GetActorEgressPolicyRequest - (*CreateActorEgressPolicyRequest)(nil), // 76: ateapi.CreateActorEgressPolicyRequest - (*UpdateActorEgressPolicyRequest)(nil), // 77: ateapi.UpdateActorEgressPolicyRequest - (*DeleteActorEgressPolicyRequest)(nil), // 78: ateapi.DeleteActorEgressPolicyRequest - (*GetTagRequest)(nil), // 79: ateapi.GetTagRequest - (*MintActorJWTRequest)(nil), // 80: ateapi.MintActorJWTRequest - (*MintActorJWTResponse)(nil), // 81: ateapi.MintActorJWTResponse - (*MintActorCertificateRequest)(nil), // 82: ateapi.MintActorCertificateRequest - (*MintActorCertificateResponse)(nil), // 83: ateapi.MintActorCertificateResponse - (*ListTagsRequest)(nil), // 84: ateapi.ListTagsRequest - (*ListTagsResponse)(nil), // 85: ateapi.ListTagsResponse - (*CreateTagRequest)(nil), // 86: ateapi.CreateTagRequest - (*UpdateTagRequest)(nil), // 87: ateapi.UpdateTagRequest - (*DeleteTagRequest)(nil), // 88: ateapi.DeleteTagRequest - (*DeleteOptions)(nil), // 89: ateapi.DeleteOptions - (*ListWorkerActorAssignmentsRequest)(nil), // 90: ateapi.ListWorkerActorAssignmentsRequest - (*ListWorkerActorAssignmentsResponse)(nil), // 91: ateapi.ListWorkerActorAssignmentsResponse - (*ListWorkersRequest)(nil), // 92: ateapi.ListWorkersRequest - (*ListWorkersResponse)(nil), // 93: ateapi.ListWorkersResponse - (*GetWorkerRequest)(nil), // 94: ateapi.GetWorkerRequest - (*CreateWorkerRequest)(nil), // 95: ateapi.CreateWorkerRequest - (*UpdateWorkerRequest)(nil), // 96: ateapi.UpdateWorkerRequest - (*DeleteWorkerRequest)(nil), // 97: ateapi.DeleteWorkerRequest - (*DrainWorkerRequest)(nil), // 98: ateapi.DrainWorkerRequest - (*ListActorsRequest)(nil), // 99: ateapi.ListActorsRequest - (*ListActorsResponse)(nil), // 100: ateapi.ListActorsResponse - (*Worker)(nil), // 101: ateapi.Worker - (*WorkerStatus)(nil), // 102: ateapi.WorkerStatus - (*HardwareIdentity)(nil), // 103: ateapi.HardwareIdentity - (*WorkerResources)(nil), // 104: ateapi.WorkerResources - (*ActorAssignment)(nil), // 105: ateapi.ActorAssignment - (*RegisterWorkerRequest)(nil), // 106: ateapi.RegisterWorkerRequest - (*RegisterWorkerResponse)(nil), // 107: ateapi.RegisterWorkerResponse - (*MintAteomActorCertificateRequest)(nil), // 108: ateapi.MintAteomActorCertificateRequest - (*MintAteomActorCertificateResponse)(nil), // 109: ateapi.MintAteomActorCertificateResponse - (*RequestActorSuspendRequest)(nil), // 110: ateapi.RequestActorSuspendRequest - (*RequestActorSuspendResponse)(nil), // 111: ateapi.RequestActorSuspendResponse - (*AccessPolicy)(nil), // 112: ateapi.AccessPolicy - (*Binding)(nil), // 113: ateapi.Binding - (*GetGlobalAccessPolicyRequest)(nil), // 114: ateapi.GetGlobalAccessPolicyRequest - (*CreateGlobalAccessPolicyRequest)(nil), // 115: ateapi.CreateGlobalAccessPolicyRequest - (*UpdateGlobalAccessPolicyRequest)(nil), // 116: ateapi.UpdateGlobalAccessPolicyRequest - (*GetAtespaceAccessPolicyRequest)(nil), // 117: ateapi.GetAtespaceAccessPolicyRequest - (*CreateAtespaceAccessPolicyRequest)(nil), // 118: ateapi.CreateAtespaceAccessPolicyRequest - (*UpdateAtespaceAccessPolicyRequest)(nil), // 119: ateapi.UpdateAtespaceAccessPolicyRequest - (*DeleteAtespaceAccessPolicyRequest)(nil), // 120: ateapi.DeleteAtespaceAccessPolicyRequest - nil, // 121: ateapi.Selector.MatchLabelsEntry - nil, // 122: ateapi.ExternalVolume.VolumeContextEntry - nil, // 123: ateapi.Worker.LabelsEntry - nil, // 124: ateapi.HardwareIdentity.AttributesEntry - (*timestamppb.Timestamp)(nil), // 125: google.protobuf.Timestamp + (SnapshotOwner)(0), // 0: ateapi.SnapshotOwner + (SnapshotDurability)(0), // 1: ateapi.SnapshotDurability + (SnapshotStorageStatus)(0), // 2: ateapi.SnapshotStorageStatus + (SnapshotFidelity)(0), // 3: ateapi.SnapshotFidelity + (TagScope)(0), // 4: ateapi.TagScope + (ActorState)(0), // 5: ateapi.ActorState + (SandboxClass)(0), // 6: ateapi.SandboxClass + (ActorMetadataField)(0), // 7: ateapi.ActorMetadataField + (WorkerState)(0), // 8: ateapi.WorkerState + (ExternalVolume_Status)(0), // 9: ateapi.ExternalVolume.Status + (*ObjectSnapshot)(nil), // 10: ateapi.ObjectSnapshot + (*LocalSnapshot)(nil), // 11: ateapi.LocalSnapshot + (*SnapshotStorage)(nil), // 12: ateapi.SnapshotStorage + (*Snapshot)(nil), // 13: ateapi.Snapshot + (*Selector)(nil), // 14: ateapi.Selector + (*ResourceMetadata)(nil), // 15: ateapi.ResourceMetadata + (*ExternalVolume)(nil), // 16: ateapi.ExternalVolume + (*Actor)(nil), // 17: ateapi.Actor + (*EgressPolicy)(nil), // 18: ateapi.EgressPolicy + (*EgressRule)(nil), // 19: ateapi.EgressRule + (*HTTPRule)(nil), // 20: ateapi.HTTPRule + (*HTTPSRule)(nil), // 21: ateapi.HTTPSRule + (*TLSPassthroughRule)(nil), // 22: ateapi.TLSPassthroughRule + (*Ports)(nil), // 23: ateapi.Ports + (*AllPorts)(nil), // 24: ateapi.AllPorts + (*HttpRuleEffects)(nil), // 25: ateapi.HttpRuleEffects + (*CredentialHeader)(nil), // 26: ateapi.CredentialHeader + (*ActorJWTSource)(nil), // 27: ateapi.ActorJWTSource + (*ActorStatus)(nil), // 28: ateapi.ActorStatus + (*ActorCrash)(nil), // 29: ateapi.ActorCrash + (*WorkerAssignment)(nil), // 30: ateapi.WorkerAssignment + (*TagStatus)(nil), // 31: ateapi.TagStatus + (*Tag)(nil), // 32: ateapi.Tag + (*Atespace)(nil), // 33: ateapi.Atespace + (*ObjectRef)(nil), // 34: ateapi.ObjectRef + (*ActorTemplate)(nil), // 35: ateapi.ActorTemplate + (*Resources)(nil), // 36: ateapi.Resources + (*Limits)(nil), // 37: ateapi.Limits + (*GoldenSnapshotStatus)(nil), // 38: ateapi.GoldenSnapshotStatus + (*ActorTemplateStatus)(nil), // 39: ateapi.ActorTemplateStatus + (*SandboxConfig)(nil), // 40: ateapi.SandboxConfig + (*SnapshotConfig)(nil), // 41: ateapi.SnapshotConfig + (*Container)(nil), // 42: ateapi.Container + (*SecurityContext)(nil), // 43: ateapi.SecurityContext + (*Capabilities)(nil), // 44: ateapi.Capabilities + (*EnvVar)(nil), // 45: ateapi.EnvVar + (*ContainerWakeupProbe)(nil), // 46: ateapi.ContainerWakeupProbe + (*HTTPGetAction)(nil), // 47: ateapi.HTTPGetAction + (*Volume)(nil), // 48: ateapi.Volume + (*ImageVolumeSource)(nil), // 49: ateapi.ImageVolumeSource + (*DurableDirVolumeSource)(nil), // 50: ateapi.DurableDirVolumeSource + (*ExternalVolumeTemplate)(nil), // 51: ateapi.ExternalVolumeTemplate + (*SystemInfoVolumeSource)(nil), // 52: ateapi.SystemInfoVolumeSource + (*SystemInfoDataSource)(nil), // 53: ateapi.SystemInfoDataSource + (*ActorMetadataDataSource)(nil), // 54: ateapi.ActorMetadataDataSource + (*ActorMetadataItem)(nil), // 55: ateapi.ActorMetadataItem + (*TrustBundleDataSource)(nil), // 56: ateapi.TrustBundleDataSource + (*VolumeMount)(nil), // 57: ateapi.VolumeMount + (*CreateAtespaceRequest)(nil), // 58: ateapi.CreateAtespaceRequest + (*GetAtespaceRequest)(nil), // 59: ateapi.GetAtespaceRequest + (*ListAtespacesRequest)(nil), // 60: ateapi.ListAtespacesRequest + (*ListAtespacesResponse)(nil), // 61: ateapi.ListAtespacesResponse + (*DeleteAtespaceRequest)(nil), // 62: ateapi.DeleteAtespaceRequest + (*CreateActorTemplateRequest)(nil), // 63: ateapi.CreateActorTemplateRequest + (*GetActorTemplateRequest)(nil), // 64: ateapi.GetActorTemplateRequest + (*ListActorTemplatesRequest)(nil), // 65: ateapi.ListActorTemplatesRequest + (*ListActorTemplatesResponse)(nil), // 66: ateapi.ListActorTemplatesResponse + (*DeleteActorTemplateRequest)(nil), // 67: ateapi.DeleteActorTemplateRequest + (*GetActorRequest)(nil), // 68: ateapi.GetActorRequest + (*CreateActorRequest)(nil), // 69: ateapi.CreateActorRequest + (*UpdateActorRequest)(nil), // 70: ateapi.UpdateActorRequest + (*SuspendActorRequest)(nil), // 71: ateapi.SuspendActorRequest + (*SuspendActorResponse)(nil), // 72: ateapi.SuspendActorResponse + (*PauseActorRequest)(nil), // 73: ateapi.PauseActorRequest + (*PauseActorResponse)(nil), // 74: ateapi.PauseActorResponse + (*ResumeActorRequest)(nil), // 75: ateapi.ResumeActorRequest + (*ResumeActorResponse)(nil), // 76: ateapi.ResumeActorResponse + (*RevertActorRequest)(nil), // 77: ateapi.RevertActorRequest + (*RevertActorResponse)(nil), // 78: ateapi.RevertActorResponse + (*DeleteActorRequest)(nil), // 79: ateapi.DeleteActorRequest + (*GetActorEgressPolicyRequest)(nil), // 80: ateapi.GetActorEgressPolicyRequest + (*CreateActorEgressPolicyRequest)(nil), // 81: ateapi.CreateActorEgressPolicyRequest + (*UpdateActorEgressPolicyRequest)(nil), // 82: ateapi.UpdateActorEgressPolicyRequest + (*DeleteActorEgressPolicyRequest)(nil), // 83: ateapi.DeleteActorEgressPolicyRequest + (*GetTagRequest)(nil), // 84: ateapi.GetTagRequest + (*MintActorJWTRequest)(nil), // 85: ateapi.MintActorJWTRequest + (*MintActorJWTResponse)(nil), // 86: ateapi.MintActorJWTResponse + (*MintActorCertificateRequest)(nil), // 87: ateapi.MintActorCertificateRequest + (*MintActorCertificateResponse)(nil), // 88: ateapi.MintActorCertificateResponse + (*ListTagsRequest)(nil), // 89: ateapi.ListTagsRequest + (*ListTagsResponse)(nil), // 90: ateapi.ListTagsResponse + (*CreateTagRequest)(nil), // 91: ateapi.CreateTagRequest + (*UpdateTagRequest)(nil), // 92: ateapi.UpdateTagRequest + (*DeleteTagRequest)(nil), // 93: ateapi.DeleteTagRequest + (*DeleteOptions)(nil), // 94: ateapi.DeleteOptions + (*ListWorkerActorAssignmentsRequest)(nil), // 95: ateapi.ListWorkerActorAssignmentsRequest + (*ListWorkerActorAssignmentsResponse)(nil), // 96: ateapi.ListWorkerActorAssignmentsResponse + (*ListWorkersRequest)(nil), // 97: ateapi.ListWorkersRequest + (*ListWorkersResponse)(nil), // 98: ateapi.ListWorkersResponse + (*GetWorkerRequest)(nil), // 99: ateapi.GetWorkerRequest + (*CreateWorkerRequest)(nil), // 100: ateapi.CreateWorkerRequest + (*UpdateWorkerRequest)(nil), // 101: ateapi.UpdateWorkerRequest + (*DeleteWorkerRequest)(nil), // 102: ateapi.DeleteWorkerRequest + (*DrainWorkerRequest)(nil), // 103: ateapi.DrainWorkerRequest + (*ListActorsRequest)(nil), // 104: ateapi.ListActorsRequest + (*ListActorsResponse)(nil), // 105: ateapi.ListActorsResponse + (*Worker)(nil), // 106: ateapi.Worker + (*WorkerStatus)(nil), // 107: ateapi.WorkerStatus + (*HardwareIdentity)(nil), // 108: ateapi.HardwareIdentity + (*WorkerResources)(nil), // 109: ateapi.WorkerResources + (*ActorAssignment)(nil), // 110: ateapi.ActorAssignment + (*RegisterWorkerRequest)(nil), // 111: ateapi.RegisterWorkerRequest + (*RegisterWorkerResponse)(nil), // 112: ateapi.RegisterWorkerResponse + (*MintAteomActorCertificateRequest)(nil), // 113: ateapi.MintAteomActorCertificateRequest + (*MintAteomActorCertificateResponse)(nil), // 114: ateapi.MintAteomActorCertificateResponse + (*RequestActorSuspendRequest)(nil), // 115: ateapi.RequestActorSuspendRequest + (*RequestActorSuspendResponse)(nil), // 116: ateapi.RequestActorSuspendResponse + (*AccessPolicy)(nil), // 117: ateapi.AccessPolicy + (*Binding)(nil), // 118: ateapi.Binding + (*GetGlobalAccessPolicyRequest)(nil), // 119: ateapi.GetGlobalAccessPolicyRequest + (*CreateGlobalAccessPolicyRequest)(nil), // 120: ateapi.CreateGlobalAccessPolicyRequest + (*UpdateGlobalAccessPolicyRequest)(nil), // 121: ateapi.UpdateGlobalAccessPolicyRequest + (*GetAtespaceAccessPolicyRequest)(nil), // 122: ateapi.GetAtespaceAccessPolicyRequest + (*CreateAtespaceAccessPolicyRequest)(nil), // 123: ateapi.CreateAtespaceAccessPolicyRequest + (*UpdateAtespaceAccessPolicyRequest)(nil), // 124: ateapi.UpdateAtespaceAccessPolicyRequest + (*DeleteAtespaceAccessPolicyRequest)(nil), // 125: ateapi.DeleteAtespaceAccessPolicyRequest + nil, // 126: ateapi.Selector.MatchLabelsEntry + nil, // 127: ateapi.ExternalVolume.VolumeContextEntry + nil, // 128: ateapi.Worker.LabelsEntry + nil, // 129: ateapi.HardwareIdentity.AttributesEntry + (*timestamppb.Timestamp)(nil), // 130: google.protobuf.Timestamp } var file_ateapi_proto_depIdxs = []int32{ - 0, // 0: ateapi.ExternalSnapshot.fidelity:type_name -> ateapi.SnapshotFidelity - 0, // 1: ateapi.LocalSnapshot.fidelity:type_name -> ateapi.SnapshotFidelity - 121, // 2: ateapi.Selector.match_labels:type_name -> ateapi.Selector.MatchLabelsEntry - 125, // 3: ateapi.ResourceMetadata.create_time:type_name -> google.protobuf.Timestamp - 125, // 4: ateapi.ResourceMetadata.update_time:type_name -> google.protobuf.Timestamp - 6, // 5: ateapi.ExternalVolume.status:type_name -> ateapi.ExternalVolume.Status - 122, // 6: ateapi.ExternalVolume.volume_context:type_name -> ateapi.ExternalVolume.VolumeContextEntry - 10, // 7: ateapi.Actor.metadata:type_name -> ateapi.ResourceMetadata - 29, // 8: ateapi.Actor.actor_template:type_name -> ateapi.ObjectRef - 9, // 9: ateapi.Actor.worker_selector:type_name -> ateapi.Selector - 29, // 10: ateapi.Actor.source_tag:type_name -> ateapi.ObjectRef - 23, // 11: ateapi.Actor.status:type_name -> ateapi.ActorStatus - 10, // 12: ateapi.EgressPolicy.metadata:type_name -> ateapi.ResourceMetadata - 14, // 13: ateapi.EgressPolicy.rules:type_name -> ateapi.EgressRule - 15, // 14: ateapi.EgressRule.http:type_name -> ateapi.HTTPRule - 16, // 15: ateapi.EgressRule.https:type_name -> ateapi.HTTPSRule - 17, // 16: ateapi.EgressRule.tls_passthrough:type_name -> ateapi.TLSPassthroughRule - 18, // 17: ateapi.HTTPRule.ports:type_name -> ateapi.Ports - 20, // 18: ateapi.HTTPRule.effects:type_name -> ateapi.HttpRuleEffects - 18, // 19: ateapi.HTTPSRule.ports:type_name -> ateapi.Ports - 20, // 20: ateapi.HTTPSRule.effects:type_name -> ateapi.HttpRuleEffects - 18, // 21: ateapi.TLSPassthroughRule.ports:type_name -> ateapi.Ports - 19, // 22: ateapi.Ports.all:type_name -> ateapi.AllPorts - 21, // 23: ateapi.HttpRuleEffects.replace_headers:type_name -> ateapi.CredentialHeader - 22, // 24: ateapi.CredentialHeader.actor_jwt:type_name -> ateapi.ActorJWTSource - 2, // 25: ateapi.ActorStatus.state:type_name -> ateapi.ActorState - 25, // 26: ateapi.ActorStatus.worker_assignment:type_name -> ateapi.WorkerAssignment - 7, // 27: ateapi.ActorStatus.external_snapshot:type_name -> ateapi.ExternalSnapshot - 8, // 28: ateapi.ActorStatus.local_snapshot:type_name -> ateapi.LocalSnapshot - 11, // 29: ateapi.ActorStatus.actor_volumes:type_name -> ateapi.ExternalVolume - 24, // 30: ateapi.ActorStatus.crash:type_name -> ateapi.ActorCrash - 125, // 31: ateapi.ActorCrash.crash_time:type_name -> google.protobuf.Timestamp - 29, // 32: ateapi.WorkerAssignment.worker:type_name -> ateapi.ObjectRef - 7, // 33: ateapi.TagStatus.snapshot:type_name -> ateapi.ExternalSnapshot - 10, // 34: ateapi.Tag.metadata:type_name -> ateapi.ResourceMetadata - 26, // 35: ateapi.Tag.status:type_name -> ateapi.TagStatus - 1, // 36: ateapi.Tag.scope:type_name -> ateapi.TagScope - 29, // 37: ateapi.Tag.source_actor:type_name -> ateapi.ObjectRef - 10, // 38: ateapi.Atespace.metadata:type_name -> ateapi.ResourceMetadata - 10, // 39: ateapi.ActorTemplate.metadata:type_name -> ateapi.ResourceMetadata - 9, // 40: ateapi.ActorTemplate.worker_selector:type_name -> ateapi.Selector - 37, // 41: ateapi.ActorTemplate.containers:type_name -> ateapi.Container - 43, // 42: ateapi.ActorTemplate.volumes:type_name -> ateapi.Volume - 36, // 43: ateapi.ActorTemplate.snapshot_config:type_name -> ateapi.SnapshotConfig - 35, // 44: ateapi.ActorTemplate.sandbox_config:type_name -> ateapi.SandboxConfig - 31, // 45: ateapi.ActorTemplate.resources:type_name -> ateapi.Resources - 34, // 46: ateapi.ActorTemplate.status:type_name -> ateapi.ActorTemplateStatus - 32, // 47: ateapi.Resources.limits:type_name -> ateapi.Limits - 29, // 48: ateapi.GoldenSnapshotStatus.golden_tag:type_name -> ateapi.ObjectRef - 125, // 49: ateapi.GoldenSnapshotStatus.take_golden_snapshot_at:type_name -> google.protobuf.Timestamp - 33, // 50: ateapi.ActorTemplateStatus.golden_snapshot_status:type_name -> ateapi.GoldenSnapshotStatus - 3, // 51: ateapi.SandboxConfig.sandbox_class:type_name -> ateapi.SandboxClass - 0, // 52: ateapi.SnapshotConfig.preferred_fidelity:type_name -> ateapi.SnapshotFidelity - 40, // 53: ateapi.Container.env:type_name -> ateapi.EnvVar - 41, // 54: ateapi.Container.wakeup_probe:type_name -> ateapi.ContainerWakeupProbe - 52, // 55: ateapi.Container.volume_mounts:type_name -> ateapi.VolumeMount - 38, // 56: ateapi.Container.security_context:type_name -> ateapi.SecurityContext - 31, // 57: ateapi.Container.resources:type_name -> ateapi.Resources - 39, // 58: ateapi.SecurityContext.capabilities:type_name -> ateapi.Capabilities - 42, // 59: ateapi.ContainerWakeupProbe.http_get:type_name -> ateapi.HTTPGetAction - 45, // 60: ateapi.Volume.durable_dir:type_name -> ateapi.DurableDirVolumeSource - 46, // 61: ateapi.Volume.external_volume_template:type_name -> ateapi.ExternalVolumeTemplate - 47, // 62: ateapi.Volume.system_info:type_name -> ateapi.SystemInfoVolumeSource - 44, // 63: ateapi.Volume.image:type_name -> ateapi.ImageVolumeSource - 48, // 64: ateapi.SystemInfoVolumeSource.data_sources:type_name -> ateapi.SystemInfoDataSource - 49, // 65: ateapi.SystemInfoDataSource.actor_metadata:type_name -> ateapi.ActorMetadataDataSource - 51, // 66: ateapi.SystemInfoDataSource.trust_bundle:type_name -> ateapi.TrustBundleDataSource - 50, // 67: ateapi.ActorMetadataDataSource.items:type_name -> ateapi.ActorMetadataItem - 4, // 68: ateapi.ActorMetadataItem.field:type_name -> ateapi.ActorMetadataField - 28, // 69: ateapi.CreateAtespaceRequest.atespace:type_name -> ateapi.Atespace - 29, // 70: ateapi.GetAtespaceRequest.atespace:type_name -> ateapi.ObjectRef - 28, // 71: ateapi.ListAtespacesResponse.atespaces:type_name -> ateapi.Atespace - 29, // 72: ateapi.DeleteAtespaceRequest.atespace:type_name -> ateapi.ObjectRef - 89, // 73: ateapi.DeleteAtespaceRequest.options:type_name -> ateapi.DeleteOptions - 30, // 74: ateapi.CreateActorTemplateRequest.actor_template:type_name -> ateapi.ActorTemplate - 29, // 75: ateapi.GetActorTemplateRequest.actor_template:type_name -> ateapi.ObjectRef - 30, // 76: ateapi.ListActorTemplatesResponse.actor_templates:type_name -> ateapi.ActorTemplate - 29, // 77: ateapi.DeleteActorTemplateRequest.actor_template:type_name -> ateapi.ObjectRef - 89, // 78: ateapi.DeleteActorTemplateRequest.options:type_name -> ateapi.DeleteOptions - 29, // 79: ateapi.GetActorRequest.actor:type_name -> ateapi.ObjectRef - 12, // 80: ateapi.CreateActorRequest.actor:type_name -> ateapi.Actor - 12, // 81: ateapi.UpdateActorRequest.actor:type_name -> ateapi.Actor - 29, // 82: ateapi.SuspendActorRequest.actor:type_name -> ateapi.ObjectRef - 12, // 83: ateapi.SuspendActorResponse.actor:type_name -> ateapi.Actor - 29, // 84: ateapi.PauseActorRequest.actor:type_name -> ateapi.ObjectRef - 12, // 85: ateapi.PauseActorResponse.actor:type_name -> ateapi.Actor - 29, // 86: ateapi.ResumeActorRequest.actor:type_name -> ateapi.ObjectRef - 12, // 87: ateapi.ResumeActorResponse.actor:type_name -> ateapi.Actor - 29, // 88: ateapi.RevertActorRequest.actor:type_name -> ateapi.ObjectRef - 12, // 89: ateapi.RevertActorResponse.actor:type_name -> ateapi.Actor - 29, // 90: ateapi.DeleteActorRequest.actor:type_name -> ateapi.ObjectRef - 89, // 91: ateapi.DeleteActorRequest.options:type_name -> ateapi.DeleteOptions - 29, // 92: ateapi.GetActorEgressPolicyRequest.actor:type_name -> ateapi.ObjectRef - 29, // 93: ateapi.CreateActorEgressPolicyRequest.actor:type_name -> ateapi.ObjectRef - 13, // 94: ateapi.CreateActorEgressPolicyRequest.egress_policy:type_name -> ateapi.EgressPolicy - 29, // 95: ateapi.UpdateActorEgressPolicyRequest.actor:type_name -> ateapi.ObjectRef - 13, // 96: ateapi.UpdateActorEgressPolicyRequest.egress_policy:type_name -> ateapi.EgressPolicy - 29, // 97: ateapi.DeleteActorEgressPolicyRequest.actor:type_name -> ateapi.ObjectRef - 89, // 98: ateapi.DeleteActorEgressPolicyRequest.options:type_name -> ateapi.DeleteOptions - 29, // 99: ateapi.GetTagRequest.tag:type_name -> ateapi.ObjectRef - 29, // 100: ateapi.MintActorJWTRequest.actor:type_name -> ateapi.ObjectRef - 125, // 101: ateapi.MintActorJWTResponse.expires_at:type_name -> google.protobuf.Timestamp - 29, // 102: ateapi.MintActorCertificateRequest.actor:type_name -> ateapi.ObjectRef - 27, // 103: ateapi.ListTagsResponse.tags:type_name -> ateapi.Tag - 27, // 104: ateapi.CreateTagRequest.tag:type_name -> ateapi.Tag - 27, // 105: ateapi.UpdateTagRequest.tag:type_name -> ateapi.Tag - 29, // 106: ateapi.DeleteTagRequest.tag:type_name -> ateapi.ObjectRef - 89, // 107: ateapi.DeleteTagRequest.options:type_name -> ateapi.DeleteOptions - 29, // 108: ateapi.ListWorkerActorAssignmentsRequest.worker:type_name -> ateapi.ObjectRef - 105, // 109: ateapi.ListWorkerActorAssignmentsResponse.actor_assignments:type_name -> ateapi.ActorAssignment - 101, // 110: ateapi.ListWorkersResponse.workers:type_name -> ateapi.Worker - 29, // 111: ateapi.GetWorkerRequest.worker:type_name -> ateapi.ObjectRef - 101, // 112: ateapi.CreateWorkerRequest.worker:type_name -> ateapi.Worker - 101, // 113: ateapi.UpdateWorkerRequest.worker:type_name -> ateapi.Worker - 29, // 114: ateapi.DeleteWorkerRequest.worker:type_name -> ateapi.ObjectRef - 89, // 115: ateapi.DeleteWorkerRequest.options:type_name -> ateapi.DeleteOptions - 29, // 116: ateapi.DrainWorkerRequest.worker:type_name -> ateapi.ObjectRef - 12, // 117: ateapi.ListActorsResponse.actors:type_name -> ateapi.Actor - 10, // 118: ateapi.Worker.metadata:type_name -> ateapi.ResourceMetadata - 123, // 119: ateapi.Worker.labels:type_name -> ateapi.Worker.LabelsEntry - 102, // 120: ateapi.Worker.status:type_name -> ateapi.WorkerStatus - 5, // 121: ateapi.WorkerStatus.state:type_name -> ateapi.WorkerState - 104, // 122: ateapi.WorkerStatus.capacity:type_name -> ateapi.WorkerResources - 104, // 123: ateapi.WorkerStatus.allocated:type_name -> ateapi.WorkerResources - 103, // 124: ateapi.WorkerStatus.hardware:type_name -> ateapi.HardwareIdentity - 124, // 125: ateapi.HardwareIdentity.attributes:type_name -> ateapi.HardwareIdentity.AttributesEntry - 31, // 126: ateapi.WorkerResources.resources:type_name -> ateapi.Resources - 10, // 127: ateapi.ActorAssignment.metadata:type_name -> ateapi.ResourceMetadata - 29, // 128: ateapi.ActorAssignment.actor:type_name -> ateapi.ObjectRef - 31, // 129: ateapi.ActorAssignment.resources:type_name -> ateapi.Resources - 29, // 130: ateapi.RegisterWorkerRequest.worker:type_name -> ateapi.ObjectRef - 104, // 131: ateapi.RegisterWorkerRequest.capacity:type_name -> ateapi.WorkerResources - 103, // 132: ateapi.RegisterWorkerRequest.hardware:type_name -> ateapi.HardwareIdentity - 101, // 133: ateapi.RegisterWorkerResponse.worker:type_name -> ateapi.Worker - 29, // 134: ateapi.MintAteomActorCertificateRequest.actor:type_name -> ateapi.ObjectRef - 29, // 135: ateapi.RequestActorSuspendRequest.worker:type_name -> ateapi.ObjectRef - 29, // 136: ateapi.RequestActorSuspendRequest.actor:type_name -> ateapi.ObjectRef - 12, // 137: ateapi.RequestActorSuspendResponse.actor:type_name -> ateapi.Actor - 10, // 138: ateapi.AccessPolicy.metadata:type_name -> ateapi.ResourceMetadata - 113, // 139: ateapi.AccessPolicy.bindings:type_name -> ateapi.Binding - 112, // 140: ateapi.CreateGlobalAccessPolicyRequest.access_policy:type_name -> ateapi.AccessPolicy - 112, // 141: ateapi.UpdateGlobalAccessPolicyRequest.access_policy:type_name -> ateapi.AccessPolicy - 29, // 142: ateapi.GetAtespaceAccessPolicyRequest.atespace:type_name -> ateapi.ObjectRef - 29, // 143: ateapi.CreateAtespaceAccessPolicyRequest.atespace:type_name -> ateapi.ObjectRef - 112, // 144: ateapi.CreateAtespaceAccessPolicyRequest.access_policy:type_name -> ateapi.AccessPolicy - 29, // 145: ateapi.UpdateAtespaceAccessPolicyRequest.atespace:type_name -> ateapi.ObjectRef - 112, // 146: ateapi.UpdateAtespaceAccessPolicyRequest.access_policy:type_name -> ateapi.AccessPolicy - 29, // 147: ateapi.DeleteAtespaceAccessPolicyRequest.atespace:type_name -> ateapi.ObjectRef - 89, // 148: ateapi.DeleteAtespaceAccessPolicyRequest.options:type_name -> ateapi.DeleteOptions - 63, // 149: ateapi.Control.GetActor:input_type -> ateapi.GetActorRequest - 64, // 150: ateapi.Control.CreateActor:input_type -> ateapi.CreateActorRequest - 65, // 151: ateapi.Control.UpdateActor:input_type -> ateapi.UpdateActorRequest - 66, // 152: ateapi.Control.SuspendActor:input_type -> ateapi.SuspendActorRequest - 68, // 153: ateapi.Control.PauseActor:input_type -> ateapi.PauseActorRequest - 70, // 154: ateapi.Control.ResumeActor:input_type -> ateapi.ResumeActorRequest - 72, // 155: ateapi.Control.RevertActor:input_type -> ateapi.RevertActorRequest - 74, // 156: ateapi.Control.DeleteActor:input_type -> ateapi.DeleteActorRequest - 75, // 157: ateapi.Control.GetActorEgressPolicy:input_type -> ateapi.GetActorEgressPolicyRequest - 76, // 158: ateapi.Control.CreateActorEgressPolicy:input_type -> ateapi.CreateActorEgressPolicyRequest - 77, // 159: ateapi.Control.UpdateActorEgressPolicy:input_type -> ateapi.UpdateActorEgressPolicyRequest - 78, // 160: ateapi.Control.DeleteActorEgressPolicy:input_type -> ateapi.DeleteActorEgressPolicyRequest - 80, // 161: ateapi.Control.MintActorJWT:input_type -> ateapi.MintActorJWTRequest - 82, // 162: ateapi.Control.MintActorCertificate:input_type -> ateapi.MintActorCertificateRequest - 86, // 163: ateapi.Control.CreateTag:input_type -> ateapi.CreateTagRequest - 79, // 164: ateapi.Control.GetTag:input_type -> ateapi.GetTagRequest - 84, // 165: ateapi.Control.ListTags:input_type -> ateapi.ListTagsRequest - 87, // 166: ateapi.Control.UpdateTag:input_type -> ateapi.UpdateTagRequest - 88, // 167: ateapi.Control.DeleteTag:input_type -> ateapi.DeleteTagRequest - 92, // 168: ateapi.Control.ListWorkers:input_type -> ateapi.ListWorkersRequest - 94, // 169: ateapi.Control.GetWorker:input_type -> ateapi.GetWorkerRequest - 95, // 170: ateapi.Control.CreateWorker:input_type -> ateapi.CreateWorkerRequest - 96, // 171: ateapi.Control.UpdateWorker:input_type -> ateapi.UpdateWorkerRequest - 97, // 172: ateapi.Control.DeleteWorker:input_type -> ateapi.DeleteWorkerRequest - 98, // 173: ateapi.Control.DrainWorker:input_type -> ateapi.DrainWorkerRequest - 90, // 174: ateapi.Control.ListWorkerActorAssignments:input_type -> ateapi.ListWorkerActorAssignmentsRequest - 99, // 175: ateapi.Control.ListActors:input_type -> ateapi.ListActorsRequest - 53, // 176: ateapi.Control.CreateAtespace:input_type -> ateapi.CreateAtespaceRequest - 54, // 177: ateapi.Control.GetAtespace:input_type -> ateapi.GetAtespaceRequest - 55, // 178: ateapi.Control.ListAtespaces:input_type -> ateapi.ListAtespacesRequest - 57, // 179: ateapi.Control.DeleteAtespace:input_type -> ateapi.DeleteAtespaceRequest - 58, // 180: ateapi.Control.CreateActorTemplate:input_type -> ateapi.CreateActorTemplateRequest - 59, // 181: ateapi.Control.GetActorTemplate:input_type -> ateapi.GetActorTemplateRequest - 60, // 182: ateapi.Control.ListActorTemplates:input_type -> ateapi.ListActorTemplatesRequest - 62, // 183: ateapi.Control.DeleteActorTemplate:input_type -> ateapi.DeleteActorTemplateRequest - 114, // 184: ateapi.Control.GetGlobalAccessPolicy:input_type -> ateapi.GetGlobalAccessPolicyRequest - 115, // 185: ateapi.Control.CreateGlobalAccessPolicy:input_type -> ateapi.CreateGlobalAccessPolicyRequest - 116, // 186: ateapi.Control.UpdateGlobalAccessPolicy:input_type -> ateapi.UpdateGlobalAccessPolicyRequest - 117, // 187: ateapi.Control.GetAtespaceAccessPolicy:input_type -> ateapi.GetAtespaceAccessPolicyRequest - 118, // 188: ateapi.Control.CreateAtespaceAccessPolicy:input_type -> ateapi.CreateAtespaceAccessPolicyRequest - 119, // 189: ateapi.Control.UpdateAtespaceAccessPolicy:input_type -> ateapi.UpdateAtespaceAccessPolicyRequest - 120, // 190: ateapi.Control.DeleteAtespaceAccessPolicy:input_type -> ateapi.DeleteAtespaceAccessPolicyRequest - 106, // 191: ateapi.WorkerService.RegisterWorker:input_type -> ateapi.RegisterWorkerRequest - 108, // 192: ateapi.WorkerService.MintAteomActorCertificate:input_type -> ateapi.MintAteomActorCertificateRequest - 110, // 193: ateapi.WorkerService.RequestActorSuspend:input_type -> ateapi.RequestActorSuspendRequest - 12, // 194: ateapi.Control.GetActor:output_type -> ateapi.Actor - 12, // 195: ateapi.Control.CreateActor:output_type -> ateapi.Actor - 12, // 196: ateapi.Control.UpdateActor:output_type -> ateapi.Actor - 67, // 197: ateapi.Control.SuspendActor:output_type -> ateapi.SuspendActorResponse - 69, // 198: ateapi.Control.PauseActor:output_type -> ateapi.PauseActorResponse - 71, // 199: ateapi.Control.ResumeActor:output_type -> ateapi.ResumeActorResponse - 73, // 200: ateapi.Control.RevertActor:output_type -> ateapi.RevertActorResponse - 12, // 201: ateapi.Control.DeleteActor:output_type -> ateapi.Actor - 13, // 202: ateapi.Control.GetActorEgressPolicy:output_type -> ateapi.EgressPolicy - 13, // 203: ateapi.Control.CreateActorEgressPolicy:output_type -> ateapi.EgressPolicy - 13, // 204: ateapi.Control.UpdateActorEgressPolicy:output_type -> ateapi.EgressPolicy - 13, // 205: ateapi.Control.DeleteActorEgressPolicy:output_type -> ateapi.EgressPolicy - 81, // 206: ateapi.Control.MintActorJWT:output_type -> ateapi.MintActorJWTResponse - 83, // 207: ateapi.Control.MintActorCertificate:output_type -> ateapi.MintActorCertificateResponse - 27, // 208: ateapi.Control.CreateTag:output_type -> ateapi.Tag - 27, // 209: ateapi.Control.GetTag:output_type -> ateapi.Tag - 85, // 210: ateapi.Control.ListTags:output_type -> ateapi.ListTagsResponse - 27, // 211: ateapi.Control.UpdateTag:output_type -> ateapi.Tag - 27, // 212: ateapi.Control.DeleteTag:output_type -> ateapi.Tag - 93, // 213: ateapi.Control.ListWorkers:output_type -> ateapi.ListWorkersResponse - 101, // 214: ateapi.Control.GetWorker:output_type -> ateapi.Worker - 101, // 215: ateapi.Control.CreateWorker:output_type -> ateapi.Worker - 101, // 216: ateapi.Control.UpdateWorker:output_type -> ateapi.Worker - 101, // 217: ateapi.Control.DeleteWorker:output_type -> ateapi.Worker - 101, // 218: ateapi.Control.DrainWorker:output_type -> ateapi.Worker - 91, // 219: ateapi.Control.ListWorkerActorAssignments:output_type -> ateapi.ListWorkerActorAssignmentsResponse - 100, // 220: ateapi.Control.ListActors:output_type -> ateapi.ListActorsResponse - 28, // 221: ateapi.Control.CreateAtespace:output_type -> ateapi.Atespace - 28, // 222: ateapi.Control.GetAtespace:output_type -> ateapi.Atespace - 56, // 223: ateapi.Control.ListAtespaces:output_type -> ateapi.ListAtespacesResponse - 28, // 224: ateapi.Control.DeleteAtespace:output_type -> ateapi.Atespace - 30, // 225: ateapi.Control.CreateActorTemplate:output_type -> ateapi.ActorTemplate - 30, // 226: ateapi.Control.GetActorTemplate:output_type -> ateapi.ActorTemplate - 61, // 227: ateapi.Control.ListActorTemplates:output_type -> ateapi.ListActorTemplatesResponse - 30, // 228: ateapi.Control.DeleteActorTemplate:output_type -> ateapi.ActorTemplate - 112, // 229: ateapi.Control.GetGlobalAccessPolicy:output_type -> ateapi.AccessPolicy - 112, // 230: ateapi.Control.CreateGlobalAccessPolicy:output_type -> ateapi.AccessPolicy - 112, // 231: ateapi.Control.UpdateGlobalAccessPolicy:output_type -> ateapi.AccessPolicy - 112, // 232: ateapi.Control.GetAtespaceAccessPolicy:output_type -> ateapi.AccessPolicy - 112, // 233: ateapi.Control.CreateAtespaceAccessPolicy:output_type -> ateapi.AccessPolicy - 112, // 234: ateapi.Control.UpdateAtespaceAccessPolicy:output_type -> ateapi.AccessPolicy - 112, // 235: ateapi.Control.DeleteAtespaceAccessPolicy:output_type -> ateapi.AccessPolicy - 107, // 236: ateapi.WorkerService.RegisterWorker:output_type -> ateapi.RegisterWorkerResponse - 109, // 237: ateapi.WorkerService.MintAteomActorCertificate:output_type -> ateapi.MintAteomActorCertificateResponse - 111, // 238: ateapi.WorkerService.RequestActorSuspend:output_type -> ateapi.RequestActorSuspendResponse - 194, // [194:239] is the sub-list for method output_type - 149, // [149:194] is the sub-list for method input_type - 149, // [149:149] is the sub-list for extension type_name - 149, // [149:149] is the sub-list for extension extendee - 0, // [0:149] is the sub-list for field type_name + 1, // 0: ateapi.SnapshotStorage.durability:type_name -> ateapi.SnapshotDurability + 2, // 1: ateapi.SnapshotStorage.status:type_name -> ateapi.SnapshotStorageStatus + 3, // 2: ateapi.SnapshotStorage.fidelity:type_name -> ateapi.SnapshotFidelity + 11, // 3: ateapi.SnapshotStorage.local:type_name -> ateapi.LocalSnapshot + 10, // 4: ateapi.SnapshotStorage.object:type_name -> ateapi.ObjectSnapshot + 0, // 5: ateapi.Snapshot.owner:type_name -> ateapi.SnapshotOwner + 12, // 6: ateapi.Snapshot.storage:type_name -> ateapi.SnapshotStorage + 126, // 7: ateapi.Selector.match_labels:type_name -> ateapi.Selector.MatchLabelsEntry + 130, // 8: ateapi.ResourceMetadata.create_time:type_name -> google.protobuf.Timestamp + 130, // 9: ateapi.ResourceMetadata.update_time:type_name -> google.protobuf.Timestamp + 9, // 10: ateapi.ExternalVolume.status:type_name -> ateapi.ExternalVolume.Status + 127, // 11: ateapi.ExternalVolume.volume_context:type_name -> ateapi.ExternalVolume.VolumeContextEntry + 15, // 12: ateapi.Actor.metadata:type_name -> ateapi.ResourceMetadata + 34, // 13: ateapi.Actor.actor_template:type_name -> ateapi.ObjectRef + 14, // 14: ateapi.Actor.worker_selector:type_name -> ateapi.Selector + 34, // 15: ateapi.Actor.source_tag:type_name -> ateapi.ObjectRef + 28, // 16: ateapi.Actor.status:type_name -> ateapi.ActorStatus + 15, // 17: ateapi.EgressPolicy.metadata:type_name -> ateapi.ResourceMetadata + 19, // 18: ateapi.EgressPolicy.rules:type_name -> ateapi.EgressRule + 20, // 19: ateapi.EgressRule.http:type_name -> ateapi.HTTPRule + 21, // 20: ateapi.EgressRule.https:type_name -> ateapi.HTTPSRule + 22, // 21: ateapi.EgressRule.tls_passthrough:type_name -> ateapi.TLSPassthroughRule + 23, // 22: ateapi.HTTPRule.ports:type_name -> ateapi.Ports + 25, // 23: ateapi.HTTPRule.effects:type_name -> ateapi.HttpRuleEffects + 23, // 24: ateapi.HTTPSRule.ports:type_name -> ateapi.Ports + 25, // 25: ateapi.HTTPSRule.effects:type_name -> ateapi.HttpRuleEffects + 23, // 26: ateapi.TLSPassthroughRule.ports:type_name -> ateapi.Ports + 24, // 27: ateapi.Ports.all:type_name -> ateapi.AllPorts + 26, // 28: ateapi.HttpRuleEffects.replace_headers:type_name -> ateapi.CredentialHeader + 27, // 29: ateapi.CredentialHeader.actor_jwt:type_name -> ateapi.ActorJWTSource + 5, // 30: ateapi.ActorStatus.state:type_name -> ateapi.ActorState + 30, // 31: ateapi.ActorStatus.worker_assignment:type_name -> ateapi.WorkerAssignment + 16, // 32: ateapi.ActorStatus.actor_volumes:type_name -> ateapi.ExternalVolume + 29, // 33: ateapi.ActorStatus.crash:type_name -> ateapi.ActorCrash + 13, // 34: ateapi.ActorStatus.snapshots:type_name -> ateapi.Snapshot + 130, // 35: ateapi.ActorCrash.crash_time:type_name -> google.protobuf.Timestamp + 34, // 36: ateapi.WorkerAssignment.worker:type_name -> ateapi.ObjectRef + 13, // 37: ateapi.TagStatus.snapshot:type_name -> ateapi.Snapshot + 15, // 38: ateapi.Tag.metadata:type_name -> ateapi.ResourceMetadata + 31, // 39: ateapi.Tag.status:type_name -> ateapi.TagStatus + 4, // 40: ateapi.Tag.scope:type_name -> ateapi.TagScope + 34, // 41: ateapi.Tag.source_actor:type_name -> ateapi.ObjectRef + 15, // 42: ateapi.Atespace.metadata:type_name -> ateapi.ResourceMetadata + 15, // 43: ateapi.ActorTemplate.metadata:type_name -> ateapi.ResourceMetadata + 14, // 44: ateapi.ActorTemplate.worker_selector:type_name -> ateapi.Selector + 42, // 45: ateapi.ActorTemplate.containers:type_name -> ateapi.Container + 48, // 46: ateapi.ActorTemplate.volumes:type_name -> ateapi.Volume + 41, // 47: ateapi.ActorTemplate.snapshot_config:type_name -> ateapi.SnapshotConfig + 40, // 48: ateapi.ActorTemplate.sandbox_config:type_name -> ateapi.SandboxConfig + 36, // 49: ateapi.ActorTemplate.resources:type_name -> ateapi.Resources + 39, // 50: ateapi.ActorTemplate.status:type_name -> ateapi.ActorTemplateStatus + 37, // 51: ateapi.Resources.limits:type_name -> ateapi.Limits + 34, // 52: ateapi.GoldenSnapshotStatus.golden_tag:type_name -> ateapi.ObjectRef + 130, // 53: ateapi.GoldenSnapshotStatus.take_golden_snapshot_at:type_name -> google.protobuf.Timestamp + 38, // 54: ateapi.ActorTemplateStatus.golden_snapshot_status:type_name -> ateapi.GoldenSnapshotStatus + 6, // 55: ateapi.SandboxConfig.sandbox_class:type_name -> ateapi.SandboxClass + 3, // 56: ateapi.SnapshotConfig.preferred_fidelity:type_name -> ateapi.SnapshotFidelity + 45, // 57: ateapi.Container.env:type_name -> ateapi.EnvVar + 46, // 58: ateapi.Container.wakeup_probe:type_name -> ateapi.ContainerWakeupProbe + 57, // 59: ateapi.Container.volume_mounts:type_name -> ateapi.VolumeMount + 43, // 60: ateapi.Container.security_context:type_name -> ateapi.SecurityContext + 36, // 61: ateapi.Container.resources:type_name -> ateapi.Resources + 44, // 62: ateapi.SecurityContext.capabilities:type_name -> ateapi.Capabilities + 47, // 63: ateapi.ContainerWakeupProbe.http_get:type_name -> ateapi.HTTPGetAction + 50, // 64: ateapi.Volume.durable_dir:type_name -> ateapi.DurableDirVolumeSource + 51, // 65: ateapi.Volume.external_volume_template:type_name -> ateapi.ExternalVolumeTemplate + 52, // 66: ateapi.Volume.system_info:type_name -> ateapi.SystemInfoVolumeSource + 49, // 67: ateapi.Volume.image:type_name -> ateapi.ImageVolumeSource + 53, // 68: ateapi.SystemInfoVolumeSource.data_sources:type_name -> ateapi.SystemInfoDataSource + 54, // 69: ateapi.SystemInfoDataSource.actor_metadata:type_name -> ateapi.ActorMetadataDataSource + 56, // 70: ateapi.SystemInfoDataSource.trust_bundle:type_name -> ateapi.TrustBundleDataSource + 55, // 71: ateapi.ActorMetadataDataSource.items:type_name -> ateapi.ActorMetadataItem + 7, // 72: ateapi.ActorMetadataItem.field:type_name -> ateapi.ActorMetadataField + 33, // 73: ateapi.CreateAtespaceRequest.atespace:type_name -> ateapi.Atespace + 34, // 74: ateapi.GetAtespaceRequest.atespace:type_name -> ateapi.ObjectRef + 33, // 75: ateapi.ListAtespacesResponse.atespaces:type_name -> ateapi.Atespace + 34, // 76: ateapi.DeleteAtespaceRequest.atespace:type_name -> ateapi.ObjectRef + 94, // 77: ateapi.DeleteAtespaceRequest.options:type_name -> ateapi.DeleteOptions + 35, // 78: ateapi.CreateActorTemplateRequest.actor_template:type_name -> ateapi.ActorTemplate + 34, // 79: ateapi.GetActorTemplateRequest.actor_template:type_name -> ateapi.ObjectRef + 35, // 80: ateapi.ListActorTemplatesResponse.actor_templates:type_name -> ateapi.ActorTemplate + 34, // 81: ateapi.DeleteActorTemplateRequest.actor_template:type_name -> ateapi.ObjectRef + 94, // 82: ateapi.DeleteActorTemplateRequest.options:type_name -> ateapi.DeleteOptions + 34, // 83: ateapi.GetActorRequest.actor:type_name -> ateapi.ObjectRef + 17, // 84: ateapi.CreateActorRequest.actor:type_name -> ateapi.Actor + 17, // 85: ateapi.UpdateActorRequest.actor:type_name -> ateapi.Actor + 34, // 86: ateapi.SuspendActorRequest.actor:type_name -> ateapi.ObjectRef + 17, // 87: ateapi.SuspendActorResponse.actor:type_name -> ateapi.Actor + 34, // 88: ateapi.PauseActorRequest.actor:type_name -> ateapi.ObjectRef + 17, // 89: ateapi.PauseActorResponse.actor:type_name -> ateapi.Actor + 34, // 90: ateapi.ResumeActorRequest.actor:type_name -> ateapi.ObjectRef + 17, // 91: ateapi.ResumeActorResponse.actor:type_name -> ateapi.Actor + 34, // 92: ateapi.RevertActorRequest.actor:type_name -> ateapi.ObjectRef + 17, // 93: ateapi.RevertActorResponse.actor:type_name -> ateapi.Actor + 34, // 94: ateapi.DeleteActorRequest.actor:type_name -> ateapi.ObjectRef + 94, // 95: ateapi.DeleteActorRequest.options:type_name -> ateapi.DeleteOptions + 34, // 96: ateapi.GetActorEgressPolicyRequest.actor:type_name -> ateapi.ObjectRef + 34, // 97: ateapi.CreateActorEgressPolicyRequest.actor:type_name -> ateapi.ObjectRef + 18, // 98: ateapi.CreateActorEgressPolicyRequest.egress_policy:type_name -> ateapi.EgressPolicy + 34, // 99: ateapi.UpdateActorEgressPolicyRequest.actor:type_name -> ateapi.ObjectRef + 18, // 100: ateapi.UpdateActorEgressPolicyRequest.egress_policy:type_name -> ateapi.EgressPolicy + 34, // 101: ateapi.DeleteActorEgressPolicyRequest.actor:type_name -> ateapi.ObjectRef + 94, // 102: ateapi.DeleteActorEgressPolicyRequest.options:type_name -> ateapi.DeleteOptions + 34, // 103: ateapi.GetTagRequest.tag:type_name -> ateapi.ObjectRef + 34, // 104: ateapi.MintActorJWTRequest.actor:type_name -> ateapi.ObjectRef + 130, // 105: ateapi.MintActorJWTResponse.expires_at:type_name -> google.protobuf.Timestamp + 34, // 106: ateapi.MintActorCertificateRequest.actor:type_name -> ateapi.ObjectRef + 32, // 107: ateapi.ListTagsResponse.tags:type_name -> ateapi.Tag + 32, // 108: ateapi.CreateTagRequest.tag:type_name -> ateapi.Tag + 32, // 109: ateapi.UpdateTagRequest.tag:type_name -> ateapi.Tag + 34, // 110: ateapi.DeleteTagRequest.tag:type_name -> ateapi.ObjectRef + 94, // 111: ateapi.DeleteTagRequest.options:type_name -> ateapi.DeleteOptions + 34, // 112: ateapi.ListWorkerActorAssignmentsRequest.worker:type_name -> ateapi.ObjectRef + 110, // 113: ateapi.ListWorkerActorAssignmentsResponse.actor_assignments:type_name -> ateapi.ActorAssignment + 106, // 114: ateapi.ListWorkersResponse.workers:type_name -> ateapi.Worker + 34, // 115: ateapi.GetWorkerRequest.worker:type_name -> ateapi.ObjectRef + 106, // 116: ateapi.CreateWorkerRequest.worker:type_name -> ateapi.Worker + 106, // 117: ateapi.UpdateWorkerRequest.worker:type_name -> ateapi.Worker + 34, // 118: ateapi.DeleteWorkerRequest.worker:type_name -> ateapi.ObjectRef + 94, // 119: ateapi.DeleteWorkerRequest.options:type_name -> ateapi.DeleteOptions + 34, // 120: ateapi.DrainWorkerRequest.worker:type_name -> ateapi.ObjectRef + 17, // 121: ateapi.ListActorsResponse.actors:type_name -> ateapi.Actor + 15, // 122: ateapi.Worker.metadata:type_name -> ateapi.ResourceMetadata + 128, // 123: ateapi.Worker.labels:type_name -> ateapi.Worker.LabelsEntry + 107, // 124: ateapi.Worker.status:type_name -> ateapi.WorkerStatus + 8, // 125: ateapi.WorkerStatus.state:type_name -> ateapi.WorkerState + 109, // 126: ateapi.WorkerStatus.capacity:type_name -> ateapi.WorkerResources + 109, // 127: ateapi.WorkerStatus.allocated:type_name -> ateapi.WorkerResources + 108, // 128: ateapi.WorkerStatus.hardware:type_name -> ateapi.HardwareIdentity + 129, // 129: ateapi.HardwareIdentity.attributes:type_name -> ateapi.HardwareIdentity.AttributesEntry + 36, // 130: ateapi.WorkerResources.resources:type_name -> ateapi.Resources + 15, // 131: ateapi.ActorAssignment.metadata:type_name -> ateapi.ResourceMetadata + 34, // 132: ateapi.ActorAssignment.actor:type_name -> ateapi.ObjectRef + 36, // 133: ateapi.ActorAssignment.resources:type_name -> ateapi.Resources + 34, // 134: ateapi.RegisterWorkerRequest.worker:type_name -> ateapi.ObjectRef + 109, // 135: ateapi.RegisterWorkerRequest.capacity:type_name -> ateapi.WorkerResources + 108, // 136: ateapi.RegisterWorkerRequest.hardware:type_name -> ateapi.HardwareIdentity + 106, // 137: ateapi.RegisterWorkerResponse.worker:type_name -> ateapi.Worker + 34, // 138: ateapi.MintAteomActorCertificateRequest.actor:type_name -> ateapi.ObjectRef + 34, // 139: ateapi.RequestActorSuspendRequest.worker:type_name -> ateapi.ObjectRef + 34, // 140: ateapi.RequestActorSuspendRequest.actor:type_name -> ateapi.ObjectRef + 17, // 141: ateapi.RequestActorSuspendResponse.actor:type_name -> ateapi.Actor + 15, // 142: ateapi.AccessPolicy.metadata:type_name -> ateapi.ResourceMetadata + 118, // 143: ateapi.AccessPolicy.bindings:type_name -> ateapi.Binding + 117, // 144: ateapi.CreateGlobalAccessPolicyRequest.access_policy:type_name -> ateapi.AccessPolicy + 117, // 145: ateapi.UpdateGlobalAccessPolicyRequest.access_policy:type_name -> ateapi.AccessPolicy + 34, // 146: ateapi.GetAtespaceAccessPolicyRequest.atespace:type_name -> ateapi.ObjectRef + 34, // 147: ateapi.CreateAtespaceAccessPolicyRequest.atespace:type_name -> ateapi.ObjectRef + 117, // 148: ateapi.CreateAtespaceAccessPolicyRequest.access_policy:type_name -> ateapi.AccessPolicy + 34, // 149: ateapi.UpdateAtespaceAccessPolicyRequest.atespace:type_name -> ateapi.ObjectRef + 117, // 150: ateapi.UpdateAtespaceAccessPolicyRequest.access_policy:type_name -> ateapi.AccessPolicy + 34, // 151: ateapi.DeleteAtespaceAccessPolicyRequest.atespace:type_name -> ateapi.ObjectRef + 94, // 152: ateapi.DeleteAtespaceAccessPolicyRequest.options:type_name -> ateapi.DeleteOptions + 68, // 153: ateapi.Control.GetActor:input_type -> ateapi.GetActorRequest + 69, // 154: ateapi.Control.CreateActor:input_type -> ateapi.CreateActorRequest + 70, // 155: ateapi.Control.UpdateActor:input_type -> ateapi.UpdateActorRequest + 71, // 156: ateapi.Control.SuspendActor:input_type -> ateapi.SuspendActorRequest + 73, // 157: ateapi.Control.PauseActor:input_type -> ateapi.PauseActorRequest + 75, // 158: ateapi.Control.ResumeActor:input_type -> ateapi.ResumeActorRequest + 77, // 159: ateapi.Control.RevertActor:input_type -> ateapi.RevertActorRequest + 79, // 160: ateapi.Control.DeleteActor:input_type -> ateapi.DeleteActorRequest + 80, // 161: ateapi.Control.GetActorEgressPolicy:input_type -> ateapi.GetActorEgressPolicyRequest + 81, // 162: ateapi.Control.CreateActorEgressPolicy:input_type -> ateapi.CreateActorEgressPolicyRequest + 82, // 163: ateapi.Control.UpdateActorEgressPolicy:input_type -> ateapi.UpdateActorEgressPolicyRequest + 83, // 164: ateapi.Control.DeleteActorEgressPolicy:input_type -> ateapi.DeleteActorEgressPolicyRequest + 85, // 165: ateapi.Control.MintActorJWT:input_type -> ateapi.MintActorJWTRequest + 87, // 166: ateapi.Control.MintActorCertificate:input_type -> ateapi.MintActorCertificateRequest + 91, // 167: ateapi.Control.CreateTag:input_type -> ateapi.CreateTagRequest + 84, // 168: ateapi.Control.GetTag:input_type -> ateapi.GetTagRequest + 89, // 169: ateapi.Control.ListTags:input_type -> ateapi.ListTagsRequest + 92, // 170: ateapi.Control.UpdateTag:input_type -> ateapi.UpdateTagRequest + 93, // 171: ateapi.Control.DeleteTag:input_type -> ateapi.DeleteTagRequest + 97, // 172: ateapi.Control.ListWorkers:input_type -> ateapi.ListWorkersRequest + 99, // 173: ateapi.Control.GetWorker:input_type -> ateapi.GetWorkerRequest + 100, // 174: ateapi.Control.CreateWorker:input_type -> ateapi.CreateWorkerRequest + 101, // 175: ateapi.Control.UpdateWorker:input_type -> ateapi.UpdateWorkerRequest + 102, // 176: ateapi.Control.DeleteWorker:input_type -> ateapi.DeleteWorkerRequest + 103, // 177: ateapi.Control.DrainWorker:input_type -> ateapi.DrainWorkerRequest + 95, // 178: ateapi.Control.ListWorkerActorAssignments:input_type -> ateapi.ListWorkerActorAssignmentsRequest + 104, // 179: ateapi.Control.ListActors:input_type -> ateapi.ListActorsRequest + 58, // 180: ateapi.Control.CreateAtespace:input_type -> ateapi.CreateAtespaceRequest + 59, // 181: ateapi.Control.GetAtespace:input_type -> ateapi.GetAtespaceRequest + 60, // 182: ateapi.Control.ListAtespaces:input_type -> ateapi.ListAtespacesRequest + 62, // 183: ateapi.Control.DeleteAtespace:input_type -> ateapi.DeleteAtespaceRequest + 63, // 184: ateapi.Control.CreateActorTemplate:input_type -> ateapi.CreateActorTemplateRequest + 64, // 185: ateapi.Control.GetActorTemplate:input_type -> ateapi.GetActorTemplateRequest + 65, // 186: ateapi.Control.ListActorTemplates:input_type -> ateapi.ListActorTemplatesRequest + 67, // 187: ateapi.Control.DeleteActorTemplate:input_type -> ateapi.DeleteActorTemplateRequest + 119, // 188: ateapi.Control.GetGlobalAccessPolicy:input_type -> ateapi.GetGlobalAccessPolicyRequest + 120, // 189: ateapi.Control.CreateGlobalAccessPolicy:input_type -> ateapi.CreateGlobalAccessPolicyRequest + 121, // 190: ateapi.Control.UpdateGlobalAccessPolicy:input_type -> ateapi.UpdateGlobalAccessPolicyRequest + 122, // 191: ateapi.Control.GetAtespaceAccessPolicy:input_type -> ateapi.GetAtespaceAccessPolicyRequest + 123, // 192: ateapi.Control.CreateAtespaceAccessPolicy:input_type -> ateapi.CreateAtespaceAccessPolicyRequest + 124, // 193: ateapi.Control.UpdateAtespaceAccessPolicy:input_type -> ateapi.UpdateAtespaceAccessPolicyRequest + 125, // 194: ateapi.Control.DeleteAtespaceAccessPolicy:input_type -> ateapi.DeleteAtespaceAccessPolicyRequest + 111, // 195: ateapi.WorkerService.RegisterWorker:input_type -> ateapi.RegisterWorkerRequest + 113, // 196: ateapi.WorkerService.MintAteomActorCertificate:input_type -> ateapi.MintAteomActorCertificateRequest + 115, // 197: ateapi.WorkerService.RequestActorSuspend:input_type -> ateapi.RequestActorSuspendRequest + 17, // 198: ateapi.Control.GetActor:output_type -> ateapi.Actor + 17, // 199: ateapi.Control.CreateActor:output_type -> ateapi.Actor + 17, // 200: ateapi.Control.UpdateActor:output_type -> ateapi.Actor + 72, // 201: ateapi.Control.SuspendActor:output_type -> ateapi.SuspendActorResponse + 74, // 202: ateapi.Control.PauseActor:output_type -> ateapi.PauseActorResponse + 76, // 203: ateapi.Control.ResumeActor:output_type -> ateapi.ResumeActorResponse + 78, // 204: ateapi.Control.RevertActor:output_type -> ateapi.RevertActorResponse + 17, // 205: ateapi.Control.DeleteActor:output_type -> ateapi.Actor + 18, // 206: ateapi.Control.GetActorEgressPolicy:output_type -> ateapi.EgressPolicy + 18, // 207: ateapi.Control.CreateActorEgressPolicy:output_type -> ateapi.EgressPolicy + 18, // 208: ateapi.Control.UpdateActorEgressPolicy:output_type -> ateapi.EgressPolicy + 18, // 209: ateapi.Control.DeleteActorEgressPolicy:output_type -> ateapi.EgressPolicy + 86, // 210: ateapi.Control.MintActorJWT:output_type -> ateapi.MintActorJWTResponse + 88, // 211: ateapi.Control.MintActorCertificate:output_type -> ateapi.MintActorCertificateResponse + 32, // 212: ateapi.Control.CreateTag:output_type -> ateapi.Tag + 32, // 213: ateapi.Control.GetTag:output_type -> ateapi.Tag + 90, // 214: ateapi.Control.ListTags:output_type -> ateapi.ListTagsResponse + 32, // 215: ateapi.Control.UpdateTag:output_type -> ateapi.Tag + 32, // 216: ateapi.Control.DeleteTag:output_type -> ateapi.Tag + 98, // 217: ateapi.Control.ListWorkers:output_type -> ateapi.ListWorkersResponse + 106, // 218: ateapi.Control.GetWorker:output_type -> ateapi.Worker + 106, // 219: ateapi.Control.CreateWorker:output_type -> ateapi.Worker + 106, // 220: ateapi.Control.UpdateWorker:output_type -> ateapi.Worker + 106, // 221: ateapi.Control.DeleteWorker:output_type -> ateapi.Worker + 106, // 222: ateapi.Control.DrainWorker:output_type -> ateapi.Worker + 96, // 223: ateapi.Control.ListWorkerActorAssignments:output_type -> ateapi.ListWorkerActorAssignmentsResponse + 105, // 224: ateapi.Control.ListActors:output_type -> ateapi.ListActorsResponse + 33, // 225: ateapi.Control.CreateAtespace:output_type -> ateapi.Atespace + 33, // 226: ateapi.Control.GetAtespace:output_type -> ateapi.Atespace + 61, // 227: ateapi.Control.ListAtespaces:output_type -> ateapi.ListAtespacesResponse + 33, // 228: ateapi.Control.DeleteAtespace:output_type -> ateapi.Atespace + 35, // 229: ateapi.Control.CreateActorTemplate:output_type -> ateapi.ActorTemplate + 35, // 230: ateapi.Control.GetActorTemplate:output_type -> ateapi.ActorTemplate + 66, // 231: ateapi.Control.ListActorTemplates:output_type -> ateapi.ListActorTemplatesResponse + 35, // 232: ateapi.Control.DeleteActorTemplate:output_type -> ateapi.ActorTemplate + 117, // 233: ateapi.Control.GetGlobalAccessPolicy:output_type -> ateapi.AccessPolicy + 117, // 234: ateapi.Control.CreateGlobalAccessPolicy:output_type -> ateapi.AccessPolicy + 117, // 235: ateapi.Control.UpdateGlobalAccessPolicy:output_type -> ateapi.AccessPolicy + 117, // 236: ateapi.Control.GetAtespaceAccessPolicy:output_type -> ateapi.AccessPolicy + 117, // 237: ateapi.Control.CreateAtespaceAccessPolicy:output_type -> ateapi.AccessPolicy + 117, // 238: ateapi.Control.UpdateAtespaceAccessPolicy:output_type -> ateapi.AccessPolicy + 117, // 239: ateapi.Control.DeleteAtespaceAccessPolicy:output_type -> ateapi.AccessPolicy + 112, // 240: ateapi.WorkerService.RegisterWorker:output_type -> ateapi.RegisterWorkerResponse + 114, // 241: ateapi.WorkerService.MintAteomActorCertificate:output_type -> ateapi.MintAteomActorCertificateResponse + 116, // 242: ateapi.WorkerService.RequestActorSuspend:output_type -> ateapi.RequestActorSuspendResponse + 198, // [198:243] is the sub-list for method output_type + 153, // [153:198] is the sub-list for method input_type + 153, // [153:153] is the sub-list for extension type_name + 153, // [153:153] is the sub-list for extension extendee + 0, // [0:153] is the sub-list for field type_name } func init() { file_ateapi_proto_init() } @@ -8937,8 +9250,8 @@ func file_ateapi_proto_init() { File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_ateapi_proto_rawDesc), len(file_ateapi_proto_rawDesc)), - NumEnums: 7, - NumMessages: 118, + NumEnums: 10, + NumMessages: 120, NumExtensions: 0, NumServices: 2, }, diff --git a/pkg/proto/ateapipb/ateapi.proto b/pkg/proto/ateapipb/ateapi.proto index 4f51e88cfa..8e82a4e7ac 100644 --- a/pkg/proto/ateapipb/ateapi.proto +++ b/pkg/proto/ateapipb/ateapi.proto @@ -171,45 +171,130 @@ service Control { rpc DeleteAtespaceAccessPolicy(DeleteAtespaceAccessPolicyRequest) returns (AccessPolicy) {} } -// ExternalSnapshot addresses a snapshot held in object storage. -message ExternalSnapshot { +// ObjectSnapshot addresses a snapshot held in object storage. +message ObjectSnapshot { // snapshot_uri addresses the snapshot's prefix in object storage. // // +k8s:required // +k8s:maxLength=2048 # the template's storage_location bound plus the owner prefix and snapshot name string snapshot_uri = 1; +} + +// LocalSnapshot records information about a node-local snapshot. +message LocalSnapshot { + // The name of the local checkpoint on the node. Checkpoint names are + // server-generated UUIDs, but any resource name is valid here. + // + // TODO: consider removing this since it's the same as the snapshot_id + // + // +k8s:required + // +k8s:format=k8s-short-name + string snapshot_name = 1; +} + +// SnapshotOwner indicates the owner of a snapshot. +enum SnapshotOwner { + SNAPSHOT_OWNER_UNSPECIFIED = 0; + // Snapshot captured directly from this Actor. + SNAPSHOT_OWNER_ACTOR = 1; + // Snapshot owned by a Tag (also includes Golden Snapshot) + SNAPSHOT_OWNER_TAG = 2; + // Keep this in sync with Snapshot.owner's maximum. +} + +// SnapshotDurability indicates the durability level of the snapshot. +// TODO: potentially rename this to survivability level. Is it at the actor or snapshot level? +enum SnapshotDurability { + SNAPSHOT_DURABILITY_UNSPECIFIED = 0; + // Node-local checkpoint on the worker node + // Survives node reboot. + SNAPSHOT_DURABILITY_LOCAL = 1; + // Durable snapshot in external storage. + // Survives node crash and deletion. + SNAPSHOT_DURABILITY_DURABLE = 2; + // Keep this in sync with SnapshotStorage.durability's maximum. +} + +// SnapshotStorageStatus indicates the lifecycle stage of a snapshot storage entry. +enum SnapshotStorageStatus { + SNAPSHOT_STORAGE_STATUS_UNSPECIFIED = 0; + // Snapshot creation or promotion is in progress. + SNAPSHOT_STORAGE_STATUS_IN_PROGRESS = 1; + // Snapshot is complete and ready for use. + SNAPSHOT_STORAGE_STATUS_COMPLETED = 2; + // Keep this in sync with SnapshotStorage.status's maximum. +} + +// SnapshotStorage records the physical storage representation of a Snapshot. +message SnapshotStorage { + // durability is the durability tier of this storage entry. Only + // SNAPSHOT_DURABILITY_LOCAL and SNAPSHOT_DURABILITY_DURABLE are allowed. + // + // +k8s:required + // +k8s:minimum=1 + // +k8s:maximum=2 # keep this in sync with the SnapshotDurability enum + SnapshotDurability durability = 1; + + // status is the lifecycle status of this storage representation. + // + // +k8s:required + // +k8s:minimum=1 + // +k8s:maximum=2 # keep this in sync with the SnapshotStorageStatus enum + SnapshotStorageStatus status = 2; // fidelity is what the snapshot holds. // - // +k8s:optional + // +k8s:required // +k8s:minimum=1 // +k8s:maximum=3 # keep this in sync with the SnapshotFidelity enum - SnapshotFidelity fidelity = 2; + SnapshotFidelity fidelity = 3; - // UID of the ActorTemplate whose sandbox this snapshot's guest state was - // captured from. + // Exactly one of local or object must be set. + // + // local snapshot is stored on the node where the actor was suspended. // // +k8s:optional - // +k8s:format=k8s-uuid - string actor_template_uid = 3; -} + // +k8s:unionMember + LocalSnapshot local = 4; -// LocalSnapshot records information about a node-local snapshot. -message LocalSnapshot { - // The name of the local checkpoint on the node. Checkpoint names are - // server-generated UUIDs, but any resource name is valid here. + // object snapshot is stored in object storage. // // +k8s:optional - // +k8s:format=k8s-short-name - string snapshot_name = 1; + // +k8s:unionMember + ObjectSnapshot object = 5; +} - // Fidelity the pause checkpoint captured: the template's preferredFidelity - // at pause time. +// Snapshot represents one generation of an Actor's or Tag's snapshot state. +message Snapshot { + // generation is the monotonically increasing generation number of this + // snapshot, starting at 1. // - // +k8s:optional + // +k8s:required // +k8s:minimum=1 - // +k8s:maximum=3 # keep this in sync with the SnapshotFidelity enum - SnapshotFidelity fidelity = 2; + int32 generation = 1; + + // owner identifies the origin and ownership of this snapshot. + // + // +k8s:required + // +k8s:minimum=1 + // +k8s:maximum=2 # keep this in sync with the SnapshotOwner enum + SnapshotOwner owner = 2; + + // UID of the ActorTemplate whose sandbox this snapshot's guest state was + // captured from. + // + // +k8s:optional + // +k8s:format=k8s-uuid + string actor_template_uid = 3; + + // storage lists the storage representations of this snapshot by durability. + // Only SNAPSHOT_DURABILITY_LOCAL and SNAPSHOT_DURABILITY_DURABLE are allowed. + // + // +k8s:required + // +k8s:maxItems=2 + // +k8s:listType=map + // +k8s:listMapKey=durability + repeated SnapshotStorage storage = 4; } // SnapshotFidelity orders the state layers a snapshot can hold, lowest to @@ -716,25 +801,6 @@ message ActorStatus { // +k8s:update=NoModify # can be set and cleared, but not changed in place WorkerAssignment worker_assignment = 2; - // in_progress_snapshot_uri is the URI in object storage of the durable - // snapshot the Actor is currently taking. - // - // +k8s:optional - // +k8s:maxLength=2048 # the template's storage_location bound plus the owner prefix and snapshot name - string in_progress_snapshot_uri = 3; - - // external_snapshot is the Actor's current external snapshot. - // If the Actor was created from a Tag this is the tag's snapshot, borrowed - // until the Actor's first suspend writes one of its own. Otherwise it is - // unset until the Actor is first suspended. - // +k8s:optional - ExternalSnapshot external_snapshot = 4; - - // Node-local state used only while the Actor is paused. - // - // +k8s:optional - LocalSnapshot local_snapshot = 5; - // Volumes attached to the actor. These volumes only live as long as the actor. // They are deleted when the actor is deleted. Each template volume // appears at most once. @@ -745,13 +811,6 @@ message ActorStatus { // +k8s:listMapKey=volume_name repeated ExternalVolume actor_volumes = 7; - // The name of the in-progress node-local checkpoint. Like durable snapshot - // names, these are server-generated UUIDs. - // - // +k8s:optional - // +k8s:format=k8s-short-name - string in_progress_local_snapshot_name = 8; - // crash records why and when the Actor entered CRASHED. It is set with the // CRASHED state and cleared when a revert returns the Actor to SUSPENDED. // @@ -765,6 +824,26 @@ message ActorStatus { // +k8s:optional // +k8s:format=k8s-long-name string assigned_node = 10; + + // snapshots holds the Actor's active and in-progress snapshot generations. + // + // +k8s:optional + // +k8s:maxItems=16 + // +k8s:listType=atomic + // +k8s:unique=map + // +k8s:listMapKey=generation + repeated Snapshot snapshots = 11; + + // last_assigned_generation tracks the last snapshot generation assigned for + // this Actor during a Suspend or Pause request. + // Even during Revert, the generation is not decreased to keep track of snapshots + // that have become stale and need to be cleaned up. + // + // +k8s:optional + // +k8s:alpha(since: "0.0")=+k8s:monotonic # TODO: get rid of alpha prefix + // +k8s:minimum=0 + // +k8s:update=NoUnset + int32 last_assigned_generation = 12; } // ActorCrash describes the failure that moved an Actor to CRASHED. @@ -851,12 +930,13 @@ message WorkerAssignment { // The tag is fixed to one external snapshot once its creation completes, so // the rest is immutable provenance. message TagStatus { - // snapshot is the external snapshot this tag points at. Unset while the tag - // is still being created, and immutable once set: a tag never moves between - // snapshots. A tag is usable only once this is set. + // snapshot is the external snapshot this tag points at. Its storage entry is + // IN_PROGRESS while the tag is being created and transitions to COMPLETED + // once the copy finishes: a tag never moves between snapshots and is usable + // only once COMPLETED. // - // +k8s:optional - ExternalSnapshot snapshot = 1; + // +k8s:required + Snapshot snapshot = 1; // UID of the ActorTemplate the snapshot was taken under. Actors can only be // seeded from a tag under the same template. @@ -864,15 +944,6 @@ message TagStatus { // +k8s:required // +k8s:format=k8s-uuid string actor_template_uid = 2; - - // storage_location is the base object-storage URI for this tag's snapshot. - // Set by the server from the source actor's template when the tag is created - // and immutable thereafter. The full snapshot URI is available in - // snapshot.snapshot_uri once tag creation completes. - // - // +k8s:required - // +k8s:maxLength=1024 - string storage_location = 3; } // Tag is an Atespace-owned, stable name for one external snapshot. The From 0519dec6cdb573e49800282c31ddeef4026ee7c0 Mon Sep 17 00:00:00 2001 From: Lucky Abolorunke <62015433+Oneimu@users.noreply.github.com> Date: Fri, 9 Oct 2026 16:22:10 +0000 Subject: [PATCH 27/44] =?UTF-8?q?benchmarking/analysis:=20aggregate=20susp?= =?UTF-8?q?end/resume=20phase=20breakdown=20log=20r=E2=80=A6=20(#2009)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## benchmarking/analysis: aggregate suspend/resume phase breakdown log records ### Why #1941 makes atelet and ateom-microvm write one structured `Checkpoint timing breakdown` / `Restore timing breakdown` record per operation per layer, so that `SuspendActor` / `ResumeActor` latency can be attributed to the phases inside it. A log line is not a benchmark result, though: something has to get those records off the cluster and turn them into percentiles. Today nothing in `benchmarking/` reads node-side logs — the runner keeps its own `logs.txt`/`traces.txt` and the orchestrator tails the locust job. This PR adds that consumer, so a run can answer "where did the 6 s go?" from data rather than from one anecdote. ### What Everything lives under `benchmarking/analysis/`; no production code changes, and **no new log output** — this only reads what the binaries already write. - **`collect_logs.sh`** — runs `kubectl logs` for every atelet pod (`ate-system`) and every worker pod (`benchmark-workloads`) for the run's window and writes them to `--dest`, next to the locust artifacts. Two records per lifecycle operation is the entire volume it collects. - **`phase_report.py`** — parses the records from those dumps (tolerant of `kubectl` line prefixes and unrelated lines) and prints: - **Phase percentiles** per layer, operation, scope and phase (count, p50, p90, p95, max), with the atelet `ateom_checkpoint` / `ateom_restore` bucket split further by the ateom rows. - An **`unattributed`** row for the two checkpoint layers: the total minus what the logged phases account for, counting the three concurrent captures once. It is the time the instrumentation does not yet name. - **Waterfalls** of the slowest N operations, with the ateom record nested under the atelet phase it decomposes and the gap between the two (RPC and queueing between the layers). - Failed operations are excluded from the percentiles via the `error.type` marker on the record. - `--csv` writes `phase_percentiles.csv` for run-over-run comparison. - **`test_phase_report.py`** — 10 unittest tests covering parsing of both layers, nanosecond timestamps, prefixed lines, failure exclusion, the residual arithmetic and the waterfall join. - **`README.md`** — the record shapes, how to collect and aggregate, how to read each section and its concurrency caveats. `benchmarking/README.md` gains a short section pointing at it. - docs/dev/suspend-resume-phase-breakdown.md — an end-to-end runbook for a dev cluster: deploy the benchmark workloads on a micro-VM pool, drive glutton suspend/resume load with a sized working set, collect (kubectl or Cloud Logging), report, read the tables, and the failure modes that come up in practice. Linked from CONTRIBUTING.md beside the local micro-VM guide. ### Scope and follow-ups - *Standalone tool*. Nothing invokes it yet. benchmarking/automation/orchestrator.py deletes the workload and ate-system pods right after each test, so the collection has to run before that teardown. The follow-up is to call collect_logs.sh and phase_report.py from run_test() before the teardown and write the phase percentiles into the run's --dest as stats.jsonl rows (the suite's existing result format: timestamp, tag, test_name, metric, measurements), so they travel with the rest of the run's artifacts. **Follow-up (#2148):** the automated runs will consume this through the locust runner, which reads the pod logs itself before the orchestrator's teardown and appends the percentiles to stats.jsonl — that lands in a separate PR stacked on this one (branch phase-logs-runner; PR link to follow), so this PR stays the tool and its docs. --- CONTRIBUTING.md | 4 +- benchmarking/README.md | 17 + benchmarking/analysis/README.md | 114 ++++++ benchmarking/analysis/collect_logs.sh | 107 +++++ benchmarking/analysis/phase_report.py | 451 +++++++++++++++++++++ benchmarking/analysis/test_phase_report.py | 300 ++++++++++++++ docs/dev/suspend-resume-phase-breakdown.md | 167 ++++++++ 7 files changed, 1159 insertions(+), 1 deletion(-) create mode 100644 benchmarking/analysis/README.md create mode 100755 benchmarking/analysis/collect_logs.sh create mode 100755 benchmarking/analysis/phase_report.py create mode 100644 benchmarking/analysis/test_phase_report.py create mode 100644 docs/dev/suspend-resume-phase-breakdown.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7668a5d809..7cb826aa83 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -31,7 +31,9 @@ The [Quickstart (Development)](README.md#quickstart-development) in the README covers bringing up a local cluster with the default (gVisor) runtime. To run the microVM runtime locally — which needs `/dev/kvm`, or Lima nested virtualization on Apple Silicon — see -[docs/dev/microvm-local.md](docs/dev/microvm-local.md). +[docs/dev/microvm-local.md](docs/dev/microvm-local.md). To measure where a +suspend or resume spends its time on such a cluster, follow +[docs/dev/suspend-resume-phase-breakdown.md](docs/dev/suspend-resume-phase-breakdown.md). ## Contribution process diff --git a/benchmarking/README.md b/benchmarking/README.md index 5ebe072fb9..154aad7737 100644 --- a/benchmarking/README.md +++ b/benchmarking/README.md @@ -463,6 +463,16 @@ Neither the Kubernetes API nor Prometheus is required. If either is unreachable, or discovery was skipped, the affected fields are written as `null` and the run still succeeds. A `null` means the value was not measured. It never means zero. +## Suspend/resume phase breakdown + +`SuspendActor` and `ResumeActor` latency can be attributed to the phases +inside them (object-storage transfer, hypervisor snapshot/restore, rootfs +assembly, ...) from the structured log records atelet and ateom-microvm +emit. `analysis/collect_logs.sh` dumps the node logs of a run and +`analysis/phase_report.py` aggregates them into per-phase percentiles and +waterfalls of the slowest operations. See +[analysis/README.md](analysis/README.md). + ## Optional: Prometheus + Grafana Locust provides graphs, statistics, etc. via the UI. However, you @@ -497,3 +507,10 @@ repository root: ```bash python3 -m unittest discover -s benchmarking/locust/unit_tests ``` + +`analysis/test_phase_report.py` covers the phase-log report's parser and +aggregation: + +```bash +python3 -m unittest discover -s benchmarking/analysis +``` diff --git a/benchmarking/analysis/README.md b/benchmarking/analysis/README.md new file mode 100644 index 0000000000..957b0f9894 --- /dev/null +++ b/benchmarking/analysis/README.md @@ -0,0 +1,114 @@ +# Suspend/resume phase analysis + +This directory turns the node side's developer-facing timing logs into the +percentiles a benchmark run needs, without making the phases metric API. The +`ate.actor.{restore,checkpoint}.duration` histograms stop at +`ateom_restore` / `ateom_checkpoint` / `persist` / `download` on purpose: +phases inside those are implementation details of a runtime, and metrics are +a contract. The finer breakdown is emitted as structured log records instead, +and this is the consumer that aggregates them. + +Two joinable JSON log records feed it, each written by both layers: + +| record (`msg`) | emitter | keys | +|---|---|---| +| `Restore timing breakdown` | atelet and ateom-microvm | `ate.actor.restore.duration.` / `ateom.actor.restore.duration.` | +| `Checkpoint timing breakdown` | atelet and ateom-microvm | `ate.actor.checkpoint.duration.` / `ateom.actor.checkpoint.duration.` | + +Every record carries the full actor identity (`ate.actor.uid`, name, +atespace, template) and the snapshot scope, which the histograms are barred +from, so records join per actor and per operation. Durations are float +seconds, the histograms' unit. A phase that never ran is absent, not zero. +A failed atelet operation still writes its record, marked with `error.type` +(the gRPC code); the report excludes those from the percentiles. + +## Making a measurement + +1. Run a suspend/resume-heavy load (e.g. the `glutton` or `sweperf` user + class; see `../README.md`). +2. Dump the node logs for the run's window while the pods still exist. + `automation/orchestrator.py` deletes the workload and ate-system pods right + after each test, and does not call this script yet, so run it before that + teardown (or against a cluster you manage yourself): + + ```bash + ./collect_logs.sh --dest /tmp/run1 --since-time 2026-09-24T18:00:00Z + ``` + + The script sources `.ate-dev-env.sh` from the repository root when present + (set `NO_DEV_ENV` to skip) and honors `KUBECTL_CONTEXT`, like the `hack/` + scripts. `--since-time` (or a relative `--since 30m`) should cover the run + and nothing before it. `--namespace` (default `ate-system`) selects the + atelet pods and `--worker-namespace` (default `benchmark-workloads`) the + worker pods; the ateom records land in the worker pod's stdout. `kubectl logs` + returns only a container's current log file, so on a long or busy run + kubelet's rotation can drop earlier records — collect soon after the run. + A container that restarted mid-run is dumped twice (`.previous.log`). + + On GKE the same records are in Cloud Logging, which keeps them past + rotation and teardown; an export is accepted as input directly, instead of + the kubectl dumps: + + ```bash + gcloud logging read '(jsonPayload.msg="Checkpoint timing breakdown" OR jsonPayload.msg="Restore timing breakdown") AND resource.labels.cluster_name="" AND timestamp>="2026-09-24T18:00:00Z"' \ + --project --format json > /tmp/run1/export.json + ``` + +3. Aggregate, from the kubectl dumps: + + ```bash + python3 phase_report.py /tmp/run1/*.log --csv /tmp/run1/report + ``` + + or from the Cloud Logging export: + + ```bash + python3 phase_report.py /tmp/run1/export.json --csv /tmp/run1/report + ``` + + Use one source per run: the dumps and an export of the same pods hold the + same records, and passing both would count each twice. + +## Reading the report + +**Phase percentiles.** Per layer, operation, sandbox class, snapshot kind, +scope and phase: count, p50, p90, p95 and max. A `golden` restore downloads +the golden image and a `latest` one the actor's own, so they are separate +rows, as are gVisor and micro-VM checkpoints. The atelet rows split a checkpoint between +`sandbox_assets`, `ateom_checkpoint` and `persist`, and a restore between +`volume_mount`, `manifest_fetch`, `sandbox_assets`, `download`, `oci_unpack` +and `ateom_restore`. The ateom rows split the `ateom_*` phase further: +`prep` / `pause` / `snapshot` / `durable_dir` / `rootfs_upper` / `teardown` +for a checkpoint, `prep` / `bundles` / `upper_join` / `lowers` / `tap` / +`vmm_launch` / `vm_restore` / `resume` / `wakeup_probe` for a restore. + +Concurrency matters when reading them: the atelet restore phases overlap +(the download runs alongside the asset fetch and OCI unpack), the three +ateom checkpoint captures run concurrently on the paused guest (the paused +window costs their max), and the ateom restore phases are sequential. For +the two checkpoint layers the report derives an `unattributed` row: the total +minus what the logged phases account for, counting the concurrent captures +once. It is the time the instrumentation does not yet name. The ateom +records carry no snapshot kind of their own; a paired one takes its kind from +the atelet record, so both layers split into the same rows. + +**Waterfalls.** The slowest operations, with the ateom record of the same +actor nested under the atelet `ateom_*` phase and that phase's gap to the +ateom total (RPC and queueing between the layers). The ateom record is the +one written inside that phase's window (for a checkpoint, before `persist` +began), and each ateom record pairs with at most one operation, so an +operation whose own record is missing prints without one rather than with +another cycle's. Failed operations (`error.type` set) are left out of both +the percentiles and the waterfalls. Tail outliers that blow up in one phase +every time are systematic; different phases each time are environmental. + +`--csv` also writes `phase_percentiles.csv` for run-over-run comparison. + +The parser is deliberately tolerant: it scans any line for a JSON object and +matches on `msg`, so raw `kubectl logs` dumps (even with prefixes) work. + +## Tests + +```bash +python3 -m unittest discover -s benchmarking/analysis +``` diff --git a/benchmarking/analysis/collect_logs.sh b/benchmarking/analysis/collect_logs.sh new file mode 100755 index 0000000000..e0555ca15b --- /dev/null +++ b/benchmarking/analysis/collect_logs.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Dump the node-side logs a benchmark run needs for phase_report.py: every +# atelet pod (the atelet-side timing breakdowns) and every worker pod +# (ateom-microvm writes its records to the worker pod's stdout). Point +# --since-time (RFC 3339) or --since at the run's start so the report covers +# exactly one run. +# +# kubectl logs returns only a container's current log file: once kubelet +# rotates it under load, earlier records are gone, so collect soon after the +# run. A container that restarted mid-run is dumped twice, its previous log +# under .previous.log. On GKE, Cloud Logging keeps every record past +# rotation and teardown; phase_report.py reads `gcloud logging read +# --format json` output directly. +# +# Like the other hack scripts, this sources .ate-dev-env.sh from the repo root +# for the cluster settings unless NO_DEV_ENV is set, and respects +# KUBECTL_CONTEXT. +# +# Usage: collect_logs.sh --dest DIR [--since 30m | --since-time 2026-09-24T18:00:00Z] +# [--namespace ate-system] [--worker-namespace benchmark-workloads] + +set -uo pipefail + +ROOT="$(git rev-parse --show-toplevel 2>/dev/null || pwd)" +if [[ -r "${ROOT}/.ate-dev-env.sh" ]] && [[ -z "${NO_DEV_ENV:-}" ]]; then + # shellcheck source=/dev/null + source "${ROOT}/.ate-dev-env.sh" +fi +run_kubectl() { kubectl ${KUBECTL_CONTEXT:+--context=${KUBECTL_CONTEXT}} "$@"; } + +DEST="" +SINCE="1h" +SINCE_TIME="" +NAMESPACE="ate-system" +WORKER_NAMESPACE="benchmark-workloads" + +while [[ $# -gt 0 ]]; do + case "$1" in + --dest) DEST="$2"; shift 2 ;; + --since) SINCE="$2"; shift 2 ;; + --since-time) SINCE_TIME="$2"; shift 2 ;; + --namespace) NAMESPACE="$2"; shift 2 ;; + --worker-namespace) WORKER_NAMESPACE="$2"; shift 2 ;; + *) echo "unknown flag: $1" >&2; exit 2 ;; + esac +done + +if [[ -z "${DEST}" ]]; then + echo "usage: $0 --dest DIR [--since 30m | --since-time RFC3339] [--namespace ate-system] [--worker-namespace benchmark-workloads]" >&2 + exit 2 +fi +mkdir -p "${DEST}" + +if [[ -n "${SINCE_TIME}" ]]; then + WINDOW=(--since-time="${SINCE_TIME}") +else + WINDOW=(--since="${SINCE}") +fi + +# One pod that cannot be read (still starting, evicted) must not cost the +# others, so every kubectl call warns and moves on rather than aborting. +collect() { + local ns="$1" selector="$2" + local pods + if ! pods=$(run_kubectl get pods -n "${ns}" -l "${selector}" -o name); then + echo "warn: could not list pods in ${ns} (${selector}); skipping" >&2 + return 0 + fi + for pod in ${pods}; do + local name="${pod#pod/}" + echo "collecting ${ns}/${name} (${WINDOW[*]})" + run_kubectl logs -n "${ns}" "${name}" "${WINDOW[@]}" --timestamps=false \ + > "${DEST}/${ns}-${name}.log" \ + || { echo "warn: kubectl logs ${ns}/${name} failed; skipping" >&2; rm -f "${DEST}/${ns}-${name}.log"; } + # --previous is only valid after a restart; kubectl rejects it otherwise. + local restarts + restarts=$(run_kubectl get pod -n "${ns}" "${name}" \ + -o jsonpath='{.status.containerStatuses[0].restartCount}' 2>/dev/null || echo 0) + if [[ "${restarts:-0}" -gt 0 ]]; then + echo "collecting ${ns}/${name} previous container (${restarts} restarts)" + run_kubectl logs -n "${ns}" "${name}" --previous "${WINDOW[@]}" --timestamps=false \ + > "${DEST}/${ns}-${name}.previous.log" \ + || { echo "warn: kubectl logs --previous ${ns}/${name} failed; skipping" >&2; rm -f "${DEST}/${ns}-${name}.previous.log"; } + fi + done +} + +collect "${NAMESPACE}" "app=atelet" +# Worker pods carry the pool label whatever the pool's name is. +collect "${WORKER_NAMESPACE}" "ate.dev/worker-pool" + +echo "logs in ${DEST}; next:" +echo " python3 benchmarking/analysis/phase_report.py ${DEST}/*.log" diff --git a/benchmarking/analysis/phase_report.py b/benchmarking/analysis/phase_report.py new file mode 100755 index 0000000000..962046f793 --- /dev/null +++ b/benchmarking/analysis/phase_report.py @@ -0,0 +1,451 @@ +#!/usr/bin/env python3 +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Aggregate the suspend/resume phase-breakdown log records into a report. + +Reads the atelet and worker pod logs, either as kubectl logs dumps (JSON +lines; unrelated lines are skipped) or as a Cloud Logging export +(`gcloud logging read --format json`: a JSON array of entries with the record +under jsonPayload), and aggregates the two joinable record kinds the node +side emits, each written by both layers: + + - "Restore timing breakdown" atelet (ate.actor.restore.duration.*) + and ateom (ateom.actor.restore.duration.*) + - "Checkpoint timing breakdown" atelet (ate.actor.checkpoint.duration.*) + and ateom (ateom.actor.checkpoint.duration.*) + +The report answers "where does the SuspendActor / ResumeActor time go": +per-phase percentiles at each layer, and the slowest operations as nested +waterfalls (the ateom record joined under the atelet record of the same +actor). + +Usage: + phase_report.py run-logs/*.log [--csv DEST_DIR] [--slowest N] + +Feed it one source per run: the kubectl dumps, or the Cloud Logging export, +not both, or every record counts twice. + +The records are developer-facing logs, not metric API; this reader is the +consumer that makes them percentiles. See benchmarking/analysis/README.md. +""" + +from __future__ import annotations + +import argparse +import csv +import json +import re +import statistics +import sys +from collections import defaultdict +from dataclasses import dataclass, field +from datetime import datetime +from pathlib import Path + +# The duration-key prefixes. Source of truth: cmd/atelet/metrics.go +# (restoreDurationMetric, checkpointDurationMetric) and +# cmd/ateom-microvm/phaselog.go. +BREAKDOWN_PREFIXES = { + ("atelet", "restore"): "ate.actor.restore.duration.", + ("atelet", "checkpoint"): "ate.actor.checkpoint.duration.", + ("ateom", "restore"): "ateom.actor.restore.duration.", + ("ateom", "checkpoint"): "ateom.actor.checkpoint.duration.", +} +BREAKDOWN_MSGS = {"Restore timing breakdown", "Checkpoint timing breakdown"} + +ACTOR_UID_KEY = "ate.actor.uid" +SCOPE_KEY = "ate.snapshot.scope" +PHASE_KEY = "ate.snapshot.phase" +KIND_KEY = "ate.snapshot.kind" +SANDBOX_CLASS_KEY = "ate.sandbox.class" +TEMPLATE_KEY = "ate.template.name" +ERROR_TYPE_KEY = "error.type" + +# Only ateom-microvm writes the ateom records, and they carry no sandbox +# class of their own; the atelet record of the same operation does. +ATEOM_SANDBOX_CLASS = "microvm" + +# The phase every record reports, and the one the report derives: the part of +# the total no logged phase accounts for. +TOTAL = "total" +UNATTRIBUTED = "unattributed" + +# Sequential order of the known phases, for display. Phases absent from a +# record simply don't print; unknown phases print after, in input order. +PHASE_ORDER = [ + # atelet restore + "volume_mount", "manifest_fetch", "sandbox_assets", "download", + "oci_unpack", "ateom_restore", + # ateom restore + "prep", "bundles", "upper_join", "lowers", "tap", "vmm_launch", + "vm_restore", "resume", "wakeup_probe", + # atelet + ateom checkpoint + "pause", "snapshot", "durable_dir", "rootfs_upper", + "ateom_checkpoint", "teardown", "persist", + UNATTRIBUTED, TOTAL, +] +_PHASE_RANK = {name: i for i, name in enumerate(PHASE_ORDER)} + +# Which atelet phase wraps the ateom record. +INNER_PHASE = {"restore": "ateom_restore", "checkpoint": "ateom_checkpoint"} + +# The ateom checkpoint captures that run concurrently on the paused guest: the +# paused window costs their max, not their sum. +CONCURRENT_CAPTURES = ("snapshot", "durable_dir", "rootfs_upper") + +# Slack on the window in which the ateom record of an operation must land: +# atelet does a little work after the ateom call returns before it writes its +# own record (unmounting, resetting the actor dirs), and the two clocks are +# the same node's but not the same goroutine's. +JOIN_SLACK_S = 1.0 + + +@dataclass +class Breakdown: + """One parsed timing-breakdown record.""" + source: str # atelet | ateom + op: str # restore | checkpoint + time: str + ts: float | None # epoch seconds parsed from time, None when unparseable + actor_uid: str + template: str + sandbox_class: str + scope: str + kind: str + phases: dict[str, float] # phase name -> seconds + failed: bool + + +@dataclass +class Parsed: + breakdowns: list[Breakdown] = field(default_factory=list) + lines_seen: int = 0 + lines_matched: int = 0 + bad_values: int = 0 # duration keys whose value was not a number + + +_TIME_RE = re.compile(r"^(.*T\d\d:\d\d:\d\d)(\.\d+)?(Z|[+-]\d\d:?\d\d)?$") + + +def parse_time(s: str) -> float | None: + """RFC 3339 -> epoch seconds. slog writes nanoseconds, which fromisoformat + rejects, so the fraction is trimmed to microseconds. None when unparseable + (a record then still counts, it just cannot be joined by time).""" + m = _TIME_RE.match(s.strip()) if s else None + if not m: + return None + base, frac, tz = m.groups() + frac = (frac or "")[:7] + tz = "+00:00" if tz in (None, "Z") else tz + try: + return datetime.fromisoformat(f"{base}{frac}{tz}").timestamp() + except ValueError: + return None + + +def unattributed(source: str, op: str, phases: dict[str, float]) -> float | None: + """The total minus what the logged phases account for, where that is + well-defined: the atelet checkpoint phases are sequential; the ateom + checkpoint is prep, pause, the concurrent captures (max), then teardown. + The atelet restore phases overlap by design (download runs alongside the + asset fetch and OCI unpack) and the ateom restore phases partition their + total by construction, so neither gets a residual. The phases are + sub-intervals of the total, so a negative result is a bug in the emitter + or in this formula; it is not clamped, so that it shows.""" + total = phases.get(TOTAL) + if total is None: + return None + if (source, op) == ("atelet", "checkpoint"): + spent = sum(phases.get(p, 0.0) for p in ("sandbox_assets", "ateom_checkpoint", "persist")) + elif (source, op) == ("ateom", "checkpoint"): + spent = (phases.get("prep", 0.0) + phases.get("pause", 0.0) + + max(phases.get(p, 0.0) for p in CONCURRENT_CAPTURES) + + phases.get("teardown", 0.0)) + else: + return None + return total - spent + + +def parse_line(obj: dict, out: Parsed) -> None: + # A Cloud Logging entry nests the record under jsonPayload and moves + # slog's time onto the entry's timestamp; a kubectl dump is the record. + payload = obj.get("jsonPayload") + if isinstance(payload, dict): + entry, obj = obj, dict(payload) + obj.setdefault("time", entry.get("timestamp", "")) + msg = obj.get("msg", "") + if msg in BREAKDOWN_MSGS: + for (source, op), prefix in BREAKDOWN_PREFIXES.items(): + phases: dict[str, float] = {} + for k, v in obj.items(): + if not k.startswith(prefix): + continue + try: + phases[k[len(prefix):]] = float(v) + except (TypeError, ValueError): + # One bad value costs that phase, not the run's report. + out.bad_values += 1 + if not phases: + continue + time_s = obj.get("time", "") + residual = unattributed(source, op, phases) + if residual is not None: + phases[UNATTRIBUTED] = residual + out.breakdowns.append(Breakdown( + source=source, + op=op, + time=time_s, + ts=parse_time(time_s), + actor_uid=obj.get(ACTOR_UID_KEY, ""), + template=obj.get(TEMPLATE_KEY, ""), + sandbox_class=obj.get(SANDBOX_CLASS_KEY, "") + or (ATEOM_SANDBOX_CLASS if source == "ateom" else ""), + scope=obj.get(SCOPE_KEY, ""), + kind=obj.get(KIND_KEY, ""), + phases=phases, + failed=ERROR_TYPE_KEY in obj, + )) + out.lines_matched += 1 + return + + +def parse_files(paths: list[str]) -> Parsed: + out = Parsed() + for path in paths: + f = sys.stdin if path == "-" else open(path, encoding="utf-8", errors="replace") + with f: + text = f.read() + if text.lstrip().startswith("["): + # A Cloud Logging export: one JSON array of entries, pretty-printed + # across lines, so it cannot be read line by line. + try: + entries = json.loads(text) + except json.JSONDecodeError as e: + # Not an export after all: a dump whose lines carry a bracket + # prefix, or a truncated export. Say so, then read it line by + # line like any other dump rather than drop the whole file. + print(f"warn: {path}: starts with '[' but is not a JSON array ({e}); " + f"reading it line by line", file=sys.stderr) + else: + if isinstance(entries, list): + for obj in entries: + if isinstance(obj, dict): + out.lines_seen += 1 + parse_line(obj, out) + continue + for line in text.splitlines(): + # kubectl log dumps may prefix each line (pod name, timestamp); + # recover the JSON object from the first brace. + brace = line.find("{") + if brace < 0: + continue + out.lines_seen += 1 + try: + obj = json.loads(line[brace:]) + except json.JSONDecodeError: + continue + if isinstance(obj, dict): + parse_line(obj, out) + return out + + +def percentile(values: list[float], q: float) -> float: + if not values: + return 0.0 + if len(values) == 1: + return values[0] + return statistics.quantiles(values, n=100, method="inclusive")[int(q) - 1] + + +def phase_sort_key(name: str) -> tuple[int, str]: + return (_PHASE_RANK.get(name, len(PHASE_ORDER)), name) + + +def fmt_s(seconds: float) -> str: + return f"{seconds * 1000:8.1f}" + + +def report_phases(breakdowns: list[Breakdown], writer) -> list[dict]: + """Per (source, op, sandbox class, kind, scope, phase) percentiles. A + golden restore (which downloads the golden image) and a latest restore, + or a gVisor and a micro-VM checkpoint, are different distributions and + must not be pooled. Returns the rows for CSV.""" + groups: dict[tuple, list[float]] = defaultdict(list) + for b in breakdowns: + if b.failed: + continue + for name, seconds in b.phases.items(): + groups[(b.source, b.op, b.sandbox_class or "-", b.kind or "-", b.scope or "-", name)].append(seconds) + + rows = [] + writer("== Phase percentiles (ms) ==") + writer(f"{'layer':7} {'op':11} {'class':8} {'kind':7} {'scope':15} {'phase':16} {'n':>5} " + f"{'p50':>8} {'p90':>8} {'p95':>8} {'max':>8}") + for key in sorted(groups, key=lambda k: (k[:5], phase_sort_key(k[5]))): + vals = sorted(groups[key]) + source, op, sandbox_class, kind, scope, name = key + row = { + "layer": source, "op": op, "class": sandbox_class, "kind": kind, + "scope": scope, "phase": name, + "count": len(vals), + "p50_ms": percentile(vals, 50) * 1000, + "p90_ms": percentile(vals, 90) * 1000, + "p95_ms": percentile(vals, 95) * 1000, + "max_ms": max(vals) * 1000, + } + rows.append(row) + writer(f"{source:7} {op:11} {sandbox_class:8} {kind:7} {scope:15} {name:16} {len(vals):5d} " + f"{fmt_s(percentile(vals, 50))} {fmt_s(percentile(vals, 90))} " + f"{fmt_s(percentile(vals, 95))} {fmt_s(max(vals))}") + failed = sum(1 for b in breakdowns if b.failed) + if failed: + writer(f"(excluded {failed} failed operation records)") + return rows + + +def ateom_window(op: Breakdown) -> tuple[float, float] | None: + """When, relative to the atelet record's time, the ateom record of the + same operation was written. ateom writes it as its RPC returns, i.e. at + the end of the atelet ateom_* phase: for a checkpoint that is before + persist runs, for a restore it is the last phase. An operation that never + reached the ateom call has no window and nothing to pair with.""" + total = op.phases.get(TOTAL) + if op.ts is None or total is None or INNER_PHASE[op.op] not in op.phases: + return None + if op.op == "checkpoint": + return (op.ts - total - JOIN_SLACK_S, + op.ts - op.phases.get("persist", 0.0) + JOIN_SLACK_S) + return (op.ts - op.phases[INNER_PHASE[op.op]] - JOIN_SLACK_S, op.ts + JOIN_SLACK_S) + + +def pair_records(breakdowns: list[Breakdown]) -> dict[int, Breakdown]: + """Match each atelet record to the ateom record of the same operation: + same actor and op, written inside the atelet ateom_* phase's window, and + not already claimed by another operation. Returns id(atelet) -> ateom. + + One cycle emits exactly one of each. An ateom record outside the window + belongs to another cycle whose partner is missing (pod gone, log rotated, + window cut); claiming it would print a gap that never happened, so an + operation without a match prints without one. Matching in time order + with a consumed set keeps rapid cycles of one actor from sharing or + swapping records. A paired ateom record also inherits the snapshot kind + its layer does not log, so the two layers' percentiles split alike.""" + by_actor: dict[tuple, list[Breakdown]] = defaultdict(list) + for b in breakdowns: + if b.source == "ateom" and b.ts is not None: + by_actor[(b.actor_uid, b.op)].append(b) + for v in by_actor.values(): + v.sort(key=lambda b: b.ts) + + pairs: dict[int, Breakdown] = {} + consumed: set[int] = set() + atelet = sorted((b for b in breakdowns if b.source == "atelet" and b.ts is not None), + key=lambda b: b.ts) + for op in atelet: + window = ateom_window(op) + if window is None: + continue + lo, hi = window + candidates = [a for a in by_actor.get((op.actor_uid, op.op), []) + if id(a) not in consumed and lo <= a.ts <= hi] + if not candidates: + continue + inner = candidates[-1] + consumed.add(id(inner)) + pairs[id(op)] = inner + if not inner.kind: + inner.kind = op.kind + return pairs + + +def report_waterfalls(breakdowns: list[Breakdown], writer, slowest: int, + pairs: dict[int, Breakdown] | None = None) -> None: + """The slowest operations: the ateom record nested under the atelet + ateom_* phase, and the gap between that phase and the ateom total (RPC + and queueing between the layers).""" + if pairs is None: + pairs = pair_records(breakdowns) + atelet = [b for b in breakdowns if b.source == "atelet" and not b.failed] + atelet.sort(key=lambda b: b.phases.get(TOTAL, 0), reverse=True) + + writer("") + writer(f"== Slowest {slowest} operations (waterfall, ms) ==") + for b in atelet[:slowest]: + inner = pairs.get(id(b)) + total = b.phases.get(TOTAL, 0) + writer(f"{b.op} actor={b.actor_uid} template={b.template} " + f"class={b.sandbox_class or '-'} scope={b.scope or '-'} kind={b.kind or '-'} " + f"total={total * 1000:.1f}") + for name in sorted(b.phases, key=phase_sort_key): + if name == TOTAL: + continue + writer(f" atelet {name:15} {fmt_s(b.phases[name])}") + if name == INNER_PHASE.get(b.op) and inner: + for iname in sorted(inner.phases, key=phase_sort_key): + if iname == TOTAL: + continue + writer(f" ateom {iname:13} {fmt_s(inner.phases[iname])}") + if TOTAL in inner.phases: + gap = b.phases[name] - inner.phases[TOTAL] + writer(f" (gap) {'rpc/queueing':13} {fmt_s(gap)}") + + +def write_csv(dest: Path, name: str, rows: list[dict]) -> None: + if not rows: + return + dest.mkdir(parents=True, exist_ok=True) + path = dest / name + with open(path, "w", newline="", encoding="utf-8") as f: + w = csv.DictWriter(f, fieldnames=list(rows[0].keys())) + w.writeheader() + w.writerows(rows) + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("logs", nargs="+", + help="kubectl logs dumps (JSON lines) or Cloud Logging exports " + "(JSON array, as gcloud logging read --format json writes); " + "'-' for stdin") + ap.add_argument("--csv", type=Path, default=None, + help="also write phase_percentiles.csv here") + ap.add_argument("--slowest", type=int, default=3, + help="number of slowest operations to print as waterfalls") + args = ap.parse_args() + + parsed = parse_files(args.logs) + print(f"parsed {parsed.lines_matched} timing breakdown records " + f"out of {parsed.lines_seen} JSON log entries") + if parsed.bad_values: + print(f"(skipped {parsed.bad_values} non-numeric duration values)") + if not parsed.breakdowns: + print("no matching records; are these the atelet and worker pod logs?", file=sys.stderr) + return 1 + + # Pair before the percentiles: a paired ateom record takes its kind from + # the atelet record, so both layers' rows split the same way. + pairs = pair_records(parsed.breakdowns) + print() + phase_rows = report_phases(parsed.breakdowns, print) + report_waterfalls(parsed.breakdowns, print, args.slowest, pairs) + + if args.csv: + write_csv(args.csv, "phase_percentiles.csv", phase_rows) + print(f"\nCSV written to {args.csv}/") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/benchmarking/analysis/test_phase_report.py b/benchmarking/analysis/test_phase_report.py new file mode 100644 index 0000000000..45e61169d6 --- /dev/null +++ b/benchmarking/analysis/test_phase_report.py @@ -0,0 +1,300 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Parser and aggregation tests for phase_report. No cluster needed: + + python3 -m unittest discover -s benchmarking/analysis +""" + +import contextlib +import io +import json +import os +import sys +import tempfile +import unittest + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +import phase_report # noqa: E402 + + +def parse(*records): + out = phase_report.Parsed() + for rec in records: + phase_report.parse_line(json.loads(json.dumps(rec)), out) + return out + + +def render(fn, *args, **kwargs): + buf = io.StringIO() + fn(*args, writer=lambda line: buf.write(line + "\n"), **kwargs) + return buf.getvalue() + + +ATELET_RESTORE = { + "time": "2026-09-23T10:00:05.500000000Z", "level": "INFO", + "msg": "Restore timing breakdown", + "ate.actor.uid": "uid-1", "ate.template.name": "swebench-astropy-7336", + "ate.snapshot.scope": "full", "ate.snapshot.kind": "latest", + "ate.actor.restore.duration.download": 2.4, + "ate.actor.restore.duration.ateom_restore": 1.1, + "ate.actor.restore.duration.total": 3.9, +} + +ATEOM_RESTORE = { + "time": "2026-09-23T10:00:05.400000000Z", "level": "INFO", + "msg": "Restore timing breakdown", + "ate.actor.uid": "uid-1", "ate.template.name": "swebench-astropy-7336", + "ate.snapshot.scope": "full", + "ateom.actor.restore.duration.vm_restore": 0.8, + "ateom.actor.restore.duration.wakeup_probe": 0.2, + "ateom.actor.restore.duration.total": 1.05, +} + +ATELET_CHECKPOINT = { + "time": "2026-09-23T10:01:00.000000000Z", "level": "INFO", + "msg": "Checkpoint timing breakdown", + "ate.actor.uid": "uid-1", "ate.template.name": "swebench-astropy-7336", + "ate.snapshot.scope": "full", "ate.snapshot.kind": "latest", + "ate.actor.checkpoint.duration.sandbox_assets": 0.01, + "ate.actor.checkpoint.duration.ateom_checkpoint": 1.14, + "ate.actor.checkpoint.duration.persist": 4.48, + "ate.actor.checkpoint.duration.total": 6.0, +} + +ATEOM_CHECKPOINT = { + "time": "2026-09-23T10:00:55.000000000Z", "level": "INFO", + "msg": "Checkpoint timing breakdown", + "ate.actor.uid": "uid-1", "ate.template.name": "swebench-astropy-7336", + "ate.snapshot.scope": "full", + "ateom.actor.checkpoint.duration.prep": 0.04, + "ateom.actor.checkpoint.duration.pause": 0.01, + "ateom.actor.checkpoint.duration.snapshot": 0.5, + "ateom.actor.checkpoint.duration.rootfs_upper": 0.9, + "ateom.actor.checkpoint.duration.teardown": 0.2, + "ateom.actor.checkpoint.duration.total": 1.2, +} + +# The same actor's ateom restore record from an hour earlier: a different +# cycle, whose own atelet partner is missing from the dump. +ATEOM_RESTORE_STALE = dict(ATEOM_RESTORE, time="2026-09-23T09:00:05.400000000Z") + +# One entry as `gcloud logging read --format json` writes it, trimmed to the +# fields that matter: the record under jsonPayload, slog's time promoted to +# the entry's timestamp, dotted keys kept as they are. +CLOUD_LOGGING_ENTRY = { + "insertId": "m9xljnzmzppup1vj", + "jsonPayload": { + "ate.actor.checkpoint.duration.ateom_checkpoint": 1.024788329, + "ate.actor.checkpoint.duration.persist": 4.18451256, + "ate.actor.checkpoint.duration.sandbox_assets": 2.4358e-05, + "ate.actor.checkpoint.duration.total": 5.385861508, + "ate.actor.name": "sb-d790f7ed", "ate.actor.uid": "cc7a2ac7", + "ate.atespace": "benchmark", "ate.sandbox.class": "microvm", + "ate.snapshot.kind": "latest", "ate.snapshot.scope": "full", + "ate.template.atespace": "benchmark-workloads", "ate.template.name": "glutton", + "level": "INFO", "msg": "Checkpoint timing breakdown", + }, + "logName": "projects/p/logs/stdout", + "resource": {"labels": {"container_name": "atelet", "namespace_name": "ate-system"}, + "type": "k8s_container"}, + "severity": "INFO", + "timestamp": "2026-09-28T17:49:47.665836237Z", +} + +FAILED = { + "time": "2026-09-23T10:02:00Z", "level": "INFO", + "msg": "Restore timing breakdown", + "ate.actor.uid": "uid-2", "ate.snapshot.scope": "full", + "error.type": "DeadlineExceeded", + "ate.actor.restore.duration.download": 30.0, + "ate.actor.restore.duration.total": 30.0, +} + + +class ParseTest(unittest.TestCase): + def test_parses_both_layers_of_the_same_msg(self): + out = parse(ATELET_RESTORE, ATEOM_RESTORE) + self.assertEqual([(b.source, b.op) for b in out.breakdowns], + [("atelet", "restore"), ("ateom", "restore")]) + self.assertEqual(out.breakdowns[0].phases["download"], 2.4) + self.assertEqual(out.breakdowns[1].phases["vm_restore"], 0.8) + self.assertEqual(out.breakdowns[1].actor_uid, "uid-1") + + def test_nanosecond_timestamps_parse(self): + out = parse(ATELET_RESTORE, FAILED) + self.assertIsNotNone(out.breakdowns[0].ts) + self.assertIsNotNone(out.breakdowns[1].ts) + self.assertAlmostEqual(out.breakdowns[0].ts % 1, 0.5, places=6) + self.assertIsNone(phase_report.parse_time("yesterday")) + + def test_ateom_checkpoint_parses(self): + b = parse(ATEOM_CHECKPOINT).breakdowns[0] + self.assertEqual((b.source, b.op), ("ateom", "checkpoint")) + self.assertEqual(b.phases["rootfs_upper"], 0.9) + + def test_cloud_logging_entry_parses_with_the_entry_timestamp(self): + b = parse(CLOUD_LOGGING_ENTRY).breakdowns[0] + self.assertEqual((b.source, b.op), ("atelet", "checkpoint")) + self.assertEqual(b.time, "2026-09-28T17:49:47.665836237Z") + self.assertIsNotNone(b.ts) + self.assertEqual((b.sandbox_class, b.kind, b.scope), ("microvm", "latest", "full")) + self.assertAlmostEqual(b.phases["persist"], 4.18451256) + + def test_cloud_logging_export_file_is_a_json_array(self): + with tempfile.TemporaryDirectory() as d: + path = os.path.join(d, "export.json") + second = dict(CLOUD_LOGGING_ENTRY, timestamp="2026-09-28T17:50:47.000000000Z") + with open(path, "w") as f: + json.dump([CLOUD_LOGGING_ENTRY, second], f, indent=2) + out = phase_report.parse_files([path]) + self.assertEqual((out.lines_seen, len(out.breakdowns)), (2, 2)) + + def test_ateom_records_default_to_the_microvm_class(self): + out = parse(ATEOM_RESTORE, ATELET_RESTORE) + self.assertEqual(out.breakdowns[0].sandbox_class, "microvm") + self.assertEqual(out.breakdowns[1].sandbox_class, "") # not on this fixture + + def test_non_numeric_duration_is_skipped_not_fatal(self): + rec = dict(ATELET_RESTORE, **{"ate.actor.restore.duration.download": None, + "ate.actor.restore.duration.oci_unpack": "fast"}) + out = parse(rec) + self.assertEqual(out.bad_values, 2) + self.assertEqual(out.breakdowns[0].phases["total"], 3.9) + self.assertNotIn("download", out.breakdowns[0].phases) + + def test_malformed_export_warns_instead_of_vanishing(self): + with tempfile.TemporaryDirectory() as d: + path = os.path.join(d, "export.json") + with open(path, "w") as f: + f.write('[{"jsonPayload": {"msg": "Checkpoint timing breakdown"') # truncated + err = io.StringIO() + with contextlib.redirect_stderr(err): + out = phase_report.parse_files([path]) + self.assertEqual(out.breakdowns, []) + self.assertIn("export.json", err.getvalue()) + self.assertIn("not a JSON array", err.getvalue()) + + def test_bracket_prefixed_dump_is_read_line_by_line(self): + # A dump whose lines start with "[INFO]" also starts with "[", but + # it is not an export; its records must not be lost. + with tempfile.TemporaryDirectory() as d: + path = os.path.join(d, "pod.log") + with open(path, "w") as f: + f.write("[INFO] " + json.dumps(ATELET_RESTORE) + "\n" + "[2026-09-23 10:00:05] " + json.dumps(ATEOM_RESTORE) + "\n") + err = io.StringIO() + with contextlib.redirect_stderr(err): + out = phase_report.parse_files([path]) + self.assertEqual(len(out.breakdowns), 2) + self.assertIn("reading it line by line", err.getvalue()) + + def test_prefixed_kubectl_lines_still_parse(self): + with tempfile.TemporaryDirectory() as d: + path = os.path.join(d, "pod.log") + with open(path, "w") as f: + f.write("pod/atelet-abc " + json.dumps(ATELET_RESTORE) + "\nnot json\n{\"msg\": \"other\"}\n") + out = phase_report.parse_files([path]) + self.assertEqual(len(out.breakdowns), 1) + self.assertEqual(out.lines_seen, 2) + + +class UnattributedTest(unittest.TestCase): + def test_atelet_checkpoint_residual_is_total_minus_sequential_phases(self): + b = parse(ATELET_CHECKPOINT).breakdowns[0] + self.assertAlmostEqual(b.phases["unattributed"], 6.0 - (0.01 + 1.14 + 4.48)) + + def test_ateom_checkpoint_counts_the_concurrent_captures_once(self): + b = parse(ATEOM_CHECKPOINT).breakdowns[0] + # prep + pause + max(snapshot, rootfs_upper) + teardown; the two + # captures overlapped, so only the slower one is spent wall time. + self.assertAlmostEqual(b.phases["unattributed"], 1.2 - (0.04 + 0.01 + 0.9 + 0.2)) + + def test_restore_layers_get_no_residual(self): + out = parse(ATELET_RESTORE, ATEOM_RESTORE) + for b in out.breakdowns: + self.assertNotIn("unattributed", b.phases) + + def test_negative_residual_is_not_clamped(self): + # Cannot happen on a well-formed record; if it does, it must show. + rec = dict(ATELET_CHECKPOINT, **{"ate.actor.checkpoint.duration.total": 1.0}) + self.assertAlmostEqual(parse(rec).breakdowns[0].phases["unattributed"], 1.0 - (0.01 + 1.14 + 4.48)) + + +class ReportTest(unittest.TestCase): + def test_failed_records_are_flagged_and_excluded_from_percentiles(self): + out = parse(ATELET_RESTORE, FAILED) + self.assertEqual([b.failed for b in out.breakdowns], [False, True]) + rows = phase_report.report_phases(out.breakdowns, lambda _: None) + downloads = [r for r in rows if r["phase"] == "download"] + self.assertEqual(len(downloads), 1) + self.assertEqual(downloads[0]["count"], 1) # the failed 30s never entered + + def test_percentiles_split_by_sandbox_class_and_kind(self): + golden = dict(ATELET_RESTORE, **{"time": "2026-09-23T09:59:00.000000000Z", + "ate.snapshot.kind": "golden", "ate.sandbox.class": "gvisor", + "ate.actor.restore.duration.download": 9.0, + "ate.actor.restore.duration.total": 10.0}) + latest = dict(ATELET_RESTORE, **{"ate.sandbox.class": "gvisor"}) + rows = phase_report.report_phases(parse(golden, latest).breakdowns, lambda _: None) + downloads = {(r["class"], r["kind"]): r["p50_ms"] for r in rows if r["phase"] == "download"} + self.assertEqual(downloads, {("gvisor", "golden"): 9000.0, ("gvisor", "latest"): 2400.0}) + + def test_paired_ateom_record_inherits_the_atelet_kind(self): + out = parse(ATEOM_RESTORE, ATELET_RESTORE) + pairs = phase_report.pair_records(out.breakdowns) + self.assertEqual(len(pairs), 1) + self.assertEqual(out.breakdowns[0].kind, "latest") + rows = phase_report.report_phases(out.breakdowns, lambda _: None) + self.assertEqual({r["kind"] for r in rows if r["layer"] == "ateom"}, {"latest"}) + + def test_checkpoint_pairing_window_ends_where_persist_starts(self): + # The ateom record is written when ateom_checkpoint ends, before the + # 4.48 s persist; a record from inside the persist window belongs to + # a later cycle of a rapidly cycling actor. + during_persist = dict(ATEOM_CHECKPOINT, time="2026-09-23T10:00:59.000000000Z") + out = parse(ATEOM_CHECKPOINT, during_persist, ATELET_CHECKPOINT) + pairs = phase_report.pair_records(out.breakdowns) + atelet = next(b for b in out.breakdowns if b.source == "atelet") + self.assertIs(pairs[id(atelet)], out.breakdowns[0]) + + def test_an_ateom_record_pairs_with_at_most_one_operation(self): + second = dict(ATELET_RESTORE, time="2026-09-23T10:00:05.900000000Z") + out = parse(ATEOM_RESTORE, ATELET_RESTORE, second) + pairs = phase_report.pair_records(out.breakdowns) + self.assertEqual(len(pairs), 1) + first = next(b for b in out.breakdowns if b.source == "atelet") + self.assertIn(id(first), pairs) + + def test_waterfall_ignores_an_ateom_record_from_another_cycle(self): + out = parse(ATEOM_RESTORE_STALE, ATELET_RESTORE) + text = render(phase_report.report_waterfalls, out.breakdowns, slowest=1) + self.assertIn("atelet ateom_restore", text) + self.assertNotIn("ateom vm_restore", text) + self.assertNotIn("(gap)", text) + + def test_waterfall_nests_ateom_and_gap_under_atelet(self): + out = parse(ATEOM_RESTORE, ATELET_RESTORE) + text = render(phase_report.report_waterfalls, out.breakdowns, slowest=1) + lines = [line.strip() for line in text.splitlines()] + self.assertTrue(any(line.startswith("atelet ateom_restore") for line in lines), text) + self.assertTrue(any(line.startswith("ateom vm_restore") for line in lines), text) + gap = [line for line in lines if line.startswith("(gap)")] + self.assertEqual(len(gap), 1, text) + self.assertIn("50.0", gap[0]) # 1.1s atelet ateom_restore - 1.05s ateom total + + +if __name__ == "__main__": + unittest.main() diff --git a/docs/dev/suspend-resume-phase-breakdown.md b/docs/dev/suspend-resume-phase-breakdown.md new file mode 100644 index 0000000000..8cc72e3b6a --- /dev/null +++ b/docs/dev/suspend-resume-phase-breakdown.md @@ -0,0 +1,167 @@ +# Suspend/resume phase breakdown: a runbook + +This guide walks through getting a per-phase breakdown of `SuspendActor` and +`ResumeActor` latency from a cluster you run yourself, from deploying the +workloads to reading the report. It strings together tools documented +elsewhere; follow the links when a step needs more than the command shown. + +The breakdown comes from two developer-facing log records, `Checkpoint timing +breakdown` and `Restore timing breakdown`, written once per operation by +atelet (the outer phases: `download`, `persist`, `ateom_*`, …) and by +ateom-microvm (the phases inside `ateom_*`: `pause`, `snapshot`, `vm_restore`, +`wakeup_probe`, …). `benchmarking/analysis/` collects the records from the pod +logs and aggregates them. See +[docs/observability.md](../observability.md#actor-attributed-component-logs) +for the record shape and +[benchmarking/analysis/README.md](../../benchmarking/analysis/README.md) for +the report. + +## What you need + +* A cluster with Agent Substrate installed from a tree that has the records + and the analysis tools: the + [Quickstart (Development)](../../README.md#quickstart-development) for kind, + or the [GKE Quickstart](../../README.md#gke-quickstart-development). For GKE + the scripts below read `.ate-dev-env.sh` from the repository root + (`collect_logs.sh` sources it itself; set `NO_DEV_ENV` to opt out). +* The **microVM** sandbox class, if you want the inner (ateom) phases: see + [microvm-local.md](microvm-local.md). ateom-gvisor writes no breakdown + record, so on a gVisor pool the report shows the atelet layer only. +* `kubectl`, `kubectl-ate` (`go install ./cmd/kubectl-ate`), and `python3` + (standard library only). + +## 1. Deploy the benchmark workloads + +```sh +./benchmarking/deploy_locust.sh --deploy \ + --sandbox-class microvm --worker-count 3 --actor-memory 1536Mi +``` + +This deploys the benchmark `WorkerPool` and `ActorTemplate`s into the +`benchmark-workloads` atespace, builds and pushes the locust image, and deploys +the locust master and workers (`benchmarking/README.md`, +[Deploy benchmarks](../../benchmarking/README.md#deploy-benchmarks)). On a kind +cluster with limited memory use smaller values, e.g. `--worker-count 1 +--actor-memory 512Mi`, and scale the load flags in step 2 down to match. + +Wait until every template has a golden snapshot — the column is a UUID once +the golden build (a full boot plus checkpoint) has finished: + +```sh +kubectl ate get actor-templates -a benchmark-workloads +``` + +Those golden builds are themselves checkpoints, so the worker pods already +carry `Checkpoint timing breakdown` records at this point: + +```sh +kubectl logs -n benchmark-workloads -l ate.dev/worker-pool --since=15m \ + | grep -c "timing breakdown" +``` + +A zero here means the running image predates the records; redeploy ate-system +from the current tree before going on. + +## 2. Run suspend/resume load + +The `glutton` user class creates an actor per user and cycles it through +suspend and resume. Give it a realistically sized working set so the memory +phases have something to measure: + +```sh +kubectl port-forward svc/locust -n benchmarking 8089:8089 +``` + +Open , pick `GluttonUser`, and set: + +| field | value | why | +|---|---|---| +| users / spawn rate | 2 / 1 | two concurrent cycles; keep it small so transfers do not contend for one NIC | +| `--mem-target` | `1Gi` | resident working set before the first suspend (below `--actor-memory`) | +| `--mem-churn` | `64Mi` | re-dirty part of it each cycle, so successive snapshots differ | +| `--mem-read` | `all` | walk the whole set after each resume, so a demand-paged restore pays for its pages | +| `--min-wait-time` / `--max-wait-time` | `1.0` / `1.0` | one cycle per second per user | + +The form fields are the flags documented in +`benchmarking/locust/common/memload_config.py`; the boomer workers fetch the +values from the master on each spawn. Let it run for about 90 seconds (≈10 +cycles per user), then stop the test. For a single operation without locust, +`kubectl ate suspend actor -a benchmark-workloads` and +`kubectl ate resume actor -a benchmark-workloads` on any actor in the +atespace produce one record pair each. + +## 3. Collect the node logs + +Collect **while the pods still exist**. `benchmarking/automation/ +orchestrator.py` deletes the workload and ate-system pods right after each +test and does not run this step yet, so on a cluster you manage yourself this +is the moment to do it: + +```sh +./benchmarking/analysis/collect_logs.sh --dest /tmp/run1 --since-time 2026-09-24T18:00:00Z +``` + +`--since-time` (or a relative `--since 30m`) should cover the run and nothing +before it. The script writes one file per atelet pod (`ate-system`) and per +worker pod (`benchmark-workloads`) under `--dest`; a pod it cannot read is +skipped with a warning. Use either these dumps or the Cloud Logging export +below for a run, not both. `kubectl logs` returns only a container's current log +file, so collect soon after the run, before kubelet rotates it. + +On GKE the same records are also in Cloud Logging, where they outlive both +rotation and teardown; an export is accepted by the report as-is: + +```sh +gcloud logging read '(jsonPayload.msg="Checkpoint timing breakdown" OR jsonPayload.msg="Restore timing breakdown") AND resource.labels.cluster_name="" AND timestamp>="2026-09-24T18:00:00Z"' \ + --project --format json > /tmp/run1/export.json +``` + +## 4. Produce the report + +```sh +python3 benchmarking/analysis/phase_report.py /tmp/run1/*.log --csv /tmp/run1/report +``` + +The first line says how many records it found; expect two per suspend and two +per resume (one per layer). If it is zero, the window in step 3 missed the run +or the images predate the records. + +## 5. Read it + +* **Phase percentiles** rank the phases. For a suspend, compare atelet's + `persist` against `ateom_checkpoint`, then the ateom rows (`snapshot`, + `teardown`, `pause`, …) against each other; for a resume, `download` against + `ateom_restore`, then `vm_restore` and `wakeup_probe`. The `unattributed` + row is the part of a checkpoint no logged phase names. +* **Waterfalls** show the slowest operations with the ateom record nested + under the atelet phase it decomposes; a tail that is always the same phase + is systematic, one that moves between phases is environmental. +* Two sanity checks on a healthy run: the atelet and ateom rows for one + operation have the same `n`, and atelet's `ateom_checkpoint` p50 is a few + tens of milliseconds above ateom's `total` p50 (the `(gap)` line in a + waterfall is that RPC overhead). + +`/tmp/run1/report/phase_percentiles.csv` holds the same table for comparing +runs, for example one at `--mem-target 1Gi` against one at `2Gi` to see which +phases scale with dirty memory. + +## 6. Tear down + +```sh +./benchmarking/deploy_locust.sh --delete +``` + +## Troubleshooting + +* **Only atelet rows, no ateom rows.** The pool is gVisor (ateom-gvisor emits + no record), or the worker image predates the records. Check + `kubectl get workerpools -A` and step 1's grep. +* **Counts differ between layers.** The collection window cut through a + cycle, or a worker pod restarted and took its log with it (the script dumps + a restarted container's previous log as `.previous.log`). On GKE the + Cloud Logging export has everything. +* **A record has `error.type`.** That operation failed; the report leaves it + out of the percentiles and the waterfalls. To see where it died, grep the + record itself: a `DeadlineExceeded` whose time sits in `download` or + `persist` with a normal-looking `ateom_*` phase points at object storage, + not the runtime. From 4621338a35f9b2b93067e38020b38f6b9c808523 Mon Sep 17 00:00:00 2001 From: Greg Castle Date: Fri, 9 Oct 2026 16:24:50 +0000 Subject: [PATCH 28/44] Reject requests with duplicate ate-target-actor headers across HTTP and CONNECT (#2308) Reject ingress requests that carry multiple `ate-target-actor` header entries across HTTP (`404 Not Found`), `CONNECT` (`%CEL(request.headers['ate-target-actor'])%` filter state + outer `CONNECT` precedence), and `atunnel` (`421 Misdirected Request`), preventing header smuggling mismatches between upstream Envoy RBAC/CEL authorization policies and `ext_proc` actor routing. Fixes #2307 - [x] Tests pass - [x] Appropriate changes to documentation are included in the PR --- .../internal/router/extproc/metadata.go | 37 ++- .../internal/router/extproc/metadata_test.go | 36 +++ cmd/atenet/internal/router/ingress/ingress.go | 14 +- .../internal/router/ingress/ingress_test.go | 288 +++++++++++++++++- cmd/atenet/internal/router/xds.go | 10 +- cmd/atenet/internal/router/xds_test.go | 4 +- internal/atenet/headers_test.go | 2 + internal/atunnel/ingress.go | 6 +- internal/atunnel/ingress_test.go | 67 ++++ internal/e2e/router_client.go | 12 +- .../suites/networking/arbitraryport_test.go | 22 ++ .../e2e/suites/networking/networking_test.go | 19 ++ 12 files changed, 500 insertions(+), 17 deletions(-) diff --git a/cmd/atenet/internal/router/extproc/metadata.go b/cmd/atenet/internal/router/extproc/metadata.go index ffee0b035d..f7ff69e168 100644 --- a/cmd/atenet/internal/router/extproc/metadata.go +++ b/cmd/atenet/internal/router/extproc/metadata.go @@ -59,7 +59,11 @@ func NewRequestMetadata(headers []*corev3.HeaderValue, attributes map[string]*st val = string(h.RawValue) } - headersMap[k] = val + if existing, ok := headersMap[k]; ok { + headersMap[k] = existing + "," + val + } else { + headersMap[k] = val + } if k == ":path" { path = val } @@ -82,9 +86,40 @@ func NewRequestMetadata(headers []*corev3.HeaderValue, attributes map[string]*st // Header returns the value of a header by name, case-insensitively, or "" when // it was not sent. func (m *RequestMetadata) Header(name string) string { + if m == nil { + return "" + } return m.Headers[strings.ToLower(name)] } +// Get returns the value associated with the passed key, implementing propagation.TextMapCarrier. +func (m *RequestMetadata) Get(key string) string { + return m.Header(key) +} + +// Set stores the key-value pair, implementing propagation.TextMapCarrier. +func (m *RequestMetadata) Set(key, val string) { + if m == nil { + return + } + if m.Headers == nil { + m.Headers = make(map[string]string) + } + m.Headers[strings.ToLower(key)] = val +} + +// Keys lists the keys stored in this carrier, implementing propagation.TextMapCarrier. +func (m *RequestMetadata) Keys() []string { + if m == nil || m.Headers == nil { + return nil + } + keys := make([]string, 0, len(m.Headers)) + for k := range m.Headers { + keys = append(keys, k) + } + return keys +} + // Attribute returns the named CEL request_attributes value, scanning every // filter's entries so callers don't need to hardcode which one reported it, // or "" if none did. A number (Envoy sends int fields as one) renders as its diff --git a/cmd/atenet/internal/router/extproc/metadata_test.go b/cmd/atenet/internal/router/extproc/metadata_test.go index 3789ec8495..09a4b2548b 100644 --- a/cmd/atenet/internal/router/extproc/metadata_test.go +++ b/cmd/atenet/internal/router/extproc/metadata_test.go @@ -103,6 +103,18 @@ func TestExtractMetadata(t *testing.T) { wantPath: "", wantHost: "", }, + { + name: "repeated headers are joined with commas", + headers: []*corev3.HeaderValue{ + {Key: "X-Repeated", Value: "first"}, + {Key: "x-repeated", Value: "second"}, + }, + wantHeaders: map[string]string{ + "x-repeated": "first,second", + }, + wantPath: "", + wantHost: "", + }, } for _, tc := range tests { @@ -122,6 +134,30 @@ func TestExtractMetadata(t *testing.T) { } } +func TestRequestMetadataHeaderCommaJoinsDuplicates(t *testing.T) { + headers := []*corev3.HeaderValue{ + {Key: "X-Single", Value: "single-val"}, + {Key: "X-Multiple", Value: "val-1"}, + {Key: "X-Multiple", Value: "val-2"}, + } + md := NewRequestMetadata(headers, nil) + + if got := md.Header("X-Single"); got != "single-val" { + t.Errorf("Header(X-Single) = %q, want %q", got, "single-val") + } + if got := md.Header("X-Multiple"); got != "val-1,val-2" { + t.Errorf("Header(X-Multiple) = %q, want %q", got, "val-1,val-2") + } + if got := md.Header("X-Missing"); got != "" { + t.Errorf("Header(X-Missing) = %q, want empty string", got) + } + + var nilMD *RequestMetadata + if got := nilMD.Header("X-Single"); got != "" { + t.Errorf("nil.Header() = %q, want empty string", got) + } +} + func TestRequestMetadataAttribute(t *testing.T) { attrs := map[string]*structpb.Struct{ "envoy.filters.http.ext_proc": { diff --git a/cmd/atenet/internal/router/ingress/ingress.go b/cmd/atenet/internal/router/ingress/ingress.go index 6c6a88e406..6b68011cea 100644 --- a/cmd/atenet/internal/router/ingress/ingress.go +++ b/cmd/atenet/internal/router/ingress/ingress.go @@ -34,7 +34,6 @@ import ( extprocv3 "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" envoy_type "github.com/envoyproxy/go-control-plane/envoy/type/v3" "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/propagation" "google.golang.org/protobuf/types/known/structpb" "github.com/agent-substrate/substrate/cmd/atenet/internal/router/extproc" @@ -83,7 +82,7 @@ func (h *Handler) HandleRequestHeaders(ctx context.Context, md *extproc.RequestM // stream's metadata — the per-request traceparent arrives in the // HTTP headers carried inside the ProcessingRequest payload. Extract // from there so our span links to the gateway's ingress span. - ctx = otel.GetTextMapPropagator().Extract(ctx, propagation.MapCarrier(md.Headers)) + ctx = otel.GetTextMapPropagator().Extract(ctx, md) ctx, span := otel.Tracer(extproc.ServiceName).Start(ctx, "ExtProc.RequestHeaders") defer span.End() @@ -177,8 +176,15 @@ func (h *Handler) HandleRequestHeaders(ctx context.Context, md *extproc.RequestM } func routingValue(md *extproc.RequestMetadata, header, attribute string) string { - if value := md.Header(header); value != "" { + if value := md.Attribute(attribute); value != "" && value != "-" { return value } - return md.Attribute(attribute) + // Requests reaching main_internal via a terminated CONNECT tunnel retain + // the tunnel's authority in ConnectAuthorityFilterStateAttribute. For such + // requests, the actor header must have arrived on the outer CONNECT request; + // inner tunneled headers are untrusted and must never be used for routing. + if md.Attribute(extproc.ConnectAuthorityFilterStateAttribute) != "" { + return "" + } + return md.Header(header) } diff --git a/cmd/atenet/internal/router/ingress/ingress_test.go b/cmd/atenet/internal/router/ingress/ingress_test.go index 97275062b1..35c940dba1 100644 --- a/cmd/atenet/internal/router/ingress/ingress_test.go +++ b/cmd/atenet/internal/router/ingress/ingress_test.go @@ -80,6 +80,121 @@ func TestHandleRequestHeadersAcceptsMixedCaseRoutingHeaders(t *testing.T) { } } +func TestHandleRequestHeadersRejectsDuplicateOrCommaHeaders(t *testing.T) { + tests := []struct { + name string + headers []*corev3.HeaderValue + attrs map[string]*structpb.Struct + }{ + { + name: "duplicate headers with different values", + headers: []*corev3.HeaderValue{ + {Key: atenet.TargetActorHeader, Value: "demo/a"}, + {Key: atenet.TargetActorHeader, Value: "secret/x"}, + }, + }, + { + name: "duplicate headers with identical values", + headers: []*corev3.HeaderValue{ + {Key: atenet.TargetActorHeader, Value: "demo/a"}, + {Key: atenet.TargetActorHeader, Value: "demo/a"}, + }, + }, + { + name: "single header with comma-joined value", + headers: []*corev3.HeaderValue{ + {Key: atenet.TargetActorHeader, Value: "demo/a,secret/x"}, + }, + }, + { + name: "filter state attribute with comma-joined value", + attrs: map[string]*structpb.Struct{ + "envoy.filters.http.ext_proc": { + Fields: map[string]*structpb.Value{ + extproc.TargetActorFilterStateAttribute: structpb.NewStringValue("demo/a,secret/x"), + }, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var resumeCalls int32 + clientMock := &mockClient{ + resumeFn: func(_ context.Context, _ *ateapipb.ResumeActorRequest, _ ...grpc.CallOption) (*ateapipb.ResumeActorResponse, error) { + atomic.AddInt32(&resumeCalls, 1) + return nil, errors.New("should not be called") + }, + } + h := New(clientMock, ParkedRequestConfig{}, nil) + md := extproc.NewRequestMetadata(tt.headers, tt.attrs) + + _, err := h.HandleRequestHeaders(context.Background(), md) + if err == nil { + t.Fatalf("HandleRequestHeaders() succeeded, want 404 error") + } + var reqErr *extproc.ReqError + if !errors.As(err, &reqErr) { + t.Fatalf("error = %v, want *extproc.ReqError", err) + } + if reqErr.StatusCode != int(envoy_type.StatusCode_NotFound) { + t.Errorf("StatusCode = %d, want %d", reqErr.StatusCode, envoy_type.StatusCode_NotFound) + } + if !strings.Contains(reqErr.Error(), "invalid actor reference") { + t.Errorf("error string = %q, want containing %q", reqErr.Error(), "invalid actor reference") + } + if calls := atomic.LoadInt32(&resumeCalls); calls != 0 { + t.Errorf("ResumeActor called %d times, want 0", calls) + } + }) + } +} + +func TestHandleRequestHeadersConnectAttributeOverridesInnerHeader(t *testing.T) { + const testUUID = "123e4567-e89b-12d3-a456-426614174000" + var gotActorRef string + clientMock := &mockClient{ + resumeFn: func(_ context.Context, in *ateapipb.ResumeActorRequest, _ ...grpc.CallOption) (*ateapipb.ResumeActorResponse, error) { + gotActorRef = in.GetActor().GetAtespace() + "/" + in.GetActor().GetName() + return &ateapipb.ResumeActorResponse{Actor: &ateapipb.Actor{ + Status: &ateapipb.ActorStatus{WorkerAssignment: &ateapipb.WorkerAssignment{WorkerPodIps: []string{"10.0.0.52"}}}, + }}, nil + }, + } + h := New(clientMock, ParkedRequestConfig{}, nil) + md := extproc.NewRequestMetadata( + []*corev3.HeaderValue{ + {Key: atenet.TargetActorHeader, Value: "secret/x"}, + }, + map[string]*structpb.Struct{ + "envoy.filters.http.ext_proc": { + Fields: map[string]*structpb.Value{ + extproc.TargetActorFilterStateAttribute: structpb.NewStringValue("team-a/" + testUUID), + }, + }, + }, + ) + + res, err := h.HandleRequestHeaders(context.Background(), md) + if err != nil { + t.Fatalf("HandleRequestHeaders() error = %v", err) + } + + if got, want := gotActorRef, "team-a/"+testUUID; got != want { + t.Errorf("ResumeActor called with %q, want %q", got, want) + } + + mutation := res.Response.GetResponse().GetHeaderMutation() + gotMutations := map[string]string{} + for _, headerOption := range mutation.GetSetHeaders() { + gotMutations[strings.ToLower(headerOption.Header.Key)] = string(headerOption.Header.RawValue) + } + if got, want := gotMutations[strings.ToLower(atenet.TargetActorHeader)], "team-a/"+testUUID; got != want { + t.Errorf("actor target mutation = %q, want %q", got, want) + } +} + // dynamicMetadataTarget extracts the resolved worker address // HandleRequestHeaders reports via OriginalDstMetadataKey/OriginalDstAddressKey. func dynamicMetadataTarget(dynamicMetadata *structpb.Struct) string { @@ -396,11 +511,11 @@ func TestHandleRequestHeadersUsesRetainedConnectAuthorityForPort(t *testing.T) { "envoy.filters.http.ext_proc": { Fields: map[string]*structpb.Value{ extproc.ConnectAuthorityFilterStateAttribute: structpb.NewStringValue("unrelated.example:9090"), + extproc.TargetActorFilterStateAttribute: structpb.NewStringValue("team-a/" + testUUID), }, }, } md := extproc.NewRequestMetadata([]*corev3.HeaderValue{ - {Key: atenet.TargetActorHeader, Value: "team-a/" + testUUID}, {Key: ":authority", Value: "inner.example"}, }, attrs) @@ -411,6 +526,177 @@ func TestHandleRequestHeadersUsesRetainedConnectAuthorityForPort(t *testing.T) { if got, want := dynamicMetadataPort(res.DynamicMetadata), "9090"; got != want { t.Errorf("target port = %q, want %q", got, want) } + + mutation := res.Response.GetResponse().GetHeaderMutation() + gotMutations := map[string]string{} + for _, headerOption := range mutation.GetSetHeaders() { + gotMutations[strings.ToLower(headerOption.Header.Key)] = string(headerOption.Header.RawValue) + } + if got, want := gotMutations[strings.ToLower(atenet.TargetActorHeader)], "team-a/"+testUUID; got != want { + t.Errorf("actor target mutation = %q, want %q", got, want) + } +} + +func TestHandleRequestHeadersConnectRequiresOuterTargetActor(t *testing.T) { + const testUUID = "123e4567-e89b-12d3-a456-426614174000" + tests := []struct { + name string + headers []*corev3.HeaderValue + attrs map[string]*structpb.Struct + expectErr bool + wantStatusCode int + wantActorRef string + }{ + { + name: "connect request with no outer target actor and inner header is rejected", + headers: []*corev3.HeaderValue{ + {Key: atenet.TargetActorHeader, Value: "team-a/" + testUUID}, + {Key: ":authority", Value: "inner.example"}, + }, + attrs: map[string]*structpb.Struct{ + "envoy.filters.http.ext_proc": { + Fields: map[string]*structpb.Value{ + extproc.ConnectAuthorityFilterStateAttribute: structpb.NewStringValue("unrelated.example:9090"), + }, + }, + }, + expectErr: true, + wantStatusCode: int(envoy_type.StatusCode_NotFound), + }, + { + name: "connect request with empty outer target actor and inner header is rejected", + headers: []*corev3.HeaderValue{ + {Key: atenet.TargetActorHeader, Value: "team-a/" + testUUID}, + {Key: ":authority", Value: "inner.example"}, + }, + attrs: map[string]*structpb.Struct{ + "envoy.filters.http.ext_proc": { + Fields: map[string]*structpb.Value{ + extproc.ConnectAuthorityFilterStateAttribute: structpb.NewStringValue("unrelated.example:9090"), + extproc.TargetActorFilterStateAttribute: structpb.NewStringValue(""), + }, + }, + }, + expectErr: true, + wantStatusCode: int(envoy_type.StatusCode_NotFound), + }, + { + name: "connect request with no outer target actor and no inner header is rejected", + headers: []*corev3.HeaderValue{ + {Key: ":authority", Value: "inner.example"}, + }, + attrs: map[string]*structpb.Struct{ + "envoy.filters.http.ext_proc": { + Fields: map[string]*structpb.Value{ + extproc.ConnectAuthorityFilterStateAttribute: structpb.NewStringValue("unrelated.example:9090"), + }, + }, + }, + expectErr: true, + wantStatusCode: int(envoy_type.StatusCode_NotFound), + }, + { + name: "connect request with comma-separated outer target actor is rejected", + headers: []*corev3.HeaderValue{ + {Key: ":authority", Value: "inner.example"}, + }, + attrs: map[string]*structpb.Struct{ + "envoy.filters.http.ext_proc": { + Fields: map[string]*structpb.Value{ + extproc.ConnectAuthorityFilterStateAttribute: structpb.NewStringValue("unrelated.example:9090"), + extproc.TargetActorFilterStateAttribute: structpb.NewStringValue("team-a/a,team-b/b"), + }, + }, + }, + expectErr: true, + wantStatusCode: int(envoy_type.StatusCode_NotFound), + }, + { + name: "connect request with outer target actor resolves even if inner header is omitted", + headers: []*corev3.HeaderValue{ + {Key: ":authority", Value: "inner.example"}, + }, + attrs: map[string]*structpb.Struct{ + "envoy.filters.http.ext_proc": { + Fields: map[string]*structpb.Value{ + extproc.ConnectAuthorityFilterStateAttribute: structpb.NewStringValue("unrelated.example:9090"), + extproc.TargetActorFilterStateAttribute: structpb.NewStringValue("team-a/" + testUUID), + }, + }, + }, + expectErr: false, + wantActorRef: "team-a/" + testUUID, + }, + { + name: "connect request with outer target actor X overrides inner header Y", + headers: []*corev3.HeaderValue{ + {Key: atenet.TargetActorHeader, Value: "team-b/untrusted-actor"}, + {Key: ":authority", Value: "inner.example"}, + }, + attrs: map[string]*structpb.Struct{ + "envoy.filters.http.ext_proc": { + Fields: map[string]*structpb.Value{ + extproc.ConnectAuthorityFilterStateAttribute: structpb.NewStringValue("unrelated.example:9090"), + extproc.TargetActorFilterStateAttribute: structpb.NewStringValue("team-a/" + testUUID), + }, + }, + }, + expectErr: false, + wantActorRef: "team-a/" + testUUID, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var gotActorRef string + var resumeCalls int32 + clientMock := &mockClient{ + resumeFn: func(_ context.Context, in *ateapipb.ResumeActorRequest, _ ...grpc.CallOption) (*ateapipb.ResumeActorResponse, error) { + atomic.AddInt32(&resumeCalls, 1) + gotActorRef = in.GetActor().GetAtespace() + "/" + in.GetActor().GetName() + return &ateapipb.ResumeActorResponse{Actor: &ateapipb.Actor{ + Status: &ateapipb.ActorStatus{WorkerAssignment: &ateapipb.WorkerAssignment{WorkerPodIps: []string{"10.0.0.52"}}}, + }}, nil + }, + } + + h := New(clientMock, ParkedRequestConfig{}, nil) + md := extproc.NewRequestMetadata(tt.headers, tt.attrs) + + res, err := h.HandleRequestHeaders(context.Background(), md) + if tt.expectErr { + if err == nil { + t.Fatalf("HandleRequestHeaders() succeeded, want error") + } + var reqErr *extproc.ReqError + if !errors.As(err, &reqErr) { + t.Fatalf("error = %v, want *extproc.ReqError", err) + } + if reqErr.StatusCode != tt.wantStatusCode { + t.Errorf("StatusCode = %d, want %d", reqErr.StatusCode, tt.wantStatusCode) + } + if calls := atomic.LoadInt32(&resumeCalls); calls != 0 { + t.Errorf("ResumeActor called %d times, want 0", calls) + } + return + } + + if err != nil { + t.Fatalf("HandleRequestHeaders() error = %v", err) + } + if got, want := gotActorRef, tt.wantActorRef; got != want { + t.Errorf("ResumeActor called with %q, want %q", got, want) + } + mutation := res.Response.GetResponse().GetHeaderMutation() + gotMutations := map[string]string{} + for _, headerOption := range mutation.GetSetHeaders() { + gotMutations[strings.ToLower(headerOption.Header.Key)] = string(headerOption.Header.RawValue) + } + if got, want := gotMutations[strings.ToLower(atenet.TargetActorHeader)], tt.wantActorRef; got != want { + t.Errorf("actor target mutation = %q, want %q", got, want) + } + }) + } } // TestHandleRequestHeaders_FullLotServesRunningActor pins the design guarantee diff --git a/cmd/atenet/internal/router/xds.go b/cmd/atenet/internal/router/xds.go index e134bbf82c..8f0973b198 100644 --- a/cmd/atenet/internal/router/xds.go +++ b/cmd/atenet/internal/router/xds.go @@ -918,12 +918,12 @@ func (x *XdsServer) buildMainInternalListener() *listenerv3.Listener { // main_internal can read it across the CONNECT internal-listener hop. func actorRoutingFilterStateFilter(captureAuthority bool) *hcmv3.HttpFilter { values := make([]*setfilterstatecommonv3.FilterStateValue, 0, 2) - routingFields := []struct{ key, header string }{ - {extproc.TargetActorFilterStateKey, atenet.TargetActorHeader}, + routingFields := []struct{ key, format string }{ + {extproc.TargetActorFilterStateKey, "%CEL(request.headers['" + atenet.TargetActorHeader + "'])%"}, } if captureAuthority { - routingFields = append(routingFields, struct{ key, header string }{ - extproc.ConnectAuthorityFilterStateKey, extproc.AuthorityHeader, + routingFields = append(routingFields, struct{ key, format string }{ + extproc.ConnectAuthorityFilterStateKey, "%REQ(" + extproc.AuthorityHeader + ")%", }) } for _, routingField := range routingFields { @@ -937,7 +937,7 @@ func actorRoutingFilterStateFilter(captureAuthority bool) *hcmv3.HttpFilter { Format: &corev3.SubstitutionFormatString_TextFormatSource{ TextFormatSource: &corev3.DataSource{ Specifier: &corev3.DataSource_InlineString{ - InlineString: "%REQ(" + routingField.header + ")%", + InlineString: routingField.format, }, }, }, diff --git a/cmd/atenet/internal/router/xds_test.go b/cmd/atenet/internal/router/xds_test.go index 2bd5a64192..3794155743 100644 --- a/cmd/atenet/internal/router/xds_test.go +++ b/cmd/atenet/internal/router/xds_test.go @@ -62,14 +62,14 @@ func TestActorRoutingFilterStateFilter(t *testing.T) { { name: "ordinary ingress", want: map[string]string{ - extproc.TargetActorFilterStateKey: "%REQ(ate-target-actor)%", + extproc.TargetActorFilterStateKey: "%CEL(request.headers['ate-target-actor'])%", }, }, { name: "CONNECT termination", captureAuthority: true, want: map[string]string{ - extproc.TargetActorFilterStateKey: "%REQ(ate-target-actor)%", + extproc.TargetActorFilterStateKey: "%CEL(request.headers['ate-target-actor'])%", extproc.ConnectAuthorityFilterStateKey: "%REQ(:authority)%", }, }, diff --git a/internal/atenet/headers_test.go b/internal/atenet/headers_test.go index b640ac8d1d..d984d1ad04 100644 --- a/internal/atenet/headers_test.go +++ b/internal/atenet/headers_test.go @@ -31,6 +31,8 @@ func TestParseTargetActor(t *testing.T) { {name: "empty actor", value: "team-a/", wantErr: true}, {name: "invalid atespace", value: "TEAM-A/actor-1", wantErr: true}, {name: "invalid actor", value: "team-a/ACTOR-1", wantErr: true}, + {name: "comma-joined actors", value: "demo/a,secret/x", wantErr: true}, + {name: "comma-separated with space", value: "demo/a, demo/b", wantErr: true}, } for _, tt := range tests { diff --git a/internal/atunnel/ingress.go b/internal/atunnel/ingress.go index fdd3e56d38..2a135a8025 100644 --- a/internal/atunnel/ingress.go +++ b/internal/atunnel/ingress.go @@ -532,7 +532,11 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { } func (s *Server) authorize(r *http.Request) (*activation, context.Context, func(), bool) { - ref, err := atenet.ParseTargetActor(r.Header.Get(atenet.TargetActorHeader)) + vals := r.Header.Values(atenet.TargetActorHeader) + if len(vals) != 1 { + return nil, nil, nil, false + } + ref, err := atenet.ParseTargetActor(vals[0]) if err != nil { return nil, nil, nil, false } diff --git a/internal/atunnel/ingress_test.go b/internal/atunnel/ingress_test.go index 983ff7f106..ffeeb3889e 100644 --- a/internal/atunnel/ingress_test.go +++ b/internal/atunnel/ingress_test.go @@ -290,6 +290,73 @@ func TestServeHTTP(t *testing.T) { } } +func TestServerRejectsDuplicateOrCommaTargetActorHeaders(t *testing.T) { + upstreamURL, err := url.Parse("http://actor.internal:80") + if err != nil { + t.Fatal(err) + } + + s := newTestServer(t, upstreamURL) + var upstreamCalled bool + actorTransport := roundTripFunc(func(r *http.Request) (*http.Response, error) { + upstreamCalled = true + return &http.Response{ + StatusCode: http.StatusNoContent, + Header: make(http.Header), + Body: http.NoBody, + }, nil + }) + if err := s.Activate("team-a", "actor-1", "uid-actor-1", testDial); err != nil { + t.Fatal(err) + } + setActorTransport(t, s, "team-a", "actor-1", actorTransport) + + tests := []struct { + name string + host string + headers []string + }{ + { + name: "duplicate different actors", + host: "actor-1.team-a.actors.resources.substrate.ate.dev", + headers: []string{"team-a/actor-1", "secret/x"}, + }, + { + name: "duplicate identical actors", + host: "actor-1.team-a.actors.resources.substrate.ate.dev", + headers: []string{"team-a/actor-1", "team-a/actor-1"}, + }, + { + name: "comma-joined actors", + host: "actor-1.team-a.actors.resources.substrate.ate.dev", + headers: []string{"team-a/actor-1,secret/x"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + upstreamCalled = false + req := httptest.NewRequest(http.MethodGet, "https://worker/hello", nil) + req.Host = tt.host + for _, h := range tt.headers { + req.Header.Add(atenet.TargetActorHeader, h) + } + rec := httptest.NewRecorder() + s.ServeHTTP(rec, req) + + if rec.Code != http.StatusMisdirectedRequest { + t.Fatalf("status = %d, want %d", rec.Code, http.StatusMisdirectedRequest) + } + if rec.Header().Get(StaleAssignmentHeader) != "true" { + t.Errorf("missing %s response header", StaleAssignmentHeader) + } + if upstreamCalled { + t.Errorf("upstream was dialed, expected request to be rejected") + } + }) + } +} + func TestServeHTTPHonorsTargetPortHeader(t *testing.T) { upstreamURL, err := url.Parse("http://actor.internal:80") if err != nil { diff --git a/internal/e2e/router_client.go b/internal/e2e/router_client.go index 300f88474e..9b8c68f10a 100644 --- a/internal/e2e/router_client.go +++ b/internal/e2e/router_client.go @@ -137,6 +137,14 @@ func (c *RouterClient) request(ctx context.Context, method string, actorRef reso // egress side. The caller owns the returned connection and must Close it; // the underlying port-forward is torn down by RouterClient.Close. func (c *RouterClient) Connect(ctx context.Context, actorRef resources.ActorRef, port int) (net.Conn, error) { + return c.ConnectWithHeaders(ctx, actorRef, port, http.Header{ + atenet.TargetActorHeader: []string{actorRef.String()}, + }) +} + +// ConnectWithHeaders is like Connect but allows specifying custom headers on +// the outer CONNECT request (e.g. testing repeated or missing headers). +func (c *RouterClient) ConnectWithHeaders(ctx context.Context, actorRef resources.ActorRef, port int, header http.Header) (net.Conn, error) { if err := c.ensureConnectPortForward(ctx); err != nil { return nil, err } @@ -151,9 +159,7 @@ func (c *RouterClient) Connect(ctx context.Context, actorRef resources.ActorRef, Method: http.MethodConnect, URL: &url.URL{Host: destination}, Host: destination, - Header: http.Header{ - atenet.TargetActorHeader: []string{actorRef.String()}, - }, + Header: header, } if err := req.Write(rawConn); err != nil { _ = rawConn.Close() diff --git a/internal/e2e/suites/networking/arbitraryport_test.go b/internal/e2e/suites/networking/arbitraryport_test.go index 3042a7b763..4ea12d5127 100644 --- a/internal/e2e/suites/networking/arbitraryport_test.go +++ b/internal/e2e/suites/networking/arbitraryport_test.go @@ -125,6 +125,28 @@ func TestActorArbitraryPortAccess(t *testing.T) { } t.Logf("tunneled request to an unlisted port correctly returned HTTP %d; body: %s", resp.StatusCode, body) }) + + t.Run("duplicate actor header rejected on CONNECT", func(t *testing.T) { + headers := http.Header{ + atenet.TargetActorHeader: []string{actorRef.String(), "untrusted/actor"}, + } + conn, err := router.ConnectWithHeaders(ctx, actorRef, counterExtraPort, headers) + if err != nil { + // A rejected CONNECT handshake is a valid rejection. + t.Logf("CONNECT with duplicate headers rejected: %v", err) + return + } + defer conn.Close() + + resp, body, err := requestTunneled(conn, actorRef.Name) + if err != nil { + t.Logf("tunneled request with duplicate CONNECT headers correctly failed: %v", err) + return + } + if resp.StatusCode != http.StatusNotFound { + t.Fatalf("request through CONNECT with duplicate actor headers returned %d, want 404; body: %s", resp.StatusCode, body) + } + }) } // waitForTunneledRouteReady retries an entire CONNECT exchange while the diff --git a/internal/e2e/suites/networking/networking_test.go b/internal/e2e/suites/networking/networking_test.go index b255f09900..ab0677d1cc 100644 --- a/internal/e2e/suites/networking/networking_test.go +++ b/internal/e2e/suites/networking/networking_test.go @@ -25,6 +25,7 @@ import ( "testing" "time" + "github.com/agent-substrate/substrate/internal/atenet" "github.com/agent-substrate/substrate/internal/e2e" "github.com/agent-substrate/substrate/internal/resources" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" @@ -50,6 +51,24 @@ func TestActorDirectAccess(t *testing.T) { }) t.Logf("Actor access through ingress succeeded; body: %s", body) }) + t.Run("duplicate actor header rejected on plain HTTP", func(t *testing.T) { + actorRef := resources.ActorRef{Atespace: networkingAtespace, Name: actorName} + req, err := http.NewRequestWithContext(ctx, http.MethodGet, router.BaseURL()+"/readyz", nil) + if err != nil { + t.Fatalf("creating request: %v", err) + } + req.Header.Add(atenet.TargetActorHeader, actorRef.String()) + req.Header.Add(atenet.TargetActorHeader, "untrusted/actor") + resp, err := (&http.Client{Timeout: 10 * time.Second}).Do(req) + if err != nil { + t.Fatalf("sending request with duplicate actor headers: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusNotFound { + body, _ := io.ReadAll(resp.Body) + t.Fatalf("duplicate actor header on plain HTTP returned %d, want 404; body: %s", resp.StatusCode, body) + } + }) } // egressHTTPTarget returns a copy of the origin TestActorEgress dials: From 5d0b419240cc30d54f30e4ee7dc413569952e6b3 Mon Sep 17 00:00:00 2001 From: Greg Castle Date: Fri, 9 Oct 2026 16:42:25 +0000 Subject: [PATCH 29/44] docs: add substrate security guide and principles (#2326) Add Substrate Security documentation covering security philosophy, scope, extensibility, and defensible multi-tenancy principles across identity, storage, networking, authorization, and auditability. Update README.md to reference the new security guide. --- README.md | 2 +- docs/security.md | 92 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+), 1 deletion(-) create mode 100644 docs/security.md diff --git a/README.md b/README.md index cbae8c646b..251188fed9 100644 --- a/README.md +++ b/README.md @@ -264,7 +264,7 @@ We provide several sample applications demonstrating Agent Substrate's capabilit * [Enabling man-in-the-middle (MITM) interception for Actor Egress policy](docs/egress-trust-bundle.md): Egress policies such as header injection depend on MITM interception of Actor traffic. This guide explains how an Actor should be configured to enable interception. * [Request Parking](docs/request-parking.md): How the router parks requests through transient worker-pool saturation. * [Rolling Upgrade Runbook](docs/upgrade.md): Upgrade a running substrate node by node without losing actor state. -* [Threat Model](docs/threat-model.md): Trust boundaries, assumptions, and known risks. +* [Security Guide](docs/security.md): Security overview, security principles, and threat modeling. * [Roadmap](docs/roadmap.md): Current limitations and what is planned next. * [Benchmarking Guide](benchmarking/README.md): Locust-based load tests, monitoring stack, and the orchestrated benchmark harness. diff --git a/docs/security.md b/docs/security.md new file mode 100644 index 0000000000..168bf42535 --- /dev/null +++ b/docs/security.md @@ -0,0 +1,92 @@ +# Substrate Security + +Agent Substrate executes untrusted actors at scale in a multi-tenant environment where workloads are dynamic and memory states migrate continuously across physical infrastructure. All security controls must operate within Substrate's sub-100ms cold-start and resume latency budget. This document defines the high-level security principles. More detailed security risks are documented [in the threat model](threat-model.md). + +## Security Scope + +Substrate provides the foundational base for agent security: + +- Sandboxing: built-in compute, storage, and networking separation. +- Identity: identity issuance is built-in and designed to federate with other identity systems. See the [Substrate Identity Critical User Journeys](https://docs.google.com/document/d/1seJNYQmsQYLCrYYzB9AmuxpSGSPCC3-u3tPNFsK2edc/edit?resourcekey=0-D6q8_0vqF6LPEk0BxWMTaA&tab=t.0#heading=h.6qjmtncwtihi). +- Gateway: built-in ingress and egress filtering that can do destination filtering and credential injection. + +We expect the application layer creating actors using substrate to be responsible for: + +- Handling human-to-actor authentication and authorization. +- Registering and maintaining information about agents in an agent registry. +- Maintaining agent hierarchy/state to be able to reconstruct agent lineage: who made this sub-sub-sub agent. + +## Security Extensibility + +More advanced security can be integrated using the ingress and egress extension points. This is where advanced prompt injection scanning, dynamic LLM-driven authorization decisions, data leak prevention (DLP), or cloud IAM policy enforcement can be added: + +- **Egress Extensibility**: The outbound path is a fully pluggable Policy Enforcement Point (PEP). Operators can either extend the existing egress proxy using Envoy's `ext_proc` callout mechanism to invoke external inspection and authorization services, or completely replace the egress gateway (`atenet-egress`) with an external solution. +- **Ingress Extensibility**: Ingress similarly supports `ext_proc` callouts for external authentication, authorization, and payload inspection. However, unlike egress, the ingress gateway (`atenet-router`) cannot be completely replaced from the outside: Substrate's core orchestration relies on ingress routing hooks to manage actor state, resolve worker assignments, and trigger on-demand cold starts or resume operations from snapshots to wake up agents. + +## Core Security Principles + +**Status Legend:** + +- Met: Fully implemented in the current codebase. +- In Progress: Partially implemented or under active development. +- Future: Planned architecture and roadmap capabilities not yet implemented. +- Out of Scope: Not substrate’s job, handled by a layer outside substrate. + +### Defensible Multi-Tenancy + +Substrate is designed to safely run mutually hostile, untrusted actors from disparate tenants side-by-side on the same physical host to maximize resource utilization and bin-packing efficiency. + +| Principle | Description | Status | +| :--- | :--- | :--- | +| **Always Sandboxed** | Every actor is required to run inside a strong compute sandbox. A linux container is not sufficient or supported. Our security design relies on strong containment. | Met (gVisor, CHV micro-VM) | +| **Per-Actor Storage Segregation** | Storage and persistent volume state must be strictly segregated per actor. Actors cannot traverse, enumerate, or access persistence belonging to other actors. Restoring a snapshot must re-validate the actor's current tenant authorization before execution resumes. | Met (Segregation)
Future (Per-Actor Encryption, Restore Authorization) | +| **Per-Actor Network Isolation** | Guest network stacks are segregated per actor. Lateral peer-to-peer network communication between co-located actors on the same host or across the cluster is blocked by default; inter-actor communication must route through authenticated, policy-enforced gateways. | Met (Private netns, atunnel) | +| **Delete Guarantee** | Reset, scrub, and sanitize local ephemeral volumes, cached state, and worker runtime environments between actor scheduling cycles to prevent cross-tenant data leakage. | In Progress (atelet resetActorDirs) | +| **Node Blast Radius Containment** | A compromise of an individual sandbox guest or node supervisor restricts impact to the actors and identities actively co-located on that node. | Future | +| **Atespace Separation** | The Substrate equivalent of a Kubernetes namespace that is used to group resources and partition object storage snapshot paths. It does not yet enforce runtime API authorization, inter-atespace network policies, or resource quotas, but an embedded OpenFGA relationship-based access control (ReBAC) model is actively being integrated. | In-Progress (OpenFGA) | + +Other forms of tenancy: + +- **[Possible, but not currently supported] Multiple Substrate installations inside a single Kubernetes cluster**, with nodes separated by taints/tolerations. This gets you Substrate API separation but gives up some node bin-packing efficiency. Care needs to be taken at the Kubernetes layer to keep everything separate. +- **[May be possible, but not currently supported] Attaching multiple Kubernetes clusters to a single Substrate installation**. This may be possible with some work today, but is not supported. We’re expecting a single large K8s cluster to be capable of running 100s of millions of mostly-idle agents. +- **[Out of Scope] Multi-tenancy inside an actor**. There is no meaningful security boundary inside an actor. Actors are designed to be cheap and efficient enough to run many, including one per human user, so that it’s possible to run them without handling content for multiple users inside the same actor. +- **[Not Supported] Per-Actor ingress/egress**. Actor-facing traffic proxies (`atenet-router` for ingress, `atenet-egress` for egress) are shared across all actors in the substrate installation by default. Due to the high cardinality and rapid lifecycle of actors, dedicated 1:1 proxies per actor are operationally and economically infeasible. While we may partition proxies further in the future to reduce sharing, by atespace for example, proxies will remain multi-tenant by default. + +### Identity + +| Principle | Description | Status | +| :--- | :--- | :--- | +| **Zero Ambient Credentials** | Default actor sandboxes execute completely credential-free. Actors may hold credential identifiers, encrypted credentials, or unprivileged tokens that need to be hydrated or decrypted out-of-band by Substrate infrastructure before accessing external services. | Met (no credentials inside sandbox) | +| **Distinct Workload Identity** | Every actor receives a short lifetime (1h) SPIFFE ID X.509 certificate. It is possible to distinguish an actor that was deleted and then re-created with the same name using their UID. | Met (X.509)
In-Progress (OIDC discovery) | +| **Universal Component mTLS** | All control plane and data plane components (`ate-api-server`, `atelet`, `ateom`, routers, tunnels) authenticate and communicate exclusively over mutual TLS using node- and service-attested identities. Unauthenticated traffic is rejected, exempting only local unauthenticated health/liveness probes (`/healthz`, `/readyz`). | Met (Mutual TLS)
In-Progress (Authz) | +| **Sub-Actor Provenance** | Delegated sub-actors inherit cryptographically signed lineage. Every tool invocation, data access, and egress transaction can be deterministically resolved to the root actor and execution chain. This provenance/hierarchy needs to be recorded in the system that calls substrate. | Out of Scope | + +### Storage + +| Principle | Description | Status | +| :--- | :--- | :--- | +| **Untrusted Storage** | Persistence layers are treated as untrusted environments. Remote storage is protected with encryption. Local snapshot caching on storage attached to the node is multi-tenant and protected from cross-tenant attacks with encryption and strict filesystem permissions. | Future (Storage is currently partitioned but is unencrypted) | +| **Dual-Party Snapshot Authorization** | Reading or writing snapshot storage requires dual-identity verification proving both the actor's identity and the node supervisor (`atelet`) operating on its behalf. Sandboxed guests cannot directly interact with snapshot storage. | Future | + +### Network + +| Principle | Description | Status | +| :--- | :--- | :--- | +| **No Access by Default** | Sandboxed actors and worker nodes launch with default-deny networking and have zero network or API access to control plane services (`ate-api-server`, Kubernetes API servers, PostgreSQL state store). DNS is currently allowed outbound by default, work on that is in progress. | Met (Loopback, private netns)
In-Progress (DNS filtering, see [#2075](https://github.com/agent-substrate/substrate/issues/2075)) | +| **Authenticated and Policy-Enforced Ingress** | Inbound invocations to actors terminate at authenticated edge routers (`atenet-router`), which authenticate callers and verify invocation policies before triggering on-demand actor cold-start or resume. | Met (atenet-router)
In-Progress (authentication) | +| **Fail-Closed Policy-Enforced Egress Gateway** | Outbound traffic is funneled through the multi-tenant egress proxy (`atenet-egress`), which validates actor identity, enforces destination host allowlists, executes TLS inspection or prompt filtering, and dynamically injects target credentials. If an egress policy enforcement point (PEP) is unconfigured/fails egress is dropped. Unsupported protocols are dropped, see [Egress Policy Table](https://docs.google.com/document/d/1XLCo9SMROddQzwsyEhdBNWuLYFtWWZWfYvGlIO1BcUI/edit?resourcekey=0-st2SqKp-O68nv5LeIUfA-g&tab=t.0#bookmark=id.pjnob1gppauk) for specifics. | Met (atenet-egress)
In-Progress (fail closed, UDP currently passthrough) | + +### Authorization + +| Principle | Description | Status | +| :--- | :--- | :--- | +| **Actor to Actor Authorization** | Built-in authorization policy for agent to agent communication will be added in the future. Before that exists users can install an authenticating reverse proxy, or implement authorization at the application level. | Future | +| **Atespace and Node-level containment** | Actors are grouped into logical Atespaces for declarative RBAC, resource quotas, and independent metadata/label authorization to prevent privilege escalation. | In Progress (embedded OpenFGA) | +| **Comprehensive Quotas & Anti-DoS Throttling** | The platform enforces strict limits across API call rates, memory/CPU allocations, and egress bandwidth to eliminate runaway actors and shared resource exhaustion. | Future | + +### Auditability and Response + +| Principle | Description | Status | +| :--- | :--- | :--- | +| **Structured Audit Trail** | All lifecycle transitions (start, suspend, resume, terminate), snapshot access, gateway routing events, and policy violations generate structured telemetry linked to cryptographic actor identity and parent lineage. Token log leaks are prevented with sanitization. | In-Progress (OTLP logging, shared principle)
Future (enhanced identity attribution, log leak protection) | +| **Dynamic Quarantine & Snapshot Tainting** | The control plane supports immediate revocation and network blackholing of misbehaving/compromised actors. Suspended states from compromised actors receive an immutable cryptographic taint, preventing resumption across the fleet. | Future | From 75e7f8b50a0ecb6799b4979ad2e617a229d5fa65 Mon Sep 17 00:00:00 2001 From: Julian Gutierrez Oschmann Date: Fri, 9 Oct 2026 17:02:46 +0000 Subject: [PATCH 30/44] Rename some `ExternalVolume` fields (#2404) Naming the field `volume_name` is redundant. Rename to to `name`. Also rename `actor_volumes` to `external_volumes`. Part of #1378 --- .../internal/apivalidation/actor_test.go | 26 ++--- .../internal/apivalidation/validation_test.go | 18 ++-- .../apivalidation/zz_generated.validation.go | 16 ++-- cmd/ateapi/internal/controlapi/actor.go | 4 +- .../controlapi/functionaltest/actor_test.go | 64 ++++++------- cmd/ateapi/internal/controlapi/volumes.go | 16 ++-- .../internal/controlapi/volumes_test.go | 96 +++++++++---------- .../internal/controlapi/workflow_delete.go | 8 +- .../internal/controlapi/workflow_resume.go | 10 +- .../controlapi/workflow_resume_test.go | 6 +- .../internal/controlapi/workload_spec.go | 4 +- .../internal/controlapi/workload_spec_test.go | 6 +- internal/e2e/suites/demo/demo_test.go | 6 +- pkg/proto/ateapipb/ateapi.pb.go | 31 +++--- pkg/proto/ateapipb/ateapi.proto | 8 +- tools/apitool/internal/model/model.go | 4 +- 16 files changed, 161 insertions(+), 162 deletions(-) diff --git a/cmd/ateapi/internal/apivalidation/actor_test.go b/cmd/ateapi/internal/apivalidation/actor_test.go index d1cd9fbf1f..7b2a2cafbd 100644 --- a/cmd/ateapi/internal/apivalidation/actor_test.go +++ b/cmd/ateapi/internal/apivalidation/actor_test.go @@ -640,43 +640,43 @@ func TestValidateActorUpdate(t *testing.T) { })), field.ErrorList{field.Invalid(field.NewPath("status", "snapshots").Index(0).Child("storage").Index(0).Child("fidelity"), nil, "").WithOrigin("maximum")}, }, { - "too many actor_volumes", + "too many external_volumes", validInput(), validOutput(withStatus(func(s *ateapipb.ActorStatus) { vols := make([]*ateapipb.ExternalVolume, 33) for i := range vols { - vols[i] = &ateapipb.ExternalVolume{VolumeName: fmt.Sprintf("vol-%d", i), VolumeType: "substrate.io/mock"} + vols[i] = &ateapipb.ExternalVolume{Name: fmt.Sprintf("vol-%d", i), VolumeType: "substrate.io/mock"} } - s.ActorVolumes = vols + s.ExternalVolumes = vols })), - field.ErrorList{field.TooMany(field.NewPath("status", "actor_volumes"), 33, 32).WithOrigin("maxItems")}, + field.ErrorList{field.TooMany(field.NewPath("status", "external_volumes"), 33, 32).WithOrigin("maxItems")}, }, { // Set-once fields permit the nil->set transition, so a volume added // in an update validates like one added at creation. "adding a volume on update is allowed", validInput(withStatus()), validOutput(withStatus(func(s *ateapipb.ActorStatus) { - s.ActorVolumes = []*ateapipb.ExternalVolume{{VolumeName: "vol-a", VolumeType: "substrate.io/mock"}} + s.ExternalVolumes = []*ateapipb.ExternalVolume{{Name: "vol-a", VolumeType: "substrate.io/mock"}} })), nil, }, { - "duplicate actor_volumes volume_name", + "duplicate external_volumes name", validInput(withStatus()), validOutput(withStatus(func(s *ateapipb.ActorStatus) { - s.ActorVolumes = []*ateapipb.ExternalVolume{ - {VolumeName: "vol-a", VolumeType: "substrate.io/mock"}, - {VolumeName: "vol-a", VolumeType: "substrate.io/mock"}, + s.ExternalVolumes = []*ateapipb.ExternalVolume{ + {Name: "vol-a", VolumeType: "substrate.io/mock"}, + {Name: "vol-a", VolumeType: "substrate.io/mock"}, } })), - field.ErrorList{field.Duplicate(field.NewPath("status", "actor_volumes").Index(1), nil)}, + field.ErrorList{field.Duplicate(field.NewPath("status", "external_volumes").Index(1), nil)}, }, { "provisioning transition on an existing volume is valid", validInput(withStatus(func(s *ateapipb.ActorStatus) { - s.ActorVolumes = []*ateapipb.ExternalVolume{{VolumeName: "vol-a", VolumeType: "substrate.io/mock", Status: ateapipb.ExternalVolume_STATUS_PENDING}} + s.ExternalVolumes = []*ateapipb.ExternalVolume{{Name: "vol-a", VolumeType: "substrate.io/mock", Status: ateapipb.ExternalVolume_STATUS_PENDING}} })), validOutput(withStatus(func(s *ateapipb.ActorStatus) { - s.ActorVolumes = []*ateapipb.ExternalVolume{{ - VolumeName: "vol-a", + s.ExternalVolumes = []*ateapipb.ExternalVolume{{ + Name: "vol-a", VolumeType: "substrate.io/mock", StorageVolumeId: "csi-426d29b7", Status: ateapipb.ExternalVolume_STATUS_CREATED, diff --git a/cmd/ateapi/internal/apivalidation/validation_test.go b/cmd/ateapi/internal/apivalidation/validation_test.go index afb373f233..f5ed650b13 100644 --- a/cmd/ateapi/internal/apivalidation/validation_test.go +++ b/cmd/ateapi/internal/apivalidation/validation_test.go @@ -913,7 +913,7 @@ func TestValidateTrustBundleDataSource(t *testing.T) { func validExternalVolume(mutate ...func(*ateapipb.ExternalVolume)) *ateapipb.ExternalVolume { v := &ateapipb.ExternalVolume{ - VolumeName: "my-vol", + Name: "my-vol", StorageVolumeId: "valid-storage-id", VolumeType: "mock", Status: ateapipb.ExternalVolume_STATUS_CREATED, @@ -936,12 +936,12 @@ func TestValidateExternalVolume(t *testing.T) { obj: valid(), }, { name: "missing volume name", - obj: valid(func(v *ateapipb.ExternalVolume) { v.VolumeName = "" }), - want: field.ErrorList{field.Required(field.NewPath("volume_name"), "")}, + obj: valid(func(v *ateapipb.ExternalVolume) { v.Name = "" }), + want: field.ErrorList{field.Required(field.NewPath("name"), "")}, }, { name: "invalid volume name", - obj: valid(func(v *ateapipb.ExternalVolume) { v.VolumeName = "NOT A VOLUME" }), - want: field.ErrorList{field.Invalid(field.NewPath("volume_name"), nil, "").WithOrigin("format=k8s-short-name")}, + obj: valid(func(v *ateapipb.ExternalVolume) { v.Name = "NOT A VOLUME" }), + want: field.ErrorList{field.Invalid(field.NewPath("name"), nil, "").WithOrigin("format=k8s-short-name")}, }, { name: "valid external volume with empty storage volume id", obj: valid(func(v *ateapipb.ExternalVolume) { v.StorageVolumeId = "" }), @@ -1058,10 +1058,10 @@ func TestValidateExternalVolume_Update(t *testing.T) { oldObj: valid(), newObj: valid(), }, { - name: "volume_name changed is invalid", - oldObj: valid(func(v *ateapipb.ExternalVolume) { v.VolumeName = "vol1" }), - newObj: valid(func(v *ateapipb.ExternalVolume) { v.VolumeName = "vol2" }), - want: field.ErrorList{field.Invalid(field.NewPath("volume_name"), nil, "").WithOrigin("update")}, + name: "name changed is invalid", + oldObj: valid(func(v *ateapipb.ExternalVolume) { v.Name = "vol1" }), + newObj: valid(func(v *ateapipb.ExternalVolume) { v.Name = "vol2" }), + want: field.ErrorList{field.Invalid(field.NewPath("name"), nil, "").WithOrigin("update")}, }, { name: "storage_volume_id transition from empty to non-empty is valid", oldObj: valid(func(v *ateapipb.ExternalVolume) { v.StorageVolumeId = "" }), diff --git a/cmd/ateapi/internal/apivalidation/zz_generated.validation.go b/cmd/ateapi/internal/apivalidation/zz_generated.validation.go index 691e4416e7..69c505dd3c 100644 --- a/cmd/ateapi/internal/apivalidation/zz_generated.validation.go +++ b/cmd/ateapi/internal/apivalidation/zz_generated.validation.go @@ -706,7 +706,7 @@ func Validate_ActorStatus( errs = append(errs, fn(fldPath.Child("worker_assignment"), obj.WorkerAssignment, oldVal, oldObj != nil)...) } - { // field ateapipb.ActorStatus.ActorVolumes + { // field ateapipb.ActorStatus.ExternalVolumes fn := func( fldPath *field.Path, obj, oldObj []*ateapipb.ExternalVolume, @@ -735,21 +735,21 @@ func Validate_ActorStatus( } // lists with map semantics require unique keys if e := validate.PtrSliceUnique(ctx, op, fldPath, obj, oldObj, - func(a *ateapipb.ExternalVolume, b *ateapipb.ExternalVolume) bool { return a.VolumeName == b.VolumeName }); len(e) != 0 { + func(a *ateapipb.ExternalVolume, b *ateapipb.ExternalVolume) bool { return a.Name == b.Name }); len(e) != 0 { errs = append(errs, e...) } // iterate the list and call the type's validation function if e := validate.EachPtrSliceVal(ctx, op, fldPath, obj, oldObj, - func(a *ateapipb.ExternalVolume, b *ateapipb.ExternalVolume) bool { return a.VolumeName == b.VolumeName }, ateDeepEqual, Validate_ExternalVolume); len(e) != 0 { + func(a *ateapipb.ExternalVolume, b *ateapipb.ExternalVolume) bool { return a.Name == b.Name }, ateDeepEqual, Validate_ExternalVolume); len(e) != 0 { errs = append(errs, e...) } return } oldVal := safe.Field(oldObj, func(oldObj *ateapipb.ActorStatus) []*ateapipb.ExternalVolume { - return oldObj.ActorVolumes + return oldObj.ExternalVolumes }) - errs = append(errs, fn(fldPath.Child("actor_volumes"), obj.ActorVolumes, oldVal, oldObj != nil)...) + errs = append(errs, fn(fldPath.Child("external_volumes"), obj.ExternalVolumes, oldVal, oldObj != nil)...) } { // field ateapipb.ActorStatus.Crash @@ -3556,7 +3556,7 @@ func Validate_ExternalVolume( ctx context.Context, op operation.Operation, fldPath *field.Path, obj, oldObj *ateapipb.ExternalVolume) (errs field.ErrorList) { - { // field ateapipb.ExternalVolume.VolumeName + { // field ateapipb.ExternalVolume.Name fn := func( fldPath *field.Path, obj, oldObj *string, @@ -3588,9 +3588,9 @@ func Validate_ExternalVolume( } oldVal := safe.Field(oldObj, func(oldObj *ateapipb.ExternalVolume) *string { - return &oldObj.VolumeName + return &oldObj.Name }) - errs = append(errs, fn(fldPath.Child("volume_name"), &obj.VolumeName, oldVal, oldObj != nil)...) + errs = append(errs, fn(fldPath.Child("name"), &obj.Name, oldVal, oldObj != nil)...) } { // field ateapipb.ExternalVolume.StorageVolumeId diff --git a/cmd/ateapi/internal/controlapi/actor.go b/cmd/ateapi/internal/controlapi/actor.go index 3c8c1aacdb..8ced1cb406 100644 --- a/cmd/ateapi/internal/controlapi/actor.go +++ b/cmd/ateapi/internal/controlapi/actor.go @@ -124,8 +124,8 @@ func (s *ServiceImpl) CreateActor(ctx context.Context, inActor *ateapipb.Actor) // Verify that the result is properly valid before storing it. outActor := proto.CloneOf(inActor) outActor.Status = &ateapipb.ActorStatus{ - State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, - ActorVolumes: initVols, + State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, + ExternalVolumes: initVols, } if sourceTag != nil { // The Actor starts out borrowing the tag's snapshot rather than diff --git a/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go b/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go index 6b9e95e187..25cb9e1f41 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go @@ -141,12 +141,12 @@ func TestCreateActor_WithExternalVolumes(t *testing.T) { t.Fatalf("CreateActor failed: %v", err) } - if len(createResp.GetStatus().GetActorVolumes()) != 1 { - t.Fatalf("expected 1 volume in CreateActor response, got %d", len(createResp.GetStatus().GetActorVolumes())) + if len(createResp.GetStatus().GetExternalVolumes()) != 1 { + t.Fatalf("expected 1 volume in CreateActor response, got %d", len(createResp.GetStatus().GetExternalVolumes())) } - vol := createResp.GetStatus().GetActorVolumes()[0] - if vol.GetVolumeName() != "ext-vol-1" { - t.Errorf("volume name = %q, want %q", vol.GetVolumeName(), "ext-vol-1") + vol := createResp.GetStatus().GetExternalVolumes()[0] + if vol.GetName() != "ext-vol-1" { + t.Errorf("volume name = %q, want %q", vol.GetName(), "ext-vol-1") } if vol.GetStatus() != ateapipb.ExternalVolume_STATUS_PENDING { t.Errorf("volume status = %v, want %v", vol.GetStatus(), ateapipb.ExternalVolume_STATUS_PENDING) @@ -162,11 +162,11 @@ func TestCreateActor_WithExternalVolumes(t *testing.T) { if err != nil { t.Fatalf("GetActor failed: %v", err) } - if len(getResp.GetStatus().GetActorVolumes()) != 1 { - t.Fatalf("expected 1 volume in GetActor response, got %d", len(getResp.GetStatus().GetActorVolumes())) + if len(getResp.GetStatus().GetExternalVolumes()) != 1 { + t.Fatalf("expected 1 volume in GetActor response, got %d", len(getResp.GetStatus().GetExternalVolumes())) } - if getResp.GetStatus().GetActorVolumes()[0].GetStatus() != ateapipb.ExternalVolume_STATUS_PENDING { - t.Errorf("GetActor status = %v, want %v", getResp.GetStatus().GetActorVolumes()[0].GetStatus(), ateapipb.ExternalVolume_STATUS_PENDING) + if getResp.GetStatus().GetExternalVolumes()[0].GetStatus() != ateapipb.ExternalVolume_STATUS_PENDING { + t.Errorf("GetActor status = %v, want %v", getResp.GetStatus().GetExternalVolumes()[0].GetStatus(), ateapipb.ExternalVolume_STATUS_PENDING) } } @@ -1372,9 +1372,9 @@ func TestDeleteActor_MultipleVolumeDeletionFailures(t *testing.T) { ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"}, Status: &ateapipb.ActorStatus{ State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, - ActorVolumes: []*ateapipb.ExternalVolume{ - {VolumeName: "vol1", StorageVolumeId: "storage-vol-1", Status: ateapipb.ExternalVolume_STATUS_CREATED, VolumeType: "substrate.io/mock"}, - {VolumeName: "vol2", StorageVolumeId: "storage-vol-2", Status: ateapipb.ExternalVolume_STATUS_CREATED, VolumeType: "substrate.io/mock"}, + ExternalVolumes: []*ateapipb.ExternalVolume{ + {Name: "vol1", StorageVolumeId: "storage-vol-1", Status: ateapipb.ExternalVolume_STATUS_CREATED, VolumeType: "substrate.io/mock"}, + {Name: "vol2", StorageVolumeId: "storage-vol-2", Status: ateapipb.ExternalVolume_STATUS_CREATED, VolumeType: "substrate.io/mock"}, }, }, } @@ -1448,8 +1448,8 @@ func TestDeleteActor_VolumeDeletionFailure_RetrySuccess(t *testing.T) { ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"}, Status: &ateapipb.ActorStatus{ State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, - ActorVolumes: []*ateapipb.ExternalVolume{ - {VolumeName: "vol1", StorageVolumeId: "storage-vol-1", Status: ateapipb.ExternalVolume_STATUS_CREATED, VolumeType: "substrate.io/mock"}, + ExternalVolumes: []*ateapipb.ExternalVolume{ + {Name: "vol1", StorageVolumeId: "storage-vol-1", Status: ateapipb.ExternalVolume_STATUS_CREATED, VolumeType: "substrate.io/mock"}, }, }, } @@ -1475,8 +1475,8 @@ func TestDeleteActor_VolumeDeletionFailure_RetrySuccess(t *testing.T) { if getResp.GetStatus().GetState() != ateapipb.ActorState_ACTOR_STATE_DELETING { t.Errorf("actor state = %v, want ACTOR_STATE_DELETING", getResp.GetStatus().GetState()) } - if len(getResp.GetStatus().GetActorVolumes()) != 1 || getResp.GetStatus().GetActorVolumes()[0].GetStatus() != ateapipb.ExternalVolume_STATUS_DELETING { - t.Errorf("actor volume status = %v, want STATUS_DELETING", getResp.GetStatus().GetActorVolumes()) + if len(getResp.GetStatus().GetExternalVolumes()) != 1 || getResp.GetStatus().GetExternalVolumes()[0].GetStatus() != ateapipb.ExternalVolume_STATUS_DELETING { + t.Errorf("actor volume status = %v, want STATUS_DELETING", getResp.GetStatus().GetExternalVolumes()) } // 4. Recover the CSI plugin to simulate backend recovery. @@ -1608,8 +1608,8 @@ func TestActorLifecycle_WithExternalVolumes(t *testing.T) { if createResp.GetStatus().GetState() != ateapipb.ActorState_ACTOR_STATE_SUSPENDED { t.Fatalf("expected initial state ACTOR_STATE_SUSPENDED, got %v", createResp.GetStatus().GetState()) } - if len(createResp.GetStatus().GetActorVolumes()) != 1 || createResp.GetStatus().GetActorVolumes()[0].GetStatus() != ateapipb.ExternalVolume_STATUS_PENDING { - t.Fatalf("expected 1 pending volume after CreateActor, got %v", createResp.GetStatus().GetActorVolumes()) + if len(createResp.GetStatus().GetExternalVolumes()) != 1 || createResp.GetStatus().GetExternalVolumes()[0].GetStatus() != ateapipb.ExternalVolume_STATUS_PENDING { + t.Fatalf("expected 1 pending volume after CreateActor, got %v", createResp.GetStatus().GetExternalVolumes()) } // 2. ResumeActor @@ -1622,10 +1622,10 @@ func TestActorLifecycle_WithExternalVolumes(t *testing.T) { if resumeResp.GetActor().GetStatus().GetState() != ateapipb.ActorState_ACTOR_STATE_RUNNING { t.Fatalf("expected state ACTOR_STATE_RUNNING after resume, got %v", resumeResp.GetActor().GetStatus().GetState()) } - if len(resumeResp.GetActor().GetStatus().GetActorVolumes()) != 1 || resumeResp.GetActor().GetStatus().GetActorVolumes()[0].GetStatus() != ateapipb.ExternalVolume_STATUS_CREATED { - t.Fatalf("expected 1 created volume after ResumeActor, got %v", resumeResp.GetActor().GetStatus().GetActorVolumes()) + if len(resumeResp.GetActor().GetStatus().GetExternalVolumes()) != 1 || resumeResp.GetActor().GetStatus().GetExternalVolumes()[0].GetStatus() != ateapipb.ExternalVolume_STATUS_CREATED { + t.Fatalf("expected 1 created volume after ResumeActor, got %v", resumeResp.GetActor().GetStatus().GetExternalVolumes()) } - if resumeResp.GetActor().GetStatus().GetActorVolumes()[0].GetStorageVolumeId() == "" { + if resumeResp.GetActor().GetStatus().GetExternalVolumes()[0].GetStorageVolumeId() == "" { t.Fatalf("expected non-empty storageVolumeId after ResumeActor") } @@ -1778,12 +1778,12 @@ func TestResumeActor_VolumeCreationFailure(t *testing.T) { } // Verify that succ-vol1 was updated to CREATED with a storageVolumeId, and fail-vol2 is still PENDING - if len(getResp.GetStatus().GetActorVolumes()) != 2 { - t.Fatalf("expected 2 volumes on actor, got %d", len(getResp.GetStatus().GetActorVolumes())) + if len(getResp.GetStatus().GetExternalVolumes()) != 2 { + t.Fatalf("expected 2 volumes on actor, got %d", len(getResp.GetStatus().GetExternalVolumes())) } volsByName := make(map[string]*ateapipb.ExternalVolume) - for _, v := range getResp.GetStatus().GetActorVolumes() { - volsByName[v.GetVolumeName()] = v + for _, v := range getResp.GetStatus().GetExternalVolumes() { + volsByName[v.GetName()] = v } if v1, ok := volsByName["succ-vol1"]; !ok || v1.GetStatus() != ateapipb.ExternalVolume_STATUS_CREATED || v1.GetStorageVolumeId() == "" { t.Errorf("succ-vol1 unexpected state: %v", v1) @@ -1916,8 +1916,8 @@ func TestResumeActor_VolumeCreationRetrySuccess(t *testing.T) { } volsByName := make(map[string]*ateapipb.ExternalVolume) - for _, v := range getResp.GetStatus().GetActorVolumes() { - volsByName[v.GetVolumeName()] = v + for _, v := range getResp.GetStatus().GetExternalVolumes() { + volsByName[v.GetName()] = v } if v1, ok := volsByName["succ-vol1"]; !ok || v1.GetStatus() != ateapipb.ExternalVolume_STATUS_CREATED || v1.GetStorageVolumeId() == "" { t.Errorf("succ-vol1 unexpected state after first resume: %v", v1) @@ -1944,9 +1944,9 @@ func TestResumeActor_VolumeCreationRetrySuccess(t *testing.T) { if getResp.GetStatus().GetState() != ateapipb.ActorState_ACTOR_STATE_RUNNING { t.Errorf("actor state after second resume = %v, want %v", getResp.GetStatus().GetState(), ateapipb.ActorState_ACTOR_STATE_RUNNING) } - for _, v := range getResp.GetStatus().GetActorVolumes() { + for _, v := range getResp.GetStatus().GetExternalVolumes() { if v.GetStatus() != ateapipb.ExternalVolume_STATUS_CREATED || v.GetStorageVolumeId() == "" { - t.Errorf("volume %s unexpected state after second resume: %v", v.GetVolumeName(), v) + t.Errorf("volume %s unexpected state after second resume: %v", v.GetName(), v) } } @@ -2066,10 +2066,10 @@ func TestResumeActor_VolumeAttachFailureAndRetry(t *testing.T) { if getResp.GetStatus().GetWorkerAssignment() == nil || getResp.GetStatus().GetWorkerAssignment().GetWorkerPod() != "worker-1" { t.Errorf("worker assignment = %v, want worker-1", getResp.GetStatus().GetWorkerAssignment()) } - if len(getResp.GetStatus().GetActorVolumes()) != 1 { - t.Fatalf("expected 1 volume on actor, got %d", len(getResp.GetStatus().GetActorVolumes())) + if len(getResp.GetStatus().GetExternalVolumes()) != 1 { + t.Fatalf("expected 1 volume on actor, got %d", len(getResp.GetStatus().GetExternalVolumes())) } - vol := getResp.GetStatus().GetActorVolumes()[0] + vol := getResp.GetStatus().GetExternalVolumes()[0] if vol.GetStatus() != ateapipb.ExternalVolume_STATUS_CREATED || vol.GetStorageVolumeId() == "" { t.Errorf("vol1 unexpected state after failed attach: %v", vol) } diff --git a/cmd/ateapi/internal/controlapi/volumes.go b/cmd/ateapi/internal/controlapi/volumes.go index efd961eaeb..b2f8292599 100644 --- a/cmd/ateapi/internal/controlapi/volumes.go +++ b/cmd/ateapi/internal/controlapi/volumes.go @@ -47,7 +47,7 @@ func initialActorVolumes(ctx context.Context, scLister storagev1listers.StorageC } volumes = append(volumes, &ateapipb.ExternalVolume{ - VolumeName: vol.GetName(), + Name: vol.GetName(), VolumeType: sc.Provisioner, Status: ateapipb.ExternalVolume_STATUS_PENDING, }) @@ -75,7 +75,7 @@ func createActorVolumes(ctx context.Context, registry VolumePluginRegistry, scLi currentIdx = idx var specVol *ateapipb.Volume - volName := vol.GetVolumeName() + volName := vol.GetName() for _, tVol := range template.GetVolumes() { if tVol.GetName() == volName { specVol = tVol @@ -128,7 +128,7 @@ func createActorVolumes(ctx context.Context, registry VolumePluginRegistry, scLi } resultVolumes = append(resultVolumes, &ateapipb.ExternalVolume{ - VolumeName: volName, + Name: volName, StorageVolumeId: resp.VolumeID, VolumeType: sc.Provisioner, Status: ateapipb.ExternalVolume_STATUS_CREATED, @@ -150,7 +150,7 @@ func deleteActorVolumes(ctx context.Context, registry VolumePluginRegistry, acto // If the volume hasn't been successfully created yet, it's possible // that it doesn't have a storage volume ID. In that case, fallback // to the original requested volID. - volID = actorVolumeID(actorUID, vol.GetVolumeName()) + volID = actorVolumeID(actorUID, vol.GetName()) } // TODO: Standardize volume plugin lookup and error handling across control plane // and worker plane (e.g. via a shared helper). @@ -177,7 +177,7 @@ func getMountedActorVolumes(ctx context.Context, ref *ateapipb.ObjectRef, volume // Find the corresponding volume in the ActorTemplate to check if it's mounted var matchedTemplateVol *ateapipb.Volume for _, tVol := range template.GetVolumes() { - if vol.GetVolumeName() == tVol.GetName() { + if vol.GetName() == tVol.GetName() { matchedTemplateVol = tVol break } @@ -214,9 +214,9 @@ func detachActorVolumes(ctx context.Context, registry VolumePluginRegistry, acto // in the template's containers. If the template is missing/deleted, fall back to // attempting detachment for all external volumes recorded on the actor so we do // not orphan attached disks on the worker node. - volumesToDetach := actor.GetStatus().GetActorVolumes() + volumesToDetach := actor.GetStatus().GetExternalVolumes() if template != nil { - volumesToDetach = getMountedActorVolumes(ctx, ref, actor.GetStatus().GetActorVolumes(), template) + volumesToDetach = getMountedActorVolumes(ctx, ref, actor.GetStatus().GetExternalVolumes(), template) } // Collect errors for all volumes to detach, but continue processing so we attempt to detach all volumes. var errs []error @@ -224,7 +224,7 @@ func detachActorVolumes(ctx context.Context, registry VolumePluginRegistry, acto // StorageVolumeId is only populated once the volume is provisioned. // Skip volumes that were never created (e.g. failed during PENDING state). if vol.GetStorageVolumeId() == "" { - slog.WarnContext(ctx, "Volume has no storage volume ID, skipping detach", slog.String("volume_name", vol.GetVolumeName()), slog.String("actor_id", actor.GetMetadata().GetName())) + slog.WarnContext(ctx, "Volume has no storage volume ID, skipping detach", slog.String("volume_name", vol.GetName()), slog.String("actor_id", actor.GetMetadata().GetName())) continue } slog.InfoContext(ctx, "Detaching volume from node", slog.String("volume_id", vol.GetStorageVolumeId()), slog.String("node", node)) diff --git a/cmd/ateapi/internal/controlapi/volumes_test.go b/cmd/ateapi/internal/controlapi/volumes_test.go index 13ba49bfab..5fb207c11d 100644 --- a/cmd/ateapi/internal/controlapi/volumes_test.go +++ b/cmd/ateapi/internal/controlapi/volumes_test.go @@ -69,7 +69,7 @@ func TestActorVolumesStorageClassErrors(t *testing.T) { }}, } volumes := []*ateapipb.ExternalVolume{{ - VolumeName: "data-vol", + Name: "data-vol", VolumeType: "mock-standard", Status: ateapipb.ExternalVolume_STATUS_PENDING, }} @@ -143,12 +143,12 @@ func TestInitialActorVolumes_PendingState(t *testing.T) { want := []*ateapipb.ExternalVolume{ { - VolumeName: "data-vol-1", + Name: "data-vol-1", VolumeType: "mock-standard", Status: ateapipb.ExternalVolume_STATUS_PENDING, }, { - VolumeName: "data-vol-2", + Name: "data-vol-2", VolumeType: "mock-fast", Status: ateapipb.ExternalVolume_STATUS_PENDING, }, @@ -225,34 +225,34 @@ func TestCreateActorVolumes(t *testing.T) { tmpl: multiVolTmpl, inputVolumes: []*ateapipb.ExternalVolume{ { - VolumeName: "vol1", + Name: "vol1", VolumeType: "mock-standard", Status: ateapipb.ExternalVolume_STATUS_PENDING, }, { - VolumeName: "vol2", - Status: ateapipb.ExternalVolume_STATUS_DELETING, + Name: "vol2", + Status: ateapipb.ExternalVolume_STATUS_DELETING, }, { - VolumeName: "vol3", - Status: ateapipb.ExternalVolume_STATUS_PENDING, + Name: "vol3", + Status: ateapipb.ExternalVolume_STATUS_PENDING, }, }, wantErr: true, wantRes: []*ateapipb.ExternalVolume{ { - VolumeName: "vol1", + Name: "vol1", StorageVolumeId: "mock-vol-substrate-actor-uid-123-vol1", VolumeType: "mock-standard", Status: ateapipb.ExternalVolume_STATUS_CREATED, }, { - VolumeName: "vol2", - Status: ateapipb.ExternalVolume_STATUS_DELETING, + Name: "vol2", + Status: ateapipb.ExternalVolume_STATUS_DELETING, }, { - VolumeName: "vol3", - Status: ateapipb.ExternalVolume_STATUS_PENDING, + Name: "vol3", + Status: ateapipb.ExternalVolume_STATUS_PENDING, }, }, }, @@ -261,7 +261,7 @@ func TestCreateActorVolumes(t *testing.T) { tmpl: standardTmpl, inputVolumes: []*ateapipb.ExternalVolume{ { - VolumeName: "data-vol", + Name: "data-vol", StorageVolumeId: "existing-vol-id", Status: ateapipb.ExternalVolume_STATUS_CREATED, }, @@ -269,7 +269,7 @@ func TestCreateActorVolumes(t *testing.T) { wantErr: false, wantRes: []*ateapipb.ExternalVolume{ { - VolumeName: "data-vol", + Name: "data-vol", StorageVolumeId: "existing-vol-id", Status: ateapipb.ExternalVolume_STATUS_CREATED, }, @@ -280,15 +280,15 @@ func TestCreateActorVolumes(t *testing.T) { tmpl: standardTmpl, inputVolumes: []*ateapipb.ExternalVolume{ { - VolumeName: "data-vol", - Status: ateapipb.ExternalVolume_STATUS_UNSPECIFIED, + Name: "data-vol", + Status: ateapipb.ExternalVolume_STATUS_UNSPECIFIED, }, }, wantErr: true, wantRes: []*ateapipb.ExternalVolume{ { - VolumeName: "data-vol", - Status: ateapipb.ExternalVolume_STATUS_UNSPECIFIED, + Name: "data-vol", + Status: ateapipb.ExternalVolume_STATUS_UNSPECIFIED, }, }, }, @@ -297,15 +297,15 @@ func TestCreateActorVolumes(t *testing.T) { tmpl: &ateapipb.ActorTemplate{}, inputVolumes: []*ateapipb.ExternalVolume{ { - VolumeName: "missing-vol", - Status: ateapipb.ExternalVolume_STATUS_PENDING, + Name: "missing-vol", + Status: ateapipb.ExternalVolume_STATUS_PENDING, }, }, wantErr: true, wantRes: []*ateapipb.ExternalVolume{ { - VolumeName: "missing-vol", - Status: ateapipb.ExternalVolume_STATUS_PENDING, + Name: "missing-vol", + Status: ateapipb.ExternalVolume_STATUS_PENDING, }, }, }, @@ -314,7 +314,7 @@ func TestCreateActorVolumes(t *testing.T) { tmpl: standardTmpl, inputVolumes: []*ateapipb.ExternalVolume{ { - VolumeName: "data-vol", + Name: "data-vol", VolumeType: "mock-standard", Status: ateapipb.ExternalVolume_STATUS_PENDING, }, @@ -332,7 +332,7 @@ func TestCreateActorVolumes(t *testing.T) { wantErr: false, wantRes: []*ateapipb.ExternalVolume{ { - VolumeName: "data-vol", + Name: "data-vol", StorageVolumeId: "mock-vol-substrate-actor-uid-123-data-vol", VolumeType: "mock-standard", Status: ateapipb.ExternalVolume_STATUS_CREATED, @@ -403,7 +403,7 @@ func TestDeleteActorVolumes(t *testing.T) { name: "uses storage volume ID when present", actorUID: "uid-abc", volumes: []*ateapipb.ExternalVolume{ - {VolumeName: "vol1", StorageVolumeId: "storage-vol-123", VolumeType: "mock"}, + {Name: "vol1", StorageVolumeId: "storage-vol-123", VolumeType: "mock"}, }, wantDeleted: []string{"storage-vol-123"}, wantErr: false, @@ -412,7 +412,7 @@ func TestDeleteActorVolumes(t *testing.T) { name: "falls back to actorVolumeID when storage volume ID is empty regardless of status", actorUID: "uid-abc", volumes: []*ateapipb.ExternalVolume{ - {VolumeName: "vol1", StorageVolumeId: "", Status: ateapipb.ExternalVolume_STATUS_CREATED, VolumeType: "mock"}, + {Name: "vol1", StorageVolumeId: "", Status: ateapipb.ExternalVolume_STATUS_CREATED, VolumeType: "mock"}, }, wantDeleted: []string{"substrate-uid-abc-vol1"}, wantErr: false, @@ -491,9 +491,9 @@ func TestDetachActorVolumes(t *testing.T) { Metadata: &ateapipb.ResourceMetadata{Name: "actor-1", Atespace: "default"}, Status: &ateapipb.ActorStatus{ AssignedNode: "node-1", - ActorVolumes: []*ateapipb.ExternalVolume{ - {VolumeName: "vol1", StorageVolumeId: "storage-vol-1", VolumeType: "mock"}, - {VolumeName: "vol2", StorageVolumeId: "storage-vol-2", VolumeType: "mock"}, + ExternalVolumes: []*ateapipb.ExternalVolume{ + {Name: "vol1", StorageVolumeId: "storage-vol-1", VolumeType: "mock"}, + {Name: "vol2", StorageVolumeId: "storage-vol-2", VolumeType: "mock"}, }, }, }, @@ -522,9 +522,9 @@ func TestDetachActorVolumes(t *testing.T) { Metadata: &ateapipb.ResourceMetadata{Name: "actor-1", Atespace: "default"}, Status: &ateapipb.ActorStatus{ AssignedNode: "node-1", - ActorVolumes: []*ateapipb.ExternalVolume{ - {VolumeName: "mounted-vol", StorageVolumeId: "storage-vol-mounted", VolumeType: "mock"}, - {VolumeName: "unmounted-vol", StorageVolumeId: "storage-vol-unmounted", VolumeType: "mock"}, + ExternalVolumes: []*ateapipb.ExternalVolume{ + {Name: "mounted-vol", StorageVolumeId: "storage-vol-mounted", VolumeType: "mock"}, + {Name: "unmounted-vol", StorageVolumeId: "storage-vol-unmounted", VolumeType: "mock"}, }, }, }, @@ -551,9 +551,9 @@ func TestDetachActorVolumes(t *testing.T) { Metadata: &ateapipb.ResourceMetadata{Name: "actor-1", Atespace: "default"}, Status: &ateapipb.ActorStatus{ AssignedNode: "node-1", - ActorVolumes: []*ateapipb.ExternalVolume{ - {VolumeName: "vol1", StorageVolumeId: "", VolumeType: "mock"}, - {VolumeName: "vol2", StorageVolumeId: "storage-vol-2", VolumeType: "mock"}, + ExternalVolumes: []*ateapipb.ExternalVolume{ + {Name: "vol1", StorageVolumeId: "", VolumeType: "mock"}, + {Name: "vol2", StorageVolumeId: "storage-vol-2", VolumeType: "mock"}, }, }, }, @@ -581,9 +581,9 @@ func TestDetachActorVolumes(t *testing.T) { Metadata: &ateapipb.ResourceMetadata{Name: "actor-1", Atespace: "default"}, Status: &ateapipb.ActorStatus{ AssignedNode: "node-1", - ActorVolumes: []*ateapipb.ExternalVolume{ - {VolumeName: "vol1", StorageVolumeId: "storage-vol-1", VolumeType: "mock"}, - {VolumeName: "vol2", StorageVolumeId: "storage-vol-2", VolumeType: "mock"}, + ExternalVolumes: []*ateapipb.ExternalVolume{ + {Name: "vol1", StorageVolumeId: "storage-vol-1", VolumeType: "mock"}, + {Name: "vol2", StorageVolumeId: "storage-vol-2", VolumeType: "mock"}, }, }, }, @@ -599,8 +599,8 @@ func TestDetachActorVolumes(t *testing.T) { Metadata: &ateapipb.ResourceMetadata{Name: "actor-1", Atespace: "default"}, Status: &ateapipb.ActorStatus{ AssignedNode: "node-1", - ActorVolumes: []*ateapipb.ExternalVolume{ - {VolumeName: "vol1", StorageVolumeId: "storage-vol-1", VolumeType: "mock"}, + ExternalVolumes: []*ateapipb.ExternalVolume{ + {Name: "vol1", StorageVolumeId: "storage-vol-1", VolumeType: "mock"}, }, }, }, @@ -620,9 +620,9 @@ func TestDetachActorVolumes(t *testing.T) { Metadata: &ateapipb.ResourceMetadata{Name: "actor-1", Atespace: "default"}, Status: &ateapipb.ActorStatus{ AssignedNode: "node-1", - ActorVolumes: []*ateapipb.ExternalVolume{ - {VolumeName: "vol1", StorageVolumeId: "storage-vol-1", VolumeType: "mock"}, - {VolumeName: "vol2", StorageVolumeId: "storage-vol-2", VolumeType: "mock"}, + ExternalVolumes: []*ateapipb.ExternalVolume{ + {Name: "vol1", StorageVolumeId: "storage-vol-1", VolumeType: "mock"}, + {Name: "vol2", StorageVolumeId: "storage-vol-2", VolumeType: "mock"}, }, }, }, @@ -644,8 +644,8 @@ func TestDetachActorVolumes(t *testing.T) { Metadata: &ateapipb.ResourceMetadata{Name: "actor-1", Atespace: "default"}, Status: &ateapipb.ActorStatus{ AssignedNode: "node-1", - ActorVolumes: []*ateapipb.ExternalVolume{ - {VolumeName: "vol1", StorageVolumeId: "storage-vol-1", VolumeType: "unknown-plugin"}, + ExternalVolumes: []*ateapipb.ExternalVolume{ + {Name: "vol1", StorageVolumeId: "storage-vol-1", VolumeType: "unknown-plugin"}, }, }, }, @@ -658,8 +658,8 @@ func TestDetachActorVolumes(t *testing.T) { Metadata: &ateapipb.ResourceMetadata{Name: "actor-1", Atespace: "default"}, Status: &ateapipb.ActorStatus{ AssignedNode: "", - ActorVolumes: []*ateapipb.ExternalVolume{ - {VolumeName: "vol1", StorageVolumeId: "storage-vol-1", VolumeType: "mock"}, + ExternalVolumes: []*ateapipb.ExternalVolume{ + {Name: "vol1", StorageVolumeId: "storage-vol-1", VolumeType: "mock"}, }, }, }, diff --git a/cmd/ateapi/internal/controlapi/workflow_delete.go b/cmd/ateapi/internal/controlapi/workflow_delete.go index 5f98de04f9..759facf0c4 100644 --- a/cmd/ateapi/internal/controlapi/workflow_delete.go +++ b/cmd/ateapi/internal/controlapi/workflow_delete.go @@ -173,12 +173,12 @@ func (w *ActorWorkflow) ensureAteletTerminated(ctx context.Context, actorRef res slog.String("templateAtespace", actor.GetActorTemplate().GetAtespace()), slog.String("templateName", actor.GetActorTemplate().GetName())) workloadSpec = &ateletpb.WorkloadSpec{} - for _, vol := range actor.GetStatus().GetActorVolumes() { + for _, vol := range actor.GetStatus().GetExternalVolumes() { // StorageVolumeId is only populated once the volume is provisioned. // Skip volumes that were never created (e.g. failed during PENDING state). if vol.GetStorageVolumeId() != "" { workloadSpec.Volumes = append(workloadSpec.Volumes, &ateletpb.Volume{ - Name: vol.GetVolumeName(), + Name: vol.GetName(), Source: &ateletpb.Volume_External{ External: &ateletpb.ExternalVolumeSource{ StorageVolumeId: vol.GetStorageVolumeId(), @@ -333,7 +333,7 @@ func (w *ActorWorkflow) ensureMarkedDeleting(ctx context.Context, actorRef resou storedActor, err := w.store.UpdateActor(ctx, actorRef, store.PreconditionFrom(actor), func(toUpdate *ateapipb.Actor) error { toUpdate.Status.State = ateapipb.ActorState_ACTOR_STATE_DELETING - for _, vol := range toUpdate.GetStatus().GetActorVolumes() { + for _, vol := range toUpdate.GetStatus().GetExternalVolumes() { vol.Status = ateapipb.ExternalVolume_STATUS_DELETING } return nil @@ -359,7 +359,7 @@ func (w *ActorWorkflow) ensureVolumesDeleted(ctx context.Context, actor *ateapip return apierror.FailedPrecondition("DeleteVolumes prerequisite not met for Actor: %s (got: %v, want %s)", actor.GetMetadata().GetName(), st, ateapipb.ActorState_ACTOR_STATE_DELETING) } - if err := deleteActorVolumes(ctx, w.pluginRegistry, actor.GetMetadata().GetUid(), actor.GetStatus().GetActorVolumes()); err != nil { + if err := deleteActorVolumes(ctx, w.pluginRegistry, actor.GetMetadata().GetUid(), actor.GetStatus().GetExternalVolumes()); err != nil { return apierror.Internal("while deleting actor volumes: %v", err) } return nil diff --git a/cmd/ateapi/internal/controlapi/workflow_resume.go b/cmd/ateapi/internal/controlapi/workflow_resume.go index 54b483b1ee..f9bd159582 100644 --- a/cmd/ateapi/internal/controlapi/workflow_resume.go +++ b/cmd/ateapi/internal/controlapi/workflow_resume.go @@ -193,7 +193,7 @@ func (w *ActorWorkflow) ensureVolumesCreated(ctx context.Context, actorRef resou defer func() { err = done(err) }() pending := false - for _, vol := range actor.GetStatus().GetActorVolumes() { + for _, vol := range actor.GetStatus().GetExternalVolumes() { if vol.GetStatus() == ateapipb.ExternalVolume_STATUS_PENDING { pending = true break @@ -204,12 +204,12 @@ func (w *ActorWorkflow) ensureVolumesCreated(ctx context.Context, actorRef resou return actor, nil } - volumes, createErr := createActorVolumes(ctx, w.pluginRegistry, w.storageClassLister, actor.GetMetadata().GetUid(), actorTemplate, actor.GetStatus().GetActorVolumes()) + volumes, createErr := createActorVolumes(ctx, w.pluginRegistry, w.storageClassLister, actor.GetMetadata().GetUid(), actorTemplate, actor.GetStatus().GetExternalVolumes()) // createActorVolumes reports the state it got to even when it fails, so both // paths persist the same field. updatePrecondition := store.PreconditionFrom(actor) persistVolumes := func(toUpdate *ateapipb.Actor) error { - toUpdate.Status.ActorVolumes = volumes + toUpdate.Status.ExternalVolumes = volumes return nil } if createErr != nil { @@ -640,7 +640,7 @@ func (w *ActorWorkflow) ensureVolumesAttached(ctx context.Context, actor *ateapi ref := &ateapipb.ObjectRef{Atespace: actor.GetMetadata().GetAtespace(), Name: actor.GetMetadata().GetName()} volumePublishContexts := make(map[string]map[string]string) - for _, vol := range getMountedActorVolumes(ctx, ref, actor.GetStatus().GetActorVolumes(), actorTemplate) { + for _, vol := range getMountedActorVolumes(ctx, ref, actor.GetStatus().GetExternalVolumes(), actorTemplate) { slog.InfoContext(ctx, "Attaching volume to node", slog.String("volume_id", vol.GetStorageVolumeId()), slog.String("node", node)) plugin, err := w.pluginRegistry.GetPlugin(ctx, vol.GetVolumeType()) if err != nil { @@ -651,7 +651,7 @@ func (w *ActorWorkflow) ensureVolumesAttached(ctx context.Context, actor *ateapi return nil, fmt.Errorf("failed to attach volume %q to node %q: %w", vol.GetStorageVolumeId(), node, err) } if len(resp.PublishContext) > 0 { - volumePublishContexts[vol.GetVolumeName()] = resp.PublishContext + volumePublishContexts[vol.GetName()] = resp.PublishContext } } return volumePublishContexts, nil diff --git a/cmd/ateapi/internal/controlapi/workflow_resume_test.go b/cmd/ateapi/internal/controlapi/workflow_resume_test.go index 9e14dad80c..02a6030fa5 100644 --- a/cmd/ateapi/internal/controlapi/workflow_resume_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_resume_test.go @@ -1549,9 +1549,9 @@ func TestEnsureVolumesAttached_ReturnsPublishContext(t *testing.T) { Metadata: &ateapipb.ResourceMetadata{Atespace: "team-a", Name: "id1"}, Status: &ateapipb.ActorStatus{ State: ateapipb.ActorState_ACTOR_STATE_RESUMING, - ActorVolumes: []*ateapipb.ExternalVolume{ - {VolumeName: "mounted", StorageVolumeId: "storage-mounted", VolumeType: "mock", Status: ateapipb.ExternalVolume_STATUS_CREATED}, - {VolumeName: "unmounted", StorageVolumeId: "storage-unmounted", VolumeType: "mock", Status: ateapipb.ExternalVolume_STATUS_CREATED}, + ExternalVolumes: []*ateapipb.ExternalVolume{ + {Name: "mounted", StorageVolumeId: "storage-mounted", VolumeType: "mock", Status: ateapipb.ExternalVolume_STATUS_CREATED}, + {Name: "unmounted", StorageVolumeId: "storage-unmounted", VolumeType: "mock", Status: ateapipb.ExternalVolume_STATUS_CREATED}, }, }, }) diff --git a/cmd/ateapi/internal/controlapi/workload_spec.go b/cmd/ateapi/internal/controlapi/workload_spec.go index a8c934195a..2c666994bf 100644 --- a/cmd/ateapi/internal/controlapi/workload_spec.go +++ b/cmd/ateapi/internal/controlapi/workload_spec.go @@ -175,8 +175,8 @@ func appendExternalVolumes(workloadSpec *ateletpb.WorkloadSpec, template *ateapi var storageVolID string var volType string var volCtx map[string]string - for _, dbVol := range actor.GetStatus().GetActorVolumes() { - if dbVol.GetVolumeName() == vol.GetName() { + for _, dbVol := range actor.GetStatus().GetExternalVolumes() { + if dbVol.GetName() == vol.GetName() { storageVolID = dbVol.GetStorageVolumeId() volType = dbVol.GetVolumeType() volCtx = dbVol.GetVolumeContext() diff --git a/cmd/ateapi/internal/controlapi/workload_spec_test.go b/cmd/ateapi/internal/controlapi/workload_spec_test.go index ec9fa9a0b6..b39a90bbfd 100644 --- a/cmd/ateapi/internal/controlapi/workload_spec_test.go +++ b/cmd/ateapi/internal/controlapi/workload_spec_test.go @@ -421,9 +421,9 @@ func TestAppendExternalVolumes(t *testing.T) { Name: "actor-123", }, Status: &ateapipb.ActorStatus{ - ActorVolumes: []*ateapipb.ExternalVolume{ + ExternalVolumes: []*ateapipb.ExternalVolume{ { - VolumeName: "vol-1", + Name: "vol-1", StorageVolumeId: "vol-gce-pd-123", VolumeType: "pd-standard", VolumeContext: map[string]string{"foo": "bar"}, @@ -480,7 +480,7 @@ func TestAppendExternalVolumes(t *testing.T) { Atespace: "space-abc", Name: "actor-123", }, - Status: &ateapipb.ActorStatus{ActorVolumes: []*ateapipb.ExternalVolume{}}, + Status: &ateapipb.ActorStatus{ExternalVolumes: []*ateapipb.ExternalVolume{}}, } if err := appendExternalVolumes(&ateletpb.WorkloadSpec{}, template, missingActor, nil); err == nil { t.Errorf("appendExternalVolumes expected error for missing volume, got nil") diff --git a/internal/e2e/suites/demo/demo_test.go b/internal/e2e/suites/demo/demo_test.go index 9fdfe062a8..2900689c6c 100644 --- a/internal/e2e/suites/demo/demo_test.go +++ b/internal/e2e/suites/demo/demo_test.go @@ -384,12 +384,12 @@ func TestDeleteActorAnyStateWithExternalVolume(t *testing.T) { if err != nil { t.Fatalf("failed to get actor: %v", err) } - if len(actor.GetStatus().GetActorVolumes()) == 0 { + if len(actor.GetStatus().GetExternalVolumes()) == 0 { t.Fatalf("expected actor to have volumes, got 0") } - for _, vol := range actor.GetStatus().GetActorVolumes() { + for _, vol := range actor.GetStatus().GetExternalVolumes() { if vol.Status != ateapipb.ExternalVolume_STATUS_CREATED { - t.Fatalf("expected volume %q to be CREATED, got %s", vol.VolumeName, vol.Status) + t.Fatalf("expected volume %q to be CREATED, got %s", vol.Name, vol.Status) } } diff --git a/pkg/proto/ateapipb/ateapi.pb.go b/pkg/proto/ateapipb/ateapi.pb.go index b87d5d48d5..02e7410462 100644 --- a/pkg/proto/ateapipb/ateapi.pb.go +++ b/pkg/proto/ateapipb/ateapi.pb.go @@ -1078,7 +1078,7 @@ type ExternalVolume struct { // +k8s:update=NoModify // +k8s:update=NoUnset # set-once; immutable would reject the create // ratchet's nil->set when the final object is validated as an update - VolumeName string `protobuf:"bytes,1,opt,name=volume_name,json=volumeName,proto3" json:"volume_name,omitempty"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` // The globally unique volume_id returned from the storage system. // This will be initially empty during volume creation. Its format is the // storage system's own, so it is only bounded and checked for control @@ -1099,7 +1099,7 @@ type ExternalVolume struct { // +k8s:required // +k8s:maxLength=253 // +k8s:update=NoModify - // +k8s:update=NoUnset # set-once, like volume_name above + // +k8s:update=NoUnset # set-once, like name above // +k8s:customValidation VolumeType string `protobuf:"bytes,3,opt,name=volume_type,json=volumeType,proto3" json:"volume_type,omitempty"` // +k8s:optional @@ -1150,9 +1150,9 @@ func (*ExternalVolume) Descriptor() ([]byte, []int) { return file_ateapi_proto_rawDescGZIP(), []int{6} } -func (x *ExternalVolume) GetVolumeName() string { +func (x *ExternalVolume) GetName() string { if x != nil { - return x.VolumeName + return x.Name } return "" } @@ -2051,8 +2051,8 @@ type ActorStatus struct { // +k8s:optional // +k8s:maxItems=32 # matches the template's volumes bound // +k8s:listType=map - // +k8s:listMapKey=volume_name - ActorVolumes []*ExternalVolume `protobuf:"bytes,7,rep,name=actor_volumes,json=actorVolumes,proto3" json:"actor_volumes,omitempty"` + // +k8s:listMapKey=name + ExternalVolumes []*ExternalVolume `protobuf:"bytes,7,rep,name=external_volumes,json=externalVolumes,proto3" json:"external_volumes,omitempty"` // crash records why and when the Actor entered CRASHED. It is set with the // CRASHED state and cleared when a revert returns the Actor to SUSPENDED. // @@ -2131,9 +2131,9 @@ func (x *ActorStatus) GetWorkerAssignment() *WorkerAssignment { return nil } -func (x *ActorStatus) GetActorVolumes() []*ExternalVolume { +func (x *ActorStatus) GetExternalVolumes() []*ExternalVolume { if x != nil { - return x.ActorVolumes + return x.ExternalVolumes } return nil } @@ -8351,10 +8351,9 @@ const file_ateapi_proto_rawDesc = "" + "\vcreate_time\x18\x05 \x01(\v2\x1a.google.protobuf.TimestampR\n" + "createTime\x12;\n" + "\vupdate_time\x18\x06 \x01(\v2\x1a.google.protobuf.TimestampR\n" + - "updateTime\"\xa8\x03\n" + - "\x0eExternalVolume\x12\x1f\n" + - "\vvolume_name\x18\x01 \x01(\tR\n" + - "volumeName\x12*\n" + + "updateTime\"\x9b\x03\n" + + "\x0eExternalVolume\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x12*\n" + "\x11storage_volume_id\x18\x02 \x01(\tR\x0fstorageVolumeId\x12\x1f\n" + "\vvolume_type\x18\x03 \x01(\tR\n" + "volumeType\x125\n" + @@ -8408,11 +8407,11 @@ const file_ateapi_proto_rawDesc = "" + "\tactor_jwt\x18\x04 \x01(\v2\x16.ateapi.ActorJWTSourceR\bactorJwt\"]\n" + "\x0eActorJWTSource\x12\x1c\n" + "\taudiences\x18\x01 \x03(\tR\taudiences\x12-\n" + - "\x12expiration_seconds\x18\x02 \x01(\x03R\x11expirationSeconds\"\xf4\x02\n" + + "\x12expiration_seconds\x18\x02 \x01(\x03R\x11expirationSeconds\"\xfa\x02\n" + "\vActorStatus\x12(\n" + "\x05state\x18\x01 \x01(\x0e2\x12.ateapi.ActorStateR\x05state\x12E\n" + - "\x11worker_assignment\x18\x02 \x01(\v2\x18.ateapi.WorkerAssignmentR\x10workerAssignment\x12;\n" + - "\ractor_volumes\x18\a \x03(\v2\x16.ateapi.ExternalVolumeR\factorVolumes\x12(\n" + + "\x11worker_assignment\x18\x02 \x01(\v2\x18.ateapi.WorkerAssignmentR\x10workerAssignment\x12A\n" + + "\x10external_volumes\x18\a \x03(\v2\x16.ateapi.ExternalVolumeR\x0fexternalVolumes\x12(\n" + "\x05crash\x18\t \x01(\v2\x12.ateapi.ActorCrashR\x05crash\x12#\n" + "\rassigned_node\x18\n" + " \x01(\tR\fassignedNode\x12.\n" + @@ -9022,7 +9021,7 @@ var file_ateapi_proto_depIdxs = []int32{ 27, // 29: ateapi.CredentialHeader.actor_jwt:type_name -> ateapi.ActorJWTSource 5, // 30: ateapi.ActorStatus.state:type_name -> ateapi.ActorState 30, // 31: ateapi.ActorStatus.worker_assignment:type_name -> ateapi.WorkerAssignment - 16, // 32: ateapi.ActorStatus.actor_volumes:type_name -> ateapi.ExternalVolume + 16, // 32: ateapi.ActorStatus.external_volumes:type_name -> ateapi.ExternalVolume 29, // 33: ateapi.ActorStatus.crash:type_name -> ateapi.ActorCrash 13, // 34: ateapi.ActorStatus.snapshots:type_name -> ateapi.Snapshot 130, // 35: ateapi.ActorCrash.crash_time:type_name -> google.protobuf.Timestamp diff --git a/pkg/proto/ateapipb/ateapi.proto b/pkg/proto/ateapipb/ateapi.proto index 8e82a4e7ac..ca0195c8d7 100644 --- a/pkg/proto/ateapipb/ateapi.proto +++ b/pkg/proto/ateapipb/ateapi.proto @@ -407,7 +407,7 @@ message ExternalVolume { // +k8s:update=NoModify // +k8s:update=NoUnset # set-once; immutable would reject the create // ratchet's nil->set when the final object is validated as an update - string volume_name = 1; + string name = 1; // The globally unique volume_id returned from the storage system. // This will be initially empty during volume creation. Its format is the @@ -430,7 +430,7 @@ message ExternalVolume { // +k8s:required // +k8s:maxLength=253 // +k8s:update=NoModify - // +k8s:update=NoUnset # set-once, like volume_name above + // +k8s:update=NoUnset # set-once, like name above // +k8s:customValidation string volume_type = 3; @@ -808,8 +808,8 @@ message ActorStatus { // +k8s:optional // +k8s:maxItems=32 # matches the template's volumes bound // +k8s:listType=map - // +k8s:listMapKey=volume_name - repeated ExternalVolume actor_volumes = 7; + // +k8s:listMapKey=name + repeated ExternalVolume external_volumes = 7; // crash records why and when the Actor entered CRASHED. It is set with the // CRASHED state and cleared when a revert returns the Actor to SUSPENDED. diff --git a/tools/apitool/internal/model/model.go b/tools/apitool/internal/model/model.go index 32c5dff669..7a664e341e 100644 --- a/tools/apitool/internal/model/model.go +++ b/tools/apitool/internal/model/model.go @@ -132,13 +132,13 @@ type Field struct { // Repeated is true for a `repeated` field, not a map - see // MapValueKind for those. For example, true for - // `repeated ExternalVolume actor_volumes = 7;` on ActorStatus; false + // `repeated ExternalVolume external_volumes = 7;` on ActorStatus; false // for ResourceMetadata.atespace. Repeated bool // TypeDisplay is always a ready-to-render string. For example, "string" // for ResourceMetadata.name, "int64" for ResourceMetadata.version, - // "repeated ExternalVolume" for ActorStatus.actor_volumes, + // "repeated ExternalVolume" for ActorStatus.external_volumes, // "map" for SandboxAssets.assets, "ActorState" for // ActorStatus.state, or "google.protobuf.Timestamp" for // ResourceMetadata.create_time. From fd00c6c81eafe576dcc2b8a370fa6d8aaf9fd443 Mon Sep 17 00:00:00 2001 From: Yu YI Date: Fri, 9 Oct 2026 17:31:56 +0000 Subject: [PATCH 31/44] kubectl-ate: add --source to select container or lifecycle log lines (#2337) Part of https://github.com/agent-substrate/substrate/issues/294. Takes over the half that #1067 left open and supersedes the stale #311. ## Summary `kubectl ate logs actors` gains `--source` to pick a class of lines from an actor's stream: | Flag | Shows | |---|---| | `--source=all` (default) | everything, unchanged | | `--source=containers` | every container's output with the lifecycle events removed | | `--source=lifecycle` | only the lifecycle events (`Actor starting/started`, `checkpointing/checkpointed`, `restoring/restored`, `terminated`) | | `-c NAME` | unchanged; implies container output, so it combines with `--source=containers` and is rejected with `--source=lifecycle` | The selection is client-side, as the issue intended: lifecycle events are the lines ateom emits with no `ate.actor.container.name` label (`ActorLogLabels` omits the key when the container name is empty). Named "lifecycle" rather than "supervisor" or "ateom" because the code and docs already use that word for these events and it leaks no implementation detail. ## Testing **Unit** (`go test ./cmd/kubectl-ate/...`): new filter table cases for every source, `TestNewLogLineFilter` for the flag validation, and `TestLogsActorRunner_Run_OneShot_SourceFilter` covering each source against an interleaved stream plus the two rejected combinations, which must fail before any logs are fetched. **End to end** on a kind cluster (`kind-ate-dev`) against the counter demo actor, using the plugin built from this branch. One-shot. Lines starting with `#` are annotations added for review; `$` lines are the commands as run. Two commands are negative cases and are **expected** to exit 1. ``` # The main commit this branch was built on; the plugin binary under test was built from the branch with make build-atectl. $ git rev-parse --short HEAD df788825 # Precondition: the actor is running, so its current worker's log stream is available. $ kubectl ate get actors -a ate-demo-counter ATESPACE NAME TEMPLATE STATE WORKER POD WORKER IPS VERSION AGE ate-demo-counter learner-one ate-demo-counter/counter ACTOR_STATE_RUNNING ate-demo-counter/counter-79b587778f-8lh4h 10.244.0.22 19 14h # Expected: only lifecycle events, no container output. Each suspend/resume cycle on this worker shows as a checkpointing/checkpointed or restoring/restored pair. $ kubectl ate logs actors learner-one -a ate-demo-counter --source=lifecycle {"time":"2026-10-07T21:31:23.367965976Z","message":"Actor restoring","span_id":"31bf9ef08d13d4d6","trace_flags":"01","trace_id":"61eb2ac56531db2b0308c31c564e6379"} {"time":"2026-10-07T21:31:23.550431306Z","message":"Actor restored","span_id":"31bf9ef08d13d4d6","trace_flags":"01","trace_id":"61eb2ac56531db2b0308c31c564e6379"} {"time":"2026-10-07T21:31:48.648599369Z","message":"Actor checkpointing","span_id":"c475c334c8df2dc4","trace_flags":"01","trace_id":"e617b6130fcc28674cc9899ec702e596"} {"time":"2026-10-07T21:31:48.791435032Z","message":"Actor checkpointed","span_id":"c475c334c8df2dc4","trace_flags":"01","trace_id":"e617b6130fcc28674cc9899ec702e596"} {"time":"2026-10-07T21:31:59.772566601Z","message":"Actor restoring","span_id":"a68c5afa10e86252","trace_flags":"01","trace_id":"b387404020eaca71d604b8f0c81d43c1"} {"time":"2026-10-07T21:31:59.96129419Z","message":"Actor restored","span_id":"a68c5afa10e86252","trace_flags":"01","trace_id":"b387404020eaca71d604b8f0c81d43c1"} {"time":"2026-10-07T21:38:25.962753328Z","message":"Actor checkpointing","span_id":"e181224bf8792b12","trace_flags":"01","trace_id":"52963b1666af63486d765bc1527fc8bc"} {"time":"2026-10-07T21:38:26.096296577Z","message":"Actor checkpointed","span_id":"e181224bf8792b12","trace_flags":"01","trace_id":"52963b1666af63486d765bc1527fc8bc"} {"time":"2026-10-07T21:43:14.718178039Z","message":"Actor restoring","span_id":"718a8f92f8735ed3","trace_flags":"00","trace_id":"42123dfac30d07fa86164199ecdbf0ac"} {"time":"2026-10-07T21:43:14.920089778Z","message":"Actor restored","span_id":"718a8f92f8735ed3","trace_flags":"00","trace_id":"42123dfac30d07fa86164199ecdbf0ac"} {"time":"2026-10-07T21:44:04.990909876Z","message":"Actor checkpointing","span_id":"107b6f12dd7f0e8b","trace_flags":"01","trace_id":"0bb09c29559dd8cfedb20f44a24bda4a"} {"time":"2026-10-07T21:44:05.134534519Z","message":"Actor checkpointed","span_id":"107b6f12dd7f0e8b","trace_flags":"01","trace_id":"0bb09c29559dd8cfedb20f44a24bda4a"} {"time":"2026-10-07T21:44:07.73397801Z","message":"Actor restoring","span_id":"7d062e8b5207afe9","trace_flags":"00","trace_id":"42f6d2926dd56687c222b1de832a072e"} {"time":"2026-10-07T21:44:07.978274791Z","message":"Actor restored","span_id":"7d062e8b5207afe9","trace_flags":"00","trace_id":"42f6d2926dd56687c222b1de832a072e"} {"time":"2026-10-07T21:44:11.552735135Z","message":"Actor checkpointing","span_id":"86954a4e96fbf250","trace_flags":"01","trace_id":"fb7199b8ee3491dcf561a25a8f60b1df"} {"time":"2026-10-07T21:44:11.69238189Z","message":"Actor checkpointed","span_id":"86954a4e96fbf250","trace_flags":"01","trace_id":"fb7199b8ee3491dcf561a25a8f60b1df"} {"time":"2026-10-07T21:44:13.26693018Z","message":"Actor restoring","span_id":"a896d4c30c7b664a","trace_flags":"00","trace_id":"607e6dbd891d7faa115e061dc4f473dc"} {"time":"2026-10-07T21:44:13.516304563Z","message":"Actor restored","span_id":"a896d4c30c7b664a","trace_flags":"00","trace_id":"607e6dbd891d7faa115e061dc4f473dc"} [exit 0] # Expected: default output is unchanged (every line). Baseline for the two counts below. $ kubectl ate logs actors learner-one -a ate-demo-counter | wc -l 5484 # Expected: baseline minus the lifecycle events. $ kubectl ate logs actors learner-one -a ate-demo-counter --source=containers | wc -l 5466 # Expected: 0, i.e. no lifecycle event survives --source=containers. $ kubectl ate logs actors learner-one -a ate-demo-counter --source=containers | grep -c '"message":"Actor ' 0 # Expected: containers + lifecycle = baseline (5466 + 18 = 5484). $ kubectl ate logs actors learner-one -a ate-demo-counter --source=lifecycle | wc -l 18 # Expected: same as --source=containers, since this actor has a single container (off by one only because the actor logs a line every 10s). $ kubectl ate logs actors learner-one -a ate-demo-counter -c counter | wc -l 5467 # Expected: ordinary application output, unchanged in shape. $ kubectl ate logs actors learner-one -a ate-demo-counter --source=containers | tail -3 {"time":"2026-10-08T12:22:35.319622976Z","count":5322,"fshash":"dorEZDUO6NCvAZmlU346thAgz9cSxuVYFZ2yOLq8rqg","level":"INFO","msg":"Count"} {"time":"2026-10-08T12:22:45.322833947Z","count":5323,"fshash":"dorEZDUO6NCvAZmlU346thAgz9cSxuVYFZ2yOLq8rqg","level":"INFO","msg":"Count"} {"time":"2026-10-08T12:22:55.316121067Z","count":5324,"fshash":"dorEZDUO6NCvAZmlU346thAgz9cSxuVYFZ2yOLq8rqg","level":"INFO","msg":"Count"} # NEGATIVE CASE, error is expected: -c and --source=lifecycle can never match the same line, so the CLI rejects the combination with exit 1 before fetching any logs. $ kubectl ate logs actors learner-one -a ate-demo-counter -c counter --source=lifecycle Error: --container cannot be combined with --source=lifecycle: lifecycle events are not emitted by a container [exit 1] # NEGATIVE CASE, error is expected: an unknown --source value is rejected with exit 1 and the list of valid values. $ kubectl ate logs actors learner-one -a ate-demo-counter --source=supervisor Error: invalid --source "supervisor": must be one of all, containers, lifecycle [exit 1] # Expected: the new flag is documented in --help next to --container. $ kubectl ate logs actors --help | grep -A1 -E -- "--source|--container" -c, --container string Show only logs from this container. -f, --follow Specify if the logs should be streamed. -- --source string Which lines to show: all, containers (every container's output, no lifecycle events), or lifecycle (only the actor's lifecycle events). (default "all") ``` Follow mode, with a suspend and a resume issued from another shell while `-f --source=lifecycle` was streaming. The stream shows only the lifecycle pair for each transition and reconnects once the actor is running again (on the same worker pod here, since the kind cluster has one). ``` $ kubectl ate logs actors learner-one -a ate-demo-counter -f --source=lifecycle # started, then suspend + resume in another shell Actor is currently running on pod ate-demo-counter/counter-79b587778f-8lh4h {"time":"2026-10-08T12:25:08.096184297Z","message":"Actor checkpointing","span_id":"b8dd6fc306a3ccf9","trace_flags":"01","trace_id":"45cae2a35d684ba8f1d6305c09333eb3"} {"time":"2026-10-08T12:25:08.291926175Z","message":"Actor checkpointed","span_id":"b8dd6fc306a3ccf9","trace_flags":"01","trace_id":"45cae2a35d684ba8f1d6305c09333eb3"} {"time":"2026-10-08T12:25:08.096184297Z","message":"Actor checkpointing","span_id":"b8dd6fc306a3ccf9","trace_flags":"01","trace_id":"45cae2a35d684ba8f1d6305c09333eb3"} {"time":"2026-10-08T12:25:08.291926175Z","message":"Actor checkpointed","span_id":"b8dd6fc306a3ccf9","trace_flags":"01","trace_id":"45cae2a35d684ba8f1d6305c09333eb3"} {"time":"2026-10-08T12:25:33.660662951Z","message":"Actor restoring","span_id":"d01fcf2c7f734ff2","trace_flags":"01","trace_id":"dfae910c43670918c5ab1cb3354863e3"} {"time":"2026-10-08T12:25:33.968364046Z","message":"Actor restored","span_id":"d01fcf2c7f734ff2","trace_flags":"01","trace_id":"dfae910c43670918c5ab1cb3354863e3"} [follow exited 124] ``` ``` [12:25:07Z] $ kubectl ate suspend actor learner-one -a ate-demo-counter ATESPACE NAME TEMPLATE STATE WORKER POD WORKER IPS VERSION AGE ate-demo-counter learner-one ate-demo-counter/counter ACTOR_STATE_SUSPENDED 21 14h [12:25:33Z] $ kubectl ate resume actor learner-one -a ate-demo-counter ATESPACE NAME TEMPLATE STATE WORKER POD WORKER IPS VERSION AGE ate-demo-counter learner-one ate-demo-counter/counter ACTOR_STATE_RUNNING ate-demo-counter/counter-79b587778f-8lh4h 10.244.0.22 23 14h [12:26:04Z] $ kubectl ate get actors -a ate-demo-counter ATESPACE NAME TEMPLATE STATE WORKER POD WORKER IPS VERSION AGE ate-demo-counter learner-one ate-demo-counter/counter ACTOR_STATE_RUNNING ate-demo-counter/counter-79b587778f-8lh4h 10.244.0.22 23 14h ``` The repeated `checkpointing`/`checkpointed` pair after the suspend is the existing follow-mode reconnect replay (the resume cursor is a second-granularity `sinceTime`); it is unchanged by this PR. - [x] Tests pass - [x] Appropriate changes to documentation are included in the PR --- cmd/kubectl-ate/README.md | 4 + cmd/kubectl-ate/internal/cmd/actor.go | 157 +++++++--- cmd/kubectl-ate/internal/cmd/actor_test.go | 315 ++++++++++++++++----- docs/observability.md | 16 +- 4 files changed, 384 insertions(+), 108 deletions(-) diff --git a/cmd/kubectl-ate/README.md b/cmd/kubectl-ate/README.md index 9da5c3ca7a..6b3a0f2df5 100644 --- a/cmd/kubectl-ate/README.md +++ b/cmd/kubectl-ate/README.md @@ -298,6 +298,10 @@ kubectl ate logs actors my-actor -a -f # Show only one container's logs with -c/--container. kubectl ate logs actors my-actor -a -c my-container + +# Pick a class of lines with --source: containers (every container's output, +# no lifecycle events) or lifecycle (only the actor's lifecycle events). +kubectl ate logs actors my-actor -a --source=lifecycle ``` Logs are streamable only while the actor is bound to a worker (i.e., `ACTOR_STATE_RUNNING`). For history across worker migrations, route through a centralized log backend (Cloud Logging, Loki, etc.); see `docs/observability.md`. diff --git a/cmd/kubectl-ate/internal/cmd/actor.go b/cmd/kubectl-ate/internal/cmd/actor.go index f063be1f60..ea74e334f6 100644 --- a/cmd/kubectl-ate/internal/cmd/actor.go +++ b/cmd/kubectl-ate/internal/cmd/actor.go @@ -55,6 +55,7 @@ var ( logsActorAtespaceFlag string logsActorFollowFlag bool logsActorContainerFlag string + logsActorSourceFlag string ) var getActorsCmd = &cobra.Command{ @@ -304,13 +305,14 @@ func (s *k8sPodLogsStreamer) StreamLogs(ctx context.Context, namespace, podName // LogsActorRunner executes the log printing or streaming. type LogsActorRunner struct { - apiClient AteAPIClient - streamer PodLogsStreamer - actorRef resources.ActorRef - stdout io.Writer - stderr io.Writer - follow bool - container string + apiClient AteAPIClient + streamer PodLogsStreamer + actorRef resources.ActorRef + stdout io.Writer + stderr io.Writer + follow bool + // filter selects the lines to display; see newLogLineFilter. + filter logLineFilter pollInterval time.Duration reconnectInterval time.Duration tickerInterval time.Duration @@ -318,6 +320,8 @@ type LogsActorRunner struct { // Run executes the logs command. func (r *LogsActorRunner) Run(ctx context.Context) error { + defer r.apiClient.Close() + if r.pollInterval <= 0 { r.pollInterval = 2 * time.Second } @@ -328,7 +332,6 @@ func (r *LogsActorRunner) Run(ctx context.Context) error { r.tickerInterval = 2 * time.Second } - defer r.apiClient.Close() if r.follow { return r.runFollow(ctx) } @@ -358,13 +361,12 @@ func (r *LogsActorRunner) runOneShot(ctx context.Context) error { } defer stream.Close() - filter := logLineFilter{target: r.actorRef, container: r.container} scanner := bufio.NewScanner(stream) buf := make([]byte, 0, 64*1024) scanner.Buffer(buf, 1024*1024) // Support up to 1MB lines for scanner.Scan() { line := scanner.Text() - filterAndDisplayLogLine(line, filter, r.stdout) + filterAndDisplayLogLine(line, r.filter, r.stdout) } if err := scanner.Err(); err != nil { return fmt.Errorf("error reading log stream: %w", err) @@ -374,6 +376,8 @@ func (r *LogsActorRunner) runOneShot(ctx context.Context) error { func (r *LogsActorRunner) runFollow(ctx context.Context) error { var lastWorkerPod string + // lastSeenTime is the kubelet's timestamp on the last line read from + // lastWorkerPod, the resume cursor for a reconnect to it. var lastSeenTime time.Time for { @@ -408,14 +412,21 @@ func (r *LogsActorRunner) runFollow(ctx context.Context) error { } } - // actor is resumed on anther worker - if podName != lastWorkerPod { + // The actor resumed on another worker. + switchedPod := podName != lastWorkerPod + if switchedPod { fmt.Fprintf(r.stderr, "Actor is currently running on pod %s/%s\n", namespace, podName) lastWorkerPod = podName } + // The kubelet's own timestamp on every line is the resume cursor. It + // is what SinceTime is compared against, and it does not depend on the + // line's content: an actor's record keeps a time the actor wrote, which + // can run ahead of the node's clock and would make a reconnect skip + // lines. opts := &corev1.PodLogOptions{ - Follow: true, + Follow: true, + Timestamps: true, } if !lastSeenTime.IsZero() { opts.SinceTime = &metav1.Time{Time: lastSeenTime} @@ -436,16 +447,22 @@ func (r *LogsActorRunner) runFollow(ctx context.Context) error { var wg sync.WaitGroup r.startMigrationMonitor(streamCtx, streamCancel, &wg, podName) - filter := logLineFilter{target: r.actorRef, container: r.container} scanner := bufio.NewScanner(stream) buf := make([]byte, 0, 64*1024) scanner.Buffer(buf, 1024*1024) // Support up to 1MB lines for scanner.Scan() { - line := scanner.Text() - logTime, _ := filterAndDisplayLogLine(line, filter, r.stdout) - if !logTime.IsZero() { - lastSeenTime = logTime + stamp, line, ok := splitKubeletTimestamp(scanner.Text()) + if ok { + // SinceTime has second precision, so a reconnect to the same pod + // re-reads the cursor's second; lines up to the cursor were + // already shown. Another pod's lines are all new, and its node's + // clock is not this one's, so they are not compared. + if !switchedPod && !stamp.After(lastSeenTime) { + continue + } + lastSeenTime = stamp } + filterAndDisplayLogLine(line, r.filter, r.stdout) } scanErr := scanner.Err() stream.Close() @@ -504,9 +521,33 @@ func (r *LogsActorRunner) startMigrationMonitor( }() } +// splitKubeletTimestamp splits the timestamp the kubelet prefixes to a log +// line when PodLogOptions.Timestamps is set (RFC 3339 with nanoseconds, then +// a space) from the line itself. ok is false for a line without one, which +// is then returned whole. +func splitKubeletTimestamp(line string) (stamp time.Time, rest string, ok bool) { + stampText, rest, found := strings.Cut(line, " ") + if !found { + return time.Time{}, line, false + } + stamp, err := time.Parse(time.RFC3339Nano, stampText) + if err != nil { + return time.Time{}, line, false + } + return stamp, rest, true +} + func runLogsActor(cmd *cobra.Command, args []string) error { ctx := cmd.Context() + // Before any connection: a bad flag combination is answered as such, not + // as an unreachable server. + actorRef := resources.ActorRef{Atespace: logsActorAtespaceFlag, Name: args[0]} + filter, err := newLogLineFilter(actorRef, logsActorSourceFlag, logsActorContainerFlag) + if err != nil { + return err + } + apiClient, err := ateclient.NewClient(ctx, kubeconfig, k8sContext, endpoint, tokenFile, traceEnabled) if err != nil { return fmt.Errorf("failed to connect to ate-api-server: %w", err) @@ -514,6 +555,7 @@ func runLogsActor(cmd *cobra.Command, args []string) error { config, err := ateclient.LoadKubeConfig(kubeconfig, k8sContext) if err != nil { + apiClient.Close() return fmt.Errorf("while loading kubeconfig: %w", err) } k8sClient, err := kubernetes.NewForConfig(config) @@ -525,11 +567,11 @@ func runLogsActor(cmd *cobra.Command, args []string) error { runner := &LogsActorRunner{ apiClient: apiClient, streamer: &k8sPodLogsStreamer{clientset: k8sClient}, - actorRef: resources.ActorRef{Atespace: logsActorAtespaceFlag, Name: args[0]}, + actorRef: actorRef, stdout: os.Stdout, stderr: os.Stderr, follow: logsActorFollowFlag, - container: logsActorContainerFlag, + filter: filter, pollInterval: 2 * time.Second, reconnectInterval: 1 * time.Second, tickerInterval: 2 * time.Second, @@ -538,13 +580,42 @@ func runLogsActor(cmd *cobra.Command, args []string) error { return runner.Run(ctx) } +// logSource names a class of lines in an actor's log stream, selected with +// --source. Container output carries the container's name; the lifecycle +// events ateom emits around a start, checkpoint, restore, or termination +// carry none, which is what tells the two apart. +type logSource string + +const ( + logSourceAll logSource = "all" + logSourceContainers logSource = "containers" + logSourceLifecycle logSource = "lifecycle" +) + // logLineFilter selects which of an actor's log lines are displayed: all of -// them by default, or only the named container's when container is set. +// them by default, one class of them when source is set, or only the named +// container's when container is set. type logLineFilter struct { target resources.ActorRef + source logSource container string } +// newLogLineFilter builds the filter for target from the --source and +// --container flag values, rejecting combinations that could never match. +func newLogLineFilter(target resources.ActorRef, source, container string) (logLineFilter, error) { + switch logSource(source) { + case logSourceAll, logSourceContainers: + case logSourceLifecycle: + if container != "" { + return logLineFilter{}, fmt.Errorf("--container cannot be combined with --source=lifecycle: lifecycle events are not emitted by a container") + } + default: + return logLineFilter{}, fmt.Errorf("invalid --source %q: must be one of %s, %s, %s", source, logSourceAll, logSourceContainers, logSourceLifecycle) + } + return logLineFilter{target: target, source: logSource(source), container: container}, nil +} + // matches reports whether a line emitted by emitter from containerName (empty // for lifecycle events) should be displayed. func (f logLineFilter) matches(emitter resources.ActorRef, containerName string) bool { @@ -553,24 +624,39 @@ func (f logLineFilter) matches(emitter resources.ActorRef, containerName string) if emitter != f.target || f.target.Atespace == "" || f.target.Name == "" { return false } + switch f.source { + case logSourceLifecycle: + // f.container is empty here: newLogLineFilter rejects it with lifecycle. + return containerName == "" + case logSourceContainers: + if containerName == "" { + return false + } + } return f.container == "" || containerName == f.container } -func filterAndDisplayLogLine(line string, filter logLineFilter, w io.Writer) (time.Time, bool) { +// containerNameKeyQuoted is the container-name label key as it appears in a +// JSON line. Every container line carries it (ateattr.ActorLogLabels) and +// no lifecycle line does; an actor cannot forge it, since actorlog strips +// the reserved keys from an actor's own labels, and the text inside a value +// only ever makes a container line skip early. +var containerNameKeyQuoted = `"` + string(ateattr.ActorContainerNameKey) + `"` + +// filterAndDisplayLogLine writes line to w if filter selects it, and +// reports whether it did. +func filterAndDisplayLogLine(line string, filter logLineFilter, w io.Writer) bool { + // Under lifecycle, nearly every line is a container's; a busy actor + // writes many, so skip them before decoding. + if filter.source == logSourceLifecycle && strings.Contains(line, containerNameKeyQuoted) { + return false + } + var m map[string]any dec := json.NewDecoder(strings.NewReader(line)) dec.UseNumber() if err := dec.Decode(&m); err != nil { - return time.Time{}, false - } - - var logTime time.Time - if tVal, ok := m["time"].(string); ok { - if t, err := time.Parse(time.RFC3339Nano, tVal); err == nil { - logTime = t - } else if t, err := time.Parse(time.RFC3339, tVal); err == nil { - logTime = t - } + return false } var emitter resources.ActorRef @@ -589,7 +675,7 @@ func filterAndDisplayLogLine(line string, filter logLineFilter, w io.Writer) (ti } if !filter.matches(emitter, emitterContainer) { - return time.Time{}, false + return false } // Remove substrate's labels from CLI output. Stripping the whole reserved @@ -619,7 +705,7 @@ func filterAndDisplayLogLine(line string, filter logLineFilter, w io.Writer) (ti enc := json.NewEncoder(&buf) enc.SetEscapeHTML(false) if err := enc.Encode(m); err != nil { - return time.Time{}, false + return false } encodedStr := strings.TrimSpace(buf.String()) @@ -636,7 +722,7 @@ func filterAndDisplayLogLine(line string, filter logLineFilter, w io.Writer) (ti fmt.Fprintln(w, encodedStr) } - return logTime, true + return true } func init() { @@ -672,5 +758,6 @@ func init() { logsActorsCmd.Flags().StringVarP(&logsActorAtespaceFlag, "atespace", "a", "", "Atespace the actor lives in") _ = logsActorsCmd.MarkFlagRequired("atespace") logsActorsCmd.Flags().StringVarP(&logsActorContainerFlag, "container", "c", "", "Show only logs from this container.") + logsActorsCmd.Flags().StringVar(&logsActorSourceFlag, "source", string(logSourceAll), "Which lines to show: all, containers (every container's output, no lifecycle events), or lifecycle (only the actor's lifecycle events).") logsCmd.AddCommand(logsActorsCmd) } diff --git a/cmd/kubectl-ate/internal/cmd/actor_test.go b/cmd/kubectl-ate/internal/cmd/actor_test.go index df93a5673d..2c3f36cf4a 100644 --- a/cmd/kubectl-ate/internal/cmd/actor_test.go +++ b/cmd/kubectl-ate/internal/cmd/actor_test.go @@ -32,6 +32,7 @@ import ( "google.golang.org/grpc/status" "google.golang.org/protobuf/testing/protocmp" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func TestActorCommandArgs(t *testing.T) { @@ -112,9 +113,9 @@ func TestFilterAndDisplayLogLine(t *testing.T) { name string line string target resources.ActorRef + source string container string wantMatched bool - wantTime string wantOutput string }{ { @@ -122,7 +123,6 @@ func TestFilterAndDisplayLogLine(t *testing.T) { line: `{"time":"2026-05-16T01:03:38.602878302Z","level":"info","msg":"Count","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-1"}}`, target: resources.ActorRef{Atespace: "space-1", Name: "act-1"}, wantMatched: true, - wantTime: "2026-05-16T01:03:38.602878302Z", wantOutput: `{"time":"2026-05-16T01:03:38.602878302Z","level":"info","msg":"Count"}`, }, { @@ -130,7 +130,6 @@ func TestFilterAndDisplayLogLine(t *testing.T) { line: `{"time":"2026-05-16T01:03:38Z","message":"Hello","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-1"}}`, target: resources.ActorRef{Atespace: "space-1", Name: "act-1"}, wantMatched: true, - wantTime: "2026-05-16T01:03:38Z", wantOutput: `{"time":"2026-05-16T01:03:38Z","message":"Hello"}`, }, { @@ -138,7 +137,6 @@ func TestFilterAndDisplayLogLine(t *testing.T) { line: `{"level":"error","msg":"Failed","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-1"}}`, target: resources.ActorRef{Atespace: "space-1", Name: "act-1"}, wantMatched: true, - wantTime: "", wantOutput: `{"level":"error","msg":"Failed"}`, }, { @@ -146,7 +144,6 @@ func TestFilterAndDisplayLogLine(t *testing.T) { line: `{"time":"2026-05-16T01:03:38.602878302Z","level":"info","msg":"Count","labels":{"ate.atespace":"space-1","ate.actor.name":"act-1"}}`, target: resources.ActorRef{Atespace: "space-1", Name: "act-1"}, wantMatched: true, - wantTime: "2026-05-16T01:03:38.602878302Z", wantOutput: `{"time":"2026-05-16T01:03:38.602878302Z","level":"info","msg":"Count"}`, }, { @@ -154,7 +151,6 @@ func TestFilterAndDisplayLogLine(t *testing.T) { line: `{"time":"2026-05-16T01:03:38Z","message":"Hello world","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-2"}}`, target: resources.ActorRef{Atespace: "space-1", Name: "act-1"}, wantMatched: false, - wantTime: "", // zero: another actor's line must not advance follow's resume cursor wantOutput: "", }, { @@ -162,7 +158,6 @@ func TestFilterAndDisplayLogLine(t *testing.T) { line: `{"time":"2026-05-16T01:03:38Z","message":"Hello world","logging.googleapis.com/labels":{"ate.atespace":"space-2","ate.actor.name":"act-1"}}`, target: resources.ActorRef{Atespace: "space-1", Name: "act-1"}, wantMatched: false, - wantTime: "", wantOutput: "", }, { @@ -170,7 +165,6 @@ func TestFilterAndDisplayLogLine(t *testing.T) { line: `{"time":"2026-05-16T01:03:38Z","message":"Hello world","logging.googleapis.com/labels":{"ate.actor.name":"act-1"}}`, target: resources.ActorRef{Atespace: "space-1", Name: "act-1"}, wantMatched: false, - wantTime: "", wantOutput: "", }, { @@ -178,7 +172,6 @@ func TestFilterAndDisplayLogLine(t *testing.T) { line: `{"time":"2026-05-16T01:03:38Z","message":"Hello world","logging.googleapis.com/labels":{"ate.atespace":"","ate.actor.name":"act-1"}}`, target: resources.ActorRef{Atespace: "", Name: "act-1"}, wantMatched: false, - wantTime: "", wantOutput: "", }, { @@ -186,7 +179,6 @@ func TestFilterAndDisplayLogLine(t *testing.T) { line: `{"time":"2026-05-16T01:03:38Z","message":"Hello world","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":""}}`, target: resources.ActorRef{Atespace: "space-1", Name: ""}, wantMatched: false, - wantTime: "", wantOutput: "", }, { @@ -194,7 +186,6 @@ func TestFilterAndDisplayLogLine(t *testing.T) { line: "not a json line", target: resources.ActorRef{Atespace: "space-1", Name: "act-1"}, wantMatched: false, - wantTime: "", wantOutput: "", }, { @@ -202,7 +193,6 @@ func TestFilterAndDisplayLogLine(t *testing.T) { line: `{"time":"2026-05-16T01:03:38Z","level":"info","msg":"Hello","traceID":"abc-123","err":"timeout","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-1"}}`, target: resources.ActorRef{Atespace: "space-1", Name: "act-1"}, wantMatched: true, - wantTime: "2026-05-16T01:03:38Z", wantOutput: `{"time":"2026-05-16T01:03:38Z","err":"timeout","level":"info","msg":"Hello","traceID":"abc-123"}`, }, { @@ -210,7 +200,6 @@ func TestFilterAndDisplayLogLine(t *testing.T) { line: `{"time":"2026-05-16T01:03:38Z","severity":"error","message":"Disk full","custom_tag":"alert","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-1"}}`, target: resources.ActorRef{Atespace: "space-1", Name: "act-1"}, wantMatched: true, - wantTime: "2026-05-16T01:03:38Z", wantOutput: `{"time":"2026-05-16T01:03:38Z","custom_tag":"alert","message":"Disk full","severity":"error"}`, }, { @@ -218,7 +207,6 @@ func TestFilterAndDisplayLogLine(t *testing.T) { line: `{"message":"login failed","code":401,"logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-1"}}`, target: resources.ActorRef{Atespace: "space-1", Name: "act-1"}, wantMatched: true, - wantTime: "", wantOutput: `{"code":401,"message":"login failed"}`, }, { @@ -226,7 +214,6 @@ func TestFilterAndDisplayLogLine(t *testing.T) { line: `{"time":"2026-05-16T01:03:38Z","level":"info","msg":"Hello","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-1","app":"my-app"}}`, target: resources.ActorRef{Atespace: "space-1", Name: "act-1"}, wantMatched: true, - wantTime: "2026-05-16T01:03:38Z", wantOutput: `{"time":"2026-05-16T01:03:38Z","level":"info","logging.googleapis.com/labels":{"app":"my-app"},"msg":"Hello"}`, }, { @@ -237,7 +224,6 @@ func TestFilterAndDisplayLogLine(t *testing.T) { line: `{"time":"2026-05-16T01:03:38Z","msg":"Hello","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-1","ate.tenant":"forged","app":"my-app"}}`, target: resources.ActorRef{Atespace: "space-1", Name: "act-1"}, wantMatched: true, - wantTime: "2026-05-16T01:03:38Z", wantOutput: `{"time":"2026-05-16T01:03:38Z","logging.googleapis.com/labels":{"app":"my-app"},"msg":"Hello"}`, }, { @@ -245,7 +231,6 @@ func TestFilterAndDisplayLogLine(t *testing.T) { line: `{"time":"2026-05-16T01:03:38Z","level":"info","msg":"hi","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-1","ate.actor.container.name":"counter"}}`, target: resources.ActorRef{Atespace: "space-1", Name: "act-1"}, wantMatched: true, - wantTime: "2026-05-16T01:03:38Z", wantOutput: `{"time":"2026-05-16T01:03:38Z","level":"info","msg":"hi"}`, }, { @@ -254,7 +239,6 @@ func TestFilterAndDisplayLogLine(t *testing.T) { target: resources.ActorRef{Atespace: "space-1", Name: "act-1"}, container: "counter", wantMatched: true, - wantTime: "2026-05-16T01:03:38Z", wantOutput: `{"time":"2026-05-16T01:03:38Z","level":"info","msg":"hi"}`, }, { @@ -263,7 +247,6 @@ func TestFilterAndDisplayLogLine(t *testing.T) { target: resources.ActorRef{Atespace: "space-1", Name: "act-1"}, container: "counter", wantMatched: false, - wantTime: "", // filtered out: only displayed lines may advance follow's resume cursor wantOutput: "", }, { @@ -272,7 +255,63 @@ func TestFilterAndDisplayLogLine(t *testing.T) { target: resources.ActorRef{Atespace: "space-1", Name: "act-1"}, container: "counter", wantMatched: false, - wantTime: "", + wantOutput: "", + }, + { + name: "source=all shows a lifecycle event", + line: `{"time":"2026-05-16T01:03:38Z","message":"Actor started","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-1"}}`, + target: resources.ActorRef{Atespace: "space-1", Name: "act-1"}, + source: "all", + wantMatched: true, + wantOutput: `{"time":"2026-05-16T01:03:38Z","message":"Actor started"}`, + }, + { + name: "source=lifecycle shows a lifecycle event", + line: `{"time":"2026-05-16T01:03:38Z","message":"Actor checkpointing","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-1"}}`, + target: resources.ActorRef{Atespace: "space-1", Name: "act-1"}, + source: "lifecycle", + wantMatched: true, + wantOutput: `{"time":"2026-05-16T01:03:38Z","message":"Actor checkpointing"}`, + }, + { + name: "source=lifecycle excludes a container line", + line: `{"time":"2026-05-16T01:03:38Z","level":"info","msg":"hi","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-1","ate.actor.container.name":"counter"}}`, + target: resources.ActorRef{Atespace: "space-1", Name: "act-1"}, + source: "lifecycle", + wantMatched: false, + wantOutput: "", + }, + { + name: "source=lifecycle excludes another actor's lifecycle event", + line: `{"time":"2026-05-16T01:03:38Z","message":"Actor started","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-2"}}`, + target: resources.ActorRef{Atespace: "space-1", Name: "act-1"}, + source: "lifecycle", + wantMatched: false, + wantOutput: "", + }, + { + name: "source=containers shows any container's line", + line: `{"time":"2026-05-16T01:03:38Z","level":"info","msg":"hi","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-1","ate.actor.container.name":"sidecar"}}`, + target: resources.ActorRef{Atespace: "space-1", Name: "act-1"}, + source: "containers", + wantMatched: true, + wantOutput: `{"time":"2026-05-16T01:03:38Z","level":"info","msg":"hi"}`, + }, + { + name: "source=containers excludes a lifecycle event", + line: `{"time":"2026-05-16T01:03:38Z","message":"Actor restored","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-1"}}`, + target: resources.ActorRef{Atespace: "space-1", Name: "act-1"}, + source: "containers", + wantMatched: false, + wantOutput: "", + }, + { + name: "source=containers with container filter keeps only the named container", + line: `{"time":"2026-05-16T01:03:38Z","level":"info","msg":"hi","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-1","ate.actor.container.name":"sidecar"}}`, + target: resources.ActorRef{Atespace: "space-1", Name: "act-1"}, + source: "containers", + container: "counter", + wantMatched: false, wantOutput: "", }, } @@ -280,26 +319,12 @@ func TestFilterAndDisplayLogLine(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { var buf bytes.Buffer - logTime, matched := filterAndDisplayLogLine(tc.line, logLineFilter{target: tc.target, container: tc.container}, &buf) + matched := filterAndDisplayLogLine(tc.line, logLineFilter{target: tc.target, source: logSource(tc.source), container: tc.container}, &buf) if matched != tc.wantMatched { t.Errorf("got matched = %v, want %v", matched, tc.wantMatched) } - if tc.wantTime != "" { - parsedTime, err := time.Parse(time.RFC3339Nano, tc.wantTime) - if err != nil { - parsedTime, _ = time.Parse(time.RFC3339, tc.wantTime) - } - if !logTime.Equal(parsedTime) { - t.Errorf("got logTime = %v, want %v", logTime, parsedTime) - } - } else { - if !logTime.IsZero() { - t.Errorf("got non-zero logTime = %v, want zero", logTime) - } - } - gotOutput := strings.TrimSpace(buf.String()) if gotOutput != tc.wantOutput { t.Errorf("got output %q, want %q", gotOutput, tc.wantOutput) @@ -308,6 +333,46 @@ func TestFilterAndDisplayLogLine(t *testing.T) { } } +func TestNewLogLineFilter(t *testing.T) { + target := resources.ActorRef{Atespace: "space-1", Name: "act-1"} + + tests := []struct { + name string + source string + container string + wantErr string + }{ + {name: "explicit empty source", source: "", wantErr: `invalid --source ""`}, + {name: "all", source: "all"}, + {name: "containers", source: "containers"}, + {name: "lifecycle", source: "lifecycle"}, + {name: "all with container", source: "all", container: "counter"}, + {name: "containers with container", source: "containers", container: "counter"}, + {name: "lifecycle with container", source: "lifecycle", container: "counter", wantErr: "--container cannot be combined with --source=lifecycle"}, + {name: "unknown source", source: "supervisor", wantErr: `invalid --source "supervisor"`}, + {name: "source is case-sensitive", source: "Lifecycle", wantErr: `invalid --source "Lifecycle"`}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got, err := newLogLineFilter(target, tc.source, tc.container) + if tc.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("got err %v, want it to contain %q", err, tc.wantErr) + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + want := logLineFilter{target: target, source: logSource(tc.source), container: tc.container} + if got != want { + t.Errorf("got %+v, want %+v", got, want) + } + }) + } +} + type mockAteAPIClient struct { GetActorFunc func(ctx context.Context, in *ateapipb.GetActorRequest, opts ...grpc.CallOption) (*ateapipb.Actor, error) CloseCalls int @@ -375,6 +440,7 @@ func TestLogsActorRunner_Run_OneShotSuccess(t *testing.T) { runner := &LogsActorRunner{ apiClient: mockAPI, actorRef: resources.ActorRef{Atespace: "space-1", Name: actorName}, + filter: logLineFilter{target: resources.ActorRef{Atespace: "space-1", Name: actorName}}, streamer: mockStreamer, stdout: &stdout, stderr: &stderr, @@ -397,30 +463,54 @@ func TestLogsActorRunner_Run_OneShotSuccess(t *testing.T) { } } -// TestLogsActorRunner_Run_OneShot_ContainerFilter exercises the --container -// selection against one stream that interleaves two containers and a -// lifecycle event. -func TestLogsActorRunner_Run_OneShot_ContainerFilter(t *testing.T) { +// TestLogsActorRunner_Run_OneShot_SourceFilter exercises the --source and +// --container selection against one stream that interleaves two containers +// and lifecycle events. +func TestLogsActorRunner_Run_OneShot_SourceFilter(t *testing.T) { actorName := "act-123" + startedLine := `{"time":"2026-05-16T01:03:37Z","message":"Actor started","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-123"}}` counterLine := `{"time":"2026-05-16T01:03:38Z","level":"info","msg":"from counter","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-123","ate.actor.container.name":"counter"}}` sidecarLine := `{"time":"2026-05-16T01:03:39Z","level":"info","msg":"from sidecar","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-123","ate.actor.container.name":"sidecar"}}` - lifecycleLine := `{"time":"2026-05-16T01:03:40Z","message":"Actor started","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-123"}}` + checkpointingLine := `{"time":"2026-05-16T01:03:40Z","message":"Actor checkpointing","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-123"}}` + stream := strings.Join([]string{startedLine, counterLine, sidecarLine, checkpointingLine}, "\n") + "\n" tests := []struct { name string + source string container string wantOutput []string }{ { name: "no filter shows everything", - wantOutput: []string{"from counter", "from sidecar", "Actor started"}, + wantOutput: []string{"Actor started", "from counter", "from sidecar", "Actor checkpointing"}, + }, + { + name: "source=all shows everything", + source: "all", + wantOutput: []string{"Actor started", "from counter", "from sidecar", "Actor checkpointing"}, }, { name: "container filter", container: "counter", wantOutput: []string{"from counter"}, }, + { + name: "source=containers drops lifecycle events", + source: "containers", + wantOutput: []string{"from counter", "from sidecar"}, + }, + { + name: "source=containers with container filter", + source: "containers", + container: "sidecar", + wantOutput: []string{"from sidecar"}, + }, + { + name: "source=lifecycle keeps only lifecycle events", + source: "lifecycle", + wantOutput: []string{"Actor started", "Actor checkpointing"}, + }, } for _, tc := range tests { @@ -439,9 +529,11 @@ func TestLogsActorRunner_Run_OneShot_ContainerFilter(t *testing.T) { }, nil }, } + streamCalls := 0 mockStreamer := &mockPodLogsStreamer{ StreamLogsFunc: func(ctx context.Context, ns, name string, opts *corev1.PodLogOptions) (io.ReadCloser, error) { - return io.NopCloser(strings.NewReader(counterLine + "\n" + sidecarLine + "\n" + lifecycleLine + "\n")), nil + streamCalls++ + return io.NopCloser(strings.NewReader(stream)), nil }, } @@ -452,12 +544,19 @@ func TestLogsActorRunner_Run_OneShot_ContainerFilter(t *testing.T) { streamer: mockStreamer, stdout: &stdout, stderr: &stderr, - container: tc.container, + filter: logLineFilter{target: resources.ActorRef{Atespace: "space-1", Name: actorName}, source: logSource(tc.source), container: tc.container}, } - if err := runner.Run(context.Background()); err != nil { + err := runner.Run(context.Background()) + if mockAPI.CloseCalls != 1 { + t.Errorf("api client Close called %d times, want 1", mockAPI.CloseCalls) + } + if err != nil { t.Fatalf("unexpected error: %v", err) } + if streamCalls != 1 { + t.Errorf("streamed %d times, want 1", streamCalls) + } var gotLines []string if got := strings.TrimSpace(stdout.String()); got != "" { @@ -497,6 +596,7 @@ func TestLogsActorRunner_Run_OneShot_ActorNotRunning(t *testing.T) { runner := &LogsActorRunner{ apiClient: mockAPI, actorRef: resources.ActorRef{Atespace: "space-1", Name: actorName}, + filter: logLineFilter{target: resources.ActorRef{Atespace: "space-1", Name: actorName}}, streamer: mockStreamer, stdout: &stdout, stderr: &stderr, @@ -554,7 +654,8 @@ func TestLogsActorRunner_Run_Follow_SuspendedToRunning(t *testing.T) { }, } - logLine := `{"time":"2026-05-16T01:03:38Z","level":"info","msg":"Follow hello","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-123"}}` + // As the kubelet streams it with Timestamps set. + logLine := `2026-05-16T01:03:38.500000000Z {"time":"2026-05-16T01:03:38Z","level":"info","msg":"Follow hello","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-123"}}` ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -564,8 +665,8 @@ func TestLogsActorRunner_Run_Follow_SuspendedToRunning(t *testing.T) { if ns != namespace || name != podName { return nil, fmt.Errorf("unexpected pod %s/%s", ns, name) } - if !opts.Follow { - return nil, fmt.Errorf("expected follow to be true in follow mode") + if !opts.Follow || !opts.Timestamps { + return nil, fmt.Errorf("expected follow and timestamps to be set in follow mode") } // Cancel main context soon to break the outer infinite loop @@ -586,6 +687,7 @@ func TestLogsActorRunner_Run_Follow_SuspendedToRunning(t *testing.T) { runner := &LogsActorRunner{ apiClient: mockAPI, actorRef: resources.ActorRef{Atespace: "space-1", Name: actorName}, + filter: logLineFilter{target: resources.ActorRef{Atespace: "space-1", Name: actorName}}, streamer: mockStreamer, stdout: &stdout, stderr: &stderr, @@ -636,6 +738,7 @@ func TestLogsActorRunner_Run_Follow_NotFoundActor(t *testing.T) { runner := &LogsActorRunner{ apiClient: mockAPI, actorRef: resources.ActorRef{Atespace: "space-1", Name: actorName}, + filter: logLineFilter{target: resources.ActorRef{Atespace: "space-1", Name: actorName}}, streamer: mockStreamer, stdout: &stdout, stderr: &stderr, @@ -751,6 +854,7 @@ func TestLogsActorRunner_Run_Follow_ActorMigration(t *testing.T) { runner := &LogsActorRunner{ apiClient: mockAPI, actorRef: resources.ActorRef{Atespace: "space-1", Name: actorName}, + filter: logLineFilter{target: resources.ActorRef{Atespace: "space-1", Name: actorName}}, streamer: mockStreamer, stdout: &stdout, stderr: &stderr, @@ -872,6 +976,7 @@ func TestLogsActorRunner_Run_Follow_ActorSuspendedMidStream(t *testing.T) { runner := &LogsActorRunner{ apiClient: mockAPI, actorRef: resources.ActorRef{Atespace: "space-1", Name: actorName}, + filter: logLineFilter{target: resources.ActorRef{Atespace: "space-1", Name: actorName}}, streamer: mockStreamer, stdout: &stdout, stderr: &stderr, @@ -895,10 +1000,14 @@ func TestLogsActorRunner_Run_Follow_ActorSuspendedMidStream(t *testing.T) { } } -// A worker pod's stream carries lines that are not displayed (other actors, -// filtered-out containers). Those must not advance the follow resume cursor -// (SinceTime), or a reconnect could skip the lines the user asked for. -func TestLogsActorRunner_Run_Follow_UndisplayedLineDoesNotAdvanceCursor(t *testing.T) { +// The follow resume cursor is the kubelet's timestamp on the last line read, +// displayed or not. Not the last displayed line: under a sparse filter such +// as --source=lifecycle a reconnect would replay everything since the last +// match. Not the line's own time either: an actor's record keeps a time the +// actor wrote, which can run ahead of the node's clock and would make a +// reconnect skip lines. On reconnect, lines up to the cursor are dropped, +// since SinceTime only has second precision. +func TestLogsActorRunner_Run_Follow_CursorIsKubeletTimestampOfLastLineRead(t *testing.T) { actorName := "act-123" mockAPI := &mockAteAPIClient{ @@ -916,45 +1025,54 @@ func TestLogsActorRunner_Run_Follow_UndisplayedLineDoesNotAdvanceCursor(t *testi }, } - // A different actor's line and a filtered-out sidecar line of our own actor: - // both carry parseable timestamps but neither is displayed. - foreignLine := `{"time":"2026-05-16T01:03:38Z","message":"not mine","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"other"}}` - sidecarLine := `{"time":"2026-05-16T01:03:39Z","level":"info","msg":"from sidecar","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-123","ate.actor.container.name":"sidecar"}}` + // Lines as the kubelet streams them with Timestamps set. Under + // --source=lifecycle only the first is displayed. The container line's + // own time is far ahead of the node's clock. + lifecycleLine := `2026-05-16T01:03:38.100000000Z {"time":"2026-05-16T01:03:38Z","message":"Actor started","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-123"}}` + foreignLine := `2026-05-16T01:03:39.200000000Z {"time":"2026-05-16T01:03:39Z","message":"not mine","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"other"}}` + counterLine := `2026-05-16T01:03:40.300000000Z {"time":"2099-01-01T00:00:00Z","level":"info","msg":"from counter","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-123","ate.actor.container.name":"counter"}}` + // After the reconnect the kubelet re-sends the cursor's second: the + // counter line again, then a new lifecycle event in the same second. + laterLine := `2026-05-16T01:03:40.900000000Z {"time":"2026-05-16T01:03:40Z","message":"Actor checkpointing","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-123"}}` + wantSince := time.Date(2026, 5, 16, 1, 3, 40, 300000000, time.UTC) ctx, cancel := context.WithCancel(context.Background()) defer cancel() var mu sync.Mutex var streamCalls int - var reconnectHadSinceTime bool - secondCall := make(chan struct{}) + var reconnectSince *metav1.Time + var reconnectTimestamps bool + thirdCall := make(chan struct{}) mockStreamer := &mockPodLogsStreamer{ StreamLogsFunc: func(streamCtx context.Context, ns, name string, opts *corev1.PodLogOptions) (io.ReadCloser, error) { mu.Lock() defer mu.Unlock() streamCalls++ - - if streamCalls == 1 { - // First connection: only undisplayed lines, then EOF to force a reconnect. - return io.NopCloser(strings.NewReader(foreignLine + "\n" + sidecarLine + "\n")), nil - } - if streamCalls == 2 { - reconnectHadSinceTime = opts.SinceTime != nil - close(secondCall) + switch streamCalls { + case 1: + return io.NopCloser(strings.NewReader(lifecycleLine + "\n" + foreignLine + "\n" + counterLine + "\n")), nil + case 2: + reconnectSince = opts.SinceTime + reconnectTimestamps = opts.Timestamps + return io.NopCloser(strings.NewReader(counterLine + "\n" + laterLine + "\n")), nil + case 3: + close(thirdCall) } return io.NopCloser(strings.NewReader("")), nil }, } + var stdout bytes.Buffer runner := &LogsActorRunner{ apiClient: mockAPI, actorRef: resources.ActorRef{Atespace: "space-1", Name: actorName}, streamer: mockStreamer, - stdout: &bytes.Buffer{}, + stdout: &stdout, stderr: &bytes.Buffer{}, follow: true, - container: "counter", + filter: logLineFilter{target: resources.ActorRef{Atespace: "space-1", Name: actorName}, source: logSourceLifecycle}, pollInterval: 1 * time.Millisecond, reconnectInterval: 1 * time.Millisecond, tickerInterval: time.Hour, // keep the migration monitor out of the way @@ -967,11 +1085,11 @@ func TestLogsActorRunner_Run_Follow_UndisplayedLineDoesNotAdvanceCursor(t *testi }() select { - case <-secondCall: + case <-thirdCall: case <-time.After(2 * time.Second): cancel() <-done - t.Fatal("reconnect (second StreamLogs call) never happened") + t.Fatal("the third StreamLogs call never happened") } cancel() @@ -979,7 +1097,62 @@ func TestLogsActorRunner_Run_Follow_UndisplayedLineDoesNotAdvanceCursor(t *testi mu.Lock() defer mu.Unlock() - if reconnectHadSinceTime { - t.Error("reconnect carried a SinceTime cursor; undisplayed lines must not advance it") + if !reconnectTimestamps { + t.Error("follow did not ask the kubelet for timestamps") + } + if reconnectSince == nil { + t.Fatal("reconnect carried no SinceTime cursor") + } + if !reconnectSince.Time.Equal(wantSince) { + t.Errorf("reconnect SinceTime = %v, want %v: the kubelet's timestamp on the last line read, not the last line displayed and not the line's own time", reconnectSince.Time, wantSince) + } + want := "{\"time\":\"2026-05-16T01:03:38Z\",\"message\":\"Actor started\"}\n" + + "{\"time\":\"2026-05-16T01:03:40Z\",\"message\":\"Actor checkpointing\"}\n" + if got := stdout.String(); got != want { + t.Errorf("output =\n%s\nwant\n%s", got, want) + } +} + +func TestSplitKubeletTimestamp(t *testing.T) { + tests := []struct { + name string + line string + wantOK bool + wantTime time.Time + wantRest string + }{ + { + name: "kubelet timestamp prefix", + line: `2026-05-16T01:03:40.300000000Z {"msg":"hi"}`, + wantOK: true, + wantTime: time.Date(2026, 5, 16, 1, 3, 40, 300000000, time.UTC), + wantRest: `{"msg":"hi"}`, + }, + { + name: "no prefix", + line: `{"msg":"hi there"}`, + wantOK: false, + wantRest: `{"msg":"hi there"}`, + }, + { + name: "no space at all", + line: `plain`, + wantOK: false, + wantRest: `plain`, + }, + { + name: "a space but not a timestamp", + line: `not-a-time {"msg":"hi"}`, + wantOK: false, + wantRest: `not-a-time {"msg":"hi"}`, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + stamp, rest, ok := splitKubeletTimestamp(tc.line) + if ok != tc.wantOK || rest != tc.wantRest || !stamp.Equal(tc.wantTime) { + t.Errorf("splitKubeletTimestamp(%q) = %v, %q, %v; want %v, %q, %v", tc.line, stamp, rest, ok, tc.wantTime, tc.wantRest, tc.wantOK) + } + }) } } diff --git a/docs/observability.md b/docs/observability.md index 12e63a3665..ae2fd83e51 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -67,13 +67,25 @@ Actor is currently running on pod ate-demo-counter/counter-ab123-x4y5z {"time":"2026-05-22T21:50:02.123456789Z","count":2,"fshash":"mCY7...","level":"INFO","msg":"Count"} ``` -#### Example 4: Filtering by Container -An actor can run several containers. By default every line is shown, including the synthetic lifecycle events (`Actor started`, `Actor checkpointing`, ...). `--container` (short form `-c`) restricts the output to the named container's logs: +#### Example 4: Filtering by Container or Source +An actor can run several containers. By default every line is shown, including the synthetic lifecycle events (`Actor starting`, `Actor started`, `Actor checkpointing`, `Actor checkpointed`, `Actor restoring`, `Actor restored`, `Actor terminated`). `--container` (short form `-c`) restricts the output to the named container's logs: ```bash kubectl ate logs actors -a -c ``` +`--source` selects a class of lines instead of a container. `containers` shows every container's output with the lifecycle events removed; `lifecycle` shows only the lifecycle events, which is the quickest way to see how often an actor has been suspended and restored on its current worker (a migration moves the actor to another pod, whose log holds none of the earlier workers' events): + +```bash +$ kubectl ate logs actors test -a demo --source=lifecycle +{"time":"2026-10-07T21:31:48.648599369Z","message":"Actor checkpointing","span_id":"c475...","trace_flags":"01","trace_id":"e617..."} +{"time":"2026-10-07T21:31:48.791435032Z","message":"Actor checkpointed","span_id":"c475...","trace_flags":"01","trace_id":"e617..."} +{"time":"2026-10-07T21:31:59.772566601Z","message":"Actor restoring","span_id":"a68c...","trace_flags":"01","trace_id":"b387..."} +{"time":"2026-10-07T21:31:59.96129419Z","message":"Actor restored","span_id":"a68c...","trace_flags":"01","trace_id":"b387..."} +``` + +`--container` already implies container output, so it can be combined with `--source=containers` but not with `--source=lifecycle`: lifecycle events are not emitted by a container, and the CLI rejects that combination. + --- ### Centralized Logging Backends (Multi-Dimensional Aggregation) From d966c8e8c059292caf1869b6aeb1e68a471321e4 Mon Sep 17 00:00:00 2001 From: Max Thompson Date: Fri, 9 Oct 2026 17:53:11 +0000 Subject: [PATCH 32/44] Egress policies can inject actor JWTs (#2141) Part of #1660. - `actor_jwt` entries in `replace_headers` are now injected on `http` and `https` rules alike, like `credential_uri` entries since #2158, when the request carries the header, instead of being denied with 501. The gateway mints a token for the actor behind the tunnel, with the policy's audiences and lifetime, and writes the prefix plus the token into the header. - Actor JWTs need no credential provider, so a gateway without one still injects them. A `credential_uri` entry still denies the request when no provider is configured. - A failed mint denies the request: 403 if the actor was deleted, 503 if ateapi is unavailable or the mint timed out, and 500 otherwise. - The `credential_uri` path moved into its own function. Each injection failure is wrapped with its header, and the credential URI for provider entries, so the router's log line says which injection denied the request. With `--experimental-enable-authz` set on ateapi, the gateway can only mint once #2026 lands. Testing: unit tests cover injection with no provider, the skip when the header is absent, and each denial on both legs. The e2e check is in #2256, stacked on this PR. - [x] Tests pass - [ ] Appropriate changes to documentation are included in the PR (the user guide lands later in the #1660 series) --- .../internal/router/egress/credentials.go | 142 ++++++++++------- .../router/egress/credentials_test.go | 149 +++++++++++++----- cmd/atenet/internal/router/egress/egress.go | 15 +- .../internal/router/egress/egress_test.go | 19 +++ 4 files changed, 226 insertions(+), 99 deletions(-) diff --git a/cmd/atenet/internal/router/egress/credentials.go b/cmd/atenet/internal/router/egress/credentials.go index 54ae689559..733ba95474 100644 --- a/cmd/atenet/internal/router/egress/credentials.go +++ b/cmd/atenet/internal/router/egress/credentials.go @@ -16,6 +16,7 @@ package egress import ( "context" + "fmt" "log/slog" "strings" @@ -49,6 +50,21 @@ func mapCredentialProviderError(err error) error { } } +// mapActorJWTError converts an actor JWT mint failure into a client-facing +// ext_proc denial. An actor that was deleted (NotFound) denies as 403, a +// transient control-plane failure as a retryable 503, and anything else, a +// gateway or policy misconfiguration that retrying cannot fix, as 500. +func mapActorJWTError(err error) error { + switch status.Code(err) { + case codes.NotFound: + return extproc.WrapReqError(envoy_type.StatusCode_Forbidden, err, deniedBody) + case codes.Unavailable, codes.DeadlineExceeded: + return extproc.WrapReqError(envoy_type.StatusCode_ServiceUnavailable, err, deniedBody) + default: + return extproc.WrapReqError(envoy_type.StatusCode_InternalServerError, err, deniedBody) + } +} + // applyEffects resolves a matched rule's credential injections and returns the // header mutations to add to the request, or an error that denies it. Only a // header the request carries is replaced (headers is keyed by lowercased @@ -56,9 +72,8 @@ func mapCredentialProviderError(err error) error { // fetched. // // Any failure to produce a credential the request needs denies it, including -// having no provider configured. -// -// This gateway cannot mint actor JWTs yet, so a request that needs one is denied. +// having no provider configured for a credential_uri entry. Actor JWTs need no +// provider. func (h *Handler) applyEffects(ctx context.Context, ref resources.ActorRef, dest egresspolicy.Destination, headers map[string]string, effects *ateapipb.HttpRuleEffects) ([]*corev3.HeaderValueOption, error) { var injections []*ateapipb.CredentialHeader for _, inj := range effects.GetReplaceHeaders() { @@ -70,24 +85,6 @@ func (h *Handler) applyEffects(ctx context.Context, ref resources.ActorRef, dest return nil, nil } - // TODO(identity): mint actor JWTs through Control.MintActorJWT. - for _, inj := range injections { - if inj.GetActorJwt() != nil { - slog.ErrorContext(ctx, "egress denied: this gateway cannot inject actor JWTs yet", - slog.Any("actor", ref), slog.String("host", dest.Hostname), slog.String("header", inj.GetHeader())) - return nil, extproc.NewReqError(envoy_type.StatusCode_NotImplemented, deniedBody) - } - } - if h.provider == nil { - slog.ErrorContext(ctx, "egress denied: policy requires credential injection but no credential provider is configured", - slog.Any("actor", ref), slog.String("host", dest.Hostname)) - return nil, extproc.NewReqError(envoy_type.StatusCode_InternalServerError, deniedBody) - } - - // Atunnel connected to us with an ateom-for-actor SPIFFE ID; translate it - // to a pure actor SPIFFE ID for plugins to make decisions on. - actorSpiffeID := resources.ActorSPIFFEID(ref).String() - setHeaders := make([]*corev3.HeaderValueOption, 0, len(injections)) for _, inj := range injections { if err := validateInjectHeader(inj.GetHeader()); err != nil { @@ -96,43 +93,20 @@ func (h *Handler) applyEffects(ctx context.Context, ref resources.ActorRef, dest return nil, extproc.WrapReqError(envoy_type.StatusCode_InternalServerError, err, deniedBody) } - // Confirm the credential URI names the provider this gateway serves - // before dialing: the configured connection fronts one provider, so a URI - // naming another cannot be resolved here and must fail closed rather than - // be sent to the wrong provider. - if h.providerName != "" { - name, err := providerNameFromURI(inj.GetCredentialUri()) + var secret []byte + var err error + if src := inj.GetActorJwt(); src != nil { + secret, err = h.actorJWT(ctx, ref, dest, src) if err != nil { - slog.ErrorContext(ctx, "egress denied: policy names an unparseable credential URI", - slog.Any("actor", ref), slog.String("host", dest.Hostname), slog.String("uri", inj.GetCredentialUri()), slog.Any("err", err)) - return nil, extproc.WrapReqError(envoy_type.StatusCode_InternalServerError, err, deniedBody) + return nil, fmt.Errorf("minting an actor JWT for header %s: %w", inj.GetHeader(), err) } - if name != h.providerName { - slog.ErrorContext(ctx, "egress denied: credential URI names a provider this gateway does not serve", - slog.Any("actor", ref), slog.String("host", dest.Hostname), slog.String("uri", inj.GetCredentialUri()), - slog.String("provider", name), slog.String("serves", h.providerName)) - return nil, extproc.NewReqError(envoy_type.StatusCode_InternalServerError, deniedBody) + } else { + secret, err = h.providerCredential(ctx, ref, dest, inj.GetCredentialUri()) + if err != nil { + return nil, fmt.Errorf("fetching credential %s for header %s: %w", inj.GetCredentialUri(), inj.GetHeader(), err) } } - resp, err := h.provider.FetchSecret(ctx, &credproviderpb.FetchSecretRequest{ - Uri: inj.GetCredentialUri(), - ActorSpiffeId: actorSpiffeID, - }) - if err != nil { - // Fail closed: a credential the policy required but we could not fetch - // must not let the request out without it. - slog.ErrorContext(ctx, "egress denied: credential fetch failed", - slog.Any("actor", ref), slog.String("host", dest.Hostname), slog.String("uri", inj.GetCredentialUri()), slog.Any("err", err)) - return nil, mapCredentialProviderError(err) - } - secret, err := sanitizeSecret(resp.GetOpaqueBytes()) - if err != nil { - slog.ErrorContext(ctx, "egress denied: unusable credential", - slog.Any("actor", ref), slog.String("host", dest.Hostname), slog.String("uri", inj.GetCredentialUri()), slog.Any("err", err)) - return nil, extproc.WrapReqError(envoy_type.StatusCode_ServiceUnavailable, err, deniedBody) - } - // Overwrite any header the actor set itself, so a client cannot pre-seed a // value that survives injection. setHeaders = append(setHeaders, &corev3.HeaderValueOption{ @@ -142,3 +116,65 @@ func (h *Handler) applyEffects(ctx context.Context, ref resources.ActorRef, dest } return setHeaders, nil } + +// actorJWT mints the JWT src asks for. Every error it returns is already a +// client-facing denial. +func (h *Handler) actorJWT(ctx context.Context, ref resources.ActorRef, dest egresspolicy.Destination, src *ateapipb.ActorJWTSource) ([]byte, error) { + jwt, err := h.actorJWTs.Token(ctx, ref, src) + if err != nil { + slog.ErrorContext(ctx, "egress denied: actor JWT mint failed", + slog.Any("actor", ref), slog.String("host", dest.Hostname), slog.Any("audiences", src.GetAudiences()), slog.Any("err", err)) + return nil, mapActorJWTError(err) + } + return []byte(jwt), nil +} + +// providerCredential fetches the secret uri names from the credential +// provider. Every error it returns is already a client-facing denial. +func (h *Handler) providerCredential(ctx context.Context, ref resources.ActorRef, dest egresspolicy.Destination, uri string) ([]byte, error) { + if h.provider == nil { + slog.ErrorContext(ctx, "egress denied: policy requires credential injection but no credential provider is configured", + slog.Any("actor", ref), slog.String("host", dest.Hostname)) + return nil, extproc.NewReqError(envoy_type.StatusCode_InternalServerError, deniedBody) + } + + // Confirm the credential URI names the provider this gateway serves + // before dialing: the configured connection fronts one provider, so a URI + // naming another cannot be resolved here and must fail closed rather than + // be sent to the wrong provider. + if h.providerName != "" { + name, err := providerNameFromURI(uri) + if err != nil { + slog.ErrorContext(ctx, "egress denied: policy names an unparseable credential URI", + slog.Any("actor", ref), slog.String("host", dest.Hostname), slog.String("uri", uri), slog.Any("err", err)) + return nil, extproc.WrapReqError(envoy_type.StatusCode_InternalServerError, err, deniedBody) + } + if name != h.providerName { + slog.ErrorContext(ctx, "egress denied: credential URI names a provider this gateway does not serve", + slog.Any("actor", ref), slog.String("host", dest.Hostname), slog.String("uri", uri), + slog.String("provider", name), slog.String("serves", h.providerName)) + return nil, extproc.NewReqError(envoy_type.StatusCode_InternalServerError, deniedBody) + } + } + + // Atunnel connected to us with an ateom-for-actor SPIFFE ID; translate it + // to a pure actor SPIFFE ID for plugins to make decisions on. + resp, err := h.provider.FetchSecret(ctx, &credproviderpb.FetchSecretRequest{ + Uri: uri, + ActorSpiffeId: resources.ActorSPIFFEID(ref).String(), + }) + if err != nil { + // Fail closed: a credential the policy required but we could not fetch + // must not let the request out without it. + slog.ErrorContext(ctx, "egress denied: credential fetch failed", + slog.Any("actor", ref), slog.String("host", dest.Hostname), slog.String("uri", uri), slog.Any("err", err)) + return nil, mapCredentialProviderError(err) + } + secret, err := sanitizeSecret(resp.GetOpaqueBytes()) + if err != nil { + slog.ErrorContext(ctx, "egress denied: unusable credential", + slog.Any("actor", ref), slog.String("host", dest.Hostname), slog.String("uri", uri), slog.Any("err", err)) + return nil, extproc.WrapReqError(envoy_type.StatusCode_ServiceUnavailable, err, deniedBody) + } + return secret, nil +} diff --git a/cmd/atenet/internal/router/egress/credentials_test.go b/cmd/atenet/internal/router/egress/credentials_test.go index d7d39166d7..4cc9656411 100644 --- a/cmd/atenet/internal/router/egress/credentials_test.go +++ b/cmd/atenet/internal/router/egress/credentials_test.go @@ -15,6 +15,7 @@ package egress import ( + "cmp" "context" "errors" "testing" @@ -24,6 +25,7 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" "github.com/agent-substrate/substrate/cmd/atenet/internal/router/extproc" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" @@ -252,47 +254,118 @@ func TestInjectionDenials(t *testing.T) { } } -func TestActorJWTInjectionNotImplemented(t *testing.T) { - effects := &ateapipb.HttpRuleEffects{ReplaceHeaders: []*ateapipb.CredentialHeader{{ - Header: "authorization", +// actorJWTHeader replaces header with an actor JWT bound to api.example.com. +func actorJWTHeader(header string) *ateapipb.CredentialHeader { + return &ateapipb.CredentialHeader{ + Header: header, Prefix: "Bearer ", ActorJwt: &ateapipb.ActorJWTSource{Audiences: []string{"https://api.example.com"}, ExpirationSeconds: 900}, - }}} - httpsPolicy := &ateapipb.EgressPolicy{Rules: []*ateapipb.EgressRule{{Https: &ateapipb.HTTPSRule{Hostnames: []string{"api.example.com"}, Effects: effects}}}} - httpPolicy := &ateapipb.EgressPolicy{Rules: []*ateapipb.EgressRule{{Http: &ateapipb.HTTPRule{Hostnames: []string{"api.example.com"}, Effects: effects}}}} - tests := []struct { - name string - leg string - policy *ateapipb.EgressPolicy - provider *fakeProvider // nil means no provider configured - }{{ - name: "https rule", - leg: extproc.EgressTLSMITMFilterChainName, - policy: httpsPolicy, - provider: &fakeProvider{resp: bearerTokenResponse("s3cr3t")}, - }, { - name: "https rule without a provider", - leg: extproc.EgressTLSMITMFilterChainName, - policy: httpsPolicy, - }, { - name: "http rule", - leg: extproc.EgressCleartextFilterChainName, - policy: httpPolicy, - provider: &fakeProvider{resp: bearerTokenResponse("s3cr3t")}, - }} - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - var h *Handler - if tc.provider == nil { - h = injectionHandlerFor(tc.policy, nil, injectionProviderName) - } else { - h = injectionHandlerFor(tc.policy, tc.provider, injectionProviderName) + } +} + +// replaceHeadersPolicy is a rule for api.example.com on leg's protocol (https +// for the MITM leg, http for the cleartext one) that replaces headers with +// entries. +func replaceHeadersPolicy(leg string, entries ...*ateapipb.CredentialHeader) *ateapipb.EgressPolicy { + hosts := []string{"api.example.com"} + effects := &ateapipb.HttpRuleEffects{ReplaceHeaders: entries} + if leg == extproc.EgressCleartextFilterChainName { + return &ateapipb.EgressPolicy{Rules: []*ateapipb.EgressRule{{Http: &ateapipb.HTTPRule{Hostnames: hosts, Effects: effects}}}} + } + return &ateapipb.EgressPolicy{Rules: []*ateapipb.EgressRule{{Https: &ateapipb.HTTPSRule{Hostnames: hosts, Effects: effects}}}} +} + +var requestLegs = []string{extproc.EgressTLSMITMFilterChainName, extproc.EgressCleartextFilterChainName} + +// An actor JWT is injected with no credential provider configured. The +// rule's credential_uri entry is for a header the request does not carry, so +// the missing provider does not deny it. +func TestActorJWTInjection(t *testing.T) { + for _, leg := range requestLegs { + t.Run(leg, func(t *testing.T) { + client := &egressMockClient{actor: runningActor(), policy: replaceHeadersPolicy(leg, + actorJWTHeader("authorization"), + &ateapipb.CredentialHeader{Header: "x-api-key", CredentialUri: "ate-secret://k8s/default/token"}, + )} + h := New(client, nil, 0, nil, "") + + res, err := h.HandleRequestHeaders(context.Background(), + withAuthorization(innerMetadata(leg, "GET", "api.example.com", nil))) + if err != nil { + t.Fatalf("HandleRequestHeaders: %v", err) } - _, err := h.HandleRequestHeaders(context.Background(), - withAuthorization(innerMetadata(tc.leg, "GET", "api.example.com", nil))) - wantStatus(t, err, envoy_type.StatusCode_NotImplemented) - if tc.provider != nil && tc.provider.got != nil { - t.Errorf("provider was asked for %v, want no fetch", tc.provider.got) + setHeaders := res.Response.GetResponse().GetHeaderMutation().GetSetHeaders() + if len(setHeaders) != 1 { + t.Fatalf("got %d header mutations, want only the actor JWT", len(setHeaders)) + } + h0 := setHeaders[0] + if got := h0.GetHeader().GetKey(); got != "authorization" { + t.Errorf("header key = %q, want authorization", got) + } + if got := string(h0.GetHeader().GetRawValue()); got != "Bearer jwt-1" { + t.Errorf("header value = %q, want %q", got, "Bearer jwt-1") + } + if h0.GetAppendAction() != corev3.HeaderValueOption_OVERWRITE_IF_EXISTS_OR_ADD { + t.Errorf("append action = %v, want OVERWRITE_IF_EXISTS_OR_ADD", h0.GetAppendAction()) + } + want := &ateapipb.MintActorJWTRequest{ + Actor: &ateapipb.ObjectRef{Atespace: testEgressAtespace, Name: testEgressActor}, + Audience: []string{"https://api.example.com"}, + ExpirationSeconds: 900, + } + if got := client.lastMint.Load(); !proto.Equal(got, want) { + t.Errorf("MintActorJWT request = %v, want %v", got, want) + } + }) + } +} + +func TestActorJWTInjectionSkippedWithoutHeader(t *testing.T) { + client := &egressMockClient{actor: runningActor(), policy: replaceHeadersPolicy(extproc.EgressTLSMITMFilterChainName, actorJWTHeader("authorization"))} + h := New(client, nil, 0, nil, "") + res, err := h.HandleRequestHeaders(context.Background(), + innerMetadata(extproc.EgressTLSMITMFilterChainName, "GET", "api.example.com", nil)) + if err != nil { + t.Fatalf("HandleRequestHeaders: %v", err) + } + if got := res.Response.GetResponse().GetHeaderMutation().GetSetHeaders(); len(got) != 0 { + t.Errorf("got %d injected headers, want 0", len(got)) + } + if calls := client.mintCalls.Load(); calls != 0 { + t.Errorf("MintActorJWT calls = %d, want 0", calls) + } +} + +func TestActorJWTInjectionDenials(t *testing.T) { + tests := []struct { + name string + header string // authorization unless set + mintErr error + want envoy_type.StatusCode + }{ + {name: "actor deleted", mintErr: status.Error(codes.NotFound, "actor not found"), want: envoy_type.StatusCode_Forbidden}, + {name: "ateapi unavailable", mintErr: status.Error(codes.Unavailable, "ateapi is down"), want: envoy_type.StatusCode_ServiceUnavailable}, + {name: "mint timed out", mintErr: status.Error(codes.DeadlineExceeded, "deadline exceeded"), want: envoy_type.StatusCode_ServiceUnavailable}, + {name: "gateway may not mint", mintErr: status.Error(codes.PermissionDenied, "denied"), want: envoy_type.StatusCode_InternalServerError}, + {name: "mint request rejected", mintErr: status.Error(codes.InvalidArgument, "bad lifetime"), want: envoy_type.StatusCode_InternalServerError}, + {name: "system header", header: ":path", want: envoy_type.StatusCode_InternalServerError}, + } + for _, leg := range requestLegs { + t.Run(leg, func(t *testing.T) { + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + header := cmp.Or(tc.header, "authorization") + mock := &egressMockClient{ + actor: runningActor(), + policy: replaceHeadersPolicy(leg, actorJWTHeader(header)), + mintErr: tc.mintErr, + } + h := New(mock, nil, 0, nil, "") + md := innerMetadata(leg, "GET", "api.example.com", nil) + md.Headers[header] = "placeholder" + _, err := h.HandleRequestHeaders(context.Background(), md) + wantStatus(t, err, tc.want) + }) } }) } diff --git a/cmd/atenet/internal/router/egress/egress.go b/cmd/atenet/internal/router/egress/egress.go index 38c678003a..01b7fcfba3 100644 --- a/cmd/atenet/internal/router/egress/egress.go +++ b/cmd/atenet/internal/router/egress/egress.go @@ -86,9 +86,8 @@ type Handler struct { policies *policyCache // actorJWTs mints the actor JWTs that credential injections ask for. actorJWTs *actorjwt.Minter - // provider resolves an egress policy's credential injections. Nil means - // credential injection is not configured, and a request that needs one is - // denied. + // provider resolves an egress policy's credential_uri injections. Nil + // means none are configured, and a request that needs one is denied. provider credproviderpb.CredentialProviderClient // providerName, when set, is the provider this gateway serves (the host of // its ate-secret:// credential URIs); a credential URI naming another @@ -100,11 +99,11 @@ type Handler struct { // trusted_ca; see verifyActorCertificate for why it is checked again here. // policyCacheTTL of 0 fetches the policy on every callout. // -// provider resolves an allowed rule's credential injections on the -// TLS-terminated MITM leg; nil leaves credential injection off, so a request -// matching a rule that requires an injection is denied. providerName, when -// set, is the provider this gateway serves; a credential URI naming another -// provider is refused. +// provider resolves an allowed rule's credential_uri injections on the +// TLS-terminated MITM leg; nil leaves them off, so a request that needs one is +// denied. Actor JWT injections need no provider. providerName, when set, is +// the provider this gateway serves; a credential URI naming another provider +// is refused. func New(apiClient ateapipb.ControlClient, actorIdentityRoots *x509.CertPool, policyCacheTTL time.Duration, provider credproviderpb.CredentialProviderClient, providerName string) *Handler { return &Handler{ apiClient: apiClient, diff --git a/cmd/atenet/internal/router/egress/egress_test.go b/cmd/atenet/internal/router/egress/egress_test.go index 72c8f33cff..3ca10de2eb 100644 --- a/cmd/atenet/internal/router/egress/egress_test.go +++ b/cmd/atenet/internal/router/egress/egress_test.go @@ -40,6 +40,7 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "google.golang.org/protobuf/types/known/structpb" + "google.golang.org/protobuf/types/known/timestamppb" "github.com/agent-substrate/substrate/cmd/atenet/internal/router/extproc" "github.com/agent-substrate/substrate/internal/egresspolicy" @@ -182,6 +183,12 @@ type egressMockClient struct { // policyGate, when non-nil, blocks each GetActorEgressPolicy until it is // closed, so a test can hold several callers on one fetch. policyGate chan struct{} + + // The nth MintActorJWT call returns "jwt-", expiring + // expiration_seconds from now. mintErr, when set, is returned instead. + mintCalls atomic.Int32 + lastMint atomic.Pointer[ateapipb.MintActorJWTRequest] + mintErr error } func (m *egressMockClient) GetActor(context.Context, *ateapipb.GetActorRequest, ...grpc.CallOption) (*ateapipb.Actor, error) { @@ -209,6 +216,18 @@ func (m *egressMockClient) GetActorEgressPolicy(ctx context.Context, _ *ateapipb return m.policy, nil } +func (m *egressMockClient) MintActorJWT(_ context.Context, req *ateapipb.MintActorJWTRequest, _ ...grpc.CallOption) (*ateapipb.MintActorJWTResponse, error) { + n := m.mintCalls.Add(1) + m.lastMint.Store(req) + if m.mintErr != nil { + return nil, m.mintErr + } + return &ateapipb.MintActorJWTResponse{ + ActorJwt: fmt.Sprintf("jwt-%d", n), + ExpiresAt: timestamppb.New(time.Now().Add(time.Duration(req.GetExpirationSeconds()) * time.Second)), + }, nil +} + // allowAllPolicy allows every name and address: cleartext HTTP on any port // and HTTPS on 443. func allowAllPolicy() *ateapipb.EgressPolicy { From bfdc7a89283d58440ebb90d62a5c4194f688b259 Mon Sep 17 00:00:00 2001 From: Bowei Du Date: Fri, 9 Oct 2026 18:05:42 +0000 Subject: [PATCH 33/44] More robust Actor DNS Relay (#2370) * Reduces go-routine and socket usage per Actor * Moves limits to per-Actor instead of across all Actors (noisy neighbor) * Clean-up of logic that can support later extension if needed. - [x] Tests pass - [x] Appropriate changes to documentation are included in the PR --- go.mod | 2 +- internal/ateomnet/dns/dnshandler.go | 283 ++ internal/ateomnet/dns/dnshandler_test.go | 364 +++ internal/ateomnet/dns/freemap/doc.go | 17 + internal/ateomnet/dns/freemap/freemap64k.go | 204 ++ .../ateomnet/dns/freemap/freemap64k_test.go | 624 ++++ internal/ateomnet/dns/limiter.go | 74 + internal/ateomnet/dns/limiter_test.go | 137 + internal/ateomnet/dns/parse.go | 54 - internal/ateomnet/dns/pendingreq.go | 242 ++ internal/ateomnet/dns/pendingreq_test.go | 367 +++ internal/ateomnet/dns/protocol/doc.go | 17 + internal/ateomnet/dns/protocol/protocol.go | 117 + .../ateomnet/dns/protocol/protocol_test.go | 287 ++ internal/ateomnet/dns/protocol/tcp.go | 52 + internal/ateomnet/dns/protocol/tcp_test.go | 120 + internal/ateomnet/dns/readme.md | 66 + internal/ateomnet/dns/relay.go | 74 +- internal/ateomnet/dns/relay_test.go | 211 +- internal/ateomnet/dns/server.go | 236 +- internal/ateomnet/dns/tcphandler.go | 254 ++ internal/ateomnet/dns/tcphandler_test.go | 293 ++ internal/ateomnet/dns/udphandler.go | 220 ++ internal/ateomnet/dns/udphandler_test.go | 169 + .../x/net/dns/dnsmessage/message.go | 2742 +++++++++++++++++ .../golang.org/x/net/dns/dnsmessage/svcb.go | 333 ++ vendor/modules.txt | 1 + 27 files changed, 7198 insertions(+), 362 deletions(-) create mode 100644 internal/ateomnet/dns/dnshandler.go create mode 100644 internal/ateomnet/dns/dnshandler_test.go create mode 100644 internal/ateomnet/dns/freemap/doc.go create mode 100644 internal/ateomnet/dns/freemap/freemap64k.go create mode 100644 internal/ateomnet/dns/freemap/freemap64k_test.go create mode 100644 internal/ateomnet/dns/limiter.go create mode 100644 internal/ateomnet/dns/limiter_test.go delete mode 100644 internal/ateomnet/dns/parse.go create mode 100644 internal/ateomnet/dns/pendingreq.go create mode 100644 internal/ateomnet/dns/pendingreq_test.go create mode 100644 internal/ateomnet/dns/protocol/doc.go create mode 100644 internal/ateomnet/dns/protocol/protocol.go create mode 100644 internal/ateomnet/dns/protocol/protocol_test.go create mode 100644 internal/ateomnet/dns/protocol/tcp.go create mode 100644 internal/ateomnet/dns/protocol/tcp_test.go create mode 100644 internal/ateomnet/dns/readme.md create mode 100644 internal/ateomnet/dns/tcphandler.go create mode 100644 internal/ateomnet/dns/tcphandler_test.go create mode 100644 internal/ateomnet/dns/udphandler.go create mode 100644 internal/ateomnet/dns/udphandler_test.go create mode 100644 vendor/golang.org/x/net/dns/dnsmessage/message.go create mode 100644 vendor/golang.org/x/net/dns/dnsmessage/svcb.go diff --git a/go.mod b/go.mod index d5699f40dd..88db75ee8a 100644 --- a/go.mod +++ b/go.mod @@ -61,6 +61,7 @@ require ( go.opentelemetry.io/otel/sdk/metric v1.45.0 go.opentelemetry.io/otel/trace v1.45.0 go.opentelemetry.io/proto/otlp v1.11.0 + golang.org/x/net v0.58.0 golang.org/x/sync v0.22.0 golang.org/x/sys v0.47.0 google.golang.org/api v0.274.0 @@ -241,7 +242,6 @@ require ( go.yaml.in/yaml/v3 v3.0.5 // indirect golang.org/x/crypto v0.56.0 // indirect golang.org/x/exp v0.0.0-20260718201538-764159d718ef // indirect - golang.org/x/net v0.58.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect golang.org/x/term v0.45.0 // indirect golang.org/x/text v0.41.0 // indirect diff --git a/internal/ateomnet/dns/dnshandler.go b/internal/ateomnet/dns/dnshandler.go new file mode 100644 index 0000000000..103b733b00 --- /dev/null +++ b/internal/ateomnet/dns/dnshandler.go @@ -0,0 +1,283 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dns + +import ( + "bytes" + "encoding/binary" + "log/slog" + "net" + "time" + + "github.com/agent-substrate/substrate/internal/ateomnet/dns/protocol" + "golang.org/x/net/dns/dnsmessage" +) + +// actionKind describes the transport-agnostic decision returned by the DNS +// protocol handler. +type actionKind string + +const ( + // actionDrop the DNS packet. + actionDrop actionKind = "Drop" + // actionReply immediately to the Actor request. + actionReply actionKind = "Reply" + // actionForward the packet upstream. + actionForward actionKind = "Forward" + // actionFailover to send the request to the next upstream. + actionFailover actionKind = "Failover" + // actionDeliver the DNS response to the Actor. + actionDeliver actionKind = "Deliver" +) + +// action is the decision returned by onUDPRequest, onUDPResponse, +// onTCPRequest, and onTCPResponse. +type action struct { + kind actionKind + payload []byte + upstreamID uint16 + clientAddr net.Addr + upstreamIdx int +} + +// queryPolicy decides whether a parsed DNS question is permitted. +type queryPolicy func(q dnsmessage.Question) bool + +// dnsHandler inspects DNS requests and responses and returns transport-agnostic +// routing decisions. +type dnsHandler struct { + pending *pendingRequests + limiter *limiter + allow queryPolicy +} + +func newDNSHandler(pending *pendingRequests, lim *limiter, allow queryPolicy) *dnsHandler { + if allow == nil { + allow = func(dnsmessage.Question) bool { return true } + } + return &dnsHandler{ + pending: pending, + limiter: lim, + allow: allow, + } +} + +// onRequestCommon checks packet structure and policy, returning actionDrop, +// actionReply (with a synthesized error response), or actionForward along with +// the parsed header and canonicalized question. +func (h *dnsHandler) onRequestCommon(raw []byte) (dnsmessage.Header, dnsmessage.Question, action) { + if !protocol.RequestSanityCheck(raw) { + return dnsmessage.Header{}, dnsmessage.Question{}, action{kind: actionDrop} + } + + var p dnsmessage.Parser + hdr, err := p.Start(raw) + if err != nil { + return hdr, dnsmessage.Question{}, errReplyAction(hdr, dnsmessage.RCodeFormatError, nil) + } + + qdCount := protocol.QDCount(raw) + + // OpCode must be QUERY (== 0). + if hdr.OpCode != 0 { + var qPtr *dnsmessage.Question + if qdCount == 1 { + if q, qErr := p.Question(); qErr == nil { + qPtr = &q + } + } + return hdr, dnsmessage.Question{}, errReplyAction(hdr, dnsmessage.RCodeNotImplemented, qPtr) + } + + // Must have one query. + if qdCount != 1 { + return hdr, dnsmessage.Question{}, errReplyAction(hdr, dnsmessage.RCodeFormatError, nil) + } + + // Question is valid. + q, err := p.Question() + if err != nil { + return hdr, dnsmessage.Question{}, errReplyAction(hdr, dnsmessage.RCodeFormatError, nil) + } + + // Question type must be supported. + if q.Type == dnsmessage.TypeAXFR || q.Type == protocol.TypeIXFR { + return hdr, dnsmessage.Question{}, errReplyAction(hdr, dnsmessage.RCodeNotImplemented, &q) + } + + // Check with query policy callout. + cq := canonicalizeQuestion(q) + if !h.allow(cq) { + return hdr, dnsmessage.Question{}, errReplyAction(hdr, dnsmessage.RCodeRefused, &q) + } + + return hdr, cq, action{ + kind: actionForward, + payload: raw, + } +} + +// onUDPRequest inspects an incoming UDP DNS query packet from the actor and +// returns whether to drop it, reply immediately with a synthesized error, or +// record and forward it to an upstream resolver. +func (h *dnsHandler) onUDPRequest(raw []byte, clientAddr net.Addr, upstreams []string) action { + if !h.limiter.inFlight.tryAcquire() { + slog.Debug("dns relay dropped query; too many in flight") + return action{kind: actionDrop} + } + + forwarded := false + defer func() { + if !forwarded { + h.limiter.inFlight.release() + } + }() + + hdr, cq, act := h.onRequestCommon(raw) + if act.kind != actionForward { + return act + } + + // Set to true before calling record, which takes ownership of the in-flight slot. + forwarded = true + upstreamID, ok := h.pending.record(raw, hdr.ID, cq, clientAddr, upstreams) + if !ok { + return action{kind: actionDrop} + } + + return action{ + kind: actionForward, + payload: raw, + upstreamID: upstreamID, + } +} + +// onUDPResponse validates an incoming UDP DNS response packet from an upstream +// resolver against pendingRequests and returns whether to drop it, fail over to +// the next upstream, or deliver it to the actor. +func (h *dnsHandler) onUDPResponse(raw []byte, from net.Addr) action { + if !protocol.ResponseSanityCheck(raw) { + return action{kind: actionDrop} + } + + // Parse response. + var p dnsmessage.Parser + hdr, err := p.Start(raw) + if err != nil { + return action{kind: actionDrop} + } + q, err := p.Question() + if err != nil { + return action{kind: actionDrop} + } + + h.pending.mu.Lock() + defer h.pending.mu.Unlock() + + // Look for request in the pending table. + entry, ok := h.pending.entries[hdr.ID] + cq := canonicalizeQuestion(q) + + if !ok || normalizeAddrString(entry.upstreams[entry.upstreamIdx]) != normalizeAddr(from) || entry.question != cq { + // Response does not match any pending request. + return action{kind: actionDrop} + } + + // Check if we need to failover to the next upstream AND there are more + // upstreams to try. + if isFailoverRCode(hdr.RCode) && entry.upstreamIdx+1 < len(entry.upstreams) { + entry.deferredResp = bytes.Clone(raw) + entry.upstreamIdx++ + entry.expiry = time.Now().Add(h.pending.timeoutForAttempt(entry.upstreamIdx, len(entry.upstreams))) + + return action{ + kind: actionFailover, + payload: bytes.Clone(entry.rawQuery), + upstreamIdx: entry.upstreamIdx, + } + } + + // Forward response to Actor. + clientID := entry.clientRequestID + clientAddr := entry.clientAddr + h.pending.deleteEntryLocked(hdr.ID) + + out := bytes.Clone(raw) + binary.BigEndian.PutUint16(out[0:2], clientID) + + return action{ + kind: actionDeliver, + payload: out, + clientAddr: clientAddr, + } +} + +// onTCPRequest inspects an incoming TCP DNS query frame from the actor and +// returns whether to drop it, reply immediately with a synthesized error, or +// forward it unmodified on the connection's dedicated upstream stream. +func (h *dnsHandler) onTCPRequest(raw []byte) action { + _, _, act := h.onRequestCommon(raw) + return act +} + +// onTCPResponse validates an incoming TCP DNS response frame from the +// connection's upstream stream and returns whether to drop or deliver it. +func (h *dnsHandler) onTCPResponse(raw []byte) action { + if !protocol.ResponseSanityCheck(raw) { + return action{kind: actionDrop} + } + var p dnsmessage.Parser + if _, err := p.Start(raw); err != nil { + return action{kind: actionDrop} + } + if _, err := p.Question(); err != nil { + return action{kind: actionDrop} + } + return action{ + kind: actionDeliver, + payload: raw, + } +} + +// errReplyAction returns an Action for an error situation. +func errReplyAction(hdr dnsmessage.Header, rcode dnsmessage.RCode, q *dnsmessage.Question) action { + return action{ + kind: actionReply, + payload: protocol.ErrorPacket(hdr, rcode, q), + } +} + +// isFailoverRCode returns true if the response error makes sense to try with +// the next DNS upstream server. +func isFailoverRCode(rcode dnsmessage.RCode) bool { + switch rcode { + case dnsmessage.RCodeServerFailure, dnsmessage.RCodeNotImplemented, dnsmessage.RCodeRefused: + return true + default: + return false + } +} + +// canonicalizeQuestion normalizes the contents of the Question. +func canonicalizeQuestion(q dnsmessage.Question) dnsmessage.Question { + for i := range int(q.Name.Length) { + c := q.Name.Data[i] + if 'A' <= c && c <= 'Z' { + q.Name.Data[i] = c + ('a' - 'A') + } + } + clear(q.Name.Data[q.Name.Length:]) + return q +} diff --git a/internal/ateomnet/dns/dnshandler_test.go b/internal/ateomnet/dns/dnshandler_test.go new file mode 100644 index 0000000000..cb7c052839 --- /dev/null +++ b/internal/ateomnet/dns/dnshandler_test.go @@ -0,0 +1,364 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dns + +import ( + "bytes" + "encoding/binary" + "net" + "testing" + + "github.com/agent-substrate/substrate/internal/ateomnet/dns/protocol" + "golang.org/x/net/dns/dnsmessage" +) + +func buildTestMessage(t *testing.T, hdr dnsmessage.Header, questions []dnsmessage.Question) []byte { + t.Helper() + b := dnsmessage.NewBuilder(nil, hdr) + if len(questions) > 0 { + if err := b.StartQuestions(); err != nil { + t.Fatal(err) + } + for _, q := range questions { + if err := b.Question(q); err != nil { + t.Fatal(err) + } + } + } + msg, err := b.Finish() + if err != nil { + t.Fatal(err) + } + return msg +} + +func TestDNSHandlerOnRequestDrop(t *testing.T) { + lim := &limiter{ + inFlight: &limiterSlots{max: 1}, + connections: &limiterSlots{max: 1}, + } + pending := newPendingRequests(lim) + h := newDNSHandler(pending, lim, nil) + upstreams := []string{"10.96.0.10:53"} + + // 1. Short packet (< 12 bytes). + if act := h.onUDPRequest([]byte{1, 2, 3, 4}, nil, upstreams); act.kind != actionDrop { + t.Errorf("short UDP packet action = %v, want actionDrop", act.kind) + } + if act := h.onTCPRequest([]byte{1, 2, 3, 4}); act.kind != actionDrop { + t.Errorf("short TCP packet action = %v, want actionDrop", act.kind) + } + + // 2. Packet with QR == 1 (response bit set). + resp := dnsAnswer(dnsQuery(0x1234), 0) + if act := h.onUDPRequest(resp, nil, upstreams); act.kind != actionDrop { + t.Errorf("QR=1 UDP packet action = %v, want actionDrop", act.kind) + } + if act := h.onTCPRequest(resp); act.kind != actionDrop { + t.Errorf("QR=1 TCP packet action = %v, want actionDrop", act.kind) + } + + // 3. Rate-limited (inFlight full) drops UDP requests. + lim.inFlight.tryAcquire() + defer lim.inFlight.release() + if act := h.onUDPRequest(dnsQuery(0x1234), nil, upstreams); act.kind != actionDrop { + t.Errorf("rate-limited UDP packet action = %v, want actionDrop", act.kind) + } +} + +func TestDNSHandlerOnRequestSynthesizedReplies(t *testing.T) { + lim := newLimiter() + pending := newPendingRequests(lim) + allow := func(q dnsmessage.Question) bool { + return q.Name.String() != "blocked.example.com." && q.Type != dnsmessage.TypeTXT + } + h := newDNSHandler(pending, lim, allow) + upstreams := []string{"10.96.0.10:53"} + + qExample := dnsmessage.Question{ + Name: dnsmessage.MustNewName("example.com."), + Type: dnsmessage.TypeA, + Class: dnsmessage.ClassINET, + } + + tests := []struct { + name string + raw []byte + wantRCode dnsmessage.RCode + }{ + { + name: "QDCOUNT == 0 -> FORMERR", + raw: buildTestMessage(t, dnsmessage.Header{ + ID: 0x1001, + RecursionDesired: true, + }, nil), + wantRCode: dnsmessage.RCodeFormatError, + }, + { + name: "QDCOUNT == 2 -> FORMERR", + raw: buildTestMessage(t, dnsmessage.Header{ + ID: 0x1002, + RecursionDesired: true, + }, []dnsmessage.Question{qExample, qExample}), + wantRCode: dnsmessage.RCodeFormatError, + }, + { + name: "malformed question name -> FORMERR", + raw: []byte{ + 0x10, 0x03, 0x01, 0x00, + 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x10, 'a', 'b', // label length 16 but truncated + }, + wantRCode: dnsmessage.RCodeFormatError, + }, + { + name: "non-zero OpCode -> NOTIMP", + raw: buildTestMessage(t, dnsmessage.Header{ + ID: 0x1004, + OpCode: 1, // IQUERY + RecursionDesired: true, + }, []dnsmessage.Question{qExample}), + wantRCode: dnsmessage.RCodeNotImplemented, + }, + { + name: "QTYPE AXFR -> NOTIMP", + raw: buildTestMessage(t, dnsmessage.Header{ + ID: 0x1005, + RecursionDesired: true, + }, []dnsmessage.Question{{ + Name: dnsmessage.MustNewName("example.com."), + Type: dnsmessage.TypeAXFR, + Class: dnsmessage.ClassINET, + }}), + wantRCode: dnsmessage.RCodeNotImplemented, + }, + { + name: "QTYPE IXFR -> NOTIMP", + raw: buildTestMessage(t, dnsmessage.Header{ + ID: 0x1006, + RecursionDesired: true, + }, []dnsmessage.Question{{ + Name: dnsmessage.MustNewName("example.com."), + Type: protocol.TypeIXFR, + Class: dnsmessage.ClassINET, + }}), + wantRCode: dnsmessage.RCodeNotImplemented, + }, + { + name: "disallowed domain -> REFUSED", + raw: buildTestMessage(t, dnsmessage.Header{ + ID: 0x1007, + RecursionDesired: true, + }, []dnsmessage.Question{{ + Name: dnsmessage.MustNewName("Blocked.Example.COM."), + Type: dnsmessage.TypeA, + Class: dnsmessage.ClassINET, + }}), + wantRCode: dnsmessage.RCodeRefused, + }, + { + name: "disallowed record type -> REFUSED", + raw: buildTestMessage(t, dnsmessage.Header{ + ID: 0x1008, + RecursionDesired: true, + }, []dnsmessage.Question{{ + Name: dnsmessage.MustNewName("example.com."), + Type: dnsmessage.TypeTXT, + Class: dnsmessage.ClassINET, + }}), + wantRCode: dnsmessage.RCodeRefused, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + wantID := binary.BigEndian.Uint16(tc.raw[0:2]) + for _, act := range []action{ + h.onUDPRequest(tc.raw, nil, upstreams), + h.onTCPRequest(tc.raw), + } { + if act.kind != actionReply { + t.Fatalf("action.kind = %v, want actionReply", act.kind) + } + var p dnsmessage.Parser + hdr, err := p.Start(act.payload) + if err != nil { + t.Fatalf("parsing synthesized reply: %v", err) + } + if hdr.ID != wantID { + t.Errorf("reply ID = %#x, want %#x", hdr.ID, wantID) + } + if !hdr.Response { + t.Error("reply QR = false, want true") + } + if hdr.RCode != tc.wantRCode { + t.Errorf("reply RCode = %v, want %v", hdr.RCode, tc.wantRCode) + } + if got := lim.inFlight.occupied(); got != 0 { + t.Errorf("inFlight slots = %d after synthesized reply, want 0", got) + } + } + }) + } +} + +func TestDNSHandlerUDPForwardAndResponseValidation(t *testing.T) { + lim := newLimiter() + pending := newPendingRequests(lim) + h := newDNSHandler(pending, lim, nil) + + upstream1 := &net.UDPAddr{IP: net.ParseIP("10.96.0.10"), Port: 53} + upstream2 := &net.UDPAddr{IP: net.ParseIP("10.96.0.11"), Port: 53} + wrongUpstream := &net.UDPAddr{IP: net.ParseIP("10.96.0.99"), Port: 53} + clientAddr := &net.UDPAddr{IP: net.ParseIP("169.254.0.2"), Port: 54321} + + // Send query with mixed-case QNAME to verify case-insensitive RFC 4343 matching. + query := buildTestMessage(t, dnsmessage.Header{ + ID: 0xbeef, + RecursionDesired: true, + }, []dnsmessage.Question{{ + Name: dnsmessage.MustNewName("ExAmPlE.CoM."), + Type: dnsmessage.TypeA, + Class: dnsmessage.ClassINET, + }}) + + act := h.onUDPRequest(bytes.Clone(query), clientAddr, []string{upstream1.String(), upstream2.String()}) + if act.kind != actionForward { + t.Fatalf("onUDPRequest kind = %v, want actionForward", act.kind) + } + rewritten := act.payload + upstreamID := act.upstreamID + if gotID := binary.BigEndian.Uint16(rewritten[0:2]); gotID != upstreamID { + t.Fatalf("payload ID = %#x, want upstreamID %#x", gotID, upstreamID) + } + + // 1. Response with QR == 0 is dropped. + if res := h.onUDPResponse(rewritten, upstream1); res.kind != actionDrop { + t.Errorf("QR=0 response kind = %v, want actionDrop", res.kind) + } + + // 2. Response from unexpected upstream address is dropped without evicting pending request. + validResp := dnsAnswer(rewritten, 0) + if res := h.onUDPResponse(validResp, wrongUpstream); res.kind != actionDrop { + t.Errorf("wrong upstream source kind = %v, want actionDrop", res.kind) + } + + // 3. Response with wrong transaction ID is dropped. + wrongIDResp := bytes.Clone(validResp) + binary.BigEndian.PutUint16(wrongIDResp[0:2], upstreamID^0xffff) + if res := h.onUDPResponse(wrongIDResp, upstream1); res.kind != actionDrop { + t.Errorf("wrong ID response kind = %v, want actionDrop", res.kind) + } + + // 4. Response with mismatched QNAME is dropped. + wrongQNameResp := buildTestMessage(t, dnsmessage.Header{ + ID: upstreamID, + Response: true, + }, []dnsmessage.Question{{ + Name: dnsmessage.MustNewName("other.com."), + Type: dnsmessage.TypeA, + Class: dnsmessage.ClassINET, + }}) + if res := h.onUDPResponse(wrongQNameResp, upstream1); res.kind != actionDrop { + t.Errorf("wrong QNAME response kind = %v, want actionDrop", res.kind) + } + + // 5. Response with mismatched QTYPE is dropped. + wrongQTypeResp := buildTestMessage(t, dnsmessage.Header{ + ID: upstreamID, + Response: true, + }, []dnsmessage.Question{{ + Name: dnsmessage.MustNewName("example.com."), + Type: dnsmessage.TypeAAAA, + Class: dnsmessage.ClassINET, + }}) + if res := h.onUDPResponse(wrongQTypeResp, upstream1); res.kind != actionDrop { + t.Errorf("wrong QTYPE response kind = %v, want actionDrop", res.kind) + } + + // 6. SERVFAIL from upstream1 triggers ActionFailover to upstream2. + servFailResp := dnsAnswer(rewritten, byte(dnsmessage.RCodeServerFailure)) + failoverAct := h.onUDPResponse(servFailResp, upstream1) + if failoverAct.kind != actionFailover { + t.Fatalf("SERVFAIL on first upstream kind = %v, want actionFailover", failoverAct.kind) + } + if failoverAct.upstreamIdx != 1 { + t.Errorf("failover upstreamIdx = %d, want 1", failoverAct.upstreamIdx) + } + if got := lim.inFlight.occupied(); got != 1 { + t.Errorf("inFlight slots during failover = %d, want 1", got) + } + + // 7. Valid response from upstream2 (with different QNAME casing) delivers and restores clientRequestID. + lowerResp := buildTestMessage(t, dnsmessage.Header{ + ID: upstreamID, + Response: true, + RCode: dnsmessage.RCodeSuccess, + }, []dnsmessage.Question{{ + Name: dnsmessage.MustNewName("example.com."), + Type: dnsmessage.TypeA, + Class: dnsmessage.ClassINET, + }}) + deliverAct := h.onUDPResponse(lowerResp, upstream2) + if deliverAct.kind != actionDeliver { + t.Fatalf("valid response on second upstream kind = %v, want actionDeliver", deliverAct.kind) + } + if gotID := binary.BigEndian.Uint16(deliverAct.payload[0:2]); gotID != 0xbeef { + t.Errorf("delivered ID = %#x, want 0xbeef", gotID) + } + if deliverAct.clientAddr != clientAddr { + t.Errorf("delivered clientAddr = %v, want %v", deliverAct.clientAddr, clientAddr) + } + if got := lim.inFlight.occupied(); got != 0 { + t.Errorf("inFlight slots after delivery = %d, want 0", got) + } +} + +func TestDNSHandlerTCPForwardAndResponseValidation(t *testing.T) { + lim := newLimiter() + pending := newPendingRequests(lim) + h := newDNSHandler(pending, lim, nil) + + query := dnsQuery(0xbeef) + act := h.onTCPRequest(bytes.Clone(query)) + if act.kind != actionForward { + t.Fatalf("onTCPRequest kind = %v, want actionForward", act.kind) + } + // TCP does not rewrite the transaction ID or record into pendingRequests. + if !bytes.Equal(act.payload, query) { + t.Errorf("onTCPRequest modified query payload") + } + if len(pending.entries) != 0 || lim.inFlight.occupied() != 0 { + t.Errorf("onTCPRequest touched pendingRequests (%d entries) or inFlight (%d slots)", len(pending.entries), lim.inFlight.occupied()) + } + + // Invalid responses are dropped. + if res := h.onTCPResponse([]byte{1, 2, 3}); res.kind != actionDrop { + t.Errorf("short TCP response kind = %v, want actionDrop", res.kind) + } + if res := h.onTCPResponse(query); res.kind != actionDrop { + t.Errorf("QR=0 TCP response kind = %v, want actionDrop", res.kind) + } + + // Valid response is delivered unmodified. + resp := dnsAnswer(query, 0) + deliverAct := h.onTCPResponse(resp) + if deliverAct.kind != actionDeliver { + t.Fatalf("valid TCP response kind = %v, want actionDeliver", deliverAct.kind) + } + if !bytes.Equal(deliverAct.payload, resp) { + t.Errorf("delivered TCP response differs from input") + } +} diff --git a/internal/ateomnet/dns/freemap/doc.go b/internal/ateomnet/dns/freemap/doc.go new file mode 100644 index 0000000000..36326a59cc --- /dev/null +++ b/internal/ateomnet/dns/freemap/doc.go @@ -0,0 +1,17 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package freemap implements an efficient mechanism for maintaining a map +// of free slots. +package freemap diff --git a/internal/ateomnet/dns/freemap/freemap64k.go b/internal/ateomnet/dns/freemap/freemap64k.go new file mode 100644 index 0000000000..1c5d7e5d56 --- /dev/null +++ b/internal/ateomnet/dns/freemap/freemap64k.go @@ -0,0 +1,204 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package freemap + +import ( + "math/bits" +) + +const ( + map64kBits = 65536 + wordSize = 64 + l0words = map64kBits / wordSize // 1024 words + l1words = l0words / wordSize // 16 words + l2Mask = (1 << l1words) - 1 // 0xFFFF +) + +// Map64k is a free map that supports 64k entries. This data structure +// uses ~8 KiB of memory. +// +// The zero value is an empty map ready to use. +// +// This data structure is not thread-safe. +// +// This is represented as a hierarchical bit-set. +type Map64k struct { + // Layer 0: Leaf-level storage (1 = occupied, 0 = empty) + l0 [l0words]uint64 + // Layer 1: Intermediate summary (1 = corresponding L0 word is fully occupied) + l1 [l1words]uint64 + // Layer 2: Root summary (1 = corresponding L1 word is fully occupied) + l2 uint64 + // count is the number of bits currently set in the map. + count int +} + +// Count returns the number of bits set in the map. +func (m *Map64k) Count() int { return m.count } + +// IsFull returns true if there is no more room in the map. +func (m *Map64k) IsFull() bool { return m.count == map64kBits } + +// bitsetIndex into the hierarchical bitset. +type bitsetIndex struct { + l0, l0Bit int + l1, l1Bit int +} + +// getIndex computes the index across all of the hierarchical bitsets. +func getIndex(index uint16) bitsetIndex { + return bitsetIndex{ + l0: int(index / wordSize), + l0Bit: int(index % wordSize), + l1: int(index / wordSize / wordSize), + l1Bit: int((index / wordSize) % wordSize), + } +} + +// Get returns true if the index is set. +func (m *Map64k) Get(index uint16) bool { + sidx := getIndex(index) + return m.l0[sidx.l0]&(1<= end { + return -1 + } + + // Fast path: nothing to find in a full map. + if m.IsFull() { + return -1 + } + + startIdx := getIndex(start) + + // 1. Scan remaining bits in the current Layer 0 word. + w0 := m.l0[startIdx.l0] | ((uint64(1) << startIdx.l0Bit) - 1) + if inv0 := ^w0; inv0 != 0 { + idx := (startIdx.l0 * wordSize) + bits.TrailingZeros64(inv0) + if idx < end { + return idx + } + return -1 + } + + // 2. Scan remaining Layer 0 words in the current Layer 1 word. + if nextL1Bit := startIdx.l1Bit + 1; nextL1Bit < wordSize && (startIdx.l0+1)*wordSize < end { + w1 := m.l1[startIdx.l1] | ((uint64(1) << nextL1Bit) - 1) + if inv1 := ^w1; inv1 != 0 { + l0WordInL1 := bits.TrailingZeros64(inv1) + targetL0 := (startIdx.l1 * wordSize) + l0WordInL1 + idx := (targetL0 * wordSize) + bits.TrailingZeros64(^m.l0[targetL0]) + if idx < end { + return idx + } + return -1 + } + } + + // 3. Scan remaining Layer 1 words in Layer 2 (Root). + if nextL1 := startIdx.l1 + 1; nextL1 < l1words && nextL1*wordSize*wordSize < end { + w2 := m.l2 | ((uint64(1) << nextL1) - 1) + if inv2 := (^w2) & l2Mask; inv2 != 0 { + targetL1 := bits.TrailingZeros64(inv2) + l0WordInL1 := bits.TrailingZeros64(^m.l1[targetL1]) + targetL0 := (targetL1 * wordSize) + l0WordInL1 + idx := (targetL0 * wordSize) + bits.TrailingZeros64(^m.l0[targetL0]) + if idx < end { + return idx + } + return -1 + } + } + + return -1 +} diff --git a/internal/ateomnet/dns/freemap/freemap64k_test.go b/internal/ateomnet/dns/freemap/freemap64k_test.go new file mode 100644 index 0000000000..4efb15136d --- /dev/null +++ b/internal/ateomnet/dns/freemap/freemap64k_test.go @@ -0,0 +1,624 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package freemap + +import ( + "math/rand/v2" + "testing" +) + +// asInt flattens a (index, ok) result into an int, with -1 for !ok, so tests +// can compare against a single expected value. +func asInt(idx uint16, ok bool) int { + if !ok { + return -1 + } + return int(idx) +} + +// fill sets every index in [start, end). +func fill(m *Map64k, start, end int) { + for i := start; i < end; i++ { + m.Set(uint16(i)) + } +} + +// refMap pairs a Map64k with a naive boolean-slice model of it. +type refMap struct { + m Map64k + occupied []bool + wantCount int +} + +func newRefMap() *refMap { + return &refMap{occupied: make([]bool, map64kBits)} +} + +func (r *refMap) set(idx uint16) { + if !r.occupied[idx] { + r.occupied[idx] = true + r.wantCount++ + } + r.m.Set(idx) +} + +func (r *refMap) clear(idx uint16) { + if r.occupied[idx] { + r.occupied[idx] = false + r.wantCount-- + } + r.m.Clear(idx) +} + +func (r *refMap) naiveFindFirstUnsetRange(start uint16, end int) int { + for idx := int(start); idx < end; idx++ { + if !r.occupied[idx] { + return idx + } + } + return -1 +} + +func (r *refMap) naiveFindFirstUnsetFrom(start uint16) int { + if idx := r.naiveFindFirstUnsetRange(start, map64kBits); idx != -1 { + return idx + } + return r.naiveFindFirstUnsetRange(0, int(start)) +} + +// checkRange compares findFirstUnsetRange against the model. +func (r *refMap) checkRange(t *testing.T, step int, start uint16, end int) { + t.Helper() + if got, want := r.m.findFirstUnsetRange(start, end), r.naiveFindFirstUnsetRange(start, end); got != want { + t.Fatalf("step %d: findFirstUnsetRange(%d, %d) = %d, want %d", step, start, end, got, want) + } +} + +// checkFrom compares FindFirstUnsetFrom and FindFirstUnset against the model. +func (r *refMap) checkFrom(t *testing.T, step int, start uint16) { + t.Helper() + if got, want := asInt(r.m.FindFirstUnsetFrom(start)), r.naiveFindFirstUnsetFrom(start); got != want { + t.Fatalf("step %d: FindFirstUnsetFrom(%d) = %d, want %d", step, start, got, want) + } + if got, want := asInt(r.m.FindFirstUnset()), r.naiveFindFirstUnsetRange(0, map64kBits); got != want { + t.Fatalf("step %d: FindFirstUnset() = %d, want %d", step, got, want) + } +} + +// checkInvariants compares Count and IsFull against the model. +func (r *refMap) checkInvariants(t *testing.T, step int) { + t.Helper() + if got := r.m.Count(); got != r.wantCount { + t.Fatalf("step %d: Count() = %d, want %d", step, got, r.wantCount) + } + if got, want := r.m.IsFull(), r.wantCount == map64kBits; got != want { + t.Fatalf("step %d: IsFull() = %v, want %v", step, got, want) + } +} + +func TestGetIndex(t *testing.T) { + tests := []struct { + index uint16 + want bitsetIndex + }{ + {index: 0, want: bitsetIndex{l0: 0, l0Bit: 0, l1: 0, l1Bit: 0}}, + {index: 63, want: bitsetIndex{l0: 0, l0Bit: 63, l1: 0, l1Bit: 0}}, + {index: 64, want: bitsetIndex{l0: 1, l0Bit: 0, l1: 0, l1Bit: 1}}, + {index: 127, want: bitsetIndex{l0: 1, l0Bit: 63, l1: 0, l1Bit: 1}}, + {index: 4095, want: bitsetIndex{l0: 63, l0Bit: 63, l1: 0, l1Bit: 63}}, + {index: 4096, want: bitsetIndex{l0: 64, l0Bit: 0, l1: 1, l1Bit: 0}}, + {index: 65535, want: bitsetIndex{l0: 1023, l0Bit: 63, l1: 15, l1Bit: 63}}, + } + + for _, tc := range tests { + got := getIndex(tc.index) + if got != tc.want { + t.Errorf("getIndex(%d) = %+v, want %+v", tc.index, got, tc.want) + } + } +} + +func TestMap64k_Basic(t *testing.T) { + var m Map64k + if got := m.Count(); got != 0 { + t.Errorf("expected count 0, got %d", got) + } + if m.IsFull() { + t.Errorf("expected IsFull() = false on empty map") + } + if got := asInt(m.FindFirstUnset()); got != 0 { + t.Errorf("expected first unset 0, got %d", got) + } + if m.Get(0) { + t.Errorf("expected Get(0) = false initially") + } + + m.Set(0) + if got := m.Count(); got != 1 { + t.Errorf("expected count 1, got %d", got) + } + if m.IsFull() { + t.Errorf("expected IsFull() = false with 1 bit set") + } + if !m.Get(0) { + t.Errorf("expected Get(0) = true after Set(0)") + } + if got := asInt(m.FindFirstUnset()); got != 1 { + t.Errorf("expected first unset 1, got %d", got) + } + + // Idempotent Set + m.Set(0) + if got := m.Count(); got != 1 { + t.Errorf("expected count 1 after redundant set, got %d", got) + } + + m.Clear(0) + if got := m.Count(); got != 0 { + t.Errorf("expected count 0, got %d", got) + } + if m.Get(0) { + t.Errorf("expected Get(0) = false after Clear(0)") + } + if got := asInt(m.FindFirstUnset()); got != 0 { + t.Errorf("expected first unset 0, got %d", got) + } + + // Idempotent Clear + m.Clear(0) + if got := m.Count(); got != 0 { + t.Errorf("expected count 0 after redundant clear, got %d", got) + } +} + +func TestMap64k_Get(t *testing.T) { + var m Map64k + testIndices := []uint16{0, 1, 63, 64, 127, 1024, 4095, 4096, 65534, 65535} + + for _, idx := range testIndices { + if m.Get(idx) { + t.Errorf("Get(%d) = true on empty map, want false", idx) + } + m.Set(idx) + if !m.Get(idx) { + t.Errorf("Get(%d) = false after Set, want true", idx) + } + m.Clear(idx) + if m.Get(idx) { + t.Errorf("Get(%d) = true after Clear, want false", idx) + } + } +} + +func TestMap64k_HierarchyPropagation(t *testing.T) { + var m Map64k + + // 1. Fill word 0 (bits 0..63). L1 bit 0 should be set, L2 should still be 0. + fill(&m, 0, 64) + if (m.l1[0] & 1) == 0 { + t.Errorf("expected l1[0] bit 0 to be set") + } + if m.l2 != 0 { + t.Errorf("expected l2 to be 0, got %x", m.l2) + } + + // Clearing one bit in word 0 should clear L1 bit 0. + m.Clear(10) + if (m.l1[0] & 1) != 0 { + t.Errorf("expected l1[0] bit 0 to be cleared after clearing bit 10") + } + + // Re-setting bit 10 should restore L1 bit 0. + m.Set(10) + if (m.l1[0] & 1) == 0 { + t.Errorf("expected l1[0] bit 0 to be set after re-setting bit 10") + } + + // 2. Fill all 4096 bits in L1 word 0 (words 0..63). L2 bit 0 should be set. + fill(&m, 64, 4096) + if (m.l2 & 1) == 0 { + t.Errorf("expected l2 bit 0 to be set after filling 4096 bits") + } + + // Clearing one bit should clear L1 bit and propagate to L2 bit 0. + m.Clear(2000) + if (m.l2 & 1) != 0 { + t.Errorf("expected l2 bit 0 to be cleared after clearing bit 2000") + } +} + +func TestMap64k_FindFirstUnsetFrom_Empty(t *testing.T) { + var m Map64k + testIndices := []uint16{0, 1, 63, 64, 100, 1023, 1024, 4095, 4096, 65535} + for _, idx := range testIndices { + if got := asInt(m.FindFirstUnsetFrom(idx)); got != int(idx) { + t.Errorf("FindFirstUnsetFrom(%d) on empty map = %d, want %d", idx, got, idx) + } + } +} + +func TestMap64k_FindFirstUnsetFrom_Full(t *testing.T) { + var m Map64k + fill(&m, 0, map64kBits) + if got := m.Count(); got != map64kBits { + t.Fatalf("expected count %d, got %d", map64kBits, got) + } + if !m.IsFull() { + t.Errorf("expected IsFull() = true on full map") + } + if _, ok := m.FindFirstUnset(); ok { + t.Errorf("FindFirstUnset on full map returned ok = true") + } + for _, idx := range []uint16{0, 1, 100, 4096, 65535} { + if _, ok := m.FindFirstUnsetFrom(idx); ok { + t.Errorf("FindFirstUnsetFrom(%d) on full map returned ok = true", idx) + } + } +} + +func TestMap64k_FindFirstUnsetFrom_SingleHole(t *testing.T) { + var m Map64k + fill(&m, 0, map64kBits) + + testHoles := []uint16{0, 1, 63, 64, 127, 128, 4095, 4096, 32768, 65534, 65535} + for _, hole := range testHoles { + m.Clear(hole) + if m.IsFull() { + t.Errorf("hole %d: expected IsFull() = false", hole) + } + if got := asInt(m.FindFirstUnset()); got != int(hole) { + t.Errorf("hole %d: FindFirstUnset = %d, want %d", hole, got, hole) + } + + // Starting from any index should find the only hole + for _, start := range []uint16{0, hole, hole + 1, 65535} { + if got := asInt(m.FindFirstUnsetFrom(start)); got != int(hole) { + t.Errorf("hole %d, start %d: FindFirstUnsetFrom = %d, want %d", hole, start, got, hole) + } + } + + m.Set(hole) + if !m.IsFull() { + t.Errorf("hole %d: expected IsFull() = true after re-setting hole", hole) + } + } +} + +func TestMap64k_FindFirstUnsetFrom_Levels(t *testing.T) { + check := func(t *testing.T, m *Map64k, start uint16, want int) { + t.Helper() + if got := asInt(m.FindFirstUnsetFrom(start)); got != want { + t.Errorf("FindFirstUnsetFrom(%d) = %d, want %d", start, got, want) + } + } + + t.Run("same L0 word", func(t *testing.T) { + var m Map64k + fill(&m, 0, 10) + for _, start := range []uint16{0, 5, 10} { + check(t, &m, start, 10) + } + }) + + t.Run("next L0 word in same L1 word", func(t *testing.T) { + var m Map64k + // Fill L0 word 0 (bits 0..63) and bits 0..4 of word 1 (bits 64..68) + fill(&m, 0, 69) + for _, start := range []uint16{0, 63, 64} { + check(t, &m, start, 69) + } + }) + + t.Run("next L1 word from L2", func(t *testing.T) { + var m Map64k + // Fill entire L1 word 0 (bits 0..4095) and bits 4096..4099 in L1 word 1. + fill(&m, 0, 4100) + for _, start := range []uint16{0, 2000, 4095, 4096} { + check(t, &m, start, 4100) + } + + // Fill the rest of L1 word 1 and bits 8192..8199 in L1 word 2. + fill(&m, 4100, 8200) + for _, start := range []uint16{0, 4095, 4096, 8191, 8192} { + check(t, &m, start, 8200) + } + }) + + t.Run("wrap around", func(t *testing.T) { + var m Map64k + fill(&m, 100, map64kBits) + fill(&m, 0, 10) + // Search from 100 should wrap around and find 10 + for _, start := range []uint16{100, 65535} { + check(t, &m, start, 10) + } + }) + + t.Run("wrap around with last bit set", func(t *testing.T) { + var m Map64k + m.Set(65535) + check(t, &m, 65535, 0) + }) +} + +func TestMap64k_FindFirstUnsetRange(t *testing.T) { + var m Map64k + + // Empty ranges + if got := m.findFirstUnsetRange(5, 5); got != -1 { + t.Errorf("range [5, 5) = %d, want -1", got) + } + if got := m.findFirstUnsetRange(10, 5); got != -1 { + t.Errorf("range [10, 5) = %d, want -1", got) + } + + // Range within single word + fill(&m, 10, 20) + if got := m.findFirstUnsetRange(10, 20); got != -1 { + t.Errorf("range [10, 20) fully set, got %d, want -1", got) + } + if got := m.findFirstUnsetRange(10, 25); got != 20 { + t.Errorf("range [10, 25), got %d, want 20", got) + } + if got := m.findFirstUnsetRange(5, 20); got != 5 { + t.Errorf("range [5, 20), got %d, want 5", got) + } + + // Range spanning word boundary + fill(&m, 60, 64) + // Bits 60..63 are set, 64 is unset + if got := m.findFirstUnsetRange(60, 64); got != -1 { + t.Errorf("range [60, 64) fully set, got %d, want -1", got) + } + if got := m.findFirstUnsetRange(60, 70); got != 64 { + t.Errorf("range [60, 70), got %d, want 64", got) + } + + // Range spanning L1 boundary (word 63 to word 64, bit 4095 to 4096) + fill(&m, 4090, 4096) + if got := m.findFirstUnsetRange(4090, 4096); got != -1 { + t.Errorf("range [4090, 4096) fully set, got %d, want -1", got) + } + if got := m.findFirstUnsetRange(4090, 4100); got != 4096 { + t.Errorf("range [4090, 4100), got %d, want 4096", got) + } + + // Full span + if got := m.findFirstUnsetRange(0, map64kBits); got != 0 { + t.Errorf("range [0, map64kBits), got %d, want 0", got) + } + + // Full map + fill(&m, 0, map64kBits) + if got := m.findFirstUnsetRange(0, map64kBits); got != -1 { + t.Errorf("range [0, map64kBits) on full map, got %d, want -1", got) + } + if got := m.findFirstUnsetRange(100, 200); got != -1 { + t.Errorf("range [100, 200) on full map, got %d, want -1", got) + } +} + +// TestMap64k_FindFirstUnsetRange_Cutoffs checks that a free bit found by the +// L1 (step 2) and L2 (step 3) scans is rejected when it lies past `end`. +func TestMap64k_FindFirstUnsetRange_Cutoffs(t *testing.T) { + t.Run("L1 scan", func(t *testing.T) { + var m Map64k + fill(&m, 0, 128) // L0 words 0 and 1 full; first free is 128. + if got := m.findFirstUnsetRange(0, 100); got != -1 { + t.Errorf("range [0, 100), got %d, want -1", got) + } + if got := m.findFirstUnsetRange(0, 128); got != -1 { + t.Errorf("range [0, 128), got %d, want -1", got) + } + if got := m.findFirstUnsetRange(0, 129); got != 128 { + t.Errorf("range [0, 129), got %d, want 128", got) + } + }) + + t.Run("L2 scan", func(t *testing.T) { + var m Map64k + fill(&m, 0, 8192) // L1 words 0 and 1 full; first free is 8192. + if got := m.findFirstUnsetRange(100, 5000); got != -1 { + t.Errorf("range [100, 5000), got %d, want -1", got) + } + if got := m.findFirstUnsetRange(100, 8192); got != -1 { + t.Errorf("range [100, 8192), got %d, want -1", got) + } + if got := m.findFirstUnsetRange(100, 9000); got != 8192 { + t.Errorf("range [100, 9000), got %d, want 8192", got) + } + }) +} + +// TestMap64k_Differential against a naive linear search on a boolean slice. +func TestMap64k_Differential(t *testing.T) { + ref := newRefMap() + r := rand.New(rand.NewPCG(42, 1337)) + + for step := range 5000 { + op := r.IntN(12) + switch { + case op < 3: // Set a random bit + idx := uint16(r.IntN(map64kBits)) + ref.set(idx) + if !ref.m.Get(idx) { + t.Fatalf("step %d: Get(%d) = false after Set", step, idx) + } + case op < 5: // Clear a random bit + idx := uint16(r.IntN(map64kBits)) + ref.clear(idx) + if ref.m.Get(idx) { + t.Fatalf("step %d: Get(%d) = true after Clear", step, idx) + } + case op == 5: // Set an entire 64-bit L0 word + base := uint16(r.IntN(l0words) * wordSize) + for i := range uint16(wordSize) { + ref.set(base + i) + } + case op == 6: // Clear an entire 64-bit L0 word + base := uint16(r.IntN(l0words) * wordSize) + for i := range uint16(wordSize) { + ref.clear(base + i) + } + case op == 7: // Set or clear an entire 4096-bit L1 block + base := uint16(r.IntN(l1words) * wordSize * wordSize) + set := r.IntN(2) == 0 + for i := range uint16(wordSize * wordSize) { + if set { + ref.set(base + i) + } else { + ref.clear(base + i) + } + } + case op < 10: // Query range + start := uint16(r.IntN(map64kBits)) + end := int(start) + r.IntN(map64kBits-int(start)+1) + ref.checkRange(t, step, start, end) + default: // Query from + ref.checkFrom(t, step, uint16(r.IntN(map64kBits))) + } + + ref.checkInvariants(t, step) + } +} + +func TestMap64k_Differential_Dense(t *testing.T) { + ref := newRefMap() + r := rand.New(rand.NewPCG(99, 12345)) + + // Start completely full so all L1 and L2 summary bits are set, then punch a few holes. + for i := range map64kBits { + ref.set(uint16(i)) + } + // holes records every cleared index. It may contain duplicates (clearing an + // already-free bit appends it again); that is harmless and keeps the + // property that an empty holes list means the map is completely full, so + // the test regularly drives the map back to full. + var holes []uint16 + for range 16 { + idx := uint16(r.IntN(map64kBits)) + ref.clear(idx) + holes = append(holes, idx) + } + + for step := range 5000 { + op := r.IntN(10) + switch { + case op < 3: // Plug an existing hole (or random bit) so L0/L1/L2 words become full again + if len(holes) > 0 { + hIdx := r.IntN(len(holes)) + ref.set(holes[hIdx]) + holes[hIdx] = holes[len(holes)-1] + holes = holes[:len(holes)-1] + } else { + ref.set(uint16(r.IntN(map64kBits))) + } + case op < 5: // Punch a new hole + idx := uint16(r.IntN(map64kBits)) + ref.clear(idx) + holes = append(holes, idx) + case op < 7: // Query range + start := uint16(r.IntN(map64kBits)) + end := int(start) + r.IntN(map64kBits-int(start)+1) + ref.checkRange(t, step, start, end) + default: // Query from and FindFirstUnset + ref.checkFrom(t, step, uint16(r.IntN(map64kBits))) + } + + ref.checkInvariants(t, step) + } +} + +// singleHoleMap returns a full map with only index 50000 free. +func singleHoleMap() *Map64k { + var m Map64k + fill(&m, 0, map64kBits) + m.Clear(50000) + return &m +} + +// sparseMap returns a map with ~37% of bits set at random. +func sparseMap() *Map64k { + var m Map64k + r := rand.New(rand.NewPCG(1, 2)) + for range 30000 { + m.Set(uint16(r.IntN(map64kBits))) + } + return &m +} + +func BenchmarkFindFirstUnsetFrom(b *testing.B) { + b.Run("sparse", func(b *testing.B) { + m := sparseMap() + var i uint16 + for b.Loop() { + m.FindFirstUnsetFrom(i) + i++ + } + }) + + b.Run("dense_single_hole", func(b *testing.B) { + m := singleHoleMap() + var i uint16 + for b.Loop() { + m.FindFirstUnsetFrom(i) + i++ + } + }) +} + +func BenchmarkFindFirstUnsetRange(b *testing.B) { + b.Run("sparse", func(b *testing.B) { + m := sparseMap() + var i int + for b.Loop() { + start := uint16(i % (map64kBits - 100)) + m.findFirstUnsetRange(start, int(start)+100) + i++ + } + }) + + b.Run("dense_single_hole", func(b *testing.B) { + m := singleHoleMap() + var i int + for b.Loop() { + start := uint16(i % (map64kBits - 100)) + m.findFirstUnsetRange(start, map64kBits) + i++ + } + }) +} + +// BenchmarkSetClear measures an allocate/free cycle, including the L1/L2 +// summary updates when a word transitions between full and not full. +func BenchmarkSetClear(b *testing.B) { + b.Run("sparse", func(b *testing.B) { + var m Map64k + var i uint16 + for b.Loop() { + m.Set(i) + m.Clear(i) + i++ + } + }) + + b.Run("dense_single_hole", func(b *testing.B) { + m := singleHoleMap() + for b.Loop() { + m.Set(50000) // Fills L0, L1 and L2. + m.Clear(50000) + } + }) +} diff --git a/internal/ateomnet/dns/limiter.go b/internal/ateomnet/dns/limiter.go new file mode 100644 index 0000000000..18f62d77fa --- /dev/null +++ b/internal/ateomnet/dns/limiter.go @@ -0,0 +1,74 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dns + +import "sync/atomic" + +const ( + // maxInFlight UDP queries. + maxInFlight = 64 + // maxConnections bounds open TCP connections. + maxConnections = 16 +) + +// limiter caps tracks DNS query limits. +type limiter struct { + inFlight *limiterSlots + connections *limiterSlots +} + +func newLimiter() *limiter { + return &limiter{ + inFlight: &limiterSlots{max: maxInFlight}, + connections: &limiterSlots{max: maxConnections}, + } +} + +type limiterSlots struct { + max int32 + used atomic.Int32 +} + +// tryAcquire a virtual slot. Returns false if this slot is at capacity. +func (s *limiterSlots) tryAcquire() bool { + // Disabled if zero. + if s == nil { + return true + } + + if s.used.Add(1) > s.max { + s.used.Add(-1) + return false + } + return true +} + +// release a use slot. MUST be paired with a successful tryAcquire. +func (s *limiterSlots) release() { + // Disabled if zero. + if s == nil { + return + } + + if s.used.Add(-1) < 0 { + panic("dns: slot released more times than acquired") + } +} + +// occupied returns the current slots occupied. Warning: this should only be +// used for testing as the count will not be accurate due to concurrency. +func (s *limiterSlots) occupied() int32 { + return s.used.Load() +} diff --git a/internal/ateomnet/dns/limiter_test.go b/internal/ateomnet/dns/limiter_test.go new file mode 100644 index 0000000000..4c77b1580f --- /dev/null +++ b/internal/ateomnet/dns/limiter_test.go @@ -0,0 +1,137 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dns + +import ( + "sync" + "sync/atomic" + "testing" +) + +func TestNewLimiter(t *testing.T) { + lim := newLimiter() + if lim.inFlight == nil || lim.inFlight.max != maxInFlight { + t.Errorf("inFlight = %+v, want max=%d", lim.inFlight, maxInFlight) + } + if lim.connections == nil || lim.connections.max != maxConnections { + t.Errorf("connections = %+v, want max=%d", lim.connections, maxConnections) + } + if got := lim.inFlight.occupied(); got != 0 { + t.Errorf("initial inFlight.occupied() = %d, want 0", got) + } + if got := lim.connections.occupied(); got != 0 { + t.Errorf("initial connections.occupied() = %d, want 0", got) + } +} + +func TestLimiterSlotsAcquireAndRelease(t *testing.T) { + const cap = 3 + s := &limiterSlots{max: cap} + + for i := int32(1); i <= cap; i++ { + if !s.tryAcquire() { + t.Fatalf("tryAcquire %d failed before reaching capacity %d", i, cap) + } + if got := s.occupied(); got != i { + t.Errorf("occupied() = %d after %d acquisitions, want %d", got, i, i) + } + } + + // At capacity: further acquisitions fail and do not change occupied count. + if s.tryAcquire() { + t.Fatal("tryAcquire succeeded at capacity") + } + if got := s.occupied(); got != cap { + t.Errorf("occupied() after rejected tryAcquire = %d, want %d", got, cap) + } + + // Releasing one slot allows one more acquisition. + s.release() + if got := s.occupied(); got != cap-1 { + t.Errorf("occupied() after release = %d, want %d", got, cap-1) + } + if !s.tryAcquire() { + t.Fatal("tryAcquire failed after releasing a slot") + } + + // Drain all slots. + for i := cap - 1; i >= 0; i-- { + s.release() + if got := s.occupied(); got != int32(i) { + t.Errorf("occupied() = %d, want %d", got, i) + } + } +} + +func TestLimiterSlotsNilReceiver(t *testing.T) { + var s *limiterSlots + if !s.tryAcquire() { + t.Error("nil limiterSlots.tryAcquire() = false, want true") + } + // Must not panic. + s.release() +} + +func TestLimiterSlotsReleaseUnderflowPanics(t *testing.T) { + s := &limiterSlots{max: 1} + defer func() { + if r := recover(); r == nil { + t.Fatal("expected panic when releasing an unacquired slot, got none") + } + }() + s.release() +} + +func TestLimiterSlotsConcurrent(t *testing.T) { + const ( + maxSlots = 8 + goroutines = 32 + iterations = 200 + ) + s := &limiterSlots{max: maxSlots} + + var active atomic.Int32 + var maxObserved atomic.Int32 + var wg sync.WaitGroup + + for range goroutines { + wg.Add(1) + go func() { + defer wg.Done() + for range iterations { + if !s.tryAcquire() { + continue + } + cur := active.Add(1) + for { + prev := maxObserved.Load() + if cur <= prev || maxObserved.CompareAndSwap(prev, cur) { + break + } + } + active.Add(-1) + s.release() + } + }() + } + wg.Wait() + + if got := maxObserved.Load(); got > maxSlots { + t.Errorf("max concurrent slots held = %d, exceeds limit %d", got, maxSlots) + } + if got := s.occupied(); got != 0 { + t.Errorf("occupied() after all goroutines finished = %d, want 0", got) + } +} diff --git a/internal/ateomnet/dns/parse.go b/internal/ateomnet/dns/parse.go deleted file mode 100644 index b3a7420155..0000000000 --- a/internal/ateomnet/dns/parse.go +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package dns - -import ( - "errors" - "fmt" -) - -// Response codes that say the resolver failed rather than answered. NXDOMAIN -// and NOERROR are answers and are passed back as they are. -const ( - rcodeServFail = 2 - rcodeNotImp = 4 - rcodeRefused = 5 -) - -// failoverRcode reports the response code when the relay should try the next -// upstream. Reads the 12-byte header only; anything shorter is passed through. -func failoverRcode(msg []byte) (byte, bool) { - if len(msg) < 12 { - return 0, false - } - rcode := msg[3] & 0x0f - switch rcode { - case rcodeServFail, rcodeNotImp, rcodeRefused: - return rcode, true - } - return 0, false -} - -func rcodeError(rcode byte) error { - switch rcode { - case rcodeServFail: - return errors.New("answered SERVFAIL") - case rcodeNotImp: - return errors.New("answered NOTIMP") - case rcodeRefused: - return errors.New("answered REFUSED") - } - return fmt.Errorf("answered rcode %d", rcode) -} diff --git a/internal/ateomnet/dns/pendingreq.go b/internal/ateomnet/dns/pendingreq.go new file mode 100644 index 0000000000..ba5146adeb --- /dev/null +++ b/internal/ateomnet/dns/pendingreq.go @@ -0,0 +1,242 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dns + +import ( + "bytes" + "math/rand/v2" + "net" + "sync" + "time" + + "github.com/agent-substrate/substrate/internal/ateomnet/dns/freemap" + "github.com/agent-substrate/substrate/internal/ateomnet/dns/protocol" + "golang.org/x/net/dns/dnsmessage" +) + +// pendingRequest holds the per-query state needed to validate UDP responses, +// restore the actor's transaction ID, and fail over across upstreams. +type pendingRequest struct { + // clientRequestID is the actor's original DNS transaction ID. + clientRequestID uint16 + // clientAddr identifies the actor UDP endpoint. + clientAddr net.Addr + // question is the canonicalized DNS question used to validate responses + // (RFC 5452). + question dnsmessage.Question + // rawQuery is a copy of the query with upstreamID written to bytes 0..1. + rawQuery []byte + // upstreams is the ordered list of upstream "host:port" addresses to try. + upstreams []string + // upstreamIdx is the index in upstreams of the current attempt. + upstreamIdx int + // deferredResp holds the last failover-eligible error response (e.g. + // SERVFAIL) to return if all remaining upstreams time out. + deferredResp []byte + // expiry is the deadline for the current upstream attempt. + expiry time.Time +} + +// pendingRequests maps upstreamID to in-flight UDP requests. +// +// TODO(bowei): move the response lookup and failover transition in +// dnsHandler.onUDPResponse into a method on pendingRequests so all lock and +// state transitions live in this file. +type pendingRequests struct { + // mu guards entries and inUse. + mu sync.Mutex + // entries maps active upstreamID keys to in-flight requests. + entries map[uint16]*pendingRequest + // inUse tracks allocated 16-bit upstream transaction IDs. + inUse freemap.Map64k + // limiter tracks in-flight request slots released when entries are removed. + limiter *limiter + + // exchangeTimeout bounds the total upstream exchange (defaults to + // dnsExchangeTimeout). + exchangeTimeout time.Duration + // attemptTimeout, if positive, overrides the per-attempt timeout (used in + // tests). + attemptTimeout time.Duration +} + +// newPendingRequests returns an empty pendingRequests table using lim for +// in-flight slot accounting. +func newPendingRequests(lim *limiter) *pendingRequests { + return &pendingRequests{ + entries: make(map[uint16]*pendingRequest), + limiter: lim, + exchangeTimeout: dnsExchangeTimeout, + } +} + +// timeoutForAttempt returns the deadline duration for attempt upstreamIdx out +// of numUpstreams. Attempts with fallback upstreams remaining get an equal +// share of exchangeTimeout capped at defaultUpstreamAttemptTimeout; the final +// upstream gets the full exchangeTimeout. +func (p *pendingRequests) timeoutForAttempt(upstreamIdx int, numUpstreams int) time.Duration { + if p.attemptTimeout > 0 { + return p.attemptTimeout + } + timeout := p.exchangeTimeout + if timeout <= 0 { + timeout = dnsExchangeTimeout + } + if numUpstreams > 1 && upstreamIdx+1 < numUpstreams { + return min(timeout/time.Duration(numUpstreams), defaultUpstreamAttemptTimeout) + } + return timeout +} + +// record allocates a unique upstreamID and stores a pendingRequest entry for +// upstreams[0]. It takes ownership of releasing the caller's in-flight slot on +// failure or entry removal. +func (p *pendingRequests) record( + raw []byte, + clientReqID uint16, + q dnsmessage.Question, + clientAddr net.Addr, + upstreams []string, +) (uint16, bool) { + p.mu.Lock() + defer p.mu.Unlock() + + // Probe from a random starting ID so upstream transaction IDs are not predictable. + start := uint16(rand.Uint32()) + upstreamID, ok := p.inUse.FindFirstUnsetFrom(start) + if !ok { + p.limiter.inFlight.release() + return 0, false + } + + // Rewrite the packet's transaction ID in place and clone it for failover retries. + protocol.SetTxnID(raw, upstreamID) + p.inUse.Set(upstreamID) + p.entries[upstreamID] = &pendingRequest{ + clientRequestID: clientReqID, + clientAddr: clientAddr, + question: q, + rawQuery: bytes.Clone(raw), + upstreams: upstreams, + upstreamIdx: 0, + expiry: time.Now().Add(p.timeoutForAttempt(0, len(upstreams))), + } + + return upstreamID, true +} + +// deleteEntryLocked removes upstreamID from entries, frees its ID in inUse, and +// releases its in-flight slot if upstreamID was present. p.mu must be held. +func (p *pendingRequests) deleteEntryLocked(upstreamID uint16) { + if _, ok := p.entries[upstreamID]; !ok { + return + } + delete(p.entries, upstreamID) + p.inUse.Clear(upstreamID) + p.limiter.inFlight.release() +} + +// failOverOnSendError advances an entry to the next upstream when sending to +// failedIdx fails synchronously, or removes the entry if no upstreams remain. +func (p *pendingRequests) failOverOnSendError( + upstreamID uint16, + failedIdx int, +) (nextIdx int, query []byte, ok bool) { + p.mu.Lock() + defer p.mu.Unlock() + + // Match both upstreamID and failedIdx so stale errors from earlier attempts + // are ignored. + entry, ok := p.entries[upstreamID] + if !ok || entry.upstreamIdx != failedIdx { + return 0, nil, false + } + // Drop the request if no fallback upstreams remain. + if entry.upstreamIdx+1 >= len(entry.upstreams) { + p.deleteEntryLocked(upstreamID) + return 0, nil, false + } + + // Advance the entry to the next upstream and reset its attempt deadline. + entry.upstreamIdx++ + entry.expiry = time.Now().Add(p.timeoutForAttempt(entry.upstreamIdx, len(entry.upstreams))) + return entry.upstreamIdx, entry.rawQuery, true +} + +// clearAll removes all pending entries and releases their in-flight slots. +func (p *pendingRequests) clearAll() { + p.mu.Lock() + defer p.mu.Unlock() + + for upstreamID := range p.entries { + p.deleteEntryLocked(upstreamID) + } +} + +// reqFailover describes a timed-out query to resend to the next upstream. +type reqFailover struct { + upstreamID uint16 + upstreamIdx int + query []byte +} + +// reqDelivery describes a deferred response to send back to the actor after all +// upstreams have been exhausted. +type reqDelivery struct { + clientAddr net.Addr + payload []byte +} + +// sweep inspects all pending entries at now, advancing expired multi-upstream +// entries to their next upstream and evicting entries that have exhausted all +// upstreams. +func (p *pendingRequests) sweep(now time.Time) ([]reqFailover, []reqDelivery) { + p.mu.Lock() + defer p.mu.Unlock() + + var failovers []reqFailover + var deliveries []reqDelivery + + for upstreamID, entry := range p.entries { + if entry.expiry.IsZero() || !now.After(entry.expiry) { + continue + } + // Advance to the next upstream if fallbacks remain. + if entry.upstreamIdx+1 < len(entry.upstreams) { + entry.upstreamIdx++ + entry.expiry = now.Add(p.timeoutForAttempt(entry.upstreamIdx, len(entry.upstreams))) + failovers = append(failovers, reqFailover{ + upstreamID: upstreamID, + upstreamIdx: entry.upstreamIdx, + query: entry.rawQuery, + }) + continue + } + + // All upstreams exhausted: evict the entry and deliver any held-back + // error response (e.g. SERVFAIL from an earlier upstream) with the + // actor's original transaction ID restored. + p.deleteEntryLocked(upstreamID) + if entry.deferredResp != nil && entry.clientAddr != nil { + resp := bytes.Clone(entry.deferredResp) + protocol.SetTxnID(resp, entry.clientRequestID) + deliveries = append(deliveries, reqDelivery{ + clientAddr: entry.clientAddr, + payload: resp, + }) + } + } + return failovers, deliveries +} diff --git a/internal/ateomnet/dns/pendingreq_test.go b/internal/ateomnet/dns/pendingreq_test.go new file mode 100644 index 0000000000..73ac25ccb0 --- /dev/null +++ b/internal/ateomnet/dns/pendingreq_test.go @@ -0,0 +1,367 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dns + +import ( + "bytes" + "encoding/binary" + "net" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "golang.org/x/net/dns/dnsmessage" +) + +// exampleQuestion is the question dnsQuery asks. +var exampleQuestion = dnsmessage.Question{ + Name: dnsmessage.MustNewName("example.com."), + Type: dnsmessage.TypeA, + Class: dnsmessage.ClassINET, +} + +// recordQuery records dnsQuery(clientID) holding an in-flight slot, as a +// forwarded query does, and returns its upstream ID and the rewritten query. +func recordQuery(t *testing.T, p *pendingRequests, clientID uint16, client net.Addr, upstreams []string) (uint16, []byte) { + t.Helper() + p.limiter.inFlight.tryAcquire() + raw := dnsQuery(clientID) + id, ok := p.record(raw, clientID, exampleQuestion, client, upstreams) + if !ok { + t.Fatalf("record(%#x) failed", clientID) + } + return id, raw +} + +// checkExpiry fails t unless expiry is timeout after an instant in [before, after]. +func checkExpiry(t *testing.T, expiry, before, after time.Time, timeout time.Duration) { + t.Helper() + if expiry.Before(before.Add(timeout)) || expiry.After(after.Add(timeout)) { + t.Errorf("expiry is %v after the call, want %v", expiry.Sub(before), timeout) + } +} + +func TestPendingRequestsTimeoutForAttempt(t *testing.T) { + // Long enough that splitting it two ways still exceeds the cap. + const exchange = 10 * defaultUpstreamAttemptTimeout + + for _, tc := range []struct { + name string + exchangeTimeout time.Duration + attemptTimeout time.Duration + upstreamIdx int + numUpstreams int + want time.Duration + }{ + { + name: "single upstream gets the whole exchange", + exchangeTimeout: exchange, + numUpstreams: 1, + want: exchange, + }, + { + name: "attempt with fallbacks left is capped", + exchangeTimeout: exchange, + numUpstreams: 2, + want: defaultUpstreamAttemptTimeout, + }, + { + name: "attempt with fallbacks left gets its share", + exchangeTimeout: defaultUpstreamAttemptTimeout, + upstreamIdx: 1, + numUpstreams: 4, + want: defaultUpstreamAttemptTimeout / 4, + }, + { + name: "last upstream gets the whole exchange", + exchangeTimeout: exchange, + upstreamIdx: 1, + numUpstreams: 2, + want: exchange, + }, + { + name: "unset exchange timeout uses the default", + numUpstreams: 1, + want: dnsExchangeTimeout, + }, + { + name: "attempt timeout overrides an attempt with fallbacks left", + exchangeTimeout: exchange, + attemptTimeout: 40 * time.Millisecond, + numUpstreams: 2, + want: 40 * time.Millisecond, + }, + { + name: "attempt timeout overrides the last attempt", + exchangeTimeout: exchange, + attemptTimeout: 40 * time.Millisecond, + upstreamIdx: 1, + numUpstreams: 2, + want: 40 * time.Millisecond, + }, + } { + t.Run(tc.name, func(t *testing.T) { + p := newPendingRequests(nil) + p.exchangeTimeout = tc.exchangeTimeout + p.attemptTimeout = tc.attemptTimeout + if got := p.timeoutForAttempt(tc.upstreamIdx, tc.numUpstreams); got != tc.want { + t.Errorf("timeoutForAttempt(%d, %d) = %v, want %v", tc.upstreamIdx, tc.numUpstreams, got, tc.want) + } + }) + } +} + +func TestPendingRequestsRecord(t *testing.T) { + lim := newLimiter() + p := newPendingRequests(lim) + client := &net.UDPAddr{IP: net.ParseIP("169.254.0.2"), Port: 54321} + // Responses are matched on their unmapped source address, so the key must + // be normalized. + upstreams := []string{"[::ffff:10.96.0.10]:53", "10.96.0.11:53"} + + lim.inFlight.tryAcquire() + raw := dnsQuery(0xbeef) + before := time.Now() + id, ok := p.record(raw, 0xbeef, exampleQuestion, client, upstreams) + after := time.Now() + if !ok { + t.Fatal("record failed") + } + if got := binary.BigEndian.Uint16(raw[0:2]); got != id { + t.Errorf("query ID = %#x, want the upstream ID %#x", got, id) + } + entry, ok := p.entries[id] + if !ok { + t.Fatal("no entry keyed by the upstream ID") + } + want := &pendingRequest{ + clientRequestID: 0xbeef, + clientAddr: client, + question: exampleQuestion, + rawQuery: raw, + upstreams: upstreams, + } + if diff := cmp.Diff(want, entry, cmp.AllowUnexported(pendingRequest{}), cmpopts.IgnoreFields(pendingRequest{}, "expiry")); diff != "" { + t.Errorf("entry mismatch (-want +got):\n%s", diff) + } + checkExpiry(t, entry.expiry, before, after, p.timeoutForAttempt(0, len(upstreams))) + if !p.inUse.Get(id) { + t.Errorf("upstream ID %#x is not marked in use", id) + } + if got := lim.inFlight.occupied(); got != 1 { + t.Errorf("%d in-flight slots held, want 1 until the request completes", got) + } + + // The entry keeps its own copy of the query to resend on failover. + raw[len(raw)-1] ^= 0xff + if bytes.Equal(entry.rawQuery, raw) { + t.Error("entry shares the caller's query buffer") + } +} + +// Responses are matched by upstream ID, so record must not hand out one still +// in flight. +func TestPendingRequestsRecordAllocatesFreeID(t *testing.T) { + lim := newLimiter() + p := newPendingRequests(lim) + upstreams := []string{"10.96.0.10:53"} + const free = 0xabcd + for id := range 1 << 16 { + if id != free { + p.inUse.Set(uint16(id)) + } + } + + if id, _ := recordQuery(t, p, 0x1234, nil, upstreams); id != free { + t.Fatalf("record allocated %#x, want the only free ID %#x", id, free) + } + + // With every ID in flight, record fails and frees the query's slot. + lim.inFlight.tryAcquire() + if _, ok := p.record(dnsQuery(0x5678), 0x5678, exampleQuestion, nil, upstreams); ok { + t.Fatal("record succeeded with every upstream ID in flight") + } + if got := lim.inFlight.occupied(); got != 1 { + t.Errorf("%d in-flight slots held, want 1 for the recorded query", got) + } +} + +func TestPendingRequestsDeleteEntryReleasesSlotOnce(t *testing.T) { + lim := newLimiter() + p := newPendingRequests(lim) + upstreams := []string{"10.96.0.10:53"} + lim.inFlight.tryAcquire() // held by another query + id, _ := recordQuery(t, p, 0x1234, nil, upstreams) + + p.mu.Lock() + p.deleteEntryLocked(id) + p.deleteEntryLocked(id) + p.mu.Unlock() + + if len(p.entries) != 0 || p.inUse.Count() != 0 { + t.Errorf("deleted entry left %d entries and %d IDs in use", len(p.entries), p.inUse.Count()) + } + if got := lim.inFlight.occupied(); got != 1 { + t.Errorf("%d in-flight slots held, want 1: deleting twice must free the slot once", got) + } +} + +func TestPendingRequestsFailOverOnSendError(t *testing.T) { + lim := newLimiter() + p := newPendingRequests(lim) + upstreams := []string{"10.96.0.10:53", "10.96.0.11:53"} + id, raw := recordQuery(t, p, 0x1234, nil, upstreams) + + // Errors for an unknown ID, or for an attempt not yet made, are ignored. + if _, _, ok := p.failOverOnSendError(id+1, 0); ok { + t.Error("failed over a request that was never recorded") + } + if _, _, ok := p.failOverOnSendError(id, 1); ok { + t.Error("failed over on an error for an attempt not yet made") + } + + // The first upstream failing moves the request to the second. + before := time.Now() + nextIdx, query, ok := p.failOverOnSendError(id, 0) + after := time.Now() + if !ok || nextIdx != 1 { + t.Fatalf("failOverOnSendError(%#x, 0) = %d, _, %v; want 1, _, true", id, nextIdx, ok) + } + if !bytes.Equal(query, raw) { + t.Error("failover query differs from the recorded query") + } + entry, ok := p.entries[id] + if !ok { + t.Fatal("entry missing after failover") + } + if entry.upstreamIdx != 1 { + t.Errorf("upstreamIdx = %d, want 1", entry.upstreamIdx) + } + checkExpiry(t, entry.expiry, before, after, p.timeoutForAttempt(1, len(upstreams))) + if got := lim.inFlight.occupied(); got != 1 { + t.Errorf("%d in-flight slots held during failover, want 1", got) + } + + // A repeated error for the first attempt is stale. + if _, _, ok := p.failOverOnSendError(id, 0); ok { + t.Error("failed over twice on errors for the same attempt") + } + + // The last upstream failing drops the request and frees its slot. + if _, _, ok := p.failOverOnSendError(id, 1); ok { + t.Error("failed over past the last upstream") + } + if len(p.entries) != 0 || p.inUse.Count() != 0 { + t.Errorf("dropped request left %d entries and %d IDs in use", len(p.entries), p.inUse.Count()) + } + if got := lim.inFlight.occupied(); got != 0 { + t.Errorf("%d in-flight slots held after the last upstream failed, want 0", got) + } +} + +func TestPendingRequestsClearAll(t *testing.T) { + lim := newLimiter() + p := newPendingRequests(lim) + upstreams := []string{"10.96.0.10:53", "10.96.0.11:53"} + for i := range 3 { + recordQuery(t, p, uint16(i), nil, upstreams) + } + + p.clearAll() + if len(p.entries) != 0 || p.inUse.Count() != 0 { + t.Errorf("clearAll left %d entries and %d IDs in use", len(p.entries), p.inUse.Count()) + } + if got := lim.inFlight.occupied(); got != 0 { + t.Errorf("%d in-flight slots held after clearAll, want 0", got) + } +} + +func TestPendingRequestsSweep(t *testing.T) { + lim := newLimiter() + p := newPendingRequests(lim) + upstreams := []string{"10.96.0.10:53", "10.96.0.11:53"} + id, raw := recordQuery(t, p, 0x1234, nil, upstreams) + entry := p.entries[id] + + if failovers, deliveries := p.sweep(entry.expiry); len(failovers) != 0 || len(deliveries) != 0 { + t.Fatalf("sweep at the expiry returned %d failovers and %d deliveries, want none until it has passed", len(failovers), len(deliveries)) + } + + // An expired attempt moves to the next upstream. + now := entry.expiry.Add(time.Nanosecond) + failovers, deliveries := p.sweep(now) + wantFailovers := []reqFailover{{upstreamID: id, upstreamIdx: 1, query: raw}} + if diff := cmp.Diff(wantFailovers, failovers, cmp.AllowUnexported(reqFailover{})); diff != "" { + t.Errorf("failovers mismatch (-want +got):\n%s", diff) + } + if len(deliveries) != 0 { + t.Errorf("sweep returned %d deliveries on failover, want 0", len(deliveries)) + } + if len(p.entries) != 1 || p.entries[id] != entry || entry.upstreamIdx != 1 { + t.Fatal("entry not advanced to the next upstream") + } + if want := now.Add(p.timeoutForAttempt(1, len(upstreams))); !entry.expiry.Equal(want) { + t.Errorf("expiry = %v, want %v", entry.expiry, want) + } + + // An expired last attempt with no answer held back is dropped silently. + failovers, deliveries = p.sweep(entry.expiry.Add(time.Nanosecond)) + if len(failovers) != 0 || len(deliveries) != 0 { + t.Errorf("sweep after the last attempt returned %d failovers and %d deliveries, want none", len(failovers), len(deliveries)) + } + if len(p.entries) != 0 || p.inUse.Count() != 0 { + t.Errorf("expired request left %d entries and %d IDs in use", len(p.entries), p.inUse.Count()) + } + if got := lim.inFlight.occupied(); got != 0 { + t.Errorf("%d in-flight slots held after expiry, want 0", got) + } +} + +// A SERVFAIL beats a timeout: when the last upstream never answers, the client +// gets the SERVFAIL held back from an earlier one. +func TestPendingRequestsSweepDeliversDeferredAnswer(t *testing.T) { + lim := newLimiter() + p := newPendingRequests(lim) + client := &net.UDPAddr{IP: net.ParseIP("169.254.0.2"), Port: 54321} + upstreams := []string{"10.96.0.10:53", "10.96.0.11:53"} + id, raw := recordQuery(t, p, 0x1234, client, upstreams) + + // Leave the request as a SERVFAIL from the first upstream does: waiting on + // the second, with the SERVFAIL held back. + if _, _, ok := p.failOverOnSendError(id, 0); !ok { + t.Fatal("failOverOnSendError failed") + } + entry := p.entries[id] + entry.deferredResp = dnsAnswer(raw, byte(dnsmessage.RCodeServerFailure)) + + failovers, deliveries := p.sweep(entry.expiry.Add(time.Nanosecond)) + if len(failovers) != 0 { + t.Errorf("sweep returned %d failovers past the last upstream, want 0", len(failovers)) + } + wantDeliveries := []reqDelivery{{ + clientAddr: client, + payload: dnsAnswer(dnsQuery(0x1234), byte(dnsmessage.RCodeServerFailure)), + }} + if diff := cmp.Diff(wantDeliveries, deliveries, cmp.AllowUnexported(reqDelivery{})); diff != "" { + t.Errorf("deliveries mismatch (-want +got):\n%s", diff) + } + if len(p.entries) != 0 || p.inUse.Count() != 0 { + t.Errorf("expired request left %d entries and %d IDs in use", len(p.entries), p.inUse.Count()) + } + if got := lim.inFlight.occupied(); got != 0 { + t.Errorf("%d in-flight slots held after expiry, want 0", got) + } +} diff --git a/internal/ateomnet/dns/protocol/doc.go b/internal/ateomnet/dns/protocol/doc.go new file mode 100644 index 0000000000..511fd5bb5b --- /dev/null +++ b/internal/ateomnet/dns/protocol/doc.go @@ -0,0 +1,17 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package protocol contains helpers to manage encoding/decoding DNS protocol +// to/from a byte array. +package protocol diff --git a/internal/ateomnet/dns/protocol/protocol.go b/internal/ateomnet/dns/protocol/protocol.go new file mode 100644 index 0000000000..d21c712842 --- /dev/null +++ b/internal/ateomnet/dns/protocol/protocol.go @@ -0,0 +1,117 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package protocol + +import ( + "encoding/binary" + + "golang.org/x/net/dns/dnsmessage" +) + +const ( + dnsHeaderLen = 12 + + // TypeIXFR is the incremental zone transfer QTYPE (RFC 1995). + TypeIXFR dnsmessage.Type = 251 + + flagQuery = 0 + flagResponse = 1 +) + +// qrFlag is the query (0) or response (1) bit +func qrFlag(raw []byte) int { + if raw[2]&0x80 == 0 { + return 0 + } + return 1 +} + +// RequestSanityCheck filters out obviously invalid packets. +func RequestSanityCheck(raw []byte) bool { + // DNS packets need to be at least 12 bytes. + if len(raw) < dnsHeaderLen { + return false + } + if qrFlag(raw) != flagQuery { + return false + } + return true +} + +// ResponseSanityCheck filters out obviously invalid packets. +func ResponseSanityCheck(raw []byte) bool { + // DNS packets need to be at least 12 bytes. + const dnsHeaderLen = 12 + if len(raw) < dnsHeaderLen { + return false + } + if qrFlag(raw) != flagResponse { + return false + } + if QDCount(raw) != 1 { + return false + } + return true +} + +// SetTxnID in the DNS packet. +func SetTxnID(raw []byte, id uint16) { + binary.BigEndian.PutUint16(raw[0:2], id) +} + +// qdCountFromRaw from the raw packet. +func QDCount(raw []byte) uint16 { + // +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ + // | QDCOUNT = 0 | bytes 4..5 + // +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ + return binary.BigEndian.Uint16(raw[4:6]) +} + +// ErrorPacket returns a serialized DNS packet signaling an error. +func ErrorPacket(hdr dnsmessage.Header, rcode dnsmessage.RCode, q *dnsmessage.Question) []byte { + respHdr := dnsmessage.Header{ + ID: hdr.ID, + Response: true, + OpCode: hdr.OpCode, + RecursionDesired: hdr.RecursionDesired, + RecursionAvailable: true, + RCode: rcode, + } + + b := dnsmessage.NewBuilder(nil, respHdr) + // Try to tack on the question, if possible. + if q != nil { + if err := b.StartQuestions(); err == nil { + _ = b.Question(*q) + } + } + + out, err := b.Finish() + if err == nil { + return out + } + + // Backstop reply: we had errors generating a response. Synthesize an empty + // DNS response that matches the request instead. + fallback := make([]byte, dnsHeaderLen) + SetTxnID(fallback, hdr.ID) + fallback[2] = 0x80 | (byte(hdr.OpCode)&0x0f)<<3 + if hdr.RecursionDesired { + fallback[2] |= 0x01 + } + fallback[3] = 0x80 | (byte(rcode) & 0x0f) + + return fallback +} diff --git a/internal/ateomnet/dns/protocol/protocol_test.go b/internal/ateomnet/dns/protocol/protocol_test.go new file mode 100644 index 0000000000..52c953dace --- /dev/null +++ b/internal/ateomnet/dns/protocol/protocol_test.go @@ -0,0 +1,287 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package protocol + +import ( + "encoding/binary" + "testing" + + "golang.org/x/net/dns/dnsmessage" +) + +func TestRequestSanityCheck(t *testing.T) { + tests := []struct { + name string + raw []byte + want bool + }{ + { + name: "nil slice", + raw: nil, + want: false, + }, + { + name: "empty slice", + raw: []byte{}, + want: false, + }, + { + name: "short packet (11 bytes)", + raw: make([]byte, 11), + want: false, + }, + { + name: "valid 12-byte query (QR=0)", + raw: make([]byte, 12), + want: true, + }, + { + name: "valid query with non-QR bits set in byte 2", + raw: []byte{ + 0x12, 0x34, 0x7f, 0x00, + 0x00, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + }, + want: true, + }, + { + name: "response packet (QR=1)", + raw: []byte{ + 0x12, 0x34, 0x80, 0x00, + 0x00, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + }, + want: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := RequestSanityCheck(tc.raw); got != tc.want { + t.Errorf("RequestSanityCheck(%x) = %v, want %v", tc.raw, got, tc.want) + } + }) + } +} + +func TestResponseSanityCheck(t *testing.T) { + tests := []struct { + name string + raw []byte + want bool + }{ + { + name: "nil slice", + raw: nil, + want: false, + }, + { + name: "short packet (11 bytes)", + raw: make([]byte, 11), + want: false, + }, + { + name: "query packet (QR=0) with QDCOUNT=1", + raw: []byte{ + 0x12, 0x34, 0x00, 0x00, + 0x00, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + }, + want: false, + }, + { + name: "response packet (QR=1) with QDCOUNT=0", + raw: []byte{ + 0x12, 0x34, 0x80, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + }, + want: false, + }, + { + name: "response packet (QR=1) with QDCOUNT=2", + raw: []byte{ + 0x12, 0x34, 0x80, 0x00, + 0x00, 0x02, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + }, + want: false, + }, + { + name: "valid response packet (QR=1, QDCOUNT=1)", + raw: []byte{ + 0x12, 0x34, 0x81, 0x80, + 0x00, 0x01, 0x00, 0x01, + 0x00, 0x00, 0x00, 0x00, + }, + want: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := ResponseSanityCheck(tc.raw); got != tc.want { + t.Errorf("ResponseSanityCheck(%x) = %v, want %v", tc.raw, got, tc.want) + } + }) + } +} + +func TestQDCount(t *testing.T) { + tests := []struct { + qdCount uint16 + }{ + {0}, + {1}, + {2}, + {0x1234}, + {0xffff}, + } + + for _, tc := range tests { + var raw [12]byte + binary.BigEndian.PutUint16(raw[4:6], tc.qdCount) + if got := QDCount(raw[:]); got != tc.qdCount { + t.Errorf("QDCount() = %#x, want %#x", got, tc.qdCount) + } + } +} + +func TestSetTxnID(t *testing.T) { + var raw [12]byte + SetTxnID(raw[:], 0xcafe) + if got := binary.BigEndian.Uint16(raw[0:2]); got != 0xcafe { + t.Errorf("SetTxnID wrote %#x, want 0xcafe", got) + } +} + +func TestErrorPacket(t *testing.T) { + validQ := dnsmessage.Question{ + Name: dnsmessage.MustNewName("example.com."), + Type: dnsmessage.TypeA, + Class: dnsmessage.ClassINET, + } + + t.Run("with valid question", func(t *testing.T) { + reqHdr := dnsmessage.Header{ + ID: 0xabcd, + OpCode: 0, + RecursionDesired: true, + } + pkt := ErrorPacket(reqHdr, dnsmessage.RCodeRefused, &validQ) + + var p dnsmessage.Parser + gotHdr, err := p.Start(pkt) + if err != nil { + t.Fatalf("p.Start() unexpected error: %v", err) + } + if gotHdr.ID != reqHdr.ID { + t.Errorf("ID = %#x, want %#x", gotHdr.ID, reqHdr.ID) + } + if !gotHdr.Response { + t.Error("Response = false, want true") + } + if gotHdr.OpCode != reqHdr.OpCode { + t.Errorf("OpCode = %v, want %v", gotHdr.OpCode, reqHdr.OpCode) + } + if !gotHdr.RecursionDesired { + t.Error("RecursionDesired = false, want true") + } + if !gotHdr.RecursionAvailable { + t.Error("RecursionAvailable = false, want true") + } + if gotHdr.RCode != dnsmessage.RCodeRefused { + t.Errorf("RCode = %v, want %v", gotHdr.RCode, dnsmessage.RCodeRefused) + } + + questions, err := p.AllQuestions() + if err != nil { + t.Fatalf("p.AllQuestions() unexpected error: %v", err) + } + if len(questions) != 1 || questions[0] != validQ { + t.Errorf("questions = %+v, want [%+v]", questions, validQ) + } + }) + + t.Run("with nil question", func(t *testing.T) { + reqHdr := dnsmessage.Header{ + ID: 0x1234, + OpCode: 1, + RecursionDesired: false, + } + pkt := ErrorPacket(reqHdr, dnsmessage.RCodeNotImplemented, nil) + + var p dnsmessage.Parser + gotHdr, err := p.Start(pkt) + if err != nil { + t.Fatalf("p.Start() unexpected error: %v", err) + } + if gotHdr.ID != reqHdr.ID { + t.Errorf("ID = %#x, want %#x", gotHdr.ID, reqHdr.ID) + } + if !gotHdr.Response { + t.Error("Response = false, want true") + } + if gotHdr.OpCode != reqHdr.OpCode { + t.Errorf("OpCode = %v, want %v", gotHdr.OpCode, reqHdr.OpCode) + } + if gotHdr.RecursionDesired { + t.Error("RecursionDesired = true, want false") + } + if !gotHdr.RecursionAvailable { + t.Error("RecursionAvailable = false, want true") + } + if gotHdr.RCode != dnsmessage.RCodeNotImplemented { + t.Errorf("RCode = %v, want %v", gotHdr.RCode, dnsmessage.RCodeNotImplemented) + } + + questions, err := p.AllQuestions() + if err != nil { + t.Fatalf("p.AllQuestions() unexpected error: %v", err) + } + if len(questions) != 0 { + t.Errorf("len(questions) = %d, want 0", len(questions)) + } + }) + + t.Run("with unencodable question falls back to empty question section", func(t *testing.T) { + // Zero-value Name (Length == 0, non-canonical) causes Builder.Question to fail, + // but Builder.Finish still succeeds with 0 questions. + badQ := dnsmessage.Question{ + Type: dnsmessage.TypeA, + Class: dnsmessage.ClassINET, + } + reqHdr := dnsmessage.Header{ + ID: 0x4321, + RecursionDesired: true, + } + pkt := ErrorPacket(reqHdr, dnsmessage.RCodeFormatError, &badQ) + + var p dnsmessage.Parser + gotHdr, err := p.Start(pkt) + if err != nil { + t.Fatalf("p.Start() unexpected error: %v", err) + } + if gotHdr.ID != reqHdr.ID { + t.Errorf("ID = %#x, want %#x", gotHdr.ID, reqHdr.ID) + } + if gotHdr.RCode != dnsmessage.RCodeFormatError { + t.Errorf("RCode = %v, want %v", gotHdr.RCode, dnsmessage.RCodeFormatError) + } + if qd := QDCount(pkt); qd != 0 { + t.Errorf("QDCount = %d, want 0", qd) + } + }) +} diff --git a/internal/ateomnet/dns/protocol/tcp.go b/internal/ateomnet/dns/protocol/tcp.go new file mode 100644 index 0000000000..c5c412f2c5 --- /dev/null +++ b/internal/ateomnet/dns/protocol/tcp.go @@ -0,0 +1,52 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package protocol + +import ( + "encoding/binary" + "errors" + "fmt" + "io" +) + +// ReadTCPFrame reads a single RFC 1035 §4.2.2 2-byte big-endian length-prefixed +// DNS message from r. +func ReadTCPFrame(r io.Reader) ([]byte, error) { + var lenBuf [2]byte + if _, err := io.ReadFull(r, lenBuf[:]); err != nil { + return nil, err + } + length := int(binary.BigEndian.Uint16(lenBuf[:])) + msg := make([]byte, length) + if _, err := io.ReadFull(r, msg); err != nil { + if errors.Is(err, io.EOF) { + return nil, io.ErrUnexpectedEOF + } + return nil, err + } + return msg, nil +} + +// WriteTCPFrame writes msg prefixed with its 2-byte big-endian length to `w“. +func WriteTCPFrame(w io.Writer, msg []byte) error { + if len(msg) > 0xffff { + return fmt.Errorf("dns: TCP message length %d exceeds uint16 maximum", len(msg)) + } + frame := make([]byte, 2+len(msg)) + binary.BigEndian.PutUint16(frame[0:2], uint16(len(msg))) + copy(frame[2:], msg) + _, err := w.Write(frame) + return err +} diff --git a/internal/ateomnet/dns/protocol/tcp_test.go b/internal/ateomnet/dns/protocol/tcp_test.go new file mode 100644 index 0000000000..4a05c43707 --- /dev/null +++ b/internal/ateomnet/dns/protocol/tcp_test.go @@ -0,0 +1,120 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package protocol + +import ( + "bytes" + "errors" + "io" + "testing" +) + +func TestTCPFrameRoundTrip(t *testing.T) { + payloads := [][]byte{ + {}, + {0x42}, + bytes.Repeat([]byte{0xab, 0xcd}, 256), + bytes.Repeat([]byte{0xff}, 0xffff), + } + + var buf bytes.Buffer + for i, p := range payloads { + if err := WriteTCPFrame(&buf, p); err != nil { + t.Fatalf("frame %d: WriteTCPFrame() unexpected error: %v", i, err) + } + } + + for i, want := range payloads { + got, err := ReadTCPFrame(&buf) + if err != nil { + t.Fatalf("frame %d: ReadTCPFrame() unexpected error: %v", i, err) + } + if !bytes.Equal(got, want) { + t.Errorf("frame %d: payload mismatch (got len %d, want len %d)", i, len(got), len(want)) + } + } + + // Subsequent read on drained buffer should return io.EOF. + if _, err := ReadTCPFrame(&buf); !errors.Is(err, io.EOF) { + t.Errorf("ReadTCPFrame on empty reader = %v, want io.EOF", err) + } +} + +func TestReadTCPFrameErrors(t *testing.T) { + tests := []struct { + name string + input []byte + wantErr error + }{ + { + name: "empty reader returns EOF", + input: nil, + wantErr: io.EOF, + }, + { + name: "1-byte truncated length prefix returns ErrUnexpectedEOF", + input: []byte{0x00}, + wantErr: io.ErrUnexpectedEOF, + }, + { + name: "length prefix with empty body returns ErrUnexpectedEOF", + input: []byte{0x00, 0x05}, + wantErr: io.ErrUnexpectedEOF, + }, + { + name: "length prefix with partial body returns ErrUnexpectedEOF", + input: []byte{0x00, 0x05, 1, 2, 3}, + wantErr: io.ErrUnexpectedEOF, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + _, err := ReadTCPFrame(bytes.NewReader(tc.input)) + if !errors.Is(err, tc.wantErr) { + t.Errorf("ReadTCPFrame(%x) error = %v, want %v", tc.input, err, tc.wantErr) + } + }) + } +} + +type failingWriter struct { + err error +} + +func (f failingWriter) Write([]byte) (int, error) { + return 0, f.err +} + +func TestWriteTCPFrameErrors(t *testing.T) { + t.Run("message exceeds uint16 max", func(t *testing.T) { + oversized := make([]byte, 0x10000) + var buf bytes.Buffer + if err := WriteTCPFrame(&buf, oversized); err == nil { + t.Error("WriteTCPFrame with 65536-byte message succeeded, want error") + } + if buf.Len() != 0 { + t.Errorf("buf.Len() = %d, want 0", buf.Len()) + } + }) + + t.Run("underlying writer error propagated", func(t *testing.T) { + wantErr := errors.New("write failed") + err := WriteTCPFrame(failingWriter{err: wantErr}, []byte{1, 2, 3}) + if !errors.Is(err, wantErr) { + t.Errorf("WriteTCPFrame error = %v, want %v", err, wantErr) + } + }) +} diff --git a/internal/ateomnet/dns/readme.md b/internal/ateomnet/dns/readme.md new file mode 100644 index 0000000000..ac742fe1ef --- /dev/null +++ b/internal/ateomnet/dns/readme.md @@ -0,0 +1,66 @@ +# Actor DNS + +Actor DNS relays DNS requests from the actor to the external world. + +## Code structure + +### Relay + +- `Relay` -- one per Ateom, holds upstream resolvers, and is used to create the per-Actor `Server`. +- `Relay.Serve()` -- creates a per-Actor `Server` that manages DNS requests for the Actor. Returns a `*Server`. + +### Server + +`Server` is the per-Actor state for the DNS traffic, including per-Actor concurrency limits (`maxInFlightDNS` and `maxDNSConnections`). + +- `Server.pendingRequests` map (UDP only): + - Allocates a rewritten `upstreamID` (`uint16`) per in-flight UDP query to avoid transaction ID collisions across different actor source ports on the shared egress socket (and prevent predictable upstream IDs). + - Maps `upstreamID` -> pending request entry: + - `clientRequestID` (`uint16`) and `clientAddr` (`net.Addr`) to restore the original ID and route the response back. + - Question metadata (`QNAME`, `QTYPE`, `QCLASS`) to validate that the response matches the query (RFC 5452). + - `upstreamIdx` and `deferredResp` to track multi-upstream failover (`SERVFAIL`, `NOTIMP`, `REFUSED`, or timeout) and return the last failure response if all upstreams fail. + - `expiry` timestamp for timing out stale entries and triggering failover via a periodic sweep ticker. +- `Server.Stop()` -- cancels in-flight work, closes sockets, and stops serving DNS requests. + +### Implementation details + +#### UDP handling + +- `udphandler.go`: `struct udpHandler` +- Two goroutines per `Server` (plus a sweep ticker for timeouts/failover): + - **Ingress reader**: reads datagrams from the actor-facing gateway `net.PacketConn`, runs `onUDPRequest()` (which validates the query, rewrites the transaction ID to `upstreamID`, and records the entry in `pendingRequests`), and sends the query on the egress socket. + - **Egress reader**: reads datagrams from an unconnected worker-namespace `net.PacketConn`, runs `onUDPResponse()` to match and validate against `pendingRequests`, either fails over to the next upstream or restores `clientRequestID` and writes the answer back to `clientAddr`. + +#### TCP handling + +- `tcphandler.go`: `struct tcpHandler` +- Framing: each DNS message on a TCP stream is prefixed with a 2-byte big-endian length (`uint16`, RFC 1035 §4.2.2). +- Dedicated upstream stream per connection (with pipelining and out-of-order response support per RFC 7766 §6.2.1.1): + - Two goroutines per accepted TCP connection sharing a per-connection downstream write mutex: + - **Downstream reader**: loops reading length-prefixed frames from the actor connection and calls `onTCPRequest()`. + - If forwarded: writes the unmodified frame directly to the dedicated upstream TCP connection. + - If rejected with a synthesized reply (`FORMERR`, `NOTIMP`, `REFUSED`): writes the framed error response directly to the actor connection under the write mutex without contacting upstream. + - On client `EOF`: half-closes the upstream write side (`CloseWrite()`) so in-flight pipelined responses can finish draining. + - **Upstream reader**: loops reading length-prefixed frames from the upstream TCP connection, validates each frame via `onTCPResponse()`, and writes valid responses back to the actor connection under the write mutex. + +#### DNS protocol layer + +`dnshandler.go`: `struct dnsHandler` + +UDP and TCP call up to the DNS protocol layer (`golang.org/x/net/dns/dnsmessage`), which inspects packets and returns transport-agnostic decisions: + +- Shared request validation (`validateRequest`): + - **Drop**: packet `< 12` bytes or `QR == 1` (response bit set on a query). + - **Reply** (synthesized response without upstream round-trip): + - `FORMERR` (`RCode = 1`): malformed header/question or `QDCOUNT != 1`. + - `NOTIMP` (`RCode = 4`): `OpCode != 0` (non-standard query) or zone transfer `QTYPE` (`AXFR`, `IXFR`). + - `REFUSED` (`RCode = 5`): disallowed by domain or record-type policy. +- `onUDPRequest(raw []byte, clientAddr net.Addr, upstreams []string)` -> `action`: + - Acquires an `inFlight` slot, runs `validateRequest()`, and on `actionForward` records the query in `pendingRequests` with a rewritten `upstreamID`. +- `onUDPResponse(raw []byte, from net.Addr)` -> `action`: + - Validates `QR == 1`, parses header + question, and matches against `pendingRequests` (verifying `upstreamID`, expected upstream source, and matching `QNAME`/`QTYPE`/`QCLASS`). + - Returns **Drop**, **Failover** (on `SERVFAIL`, `NOTIMP`, or `REFUSED` when fallback upstreams remain), or **Deliver** (restoring `clientRequestID` and returning `clientAddr`). +- `onTCPRequest(raw []byte)` -> `action`: + - Runs `validateRequest()` and returns **Drop**, **Reply**, or **Forward** (unmodified frame). +- `onTCPResponse(raw []byte)` -> `action`: + - Validates `QR == 1` and parses header + question, returning **Drop** or **Deliver**. diff --git a/internal/ateomnet/dns/relay.go b/internal/ateomnet/dns/relay.go index 804b5ff47b..81e2854399 100644 --- a/internal/ateomnet/dns/relay.go +++ b/internal/ateomnet/dns/relay.go @@ -19,6 +19,7 @@ import ( "fmt" "log/slog" "net" + "net/netip" "strconv" "time" @@ -29,38 +30,19 @@ const ( // dnsPort is the relay port on the sandbox's gateway. dnsPort = 53 - // Read complete UDP datagrams without truncating EDNS responses. - maxDNSDatagram = 65535 - - // Drop excess UDP queries to bound goroutines and upstream sockets. - maxInFlightDNS = 64 - - // maxDNSConnections bounds open TCP connections. - maxDNSConnections = 16 - - // dnsTCPTimeout limits connection lifetime, including idle clients. - dnsTCPTimeout = 30 * time.Second - // dnsExchangeTimeout bounds each upstream attempt. dnsExchangeTimeout = 5 * time.Second ) -// limiter caps UDP queries in flight and open TCP connections across every -// sandbox the relay serves, so each Server must release its slots on Stop. -type limiter struct { - inFlight chan struct{} - connections chan struct{} -} - -// Relay forwards UDP and TCP DNS unchanged to the worker pod's resolvers. -// It listens in the sandbox's gateway namespace and dials from the worker's. -// DNS bypasses the egress tunnel and is not checked against egress policy. +// Relay validates and forwards UDP and TCP DNS queries to the worker pod's +// resolvers. It listens in the sandbox's gateway namespace and dials from the +// worker's. DNS bypasses the egress tunnel and is not checked against egress +// policy. type Relay struct { upstreams []string // dialer reaches upstream resolvers from the worker namespace. - dialer *net.Dialer - limiter limiter + dialer *net.Dialer } // NewRelay reads nameservers from resolvConfPath and forwards to each on port 53. @@ -72,6 +54,29 @@ func NewRelay(resolvConfPath string) (*Relay, error) { return NewRelayForUpstreams(upstreams) } +func normalizeAddr(addr net.Addr) string { + if addr == nil { + return "" + } + switch a := addr.(type) { + case *net.UDPAddr: + ap := a.AddrPort() + return netip.AddrPortFrom(ap.Addr().Unmap(), ap.Port()).String() + case *net.TCPAddr: + ap := a.AddrPort() + return netip.AddrPortFrom(ap.Addr().Unmap(), ap.Port()).String() + default: + return normalizeAddrString(addr.String()) + } +} + +func normalizeAddrString(s string) string { + if ap, err := netip.ParseAddrPort(s); err == nil { + return netip.AddrPortFrom(ap.Addr().Unmap(), ap.Port()).String() + } + return s +} + // NewRelayForUpstreams forwards to upstreams, each "host:port", tried in order. func NewRelayForUpstreams(upstreams []string) (*Relay, error) { if len(upstreams) == 0 { @@ -81,6 +86,9 @@ func NewRelayForUpstreams(upstreams []string) (*Relay, error) { if _, _, err := net.SplitHostPort(u); err != nil { return nil, fmt.Errorf("dns: invalid upstream resolver %q: %w", u, err) } + if _, err := net.ResolveUDPAddr("udp", u); err != nil { + return nil, fmt.Errorf("dns: invalid upstream resolver %q: %w", u, err) + } } slog.Info("DNS relay configured", slog.Any("upstreams", upstreams)) @@ -88,10 +96,6 @@ func NewRelayForUpstreams(upstreams []string) (*Relay, error) { return &Relay{ upstreams: upstreams, dialer: &net.Dialer{Timeout: dnsExchangeTimeout}, - limiter: limiter{ - inFlight: make(chan struct{}, maxInFlightDNS), - connections: make(chan struct{}, maxDNSConnections), - }, }, nil } @@ -121,11 +125,19 @@ func (r *Relay) Serve(ctx context.Context, ns netns.Handle) (*Server, error) { return nil, err } - return r.serveOn(ctx, &netC), nil + egressUDP, err := net.ListenPacket("udp", ":0") + if err != nil { + _ = netC.udp.Close() + _ = netC.tcpListener.Close() + return nil, fmt.Errorf("while opening the worker DNS egress socket: %w", err) + } + netC.egressUDP = egressUDP + + return r.serveOn(ctx, &netC) } // serveOn serves DNS on netC's sockets, which the caller has already bound, // and dials upstreams with netC's dialer. -func (r *Relay) serveOn(ctx context.Context, netC *netConn) *Server { - return newServer(ctx, &serverConfig{upstreams: r.upstreams}, netC, &r.limiter) +func (r *Relay) serveOn(ctx context.Context, netC *netConn) (*Server, error) { + return newServer(ctx, &serverConfig{upstreams: r.upstreams}, netC) } diff --git a/internal/ateomnet/dns/relay_test.go b/internal/ateomnet/dns/relay_test.go index ef9a3333f1..104622c033 100644 --- a/internal/ateomnet/dns/relay_test.go +++ b/internal/ateomnet/dns/relay_test.go @@ -28,46 +28,54 @@ import ( "time" "github.com/google/go-cmp/cmp" + "golang.org/x/net/dns/dnsmessage" ) func TestRelayCancelsUDPExchange(t *testing.T) { // A resolver that receives the query and never answers, so the exchange is - // blocked on the read when the context is canceled. + // in flight when the server stops. silent, asked := newSilentResolver(t) - // Two upstreams: a canceled exchange must not move on to the second. - second := newFakeResolver(t, func(query []byte) []byte { return query }) + // Two upstreams: stopping the server must not move on to the second. + var secondAsked atomic.Bool + second := newFakeResolver(t, func(query []byte) []byte { + secondAsked.Store(true) + return dnsAnswer(query, 0) + }) relay, err := NewRelayForUpstreams([]string{silent, second}) if err != nil { t.Fatal(err) } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - done := make(chan error, 1) - go func() { - _, err := unstartedServer(relay).exchangeUDP(ctx, dnsQuery(0x1234)) - done <- err - }() + srv, udpAddr, _ := serveLoopback(t, relay) + client, err := net.Dial("udp", udpAddr.String()) + if err != nil { + t.Fatal(err) + } + defer client.Close() + + if _, err := client.Write(dnsQuery(0x1234)); err != nil { + t.Fatal(err) + } select { case <-asked: case <-time.After(5 * time.Second): t.Fatal("the resolver never saw the query") } - cancel() - select { - case err := <-done: - if !errors.Is(err, context.Canceled) { - t.Fatalf("exchange returned %v, want cancellation", err) - } - case <-time.After(5 * time.Second): - t.Fatal("canceled exchange remained blocked reading upstream") + + stopCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := srv.Stop(stopCtx); err != nil { + t.Fatalf("Stop: %v", err) + } + if secondAsked.Load() { + t.Error("stopping the server moved on to the second upstream") } } func TestRelayForwardsUDPVerbatim(t *testing.T) { upstream := newFakeResolver(t, func(query []byte) []byte { - return append([]byte{0xff}, query...) + return dnsAnswer(query, 0) }) relay, err := NewRelayForUpstreams([]string{upstream}) @@ -76,12 +84,12 @@ func TestRelayForwardsUDPVerbatim(t *testing.T) { } client := serveRelayUDP(t, relay) - query := []byte{0xab, 0xcd, 0x01, 0x00, 0x00, 0x01} + query := dnsQuery(0xabcd) if _, err := client.Write(query); err != nil { t.Fatal(err) } answer := readWithin(t, client) - if diff := cmp.Diff(append([]byte{0xff}, query...), answer); diff != "" { + if diff := cmp.Diff(dnsAnswer(query, 0), answer); diff != "" { t.Errorf("answer mismatch (-want +got):\n%s", diff) } } @@ -92,21 +100,22 @@ func TestRelayFallsBackToTheNextResolver(t *testing.T) { t.Fatal(err) } deadAddress := dead.LocalAddr().String() - // Closed, so the exchange fails rather than hanging to its deadline. + // Closed, so the exchange times out on the first upstream and fails over. dead.Close() - live := newFakeResolver(t, func(query []byte) []byte { return []byte("answered") }) + live := newFakeResolver(t, func(query []byte) []byte { return dnsAnswer(query, 0) }) relay, err := NewRelayForUpstreams([]string{deadAddress, live}) if err != nil { t.Fatal(err) } client := serveRelayUDP(t, relay) - if _, err := client.Write([]byte("query")); err != nil { + query := dnsQuery(0x2468) + if _, err := client.Write(query); err != nil { t.Fatal(err) } - if got := string(readWithin(t, client)); got != "answered" { - t.Errorf("answer = %q, want %q", got, "answered") + if diff := cmp.Diff(dnsAnswer(query, 0), readWithin(t, client)); diff != "" { + t.Errorf("answer mismatch (-want +got):\n%s", diff) } } @@ -135,7 +144,7 @@ func newFakeResolver(t *testing.T, respond func([]byte) []byte) string { } t.Cleanup(func() { pc.Close() }) go func() { - buf := make([]byte, maxDNSDatagram) + buf := make([]byte, maxDatagramSize) for { n, from, err := pc.ReadFrom(buf) if err != nil { @@ -160,7 +169,7 @@ func newSilentResolver(t *testing.T) (address string, asked <-chan struct{}) { t.Cleanup(func() { pc.Close() }) seen := make(chan struct{}, 1) go func() { - buf := make([]byte, maxDNSDatagram) + buf := make([]byte, maxDatagramSize) for { if _, _, err := pc.ReadFrom(buf); err != nil { return @@ -186,25 +195,19 @@ func serveLoopback(t *testing.T, relay *Relay) (*Server, net.Addr, net.Addr) { pc.Close() t.Fatal(err) } - srv := relay.serveOn(context.Background(), &netConn{ + srv, err := relay.serveOn(context.Background(), &netConn{ dialer: *relay.dialer, udp: pc, tcpListener: lis, }) + if err != nil { + pc.Close() + t.Fatal(err) + } t.Cleanup(func() { _ = srv.Stop(context.Background()) }) return srv, pc.LocalAddr(), lis.Addr() } -// unstartedServer builds a Server for relay without serving on any socket, so -// a test can call its methods directly. -func unstartedServer(relay *Relay) *Server { - return &Server{ - config: &serverConfig{upstreams: relay.upstreams}, - n: &netConn{dialer: *relay.dialer}, - limiter: &relay.limiter, - } -} - // serveRelayUDP runs the relay on loopback sockets and returns a connection to it. func serveRelayUDP(t *testing.T, relay *Relay) net.Conn { t.Helper() @@ -222,7 +225,7 @@ func readWithin(t *testing.T, conn net.Conn) []byte { if err := conn.SetReadDeadline(time.Now().Add(5 * time.Second)); err != nil { t.Fatal(err) } - buf := make([]byte, maxDNSDatagram) + buf := make([]byte, maxDatagramSize) n, err := conn.Read(buf) if err != nil { t.Fatalf("reading the relay's answer: %v", err) @@ -231,27 +234,32 @@ func readWithin(t *testing.T, conn net.Conn) []byte { } func TestRelayForwardsAnswersLargerThanTheCommonBuffer(t *testing.T) { - const size = 9000 - answer := make([]byte, size) - for i := range answer { - answer[i] = byte(i) - } - upstream := newFakeResolver(t, func([]byte) []byte { return answer }) + const size = maxDatagramSize + query := dnsQuery(0x1357) + baseAnswer := dnsAnswer(query, 0) + padding := make([]byte, size-len(baseAnswer)) + for i := range padding { + padding[i] = byte(i) + } + upstream := newFakeResolver(t, func(q []byte) []byte { + return append(dnsAnswer(q, 0), padding...) + }) relay, err := NewRelayForUpstreams([]string{upstream}) if err != nil { t.Fatal(err) } client := serveRelayUDP(t, relay) - if _, err := client.Write([]byte("query")); err != nil { + if _, err := client.Write(query); err != nil { t.Fatal(err) } + want := append(baseAnswer, padding...) got := readWithin(t, client) if len(got) != size { t.Fatalf("answer is %d bytes, want %d: it was cut down in the relay", len(got), size) } - if !bytes.Equal(got, answer) { + if !bytes.Equal(got, want) { t.Error("answer differs from what the resolver sent") } } @@ -267,14 +275,13 @@ func TestRelayDropsQueriesBeyondItsInFlightLimit(t *testing.T) { t.Cleanup(func() { close(release) }) var inFlight atomic.Int64 go func() { - buf := make([]byte, maxDNSDatagram) + buf := make([]byte, maxDatagramSize) for { n, from, err := pc.ReadFrom(buf) if err != nil { return } - answer := make([]byte, n) - copy(answer, buf[:n]) + answer := dnsAnswer(buf[:n], 0) go func() { inFlight.Add(1) <-release @@ -289,8 +296,8 @@ func TestRelayDropsQueriesBeyondItsInFlightLimit(t *testing.T) { } client := serveRelayUDP(t, relay) - for range maxInFlightDNS * 4 { - if _, err := client.Write([]byte("query")); err != nil { + for i := range maxInFlight * 4 { + if _, err := client.Write(dnsQuery(uint16(i))); err != nil { t.Fatal(err) } } @@ -305,8 +312,8 @@ func TestRelayDropsQueriesBeyondItsInFlightLimit(t *testing.T) { last = n } } - if got := inFlight.Load(); got > maxInFlightDNS { - t.Errorf("the relay had %d queries in flight, want at most %d", got, maxInFlightDNS) + if got := inFlight.Load(); got > maxInFlight { + t.Errorf("the relay had %d queries in flight, want at most %d", got, maxInFlight) } } @@ -366,14 +373,14 @@ func TestRelayRefusesTCPConnectionsBeyondItsLimit(t *testing.T) { } _, _, relayAddr := serveLoopback(t, relay) - for range maxDNSConnections { + for range maxConnections { conn, err := net.Dial("tcp", relayAddr.String()) if err != nil { t.Fatal(err) } t.Cleanup(func() { conn.Close() }) } - waitFor(t, "the relay to fill up", func() bool { return held.Load() == maxDNSConnections }) + waitFor(t, "the relay to fill up", func() bool { return held.Load() == maxConnections }) // The relay should close a connection accepted beyond its limit. extra, err := net.Dial("tcp", relayAddr.String()) @@ -387,8 +394,8 @@ func TestRelayRefusesTCPConnectionsBeyondItsLimit(t *testing.T) { if _, err := extra.Read(make([]byte, 1)); !errors.Is(err, io.EOF) { t.Errorf("reading the refused connection = %v, want %v", err, io.EOF) } - if got := held.Load(); got != maxDNSConnections { - t.Errorf("the relay holds %d upstream connections, want %d", got, maxDNSConnections) + if got := held.Load(); got != maxConnections { + t.Errorf("the relay holds %d upstream connections, want %d", got, maxConnections) } } @@ -406,7 +413,7 @@ func TestRelayClosesTCPConnectionsWhenServingEnds(t *testing.T) { } defer conn.Close() waitFor(t, "the connection to reach an upstream", func() bool { return held.Load() == 1 }) - if got := len(relay.limiter.connections); got != 1 { + if got := srv.limiter.connections.occupied(); got != 1 { t.Fatalf("%d connection slots held before Stop, want 1", got) } @@ -415,7 +422,7 @@ func TestRelayClosesTCPConnectionsWhenServingEnds(t *testing.T) { if err := srv.Stop(stopCtx); err != nil { t.Fatalf("Stop: %v", err) } - if got := len(relay.limiter.connections); got != 0 { + if got := srv.limiter.connections.occupied(); got != 0 { t.Errorf("%d connection slots still held after Stop, want 0", got) } @@ -432,7 +439,7 @@ func TestRelayClosesTCPConnectionsWhenServingEnds(t *testing.T) { } // Stop must cancel queries still being resolved rather than wait out -// dnsExchangeTimeout, and return their slots to the worker's limiter. +// dnsExchangeTimeout, and return their slots to the server's limiter. func TestStopCancelsUDPQueriesInFlight(t *testing.T) { silent, asked := newSilentResolver(t) relay, err := NewRelayForUpstreams([]string{silent}) @@ -454,7 +461,7 @@ func TestStopCancelsUDPQueriesInFlight(t *testing.T) { case <-time.After(5 * time.Second): t.Fatal("the resolver never saw the query") } - if got := len(relay.limiter.inFlight); got != 1 { + if got := srv.limiter.inFlight.occupied(); got != 1 { t.Fatalf("%d queries in flight before Stop, want 1", got) } @@ -464,11 +471,49 @@ func TestStopCancelsUDPQueriesInFlight(t *testing.T) { if err := srv.Stop(stopCtx); err != nil { t.Fatalf("Stop: %v", err) } - if got := len(relay.limiter.inFlight); got != 0 { + if got := srv.limiter.inFlight.occupied(); got != 0 { t.Errorf("%d in-flight slots still held after Stop, want 0", got) } } +func TestRelayLimitsArePerActor(t *testing.T) { + upstream, held := newHeldTCPResolver(t) + relay, err := NewRelayForUpstreams([]string{upstream}) + if err != nil { + t.Fatal(err) + } + + srvA, _, addrA := serveLoopback(t, relay) + srvB, _, addrB := serveLoopback(t, relay) + + // Saturate Actor A's TCP connection limit. + for range maxConnections { + conn, err := net.Dial("tcp", addrA.String()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { conn.Close() }) + } + waitFor(t, "actor A to fill up", func() bool { return held.Load() == maxConnections }) + if got := srvA.limiter.connections.occupied(); got != maxConnections { + t.Fatalf("actor A connection slots = %d, want %d", got, maxConnections) + } + if got := srvB.limiter.connections.occupied(); got != 0 { + t.Fatalf("actor B connection slots = %d, want 0", got) + } + + // Actor B must still accept a TCP connection even though Actor A is full. + connB, err := net.Dial("tcp", addrB.String()) + if err != nil { + t.Fatal(err) + } + defer connB.Close() + waitFor(t, "actor B connection to reach upstream", func() bool { return held.Load() == maxConnections+1 }) + if got := srvB.limiter.connections.occupied(); got != 1 { + t.Errorf("actor B connection slots = %d, want 1", got) + } +} + // dnsQuery builds a real query for example.com A, so tests exercise messages a // resolver would actually send rather than an arbitrary string. func dnsQuery(id uint16) []byte { @@ -496,7 +541,7 @@ func TestRelayFailsOverOnServerFailure(t *testing.T) { var sickCalls, healthyCalls atomic.Int32 sick := newFakeResolver(t, func(query []byte) []byte { sickCalls.Add(1) - return dnsAnswer(query, rcodeServFail) + return dnsAnswer(query, byte(dnsmessage.RCodeServerFailure)) }) healthy := newFakeResolver(t, func(query []byte) []byte { healthyCalls.Add(1) @@ -507,10 +552,11 @@ func TestRelayFailsOverOnServerFailure(t *testing.T) { if err != nil { t.Fatal(err) } - answer, err := unstartedServer(relay).exchangeUDP(context.Background(), dnsQuery(0x1234)) - if err != nil { - t.Fatalf("exchange: %v", err) + client := serveRelayUDP(t, relay) + if _, err := client.Write(dnsQuery(0x1234)); err != nil { + t.Fatal(err) } + answer := readWithin(t, client) if got := answer[3] & 0x0f; got != 0 { t.Errorf("answer rcode = %d, want 0: the relay returned the failing resolver's answer", got) } @@ -522,28 +568,28 @@ func TestRelayFailsOverOnServerFailure(t *testing.T) { // With every resolver failing there is nothing better to return, and a real // SERVFAIL beats a timeout: the sandbox's resolver can act on it. func TestRelayReturnsServerFailureWhenAllFail(t *testing.T) { - first := newFakeResolver(t, func(query []byte) []byte { return dnsAnswer(query, rcodeServFail) }) - second := newFakeResolver(t, func(query []byte) []byte { return dnsAnswer(query, rcodeRefused) }) + first := newFakeResolver(t, func(query []byte) []byte { return dnsAnswer(query, byte(dnsmessage.RCodeServerFailure)) }) + second := newFakeResolver(t, func(query []byte) []byte { return dnsAnswer(query, byte(dnsmessage.RCodeRefused)) }) relay, err := NewRelayForUpstreams([]string{first, second}) if err != nil { t.Fatal(err) } - answer, err := unstartedServer(relay).exchangeUDP(context.Background(), dnsQuery(0x2345)) - if err != nil { - t.Fatalf("exchange: %v", err) + client := serveRelayUDP(t, relay) + if _, err := client.Write(dnsQuery(0x2345)); err != nil { + t.Fatal(err) } - if got := answer[3] & 0x0f; got != rcodeRefused { - t.Errorf("answer rcode = %d, want the last resolver's %d", got, rcodeRefused) + answer := readWithin(t, client) + if got := dnsmessage.RCode(answer[3] & 0x0f); got != dnsmessage.RCodeRefused { + t.Errorf("answer rcode = %v, want the last resolver's %v", got, dnsmessage.RCodeRefused) } } // NXDOMAIN is an answer, not a failure: failing over would ask every resolver // about a name that does not exist. func TestRelayReturnsNXDomainWithoutFailover(t *testing.T) { - const rcodeNXDomain = 3 var secondCalls atomic.Int32 - first := newFakeResolver(t, func(query []byte) []byte { return dnsAnswer(query, rcodeNXDomain) }) + first := newFakeResolver(t, func(query []byte) []byte { return dnsAnswer(query, byte(dnsmessage.RCodeNameError)) }) second := newFakeResolver(t, func(query []byte) []byte { secondCalls.Add(1) return dnsAnswer(query, 0) @@ -553,12 +599,13 @@ func TestRelayReturnsNXDomainWithoutFailover(t *testing.T) { if err != nil { t.Fatal(err) } - answer, err := unstartedServer(relay).exchangeUDP(context.Background(), dnsQuery(0x3456)) - if err != nil { - t.Fatalf("exchange: %v", err) + client := serveRelayUDP(t, relay) + if _, err := client.Write(dnsQuery(0x3456)); err != nil { + t.Fatal(err) } - if got := answer[3] & 0x0f; got != rcodeNXDomain { - t.Errorf("answer rcode = %d, want NXDOMAIN %d", got, rcodeNXDomain) + answer := readWithin(t, client) + if got := dnsmessage.RCode(answer[3] & 0x0f); got != dnsmessage.RCodeNameError { + t.Errorf("answer rcode = %v, want NXDOMAIN %v", got, dnsmessage.RCodeNameError) } if secondCalls.Load() != 0 { t.Error("the relay failed over on NXDOMAIN, which is a valid answer") diff --git a/internal/ateomnet/dns/server.go b/internal/ateomnet/dns/server.go index dab647d3f1..23987fbb54 100644 --- a/internal/ateomnet/dns/server.go +++ b/internal/ateomnet/dns/server.go @@ -22,7 +22,6 @@ import ( "log/slog" "net" "sync" - "time" ) type serverConfig struct { @@ -32,15 +31,19 @@ type serverConfig struct { type netConn struct { dialer net.Dialer udp net.PacketConn + egressUDP net.PacketConn tcpListener net.Listener } // Server is one sandbox's running DNS relay, returned by [Relay.Serve]. type Server struct { - config *serverConfig + n *netConn - n *netConn - limiter *limiter + limiter *limiter + pendingRequests *pendingRequests + dns *dnsHandler + udp *udpHandler + tcp *tcpHandler stopServing context.CancelFunc closeOnce sync.Once @@ -48,36 +51,58 @@ type Server struct { serving sync.WaitGroup } +// newServer binds the sockets and starts serving goroutines. func newServer( ctx context.Context, config *serverConfig, netC *netConn, - limiter *limiter, -) *Server { - // Detached from the activation RPC's context but cancelable: the relay's - // capacity is the worker's, so teardown must drop queries still in flight. +) (*Server, error) { + if netC.egressUDP == nil { + egressUDP, err := net.ListenPacket("udp", ":0") + if err != nil { + slog.WarnContext(ctx, "Failed to open worker DNS egress socket", slog.Any("err", err)) + return nil, fmt.Errorf("DNS egress socket ListenPacket: %w", err) + } + netC.egressUDP = egressUDP + } + + // Detached from the activation RPC's context but cancelable so teardown + // drops queries still in flight and releases the actor's sockets. serveCtx, stopServing := context.WithCancel(context.WithoutCancel(ctx)) + + udpAddrs := make([]*net.UDPAddr, 0, len(config.upstreams)) + for _, u := range config.upstreams { + if addr, err := net.ResolveUDPAddr("udp", u); err == nil { + udpAddrs = append(udpAddrs, addr) + } + } + s := &Server{ - config: config, stopServing: stopServing, n: netC, - limiter: limiter, + limiter: newLimiter(), } + + s.pendingRequests = newPendingRequests(s.limiter) + s.dns = newDNSHandler(s.pendingRequests, s.limiter, nil) + s.udp = newUDPHandler(netC.udp, netC.egressUDP, udpAddrs, s.dns, s.pendingRequests) + s.tcp = newTCPHandler(config.upstreams, netC.tcpListener, netC.dialer, s.dns, s.limiter) + s.serving.Add(2) go func() { defer s.serving.Done() - if err := s.servePacket(serveCtx); err != nil { + if err := s.udp.serve(serveCtx); err != nil { slog.WarnContext(ctx, "Actor DNS socket stopped", slog.Any("err", err)) } }() go func() { defer s.serving.Done() - if err := s.serveTCP(serveCtx); err != nil { + if err := s.tcp.serve(serveCtx); err != nil { slog.WarnContext(ctx, "Actor DNS listener stopped", slog.Any("err", err)) } }() - return s + return s, nil } // Stop cancels in-flight queries and connections, closes the sockets, and @@ -87,7 +112,10 @@ func (s *Server) Stop(ctx context.Context) error { // Cancel first: closing the sockets alone leaves the queries already // being resolved holding the relay. s.stopServing() - for _, c := range []io.Closer{s.n.udp, s.n.tcpListener} { + for _, c := range []io.Closer{s.n.udp, s.n.egressUDP, s.n.tcpListener} { + if c == nil { + continue + } if err := c.Close(); err != nil && !errors.Is(err, net.ErrClosed) { s.closeErr = errors.Join(s.closeErr, err) } @@ -105,183 +133,3 @@ func (s *Server) Stop(ctx context.Context) error { return errors.Join(s.closeErr, fmt.Errorf("dns: waiting for relay to stop: %w", ctx.Err())) } } - -// servePacket answers UDP queries until ctx is canceled or the socket fails. -func (s *Server) servePacket(ctx context.Context) error { - var wg sync.WaitGroup - defer wg.Wait() - - buf := make([]byte, maxDNSDatagram) - for { - n, from, err := s.n.udp.ReadFrom(buf) - if err != nil { - if ctx.Err() != nil || errors.Is(err, net.ErrClosed) { - return nil - } - return fmt.Errorf("dns: reading actor DNS query: %w", err) - } - // Copied: the buffer is reused by the next read. - query := make([]byte, n) - copy(query, buf[:n]) - - select { - case s.limiter.inFlight <- struct{}{}: - default: - slog.DebugContext(ctx, "dns relay dropped a DNS query; too many in flight") - continue - } - wg.Add(1) - go func() { - defer wg.Done() - defer func() { <-s.limiter.inFlight }() - answer, err := s.exchangeUDP(ctx, query) - if err != nil { - slog.WarnContext(ctx, "dns relay could not resolve an actor DNS query", slog.Any("err", err)) - return - } - if _, err := s.n.udp.WriteTo(answer, from); err != nil && ctx.Err() == nil { - slog.WarnContext(ctx, "dns relay could not return a DNS answer", slog.Any("err", err)) - } - }() - } -} - -// serveTCP relays TCP DNS connections until ctx is canceled or the listener closes. -func (s *Server) serveTCP(ctx context.Context) error { - // Wait for the relays to drain before returning, so a closed listener - // leaves no goroutine still holding a connection slot. - var wg sync.WaitGroup - defer wg.Wait() - - for { - conn, err := s.n.tcpListener.Accept() - if err != nil { - if ctx.Err() != nil || errors.Is(err, net.ErrClosed) { - return nil - } - return fmt.Errorf("dns: accepting actor DNS connection: %w", err) - } - select { - case s.limiter.connections <- struct{}{}: - default: - slog.DebugContext(ctx, "dns relay refused a DNS connection; too many open") - _ = conn.Close() - continue - } - wg.Add(1) - go func() { - defer wg.Done() - defer func() { <-s.limiter.connections }() - s.relayTCP(ctx, conn) - }() - } -} - -func (s *Server) exchangeUDP(ctx context.Context, query []byte) ([]byte, error) { - var errs error - // deferred holds a server-failure answer to fall back on, see below. - var deferred []byte - for _, upstream := range s.config.upstreams { - if err := ctx.Err(); err != nil { - return nil, err - } - conn, err := s.n.dialer.DialContext(ctx, "udp", upstream) - if err != nil { - errs = errors.Join(errs, err) - continue - } - answer, err := func() ([]byte, error) { - defer conn.Close() - stop := context.AfterFunc(ctx, func() { _ = conn.Close() }) - defer stop() - if err := conn.SetDeadline(time.Now().Add(dnsExchangeTimeout)); err != nil { - return nil, err - } - if _, err := conn.Write(query); err != nil { - return nil, err - } - buf := make([]byte, maxDNSDatagram) - n, err := conn.Read(buf) - if err != nil { - return nil, err - } - return buf[:n], nil - }() - if err != nil { - errs = errors.Join(errs, fmt.Errorf("upstream %s: %w", upstream, err)) - continue - } - // SERVFAIL is not an answer, and the sandbox sees only the gateway, so - // it cannot try the pod's other resolvers itself. Keep the last one to - // return if none does better: a real response beats a timeout. - if rcode, ok := failoverRcode(answer); ok { - errs = errors.Join(errs, fmt.Errorf("upstream %s: %w", upstream, rcodeError(rcode))) - deferred = answer - continue - } - return answer, nil - } - if deferred != nil { - return deferred, nil - } - return nil, fmt.Errorf("dns: no upstream resolver answered: %w", errs) -} - -// relayTCP copies a DNS stream without parsing its length-prefixed messages. -func (s *Server) relayTCP(ctx context.Context, downstream net.Conn) { - defer downstream.Close() - - var upstream net.Conn - var errs error - for _, address := range s.config.upstreams { - conn, err := s.n.dialer.DialContext(ctx, "tcp", address) - if err != nil { - errs = errors.Join(errs, err) - continue - } - upstream = conn - break - } - if upstream == nil { - slog.WarnContext(ctx, "dns relay could not reach any resolver for an actor DNS connection", slog.Any("err", errs)) - return - } - defer upstream.Close() - - // Cancel active copies on teardown to release the worker's connection slots. - relayDone := make(chan struct{}) - defer close(relayDone) - go func() { - select { - case <-ctx.Done(): - _ = downstream.Close() - _ = upstream.Close() - case <-relayDone: - } - }() - - deadline := time.Now().Add(dnsTCPTimeout) - if ctxDeadline, ok := ctx.Deadline(); ok && ctxDeadline.Before(deadline) { - deadline = ctxDeadline - } - _ = downstream.SetDeadline(deadline) - _ = upstream.SetDeadline(deadline) - - var wg sync.WaitGroup - wg.Add(2) - go func() { - defer wg.Done() - _, _ = io.Copy(upstream, downstream) - if c, ok := upstream.(*net.TCPConn); ok { - _ = c.CloseWrite() - } - }() - go func() { - defer wg.Done() - _, _ = io.Copy(downstream, upstream) - if c, ok := downstream.(*net.TCPConn); ok { - _ = c.CloseWrite() - } - }() - wg.Wait() -} diff --git a/internal/ateomnet/dns/tcphandler.go b/internal/ateomnet/dns/tcphandler.go new file mode 100644 index 0000000000..93ca9883dd --- /dev/null +++ b/internal/ateomnet/dns/tcphandler.go @@ -0,0 +1,254 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dns + +import ( + "context" + "errors" + "fmt" + "io" + "log/slog" + "net" + "sync" + "time" + + "github.com/agent-substrate/substrate/internal/ateomnet/dns/protocol" +) + +const ( + // defaultConnectionTimeout limits idle connection time between reads/writes. + defaultConnectionTimeout = 30 * time.Second +) + +// tcpClientConn wraps a downstream actor TCP connection with a mutex to +// serialize concurrent frame writes from replies and out-of-order +// upstream responses. +type tcpClientConn struct { + conn net.Conn + writeMu sync.Mutex +} + +// tcpHandler accepts actor TCP DNS connections and relays length-prefixed DNS +// frames with pipelining and out-of-order response support. +type tcpHandler struct { + upstreams []string + + listener net.Listener + dialer net.Dialer + dns *dnsHandler + limiter *limiter + + tcpTimeout time.Duration +} + +func newTCPHandler( + upstreams []string, + listener net.Listener, + dialer net.Dialer, + dns *dnsHandler, + lim *limiter, +) *tcpHandler { + return &tcpHandler{ + upstreams: upstreams, + listener: listener, + dialer: dialer, + dns: dns, + limiter: lim, + tcpTimeout: defaultConnectionTimeout, + } +} + +// serve accepts and relays actor TCP DNS connections until ctx is canceled or +// the listener closes. +func (h *tcpHandler) serve(ctx context.Context) error { + var wg sync.WaitGroup + defer wg.Wait() + + for { + conn, err := h.listener.Accept() + if err != nil { + if ctx.Err() != nil || errors.Is(err, net.ErrClosed) { + return nil + } + return fmt.Errorf("dns: accepting actor DNS connection: %w", err) + } + if !h.limiter.connections.tryAcquire() { + slog.DebugContext(ctx, "dns relay: refusing connection, at limit") + _ = conn.Close() + continue + + } + wg.Add(1) + go func() { + defer wg.Done() + defer func() { h.limiter.connections.release() }() + + h.onConnect(ctx, conn) + }() + } +} + +// refreshDeadline for extending the TCP connection closure on idle. +func (h *tcpHandler) refreshDeadline(ctx context.Context, downstream, upstream net.Conn) { + deadline := time.Now().Add(h.tcpTimeout) + if ctxDeadline, ok := ctx.Deadline(); ok && ctxDeadline.Before(deadline) { + deadline = ctxDeadline + } + _ = downstream.SetDeadline(deadline) + _ = upstream.SetDeadline(deadline) +} + +func (h *tcpHandler) onConnect(ctx context.Context, downstream net.Conn) { + defer downstream.Close() + + var upstream net.Conn + var errs error + + // Get a connection to a healthy upstream. + for _, address := range h.upstreams { + conn, err := h.dialer.DialContext(ctx, "tcp", address) + if err != nil { + errs = errors.Join(errs, err) + continue + } + upstream = conn + break + } + if upstream == nil { + slog.WarnContext(ctx, "dns relay: could not reach any upstream", slog.Any("err", errs)) + return + } + defer upstream.Close() + + stop := context.AfterFunc(ctx, func() { + _ = downstream.Close() + _ = upstream.Close() + }) + defer stop() + + // Set initial idle close deadline for the connections. + h.refreshDeadline(ctx, downstream, upstream) + + client := &tcpClientConn{conn: downstream} + + // Forward request/responses. + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + h.processRequests(ctx, client, upstream) + }() + go func() { + defer wg.Done() + h.processResponses(ctx, client, upstream) + }() + wg.Wait() +} + +func (h *tcpHandler) processRequests( + ctx context.Context, + client *tcpClientConn, + upstream net.Conn, +) { + for { + raw, err := protocol.ReadTCPFrame(client.conn) + if err != nil { + if errors.Is(err, io.EOF) { + if c, ok := upstream.(*net.TCPConn); ok { + _ = c.CloseWrite() + } + return + } + _ = upstream.Close() + _ = client.conn.Close() + return + } + h.refreshDeadline(ctx, client.conn, upstream) + + act := h.dns.onTCPRequest(raw) + + switch act.kind { + case actionDrop: + slog.DebugContext(ctx, "dns relay: dropping request and closing stream") + _ = upstream.Close() + _ = client.conn.Close() + return + case actionReply: + if err := client.writeFrame(act.payload); err != nil { + if ctx.Err() == nil && !errors.Is(err, net.ErrClosed) { + slog.DebugContext(ctx, "dns relay: error on TCP reply", slog.Any("err", err)) + } + _ = upstream.Close() + _ = client.conn.Close() + return + } + h.refreshDeadline(ctx, client.conn, upstream) + case actionForward: + if err := protocol.WriteTCPFrame(upstream, act.payload); err != nil { + if ctx.Err() == nil && !errors.Is(err, net.ErrClosed) { + slog.WarnContext(ctx, "dns relay: error in forward", slog.Any("err", err)) + } + _ = upstream.Close() + _ = client.conn.Close() + return + } + h.refreshDeadline(ctx, client.conn, upstream) + } + } +} + +func (h *tcpHandler) processResponses( + ctx context.Context, + client *tcpClientConn, + upstream net.Conn, +) { + for { + raw, err := protocol.ReadTCPFrame(upstream) + if err != nil { + if errors.Is(err, io.EOF) { + if c, ok := client.conn.(*net.TCPConn); ok { + _ = c.CloseWrite() + } + } + _ = client.conn.Close() + return + } + h.refreshDeadline(ctx, client.conn, upstream) + + act := h.dns.onTCPResponse(raw) + switch act.kind { + case actionDrop: + _ = upstream.Close() + _ = client.conn.Close() + return + case actionDeliver: + if err := client.writeFrame(act.payload); err != nil { + if ctx.Err() == nil && !errors.Is(err, net.ErrClosed) { + slog.WarnContext(ctx, "dns relay could not return TCP DNS answer", slog.Any("err", err)) + } + _ = upstream.Close() + _ = client.conn.Close() + return + } + h.refreshDeadline(ctx, client.conn, upstream) + } + } +} + +func (c *tcpClientConn) writeFrame(msg []byte) error { + c.writeMu.Lock() + defer c.writeMu.Unlock() + return protocol.WriteTCPFrame(c.conn, msg) +} diff --git a/internal/ateomnet/dns/tcphandler_test.go b/internal/ateomnet/dns/tcphandler_test.go new file mode 100644 index 0000000000..6322342bdd --- /dev/null +++ b/internal/ateomnet/dns/tcphandler_test.go @@ -0,0 +1,293 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dns + +import ( + "context" + "encoding/binary" + "net" + "sync" + "testing" + "time" + + "github.com/agent-substrate/substrate/internal/ateomnet/dns/protocol" + "golang.org/x/net/dns/dnsmessage" +) + +func startTestTCPHandler(t *testing.T, upstreams []string, configure func(*tcpHandler, *dnsHandler)) net.Addr { + t.Helper() + lis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + + lim := newLimiter() + dnsH := newDNSHandler(nil, lim, nil) + h := newTCPHandler(upstreams, lis, net.Dialer{Timeout: 5 * time.Second}, dnsH, lim) + if configure != nil { + configure(h, dnsH) + } + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + defer close(done) + _ = h.serve(ctx) + }() + t.Cleanup(func() { + cancel() + _ = lis.Close() + <-done + }) + + return lis.Addr() +} + +// newOutOfOrderTCPResolver reads 2 queries on a single TCP connection and +// answers the second query before the first to exercise RFC 7766 pipelining. +func newOutOfOrderTCPResolver(t *testing.T) string { + t.Helper() + lis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = lis.Close() }) + + go func() { + for { + conn, err := lis.Accept() + if err != nil { + return + } + go func() { + defer conn.Close() + q1, err := protocol.ReadTCPFrame(conn) + if err != nil { + return + } + q2, err := protocol.ReadTCPFrame(conn) + if err != nil { + return + } + // Respond to q2 first, then q1. + _ = protocol.WriteTCPFrame(conn, dnsAnswer(q2, 0)) + _ = protocol.WriteTCPFrame(conn, dnsAnswer(q1, 0)) + }() + } + }() + return lis.Addr().String() +} + +func TestTCPHandlerPipeliningAndOutOfOrderResponses(t *testing.T) { + upstream := newOutOfOrderTCPResolver(t) + addr := startTestTCPHandler(t, []string{upstream}, nil) + + conn, err := net.Dial("tcp", addr.String()) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + _ = conn.SetDeadline(time.Now().Add(5 * time.Second)) + + if err := protocol.WriteTCPFrame(conn, dnsQuery(0x1111)); err != nil { + t.Fatal(err) + } + if err := protocol.WriteTCPFrame(conn, dnsQuery(0x2222)); err != nil { + t.Fatal(err) + } + + resp1, err := protocol.ReadTCPFrame(conn) + if err != nil { + t.Fatalf("reading first response frame: %v", err) + } + resp2, err := protocol.ReadTCPFrame(conn) + if err != nil { + t.Fatalf("reading second response frame: %v", err) + } + + if got1 := binary.BigEndian.Uint16(resp1[0:2]); got1 != 0x2222 { + t.Errorf("first response ID = %#x, want 0x2222 (out-of-order)", got1) + } + if got2 := binary.BigEndian.Uint16(resp2[0:2]); got2 != 0x1111 { + t.Errorf("second response ID = %#x, want 0x1111", got2) + } +} + +func TestTCPHandlerSynthesizedReplyAndHalfClose(t *testing.T) { + lis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = lis.Close() }) + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + conn, err := lis.Accept() + if err != nil { + return + } + defer conn.Close() + q, err := protocol.ReadTCPFrame(conn) + if err != nil { + return + } + _ = protocol.WriteTCPFrame(conn, dnsAnswer(q, 0)) + }() + + addr := startTestTCPHandler(t, []string{lis.Addr().String()}, func(_ *tcpHandler, dnsH *dnsHandler) { + dnsH.allow = func(q dnsmessage.Question) bool { + return q.Name.String() != "blocked.example.com." + } + }) + + conn, err := net.Dial("tcp", addr.String()) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + _ = conn.SetDeadline(time.Now().Add(5 * time.Second)) + + blockedQuery := buildTestMessage(t, dnsmessage.Header{ + ID: 0xbad0, + RecursionDesired: true, + }, []dnsmessage.Question{{ + Name: dnsmessage.MustNewName("blocked.example.com."), + Type: dnsmessage.TypeA, + Class: dnsmessage.ClassINET, + }}) + + // First frame triggers a synthesized reply; second frame is forwarded, and + // then client half-closes its write side. + if err := protocol.WriteTCPFrame(conn, blockedQuery); err != nil { + t.Fatal(err) + } + if err := protocol.WriteTCPFrame(conn, dnsQuery(0x1234)); err != nil { + t.Fatal(err) + } + if tc, ok := conn.(*net.TCPConn); ok { + _ = tc.CloseWrite() + } + + r1, err := protocol.ReadTCPFrame(conn) + if err != nil { + t.Fatalf("reading first frame: %v", err) + } + r2, err := protocol.ReadTCPFrame(conn) + if err != nil { + t.Fatalf("reading second frame: %v", err) + } + + got := map[uint16]dnsmessage.RCode{ + binary.BigEndian.Uint16(r1[0:2]): dnsmessage.RCode(r1[3] & 0x0f), + binary.BigEndian.Uint16(r2[0:2]): dnsmessage.RCode(r2[3] & 0x0f), + } + if got[0xbad0] != dnsmessage.RCodeRefused { + t.Errorf("0xbad0 rcode = %v, want %v", got[0xbad0], dnsmessage.RCodeRefused) + } + if got[0x1234] != dnsmessage.RCodeSuccess { + t.Errorf("0x1234 rcode = %v, want %v", got[0x1234], dnsmessage.RCodeSuccess) + } + wg.Wait() +} + +func TestTCPHandlerSlidingIdleTimeout(t *testing.T) { + lis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = lis.Close() }) + + go func() { + for { + conn, err := lis.Accept() + if err != nil { + return + } + go func() { + defer conn.Close() + for { + q, err := protocol.ReadTCPFrame(conn) + if err != nil { + return + } + if err := protocol.WriteTCPFrame(conn, dnsAnswer(q, 0)); err != nil { + return + } + } + }() + } + }() + + const idleTimeout = 80 * time.Millisecond + addr := startTestTCPHandler(t, []string{lis.Addr().String()}, func(h *tcpHandler, _ *dnsHandler) { + h.tcpTimeout = idleTimeout + }) + + conn, err := net.Dial("tcp", addr.String()) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + _ = conn.SetDeadline(time.Now().Add(5 * time.Second)) + + // Send 3 queries spaced by half the idleTimeout (total elapsed ~1.5x idleTimeout). + // Because each read/write refreshes the deadline, the connection stays alive. + for i := range 3 { + if i > 0 { + time.Sleep(idleTimeout / 2) + } + id := uint16(0x3000 + i) + if err := protocol.WriteTCPFrame(conn, dnsQuery(id)); err != nil { + t.Fatalf("query %d write failed: %v", i, err) + } + resp, err := protocol.ReadTCPFrame(conn) + if err != nil { + t.Fatalf("query %d read failed: %v", i, err) + } + if got := binary.BigEndian.Uint16(resp[0:2]); got != id { + t.Errorf("query %d response ID = %#x, want %#x", i, got, id) + } + } + + // Now leave the connection idle past idleTimeout; the handler should close it. + var buf [1]byte + if _, err := conn.Read(buf[:]); err == nil { + t.Error("expected idle connection to be closed after timeout, but Read succeeded") + } +} + +func TestTCPHandlerClosesOnDroppedFrame(t *testing.T) { + upstream, _ := newHeldTCPResolver(t) + addr := startTestTCPHandler(t, []string{upstream}, nil) + + conn, err := net.Dial("tcp", addr.String()) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + _ = conn.SetDeadline(time.Now().Add(5 * time.Second)) + + // Send an invalid (< 12 byte) DNS frame that triggers actionDrop. + if err := protocol.WriteTCPFrame(conn, []byte{1, 2, 3}); err != nil { + t.Fatal(err) + } + + var buf [1]byte + if _, err := conn.Read(buf[:]); err == nil { + t.Error("expected connection to be closed after dropped frame, but Read succeeded") + } +} diff --git a/internal/ateomnet/dns/udphandler.go b/internal/ateomnet/dns/udphandler.go new file mode 100644 index 0000000000..495e079382 --- /dev/null +++ b/internal/ateomnet/dns/udphandler.go @@ -0,0 +1,220 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dns + +import ( + "bytes" + "context" + "encoding/binary" + "errors" + "fmt" + "log/slog" + "net" + "sync" + "time" +) + +const ( + // defaultSweepInterval is how often pendingRequests is checked for expired + // UDP queries. + defaultSweepInterval = 50 * time.Millisecond + // defaultUpstreamAttemptTimeout bounds a single UDP upstream attempt + // when fallback upstreams remain. + defaultUpstreamAttemptTimeout = 1 * time.Second + // maxDatagramSize that we accept. Raising this consumes more memory while + // processing packets. + // + // References: + // - 4096: https://www.rfc-editor.org/info/rfc6891/#section-6.2.5 + // - 1232: CoreDNS default, https://www.dnsflagday.net/2020/#dns-flag-day-2020 + maxDatagramSize = 4 * 1024 +) + +// udpHandler manages actor UDP DNS traffic using an actorSock from +// the Actor and upstreamSock out to the upstream DNS server. +type udpHandler struct { + // actorSock that receives DNS requests from the Actor. + actorSock net.PacketConn + // upstreamSock for the outbound DNS requests. + upstreamSock net.PacketConn + + upstreams []*net.UDPAddr + upstreamStrs []string + + dns *dnsHandler + pending *pendingRequests + sweepInterval time.Duration +} + +func newUDPHandler( + actorSock net.PacketConn, + upstreamSock net.PacketConn, + upstreams []*net.UDPAddr, + dns *dnsHandler, + pending *pendingRequests, +) *udpHandler { + strs := make([]string, len(upstreams)) + for i, u := range upstreams { + strs[i] = normalizeAddr(u) + } + return &udpHandler{ + actorSock: actorSock, + upstreamSock: upstreamSock, + upstreams: upstreams, + upstreamStrs: strs, + dns: dns, + pending: pending, + sweepInterval: defaultSweepInterval, + } +} + +// serve runs the readers, and timeout sweep ticker until ctx is canceled +// or the sockets close. +func (h *udpHandler) serve(ctx context.Context) error { + var wg sync.WaitGroup + errCh := make(chan error, 2) + + wg.Add(3) + go func() { + defer wg.Done() + if err := h.processRequests(ctx); err != nil { + errCh <- err + } + }() + go func() { + defer wg.Done() + if err := h.processResponses(ctx); err != nil { + errCh <- err + } + }() + go func() { + defer wg.Done() + h.sweepLoop(ctx) + }() + + wg.Wait() + h.pending.clearAll() + close(errCh) + + var errs error + for err := range errCh { + errs = errors.Join(errs, err) + } + + return errs +} + +func (h *udpHandler) processRequests(ctx context.Context) error { + buf := make([]byte, maxDatagramSize) + for { + n, from, err := h.actorSock.ReadFrom(buf) + if err != nil { + if ctx.Err() != nil || errors.Is(err, net.ErrClosed) { + return nil + } + return fmt.Errorf("dns relay: reading actor DNS query: %w", err) + } + raw := bytes.Clone(buf[:n]) + act := h.dns.onUDPRequest(raw, from, h.upstreamStrs) + switch act.kind { + case actionDrop: + continue + case actionReply: + if _, err := h.actorSock.WriteTo(act.payload, from); err != nil && ctx.Err() == nil && !errors.Is(err, net.ErrClosed) { + slog.WarnContext(ctx, "dns relay: could not return a synthesized DNS reply", slog.Any("err", err)) + } + case actionForward: + h.sendToUpstream(ctx, act.upstreamID, 0, act.payload) + } + } +} + +func (h *udpHandler) sendToUpstream(ctx context.Context, upstreamID uint16, idx int, query []byte) { + for idx < len(h.upstreams) { + _, err := h.upstreamSock.WriteTo(query, h.upstreams[idx]) + if err == nil { + return + } + if ctx.Err() != nil || errors.Is(err, net.ErrClosed) { + slog.ErrorContext(ctx, "dns relay: upstream UDP socket closed, stopping forwarding") + h.pending.clearAll() + return + } + slog.WarnContext(ctx, "dns relay: could not send query to upstream", slog.String("upstream", h.upstreamStrs[idx]), slog.Any("err", err)) + nextIdx, nextQuery, ok := h.pending.failOverOnSendError(upstreamID, idx) + if !ok { + return + } + idx = nextIdx + query = nextQuery + } +} + +func (h *udpHandler) processResponses(ctx context.Context) error { + buf := make([]byte, maxDatagramSize) + for { + n, from, err := h.upstreamSock.ReadFrom(buf) + if err != nil { + if ctx.Err() != nil || errors.Is(err, net.ErrClosed) { + return nil + } + slog.DebugContext(ctx, "dns relay: upstream socket read error", slog.Any("err", err)) + continue + } + raw := bytes.Clone(buf[:n]) + act := h.dns.onUDPResponse(raw, from) + switch act.kind { + case actionDrop: + continue + case actionFailover: + if act.upstreamIdx >= 0 && act.upstreamIdx < len(h.upstreams) && len(act.payload) >= 2 { + upstreamID := binary.BigEndian.Uint16(act.payload[0:2]) + h.sendToUpstream(ctx, upstreamID, act.upstreamIdx, act.payload) + } + case actionDeliver: + if act.clientAddr != nil { + if _, err := h.actorSock.WriteTo(act.payload, act.clientAddr); err != nil && ctx.Err() == nil && !errors.Is(err, net.ErrClosed) { + slog.WarnContext(ctx, "dns relay: could not return a DNS answer", slog.Any("err", err)) + } + } + } + } +} + +func (h *udpHandler) sweepLoop(ctx context.Context) { + interval := h.sweepInterval + if interval <= 0 { + interval = defaultSweepInterval + } + ticker := time.NewTicker(interval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case now := <-ticker.C: + failovers, deliveries := h.pending.sweep(now) + for _, f := range failovers { + h.sendToUpstream(ctx, f.upstreamID, f.upstreamIdx, f.query) + } + for _, d := range deliveries { + if _, err := h.actorSock.WriteTo(d.payload, d.clientAddr); err != nil && ctx.Err() == nil && !errors.Is(err, net.ErrClosed) { + slog.WarnContext(ctx, "dns relay: could not return a deferred DNS answer", slog.Any("err", err)) + } + } + } + } +} diff --git a/internal/ateomnet/dns/udphandler_test.go b/internal/ateomnet/dns/udphandler_test.go new file mode 100644 index 0000000000..ca7854a260 --- /dev/null +++ b/internal/ateomnet/dns/udphandler_test.go @@ -0,0 +1,169 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dns + +import ( + "context" + "encoding/binary" + "net" + "sync/atomic" + "testing" + "time" + + "golang.org/x/net/dns/dnsmessage" +) + +func startTestUDPHandler(t *testing.T, upstreams []string, configure func(*udpHandler, *dnsHandler)) net.Conn { + t.Helper() + ingress, err := net.ListenPacket("udp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + egress, err := net.ListenPacket("udp", "127.0.0.1:0") + if err != nil { + ingress.Close() + t.Fatal(err) + } + + udpAddrs := make([]*net.UDPAddr, len(upstreams)) + for i, u := range upstreams { + addr, err := net.ResolveUDPAddr("udp", u) + if err != nil { + ingress.Close() + egress.Close() + t.Fatal(err) + } + udpAddrs[i] = addr + } + + lim := newLimiter() + pending := newPendingRequests(lim) + dnsH := newDNSHandler(pending, lim, nil) + h := newUDPHandler(ingress, egress, udpAddrs, dnsH, pending) + if configure != nil { + configure(h, dnsH) + } + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + defer close(done) + _ = h.serve(ctx) + }() + t.Cleanup(func() { + cancel() + _ = ingress.Close() + _ = egress.Close() + <-done + }) + + client, err := net.Dial("udp", ingress.LocalAddr().String()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = client.Close() }) + return client +} + +func TestUDPHandlerForwardsAndRestoresID(t *testing.T) { + var seenUpstreamID atomic.Uint32 + upstream := newFakeResolver(t, func(query []byte) []byte { + seenUpstreamID.Store(uint32(binary.BigEndian.Uint16(query[0:2]))) + return dnsAnswer(query, 0) + }) + + client := startTestUDPHandler(t, []string{upstream}, nil) + const clientID = 0x4242 + query := dnsQuery(clientID) + if _, err := client.Write(query); err != nil { + t.Fatal(err) + } + got := readWithin(t, client) + if gotID := binary.BigEndian.Uint16(got[0:2]); gotID != clientID { + t.Errorf("response ID = %#x, want %#x", gotID, clientID) + } + if got[2]&0x80 == 0 { + t.Error("expected response bit QR=1 to be set") + } +} + +func TestUDPHandlerSynthesizedReply(t *testing.T) { + var upstreamCalls atomic.Int32 + upstream := newFakeResolver(t, func(query []byte) []byte { + upstreamCalls.Add(1) + return dnsAnswer(query, 0) + }) + + client := startTestUDPHandler(t, []string{upstream}, func(_ *udpHandler, dnsH *dnsHandler) { + dnsH.allow = func(dnsmessage.Question) bool { return false } + }) + + query := dnsQuery(0x9999) + if _, err := client.Write(query); err != nil { + t.Fatal(err) + } + got := readWithin(t, client) + if rcode := dnsmessage.RCode(got[3] & 0x0f); rcode != dnsmessage.RCodeRefused { + t.Errorf("synthesized rcode = %v, want %v", rcode, dnsmessage.RCodeRefused) + } + if upstreamCalls.Load() != 0 { + t.Errorf("upstream was called %d times on synthesized reply, want 0", upstreamCalls.Load()) + } +} + +func TestUDPHandlerFailoverOnSweepTimeoutAndDeferredDelivery(t *testing.T) { + silent1, _ := newSilentResolver(t) + healthy := newFakeResolver(t, func(query []byte) []byte { + return dnsAnswer(query, 0) + }) + + client := startTestUDPHandler(t, []string{silent1, healthy}, func(h *udpHandler, _ *dnsHandler) { + h.pending.attemptTimeout = 40 * time.Millisecond + h.sweepInterval = 10 * time.Millisecond + }) + + if _, err := client.Write(dnsQuery(0x7777)); err != nil { + t.Fatal(err) + } + got := readWithin(t, client) + if gotID := binary.BigEndian.Uint16(got[0:2]); gotID != 0x7777 { + t.Errorf("response ID = %#x, want 0x7777", gotID) + } + if rcode := got[3] & 0x0f; rcode != 0 { + t.Errorf("rcode = %d, want 0 after timeout failover", rcode) + } + + // Also verify that if the first upstream returns SERVFAIL and the second + // upstream times out, the deferred SERVFAIL is delivered on expiry. + sick := newFakeResolver(t, func(query []byte) []byte { + return dnsAnswer(query, byte(dnsmessage.RCodeServerFailure)) + }) + silent2, _ := newSilentResolver(t) + client2 := startTestUDPHandler(t, []string{sick, silent2}, func(h *udpHandler, _ *dnsHandler) { + h.pending.attemptTimeout = 40 * time.Millisecond + h.sweepInterval = 10 * time.Millisecond + }) + + if _, err := client2.Write(dnsQuery(0x8888)); err != nil { + t.Fatal(err) + } + got2 := readWithin(t, client2) + if gotID := binary.BigEndian.Uint16(got2[0:2]); gotID != 0x8888 { + t.Errorf("deferred response ID = %#x, want 0x8888", gotID) + } + if rcode := dnsmessage.RCode(got2[3] & 0x0f); rcode != dnsmessage.RCodeServerFailure { + t.Errorf("deferred rcode = %v, want %v", rcode, dnsmessage.RCodeServerFailure) + } +} diff --git a/vendor/golang.org/x/net/dns/dnsmessage/message.go b/vendor/golang.org/x/net/dns/dnsmessage/message.go new file mode 100644 index 0000000000..cbe94a38f5 --- /dev/null +++ b/vendor/golang.org/x/net/dns/dnsmessage/message.go @@ -0,0 +1,2742 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package dnsmessage provides a mostly RFC 1035 compliant implementation of +// DNS message packing and unpacking. +// +// The package also supports messages with Extension Mechanisms for DNS +// (EDNS(0)) as defined in RFC 6891. +// +// This implementation is designed to minimize heap allocations and avoid +// unnecessary packing and unpacking as much as possible. +package dnsmessage + +import ( + "errors" +) + +// Message formats +// +// To add a new Resource Record type: +// 1. Create Resource Record types +// 1.1. Add a Type constant named "Type" +// 1.2. Add the corresponding entry to the typeNames map +// 1.3. Add a [ResourceBody] implementation named "Resource" +// 2. Implement packing +// 2.1. Implement Builder.Resource() +// 3. Implement unpacking +// 3.1. Add the unpacking code to unpackResourceBody() +// 3.2. Implement Parser.Resource() + +// A Type is the type of a DNS Resource Record, as defined in the [IANA registry]. +// +// [IANA registry]: https://www.iana.org/assignments/dns-parameters/dns-parameters.xhtml#dns-parameters-4 +type Type uint16 + +const ( + // ResourceHeader.Type and Question.Type + TypeA Type = 1 + TypeNS Type = 2 + TypeCNAME Type = 5 + TypeSOA Type = 6 + TypePTR Type = 12 + TypeMX Type = 15 + TypeTXT Type = 16 + TypeAAAA Type = 28 + TypeSRV Type = 33 + TypeOPT Type = 41 + TypeSVCB Type = 64 + TypeHTTPS Type = 65 + + // Question.Type + TypeWKS Type = 11 + TypeHINFO Type = 13 + TypeMINFO Type = 14 + TypeAXFR Type = 252 + TypeALL Type = 255 +) + +var typeNames = map[Type]string{ + TypeA: "TypeA", + TypeNS: "TypeNS", + TypeCNAME: "TypeCNAME", + TypeSOA: "TypeSOA", + TypePTR: "TypePTR", + TypeMX: "TypeMX", + TypeTXT: "TypeTXT", + TypeAAAA: "TypeAAAA", + TypeSRV: "TypeSRV", + TypeOPT: "TypeOPT", + TypeSVCB: "TypeSVCB", + TypeHTTPS: "TypeHTTPS", + TypeWKS: "TypeWKS", + TypeHINFO: "TypeHINFO", + TypeMINFO: "TypeMINFO", + TypeAXFR: "TypeAXFR", + TypeALL: "TypeALL", +} + +// String implements fmt.Stringer.String. +func (t Type) String() string { + if n, ok := typeNames[t]; ok { + return n + } + return printUint16(uint16(t)) +} + +// GoString implements fmt.GoStringer.GoString. +func (t Type) GoString() string { + if n, ok := typeNames[t]; ok { + return "dnsmessage." + n + } + return printUint16(uint16(t)) +} + +// A Class is a type of network. +type Class uint16 + +const ( + // ResourceHeader.Class and Question.Class + ClassINET Class = 1 + ClassCSNET Class = 2 + ClassCHAOS Class = 3 + ClassHESIOD Class = 4 + + // Question.Class + ClassANY Class = 255 +) + +var classNames = map[Class]string{ + ClassINET: "ClassINET", + ClassCSNET: "ClassCSNET", + ClassCHAOS: "ClassCHAOS", + ClassHESIOD: "ClassHESIOD", + ClassANY: "ClassANY", +} + +// String implements fmt.Stringer.String. +func (c Class) String() string { + if n, ok := classNames[c]; ok { + return n + } + return printUint16(uint16(c)) +} + +// GoString implements fmt.GoStringer.GoString. +func (c Class) GoString() string { + if n, ok := classNames[c]; ok { + return "dnsmessage." + n + } + return printUint16(uint16(c)) +} + +// An OpCode is a DNS operation code. +type OpCode uint16 + +// GoString implements fmt.GoStringer.GoString. +func (o OpCode) GoString() string { + return printUint16(uint16(o)) +} + +// An RCode is a DNS response status code. +type RCode uint16 + +// Header.RCode values. +const ( + RCodeSuccess RCode = 0 // NoError + RCodeFormatError RCode = 1 // FormErr + RCodeServerFailure RCode = 2 // ServFail + RCodeNameError RCode = 3 // NXDomain + RCodeNotImplemented RCode = 4 // NotImp + RCodeRefused RCode = 5 // Refused +) + +var rCodeNames = map[RCode]string{ + RCodeSuccess: "RCodeSuccess", + RCodeFormatError: "RCodeFormatError", + RCodeServerFailure: "RCodeServerFailure", + RCodeNameError: "RCodeNameError", + RCodeNotImplemented: "RCodeNotImplemented", + RCodeRefused: "RCodeRefused", +} + +// String implements fmt.Stringer.String. +func (r RCode) String() string { + if n, ok := rCodeNames[r]; ok { + return n + } + return printUint16(uint16(r)) +} + +// GoString implements fmt.GoStringer.GoString. +func (r RCode) GoString() string { + if n, ok := rCodeNames[r]; ok { + return "dnsmessage." + n + } + return printUint16(uint16(r)) +} + +func printPaddedUint8(i uint8) string { + b := byte(i) + return string([]byte{ + b/100 + '0', + b/10%10 + '0', + b%10 + '0', + }) +} + +func printUint8Bytes(buf []byte, i uint8) []byte { + b := byte(i) + if i >= 100 { + buf = append(buf, b/100+'0') + } + if i >= 10 { + buf = append(buf, b/10%10+'0') + } + return append(buf, b%10+'0') +} + +func printByteSlice(b []byte) string { + if len(b) == 0 { + return "" + } + buf := make([]byte, 0, 5*len(b)) + buf = printUint8Bytes(buf, uint8(b[0])) + for _, n := range b[1:] { + buf = append(buf, ',', ' ') + buf = printUint8Bytes(buf, uint8(n)) + } + return string(buf) +} + +const hexDigits = "0123456789abcdef" + +func printString(str []byte) string { + buf := make([]byte, 0, len(str)) + for i := 0; i < len(str); i++ { + c := str[i] + if c == '.' || c == '-' || c == ' ' || + 'A' <= c && c <= 'Z' || + 'a' <= c && c <= 'z' || + '0' <= c && c <= '9' { + buf = append(buf, c) + continue + } + + upper := c >> 4 + lower := (c << 4) >> 4 + buf = append( + buf, + '\\', + 'x', + hexDigits[upper], + hexDigits[lower], + ) + } + return string(buf) +} + +func printUint16(i uint16) string { + return printUint32(uint32(i)) +} + +func printUint32(i uint32) string { + // Max value is 4294967295. + buf := make([]byte, 10) + for b, d := buf, uint32(1000000000); d > 0; d /= 10 { + b[0] = byte(i/d%10 + '0') + if b[0] == '0' && len(b) == len(buf) && len(buf) > 1 { + buf = buf[1:] + } + b = b[1:] + i %= d + } + return string(buf) +} + +func printBool(b bool) string { + if b { + return "true" + } + return "false" +} + +var ( + // ErrNotStarted indicates that the prerequisite information isn't + // available yet because the previous records haven't been appropriately + // parsed, skipped or finished. + ErrNotStarted = errors.New("parsing/packing of this type isn't available yet") + + // ErrSectionDone indicated that all records in the section have been + // parsed or finished. + ErrSectionDone = errors.New("parsing/packing of this section has completed") + + errBaseLen = errors.New("insufficient data for base length type") + errCalcLen = errors.New("insufficient data for calculated length type") + errReserved = errors.New("segment prefix is reserved") + errTooManyPtr = errors.New("too many pointers (>10)") + errInvalidPtr = errors.New("invalid pointer") + errInvalidName = errors.New("invalid dns name") + errNilResouceBody = errors.New("nil resource body") + errResourceLen = errors.New("insufficient data for resource body length") + errSegTooLong = errors.New("segment length too long") + errNameTooLong = errors.New("name too long") + errZeroSegLen = errors.New("zero length segment") + errResTooLong = errors.New("resource length too long") + errTooManyQuestions = errors.New("too many Questions to pack (>65535)") + errTooManyAnswers = errors.New("too many Answers to pack (>65535)") + errTooManyAuthorities = errors.New("too many Authorities to pack (>65535)") + errTooManyAdditionals = errors.New("too many Additionals to pack (>65535)") + errNonCanonicalName = errors.New("name is not in canonical format (it must end with a .)") + errStringTooLong = errors.New("character string exceeds maximum length (255)") + errParamOutOfOrder = errors.New("parameter out of order") + errTooLongSVCBValue = errors.New("value too long (>65535 bytes)") +) + +// Internal constants. +const ( + // packStartingCap is the default initial buffer size allocated during + // packing. + // + // The starting capacity doesn't matter too much, but most DNS responses + // Will be <= 512 bytes as it is the limit for DNS over UDP. + packStartingCap = 512 + + // uint16Len is the length (in bytes) of a uint16. + uint16Len = 2 + + // uint32Len is the length (in bytes) of a uint32. + uint32Len = 4 + + // headerLen is the length (in bytes) of a DNS header. + // + // A header is comprised of 6 uint16s and no padding. + headerLen = 6 * uint16Len +) + +type nestedError struct { + // s is the current level's error message. + s string + + // err is the nested error. + err error +} + +// nestedError implements error.Error. +func (e *nestedError) Error() string { + return e.s + ": " + e.err.Error() +} + +// Header is a representation of a DNS message header. +type Header struct { + ID uint16 + Response bool + OpCode OpCode + Authoritative bool + Truncated bool + RecursionDesired bool + RecursionAvailable bool + AuthenticData bool + CheckingDisabled bool + RCode RCode +} + +func (m *Header) pack() (id uint16, bits uint16) { + id = m.ID + bits = uint16(m.OpCode)<<11 | uint16(m.RCode) + if m.RecursionAvailable { + bits |= headerBitRA + } + if m.RecursionDesired { + bits |= headerBitRD + } + if m.Truncated { + bits |= headerBitTC + } + if m.Authoritative { + bits |= headerBitAA + } + if m.Response { + bits |= headerBitQR + } + if m.AuthenticData { + bits |= headerBitAD + } + if m.CheckingDisabled { + bits |= headerBitCD + } + return +} + +// GoString implements fmt.GoStringer.GoString. +func (m *Header) GoString() string { + return "dnsmessage.Header{" + + "ID: " + printUint16(m.ID) + ", " + + "Response: " + printBool(m.Response) + ", " + + "OpCode: " + m.OpCode.GoString() + ", " + + "Authoritative: " + printBool(m.Authoritative) + ", " + + "Truncated: " + printBool(m.Truncated) + ", " + + "RecursionDesired: " + printBool(m.RecursionDesired) + ", " + + "RecursionAvailable: " + printBool(m.RecursionAvailable) + ", " + + "AuthenticData: " + printBool(m.AuthenticData) + ", " + + "CheckingDisabled: " + printBool(m.CheckingDisabled) + ", " + + "RCode: " + m.RCode.GoString() + "}" +} + +// Message is a representation of a DNS message. +type Message struct { + Header + Questions []Question + Answers []Resource + Authorities []Resource + Additionals []Resource +} + +type section uint8 + +const ( + sectionNotStarted section = iota + sectionHeader + sectionQuestions + sectionAnswers + sectionAuthorities + sectionAdditionals + sectionDone + + headerBitQR = 1 << 15 // query/response (response=1) + headerBitAA = 1 << 10 // authoritative + headerBitTC = 1 << 9 // truncated + headerBitRD = 1 << 8 // recursion desired + headerBitRA = 1 << 7 // recursion available + headerBitAD = 1 << 5 // authentic data + headerBitCD = 1 << 4 // checking disabled +) + +var sectionNames = map[section]string{ + sectionHeader: "header", + sectionQuestions: "Question", + sectionAnswers: "Answer", + sectionAuthorities: "Authority", + sectionAdditionals: "Additional", +} + +// header is the wire format for a DNS message header. +type header struct { + id uint16 + bits uint16 + questions uint16 + answers uint16 + authorities uint16 + additionals uint16 +} + +func (h *header) count(sec section) uint16 { + switch sec { + case sectionQuestions: + return h.questions + case sectionAnswers: + return h.answers + case sectionAuthorities: + return h.authorities + case sectionAdditionals: + return h.additionals + } + return 0 +} + +// pack appends the wire format of the header to msg. +func (h *header) pack(msg []byte) []byte { + msg = packUint16(msg, h.id) + msg = packUint16(msg, h.bits) + msg = packUint16(msg, h.questions) + msg = packUint16(msg, h.answers) + msg = packUint16(msg, h.authorities) + return packUint16(msg, h.additionals) +} + +func (h *header) unpack(msg []byte, off int) (int, error) { + newOff := off + var err error + if h.id, newOff, err = unpackUint16(msg, newOff); err != nil { + return off, &nestedError{"id", err} + } + if h.bits, newOff, err = unpackUint16(msg, newOff); err != nil { + return off, &nestedError{"bits", err} + } + if h.questions, newOff, err = unpackUint16(msg, newOff); err != nil { + return off, &nestedError{"questions", err} + } + if h.answers, newOff, err = unpackUint16(msg, newOff); err != nil { + return off, &nestedError{"answers", err} + } + if h.authorities, newOff, err = unpackUint16(msg, newOff); err != nil { + return off, &nestedError{"authorities", err} + } + if h.additionals, newOff, err = unpackUint16(msg, newOff); err != nil { + return off, &nestedError{"additionals", err} + } + return newOff, nil +} + +func (h *header) header() Header { + return Header{ + ID: h.id, + Response: (h.bits & headerBitQR) != 0, + OpCode: OpCode(h.bits>>11) & 0xF, + Authoritative: (h.bits & headerBitAA) != 0, + Truncated: (h.bits & headerBitTC) != 0, + RecursionDesired: (h.bits & headerBitRD) != 0, + RecursionAvailable: (h.bits & headerBitRA) != 0, + AuthenticData: (h.bits & headerBitAD) != 0, + CheckingDisabled: (h.bits & headerBitCD) != 0, + RCode: RCode(h.bits & 0xF), + } +} + +// A Resource is a DNS resource record. +type Resource struct { + Header ResourceHeader + Body ResourceBody +} + +func (r *Resource) GoString() string { + return "dnsmessage.Resource{" + + "Header: " + r.Header.GoString() + + ", Body: &" + r.Body.GoString() + + "}" +} + +// A ResourceBody is a DNS resource record minus the header. +type ResourceBody interface { + // pack packs a Resource except for its header. + pack(msg []byte, compression map[string]uint16, compressionOff int) ([]byte, error) + + // realType returns the actual type of the Resource. This is used to + // fill in the header Type field. + realType() Type + + // GoString implements fmt.GoStringer.GoString. + GoString() string +} + +// pack appends the wire format of the Resource to msg. +func (r *Resource) pack(msg []byte, compression map[string]uint16, compressionOff int) ([]byte, error) { + if r.Body == nil { + return msg, errNilResouceBody + } + oldMsg := msg + r.Header.Type = r.Body.realType() + msg, lenOff, err := r.Header.pack(msg, compression, compressionOff) + if err != nil { + return msg, &nestedError{"ResourceHeader", err} + } + preLen := len(msg) + msg, err = r.Body.pack(msg, compression, compressionOff) + if err != nil { + return msg, &nestedError{"content", err} + } + if err := r.Header.fixLen(msg, lenOff, preLen); err != nil { + return oldMsg, err + } + return msg, nil +} + +// A Parser allows incrementally parsing a DNS message. +// +// When parsing is started, the Header is parsed. Next, each Question can be +// either parsed or skipped. Alternatively, all Questions can be skipped at +// once. When all Questions have been parsed, attempting to parse Questions +// will return the [ErrSectionDone] error. +// After all Questions have been either parsed or skipped, all +// Answers, Authorities and Additionals can be either parsed or skipped in the +// same way, and each type of Resource must be fully parsed or skipped before +// proceeding to the next type of Resource. +// +// Parser is safe to copy to preserve the parsing state. +// +// Note that there is no requirement to fully skip or parse the message. +type Parser struct { + msg []byte + header header + + section section + off int + index int + resHeaderValid bool + resHeaderOffset int + resHeaderType Type + resHeaderLength uint16 +} + +// Start parses the header and enables the parsing of Questions. +func (p *Parser) Start(msg []byte) (Header, error) { + if p.msg != nil { + *p = Parser{} + } + p.msg = msg + var err error + if p.off, err = p.header.unpack(msg, 0); err != nil { + return Header{}, &nestedError{"unpacking header", err} + } + p.section = sectionQuestions + return p.header.header(), nil +} + +func (p *Parser) checkAdvance(sec section) error { + if p.section < sec { + return ErrNotStarted + } + if p.section > sec { + return ErrSectionDone + } + p.resHeaderValid = false + if p.index == int(p.header.count(sec)) { + p.index = 0 + p.section++ + return ErrSectionDone + } + return nil +} + +func (p *Parser) resource(sec section) (Resource, error) { + var r Resource + var err error + r.Header, err = p.resourceHeader(sec) + if err != nil { + return r, err + } + p.resHeaderValid = false + r.Body, p.off, err = unpackResourceBody(p.msg, p.off, r.Header) + if err != nil { + return Resource{}, &nestedError{"unpacking " + sectionNames[sec], err} + } + p.index++ + return r, nil +} + +func (p *Parser) resourceHeader(sec section) (ResourceHeader, error) { + if p.resHeaderValid { + p.off = p.resHeaderOffset + } + + if err := p.checkAdvance(sec); err != nil { + return ResourceHeader{}, err + } + var hdr ResourceHeader + off, err := hdr.unpack(p.msg, p.off) + if err != nil { + return ResourceHeader{}, err + } + p.resHeaderValid = true + p.resHeaderOffset = p.off + p.resHeaderType = hdr.Type + p.resHeaderLength = hdr.Length + p.off = off + return hdr, nil +} + +func (p *Parser) skipResource(sec section) error { + if p.resHeaderValid && p.section == sec { + newOff := p.off + int(p.resHeaderLength) + if newOff > len(p.msg) { + return errResourceLen + } + p.off = newOff + p.resHeaderValid = false + p.index++ + return nil + } + if err := p.checkAdvance(sec); err != nil { + return err + } + var err error + p.off, err = skipResource(p.msg, p.off) + if err != nil { + return &nestedError{"skipping: " + sectionNames[sec], err} + } + p.index++ + return nil +} + +// Question parses a single Question. +func (p *Parser) Question() (Question, error) { + if err := p.checkAdvance(sectionQuestions); err != nil { + return Question{}, err + } + var name Name + off, err := name.unpack(p.msg, p.off) + if err != nil { + return Question{}, &nestedError{"unpacking Question.Name", err} + } + typ, off, err := unpackType(p.msg, off) + if err != nil { + return Question{}, &nestedError{"unpacking Question.Type", err} + } + class, off, err := unpackClass(p.msg, off) + if err != nil { + return Question{}, &nestedError{"unpacking Question.Class", err} + } + p.off = off + p.index++ + return Question{name, typ, class}, nil +} + +// AllQuestions parses all Questions. +func (p *Parser) AllQuestions() ([]Question, error) { + // Multiple questions are valid according to the spec, + // but servers don't actually support them. There will + // be at most one question here. + // + // Do not pre-allocate based on info in p.header, since + // the data is untrusted. + qs := []Question{} + for { + q, err := p.Question() + if err == ErrSectionDone { + return qs, nil + } + if err != nil { + return nil, err + } + qs = append(qs, q) + } +} + +// SkipQuestion skips a single Question. +func (p *Parser) SkipQuestion() error { + if err := p.checkAdvance(sectionQuestions); err != nil { + return err + } + off, err := skipName(p.msg, p.off) + if err != nil { + return &nestedError{"skipping Question Name", err} + } + if off, err = skipType(p.msg, off); err != nil { + return &nestedError{"skipping Question Type", err} + } + if off, err = skipClass(p.msg, off); err != nil { + return &nestedError{"skipping Question Class", err} + } + p.off = off + p.index++ + return nil +} + +// SkipAllQuestions skips all Questions. +func (p *Parser) SkipAllQuestions() error { + for { + if err := p.SkipQuestion(); err == ErrSectionDone { + return nil + } else if err != nil { + return err + } + } +} + +// AnswerHeader parses a single Answer ResourceHeader. +func (p *Parser) AnswerHeader() (ResourceHeader, error) { + return p.resourceHeader(sectionAnswers) +} + +// Answer parses a single Answer Resource. +func (p *Parser) Answer() (Resource, error) { + return p.resource(sectionAnswers) +} + +// AllAnswers parses all Answer Resources. +func (p *Parser) AllAnswers() ([]Resource, error) { + // The most common query is for A/AAAA, which usually returns + // a handful of IPs. + // + // Pre-allocate up to a certain limit, since p.header is + // untrusted data. + n := int(p.header.answers) + if n > 20 { + n = 20 + } + as := make([]Resource, 0, n) + for { + a, err := p.Answer() + if err == ErrSectionDone { + return as, nil + } + if err != nil { + return nil, err + } + as = append(as, a) + } +} + +// SkipAnswer skips a single Answer Resource. +// +// It does not perform a complete validation of the resource header, which means +// it may return a nil error when the [AnswerHeader] would actually return an error. +func (p *Parser) SkipAnswer() error { + return p.skipResource(sectionAnswers) +} + +// SkipAllAnswers skips all Answer Resources. +func (p *Parser) SkipAllAnswers() error { + for { + if err := p.SkipAnswer(); err == ErrSectionDone { + return nil + } else if err != nil { + return err + } + } +} + +// AuthorityHeader parses a single Authority ResourceHeader. +func (p *Parser) AuthorityHeader() (ResourceHeader, error) { + return p.resourceHeader(sectionAuthorities) +} + +// Authority parses a single Authority Resource. +func (p *Parser) Authority() (Resource, error) { + return p.resource(sectionAuthorities) +} + +// AllAuthorities parses all Authority Resources. +func (p *Parser) AllAuthorities() ([]Resource, error) { + // Authorities contains SOA in case of NXDOMAIN and friends, + // otherwise it is empty. + // + // Pre-allocate up to a certain limit, since p.header is + // untrusted data. + n := int(p.header.authorities) + if n > 10 { + n = 10 + } + as := make([]Resource, 0, n) + for { + a, err := p.Authority() + if err == ErrSectionDone { + return as, nil + } + if err != nil { + return nil, err + } + as = append(as, a) + } +} + +// SkipAuthority skips a single Authority Resource. +// +// It does not perform a complete validation of the resource header, which means +// it may return a nil error when the [AuthorityHeader] would actually return an error. +func (p *Parser) SkipAuthority() error { + return p.skipResource(sectionAuthorities) +} + +// SkipAllAuthorities skips all Authority Resources. +func (p *Parser) SkipAllAuthorities() error { + for { + if err := p.SkipAuthority(); err == ErrSectionDone { + return nil + } else if err != nil { + return err + } + } +} + +// AdditionalHeader parses a single Additional ResourceHeader. +func (p *Parser) AdditionalHeader() (ResourceHeader, error) { + return p.resourceHeader(sectionAdditionals) +} + +// Additional parses a single Additional Resource. +func (p *Parser) Additional() (Resource, error) { + return p.resource(sectionAdditionals) +} + +// AllAdditionals parses all Additional Resources. +func (p *Parser) AllAdditionals() ([]Resource, error) { + // Additionals usually contain OPT, and sometimes A/AAAA + // glue records. + // + // Pre-allocate up to a certain limit, since p.header is + // untrusted data. + n := int(p.header.additionals) + if n > 10 { + n = 10 + } + as := make([]Resource, 0, n) + for { + a, err := p.Additional() + if err == ErrSectionDone { + return as, nil + } + if err != nil { + return nil, err + } + as = append(as, a) + } +} + +// SkipAdditional skips a single Additional Resource. +// +// It does not perform a complete validation of the resource header, which means +// it may return a nil error when the [AdditionalHeader] would actually return an error. +func (p *Parser) SkipAdditional() error { + return p.skipResource(sectionAdditionals) +} + +// SkipAllAdditionals skips all Additional Resources. +func (p *Parser) SkipAllAdditionals() error { + for { + if err := p.SkipAdditional(); err == ErrSectionDone { + return nil + } else if err != nil { + return err + } + } +} + +// CNAMEResource parses a single CNAMEResource. +// +// One of the XXXHeader methods must have been called before calling this +// method. +func (p *Parser) CNAMEResource() (CNAMEResource, error) { + if !p.resHeaderValid || p.resHeaderType != TypeCNAME { + return CNAMEResource{}, ErrNotStarted + } + r, err := unpackCNAMEResource(p.msg, p.off) + if err != nil { + return CNAMEResource{}, err + } + p.off += int(p.resHeaderLength) + p.resHeaderValid = false + p.index++ + return r, nil +} + +// MXResource parses a single MXResource. +// +// One of the XXXHeader methods must have been called before calling this +// method. +func (p *Parser) MXResource() (MXResource, error) { + if !p.resHeaderValid || p.resHeaderType != TypeMX { + return MXResource{}, ErrNotStarted + } + r, err := unpackMXResource(p.msg, p.off) + if err != nil { + return MXResource{}, err + } + p.off += int(p.resHeaderLength) + p.resHeaderValid = false + p.index++ + return r, nil +} + +// NSResource parses a single NSResource. +// +// One of the XXXHeader methods must have been called before calling this +// method. +func (p *Parser) NSResource() (NSResource, error) { + if !p.resHeaderValid || p.resHeaderType != TypeNS { + return NSResource{}, ErrNotStarted + } + r, err := unpackNSResource(p.msg, p.off) + if err != nil { + return NSResource{}, err + } + p.off += int(p.resHeaderLength) + p.resHeaderValid = false + p.index++ + return r, nil +} + +// PTRResource parses a single PTRResource. +// +// One of the XXXHeader methods must have been called before calling this +// method. +func (p *Parser) PTRResource() (PTRResource, error) { + if !p.resHeaderValid || p.resHeaderType != TypePTR { + return PTRResource{}, ErrNotStarted + } + r, err := unpackPTRResource(p.msg, p.off) + if err != nil { + return PTRResource{}, err + } + p.off += int(p.resHeaderLength) + p.resHeaderValid = false + p.index++ + return r, nil +} + +// SOAResource parses a single SOAResource. +// +// One of the XXXHeader methods must have been called before calling this +// method. +func (p *Parser) SOAResource() (SOAResource, error) { + if !p.resHeaderValid || p.resHeaderType != TypeSOA { + return SOAResource{}, ErrNotStarted + } + r, err := unpackSOAResource(p.msg, p.off) + if err != nil { + return SOAResource{}, err + } + p.off += int(p.resHeaderLength) + p.resHeaderValid = false + p.index++ + return r, nil +} + +// TXTResource parses a single TXTResource. +// +// One of the XXXHeader methods must have been called before calling this +// method. +func (p *Parser) TXTResource() (TXTResource, error) { + if !p.resHeaderValid || p.resHeaderType != TypeTXT { + return TXTResource{}, ErrNotStarted + } + r, err := unpackTXTResource(p.msg, p.off, p.resHeaderLength) + if err != nil { + return TXTResource{}, err + } + p.off += int(p.resHeaderLength) + p.resHeaderValid = false + p.index++ + return r, nil +} + +// SRVResource parses a single SRVResource. +// +// One of the XXXHeader methods must have been called before calling this +// method. +func (p *Parser) SRVResource() (SRVResource, error) { + if !p.resHeaderValid || p.resHeaderType != TypeSRV { + return SRVResource{}, ErrNotStarted + } + r, err := unpackSRVResource(p.msg, p.off) + if err != nil { + return SRVResource{}, err + } + p.off += int(p.resHeaderLength) + p.resHeaderValid = false + p.index++ + return r, nil +} + +// AResource parses a single AResource. +// +// One of the XXXHeader methods must have been called before calling this +// method. +func (p *Parser) AResource() (AResource, error) { + if !p.resHeaderValid || p.resHeaderType != TypeA { + return AResource{}, ErrNotStarted + } + r, err := unpackAResource(p.msg, p.off) + if err != nil { + return AResource{}, err + } + p.off += int(p.resHeaderLength) + p.resHeaderValid = false + p.index++ + return r, nil +} + +// AAAAResource parses a single AAAAResource. +// +// One of the XXXHeader methods must have been called before calling this +// method. +func (p *Parser) AAAAResource() (AAAAResource, error) { + if !p.resHeaderValid || p.resHeaderType != TypeAAAA { + return AAAAResource{}, ErrNotStarted + } + r, err := unpackAAAAResource(p.msg, p.off) + if err != nil { + return AAAAResource{}, err + } + p.off += int(p.resHeaderLength) + p.resHeaderValid = false + p.index++ + return r, nil +} + +// OPTResource parses a single OPTResource. +// +// One of the XXXHeader methods must have been called before calling this +// method. +func (p *Parser) OPTResource() (OPTResource, error) { + if !p.resHeaderValid || p.resHeaderType != TypeOPT { + return OPTResource{}, ErrNotStarted + } + r, err := unpackOPTResource(p.msg, p.off, p.resHeaderLength) + if err != nil { + return OPTResource{}, err + } + p.off += int(p.resHeaderLength) + p.resHeaderValid = false + p.index++ + return r, nil +} + +// UnknownResource parses a single UnknownResource. +// +// One of the XXXHeader methods must have been called before calling this +// method. +func (p *Parser) UnknownResource() (UnknownResource, error) { + if !p.resHeaderValid { + return UnknownResource{}, ErrNotStarted + } + r, err := unpackUnknownResource(p.resHeaderType, p.msg, p.off, p.resHeaderLength) + if err != nil { + return UnknownResource{}, err + } + p.off += int(p.resHeaderLength) + p.resHeaderValid = false + p.index++ + return r, nil +} + +// Unpack parses a full Message. +func (m *Message) Unpack(msg []byte) error { + var p Parser + var err error + if m.Header, err = p.Start(msg); err != nil { + return err + } + if m.Questions, err = p.AllQuestions(); err != nil { + return err + } + if m.Answers, err = p.AllAnswers(); err != nil { + return err + } + if m.Authorities, err = p.AllAuthorities(); err != nil { + return err + } + if m.Additionals, err = p.AllAdditionals(); err != nil { + return err + } + return nil +} + +// Pack packs a full Message. +func (m *Message) Pack() ([]byte, error) { + return m.AppendPack(make([]byte, 0, packStartingCap)) +} + +// AppendPack is like Pack but appends the full Message to b and returns the +// extended buffer. +func (m *Message) AppendPack(b []byte) ([]byte, error) { + // Validate the lengths. It is very unlikely that anyone will try to + // pack more than 65535 of any particular type, but it is possible and + // we should fail gracefully. + if len(m.Questions) > int(^uint16(0)) { + return nil, errTooManyQuestions + } + if len(m.Answers) > int(^uint16(0)) { + return nil, errTooManyAnswers + } + if len(m.Authorities) > int(^uint16(0)) { + return nil, errTooManyAuthorities + } + if len(m.Additionals) > int(^uint16(0)) { + return nil, errTooManyAdditionals + } + + var h header + h.id, h.bits = m.Header.pack() + + h.questions = uint16(len(m.Questions)) + h.answers = uint16(len(m.Answers)) + h.authorities = uint16(len(m.Authorities)) + h.additionals = uint16(len(m.Additionals)) + + compressionOff := len(b) + msg := h.pack(b) + + // RFC 1035 allows (but does not require) compression for packing. RFC + // 1035 requires unpacking implementations to support compression, so + // unconditionally enabling it is fine. + // + // DNS lookups are typically done over UDP, and RFC 1035 states that UDP + // DNS messages can be a maximum of 512 bytes long. Without compression, + // many DNS response messages are over this limit, so enabling + // compression will help ensure compliance. + compression := map[string]uint16{} + + for i := range m.Questions { + var err error + if msg, err = m.Questions[i].pack(msg, compression, compressionOff); err != nil { + return nil, &nestedError{"packing Question", err} + } + } + for i := range m.Answers { + var err error + if msg, err = m.Answers[i].pack(msg, compression, compressionOff); err != nil { + return nil, &nestedError{"packing Answer", err} + } + } + for i := range m.Authorities { + var err error + if msg, err = m.Authorities[i].pack(msg, compression, compressionOff); err != nil { + return nil, &nestedError{"packing Authority", err} + } + } + for i := range m.Additionals { + var err error + if msg, err = m.Additionals[i].pack(msg, compression, compressionOff); err != nil { + return nil, &nestedError{"packing Additional", err} + } + } + + return msg, nil +} + +// GoString implements fmt.GoStringer.GoString. +func (m *Message) GoString() string { + s := "dnsmessage.Message{Header: " + m.Header.GoString() + ", " + + "Questions: []dnsmessage.Question{" + if len(m.Questions) > 0 { + s += m.Questions[0].GoString() + for _, q := range m.Questions[1:] { + s += ", " + q.GoString() + } + } + s += "}, Answers: []dnsmessage.Resource{" + if len(m.Answers) > 0 { + s += m.Answers[0].GoString() + for _, a := range m.Answers[1:] { + s += ", " + a.GoString() + } + } + s += "}, Authorities: []dnsmessage.Resource{" + if len(m.Authorities) > 0 { + s += m.Authorities[0].GoString() + for _, a := range m.Authorities[1:] { + s += ", " + a.GoString() + } + } + s += "}, Additionals: []dnsmessage.Resource{" + if len(m.Additionals) > 0 { + s += m.Additionals[0].GoString() + for _, a := range m.Additionals[1:] { + s += ", " + a.GoString() + } + } + return s + "}}" +} + +// A Builder allows incrementally packing a DNS message. +// +// Example usage: +// +// buf := make([]byte, 2, 514) +// b := NewBuilder(buf, Header{...}) +// b.EnableCompression() +// // Optionally start a section and add things to that section. +// // Repeat adding sections as necessary. +// buf, err := b.Finish() +// // If err is nil, buf[2:] will contain the built bytes. +type Builder struct { + // msg is the storage for the message being built. + msg []byte + + // section keeps track of the current section being built. + section section + + // header keeps track of what should go in the header when Finish is + // called. + header header + + // start is the starting index of the bytes allocated in msg for header. + start int + + // compression is a mapping from name suffixes to their starting index + // in msg. + compression map[string]uint16 +} + +// NewBuilder creates a new builder with compression disabled. +// +// Note: Most users will want to immediately enable compression with the +// EnableCompression method. See that method's comment for why you may or may +// not want to enable compression. +// +// The DNS message is appended to the provided initial buffer buf (which may be +// nil) as it is built. The final message is returned by the (*Builder).Finish +// method, which includes buf[:len(buf)] and may return the same underlying +// array if there was sufficient capacity in the slice. +func NewBuilder(buf []byte, h Header) Builder { + if buf == nil { + buf = make([]byte, 0, packStartingCap) + } + b := Builder{msg: buf, start: len(buf)} + b.header.id, b.header.bits = h.pack() + var hb [headerLen]byte + b.msg = append(b.msg, hb[:]...) + b.section = sectionHeader + return b +} + +// EnableCompression enables compression in the Builder. +// +// Leaving compression disabled avoids compression related allocations, but can +// result in larger message sizes. Be careful with this mode as it can cause +// messages to exceed the UDP size limit. +// +// According to RFC 1035, section 4.1.4, the use of compression is optional, but +// all implementations must accept both compressed and uncompressed DNS +// messages. +// +// Compression should be enabled before any sections are added for best results. +func (b *Builder) EnableCompression() { + b.compression = map[string]uint16{} +} + +func (b *Builder) startCheck(s section) error { + if b.section <= sectionNotStarted { + return ErrNotStarted + } + if b.section > s { + return ErrSectionDone + } + return nil +} + +// StartQuestions prepares the builder for packing Questions. +func (b *Builder) StartQuestions() error { + if err := b.startCheck(sectionQuestions); err != nil { + return err + } + b.section = sectionQuestions + return nil +} + +// StartAnswers prepares the builder for packing Answers. +func (b *Builder) StartAnswers() error { + if err := b.startCheck(sectionAnswers); err != nil { + return err + } + b.section = sectionAnswers + return nil +} + +// StartAuthorities prepares the builder for packing Authorities. +func (b *Builder) StartAuthorities() error { + if err := b.startCheck(sectionAuthorities); err != nil { + return err + } + b.section = sectionAuthorities + return nil +} + +// StartAdditionals prepares the builder for packing Additionals. +func (b *Builder) StartAdditionals() error { + if err := b.startCheck(sectionAdditionals); err != nil { + return err + } + b.section = sectionAdditionals + return nil +} + +func (b *Builder) incrementSectionCount() error { + var count *uint16 + var err error + switch b.section { + case sectionQuestions: + count = &b.header.questions + err = errTooManyQuestions + case sectionAnswers: + count = &b.header.answers + err = errTooManyAnswers + case sectionAuthorities: + count = &b.header.authorities + err = errTooManyAuthorities + case sectionAdditionals: + count = &b.header.additionals + err = errTooManyAdditionals + } + if *count == ^uint16(0) { + return err + } + *count++ + return nil +} + +// Question adds a single Question. +func (b *Builder) Question(q Question) error { + if b.section < sectionQuestions { + return ErrNotStarted + } + if b.section > sectionQuestions { + return ErrSectionDone + } + msg, err := q.pack(b.msg, b.compression, b.start) + if err != nil { + return err + } + if err := b.incrementSectionCount(); err != nil { + return err + } + b.msg = msg + return nil +} + +func (b *Builder) checkResourceSection() error { + if b.section < sectionAnswers { + return ErrNotStarted + } + if b.section > sectionAdditionals { + return ErrSectionDone + } + return nil +} + +// CNAMEResource adds a single CNAMEResource. +func (b *Builder) CNAMEResource(h ResourceHeader, r CNAMEResource) error { + if err := b.checkResourceSection(); err != nil { + return err + } + h.Type = r.realType() + msg, lenOff, err := h.pack(b.msg, b.compression, b.start) + if err != nil { + return &nestedError{"ResourceHeader", err} + } + preLen := len(msg) + if msg, err = r.pack(msg, b.compression, b.start); err != nil { + return &nestedError{"CNAMEResource body", err} + } + if err := h.fixLen(msg, lenOff, preLen); err != nil { + return err + } + if err := b.incrementSectionCount(); err != nil { + return err + } + b.msg = msg + return nil +} + +// MXResource adds a single MXResource. +func (b *Builder) MXResource(h ResourceHeader, r MXResource) error { + if err := b.checkResourceSection(); err != nil { + return err + } + h.Type = r.realType() + msg, lenOff, err := h.pack(b.msg, b.compression, b.start) + if err != nil { + return &nestedError{"ResourceHeader", err} + } + preLen := len(msg) + if msg, err = r.pack(msg, b.compression, b.start); err != nil { + return &nestedError{"MXResource body", err} + } + if err := h.fixLen(msg, lenOff, preLen); err != nil { + return err + } + if err := b.incrementSectionCount(); err != nil { + return err + } + b.msg = msg + return nil +} + +// NSResource adds a single NSResource. +func (b *Builder) NSResource(h ResourceHeader, r NSResource) error { + if err := b.checkResourceSection(); err != nil { + return err + } + h.Type = r.realType() + msg, lenOff, err := h.pack(b.msg, b.compression, b.start) + if err != nil { + return &nestedError{"ResourceHeader", err} + } + preLen := len(msg) + if msg, err = r.pack(msg, b.compression, b.start); err != nil { + return &nestedError{"NSResource body", err} + } + if err := h.fixLen(msg, lenOff, preLen); err != nil { + return err + } + if err := b.incrementSectionCount(); err != nil { + return err + } + b.msg = msg + return nil +} + +// PTRResource adds a single PTRResource. +func (b *Builder) PTRResource(h ResourceHeader, r PTRResource) error { + if err := b.checkResourceSection(); err != nil { + return err + } + h.Type = r.realType() + msg, lenOff, err := h.pack(b.msg, b.compression, b.start) + if err != nil { + return &nestedError{"ResourceHeader", err} + } + preLen := len(msg) + if msg, err = r.pack(msg, b.compression, b.start); err != nil { + return &nestedError{"PTRResource body", err} + } + if err := h.fixLen(msg, lenOff, preLen); err != nil { + return err + } + if err := b.incrementSectionCount(); err != nil { + return err + } + b.msg = msg + return nil +} + +// SOAResource adds a single SOAResource. +func (b *Builder) SOAResource(h ResourceHeader, r SOAResource) error { + if err := b.checkResourceSection(); err != nil { + return err + } + h.Type = r.realType() + msg, lenOff, err := h.pack(b.msg, b.compression, b.start) + if err != nil { + return &nestedError{"ResourceHeader", err} + } + preLen := len(msg) + if msg, err = r.pack(msg, b.compression, b.start); err != nil { + return &nestedError{"SOAResource body", err} + } + if err := h.fixLen(msg, lenOff, preLen); err != nil { + return err + } + if err := b.incrementSectionCount(); err != nil { + return err + } + b.msg = msg + return nil +} + +// TXTResource adds a single TXTResource. +func (b *Builder) TXTResource(h ResourceHeader, r TXTResource) error { + if err := b.checkResourceSection(); err != nil { + return err + } + h.Type = r.realType() + msg, lenOff, err := h.pack(b.msg, b.compression, b.start) + if err != nil { + return &nestedError{"ResourceHeader", err} + } + preLen := len(msg) + if msg, err = r.pack(msg, b.compression, b.start); err != nil { + return &nestedError{"TXTResource body", err} + } + if err := h.fixLen(msg, lenOff, preLen); err != nil { + return err + } + if err := b.incrementSectionCount(); err != nil { + return err + } + b.msg = msg + return nil +} + +// SRVResource adds a single SRVResource. +func (b *Builder) SRVResource(h ResourceHeader, r SRVResource) error { + if err := b.checkResourceSection(); err != nil { + return err + } + h.Type = r.realType() + msg, lenOff, err := h.pack(b.msg, b.compression, b.start) + if err != nil { + return &nestedError{"ResourceHeader", err} + } + preLen := len(msg) + if msg, err = r.pack(msg, b.compression, b.start); err != nil { + return &nestedError{"SRVResource body", err} + } + if err := h.fixLen(msg, lenOff, preLen); err != nil { + return err + } + if err := b.incrementSectionCount(); err != nil { + return err + } + b.msg = msg + return nil +} + +// AResource adds a single AResource. +func (b *Builder) AResource(h ResourceHeader, r AResource) error { + if err := b.checkResourceSection(); err != nil { + return err + } + h.Type = r.realType() + msg, lenOff, err := h.pack(b.msg, b.compression, b.start) + if err != nil { + return &nestedError{"ResourceHeader", err} + } + preLen := len(msg) + if msg, err = r.pack(msg, b.compression, b.start); err != nil { + return &nestedError{"AResource body", err} + } + if err := h.fixLen(msg, lenOff, preLen); err != nil { + return err + } + if err := b.incrementSectionCount(); err != nil { + return err + } + b.msg = msg + return nil +} + +// AAAAResource adds a single AAAAResource. +func (b *Builder) AAAAResource(h ResourceHeader, r AAAAResource) error { + if err := b.checkResourceSection(); err != nil { + return err + } + h.Type = r.realType() + msg, lenOff, err := h.pack(b.msg, b.compression, b.start) + if err != nil { + return &nestedError{"ResourceHeader", err} + } + preLen := len(msg) + if msg, err = r.pack(msg, b.compression, b.start); err != nil { + return &nestedError{"AAAAResource body", err} + } + if err := h.fixLen(msg, lenOff, preLen); err != nil { + return err + } + if err := b.incrementSectionCount(); err != nil { + return err + } + b.msg = msg + return nil +} + +// OPTResource adds a single OPTResource. +func (b *Builder) OPTResource(h ResourceHeader, r OPTResource) error { + if err := b.checkResourceSection(); err != nil { + return err + } + h.Type = r.realType() + msg, lenOff, err := h.pack(b.msg, b.compression, b.start) + if err != nil { + return &nestedError{"ResourceHeader", err} + } + preLen := len(msg) + if msg, err = r.pack(msg, b.compression, b.start); err != nil { + return &nestedError{"OPTResource body", err} + } + if err := h.fixLen(msg, lenOff, preLen); err != nil { + return err + } + if err := b.incrementSectionCount(); err != nil { + return err + } + b.msg = msg + return nil +} + +// UnknownResource adds a single UnknownResource. +func (b *Builder) UnknownResource(h ResourceHeader, r UnknownResource) error { + if err := b.checkResourceSection(); err != nil { + return err + } + h.Type = r.realType() + msg, lenOff, err := h.pack(b.msg, b.compression, b.start) + if err != nil { + return &nestedError{"ResourceHeader", err} + } + preLen := len(msg) + if msg, err = r.pack(msg, b.compression, b.start); err != nil { + return &nestedError{"UnknownResource body", err} + } + if err := h.fixLen(msg, lenOff, preLen); err != nil { + return err + } + if err := b.incrementSectionCount(); err != nil { + return err + } + b.msg = msg + return nil +} + +// Finish ends message building and generates a binary message. +func (b *Builder) Finish() ([]byte, error) { + if b.section < sectionHeader { + return nil, ErrNotStarted + } + b.section = sectionDone + // Space for the header was allocated in NewBuilder. + b.header.pack(b.msg[b.start:b.start]) + return b.msg, nil +} + +// A ResourceHeader is the header of a DNS resource record. There are +// many types of DNS resource records, but they all share the same header. +type ResourceHeader struct { + // Name is the domain name for which this resource record pertains. + Name Name + + // Type is the type of DNS resource record. + // + // This field will be set automatically during packing. + Type Type + + // Class is the class of network to which this DNS resource record + // pertains. + Class Class + + // TTL is the length of time (measured in seconds) which this resource + // record is valid for (time to live). All Resources in a set should + // have the same TTL (RFC 2181 Section 5.2). + TTL uint32 + + // Length is the length of data in the resource record after the header. + // + // This field will be set automatically during packing. + Length uint16 +} + +// GoString implements fmt.GoStringer.GoString. +func (h *ResourceHeader) GoString() string { + return "dnsmessage.ResourceHeader{" + + "Name: " + h.Name.GoString() + ", " + + "Type: " + h.Type.GoString() + ", " + + "Class: " + h.Class.GoString() + ", " + + "TTL: " + printUint32(h.TTL) + ", " + + "Length: " + printUint16(h.Length) + "}" +} + +// pack appends the wire format of the ResourceHeader to oldMsg. +// +// lenOff is the offset in msg where the Length field was packed. +func (h *ResourceHeader) pack(oldMsg []byte, compression map[string]uint16, compressionOff int) (msg []byte, lenOff int, err error) { + msg = oldMsg + if msg, err = h.Name.pack(msg, compression, compressionOff); err != nil { + return oldMsg, 0, &nestedError{"Name", err} + } + msg = packType(msg, h.Type) + msg = packClass(msg, h.Class) + msg = packUint32(msg, h.TTL) + lenOff = len(msg) + msg = packUint16(msg, h.Length) + return msg, lenOff, nil +} + +func (h *ResourceHeader) unpack(msg []byte, off int) (int, error) { + newOff := off + var err error + if newOff, err = h.Name.unpack(msg, newOff); err != nil { + return off, &nestedError{"Name", err} + } + if h.Type, newOff, err = unpackType(msg, newOff); err != nil { + return off, &nestedError{"Type", err} + } + if h.Class, newOff, err = unpackClass(msg, newOff); err != nil { + return off, &nestedError{"Class", err} + } + if h.TTL, newOff, err = unpackUint32(msg, newOff); err != nil { + return off, &nestedError{"TTL", err} + } + if h.Length, newOff, err = unpackUint16(msg, newOff); err != nil { + return off, &nestedError{"Length", err} + } + return newOff, nil +} + +// fixLen updates a packed ResourceHeader to include the length of the +// ResourceBody. +// +// lenOff is the offset of the ResourceHeader.Length field in msg. +// +// preLen is the length that msg was before the ResourceBody was packed. +func (h *ResourceHeader) fixLen(msg []byte, lenOff int, preLen int) error { + conLen := len(msg) - preLen + if conLen > int(^uint16(0)) { + return errResTooLong + } + + // Fill in the length now that we know how long the content is. + packUint16(msg[lenOff:lenOff], uint16(conLen)) + h.Length = uint16(conLen) + + return nil +} + +// EDNS(0) wire constants. +const ( + edns0Version = 0 + + edns0DNSSECOK = 0x00008000 + ednsVersionMask = 0x00ff0000 + edns0DNSSECOKMask = 0x00ff8000 +) + +// SetEDNS0 configures h for EDNS(0). +// +// The provided extRCode must be an extended RCode. +func (h *ResourceHeader) SetEDNS0(udpPayloadLen int, extRCode RCode, dnssecOK bool) error { + h.Name = Name{Data: [255]byte{'.'}, Length: 1} // RFC 6891 section 6.1.2 + h.Type = TypeOPT + h.Class = Class(udpPayloadLen) + h.TTL = uint32(extRCode) >> 4 << 24 + if dnssecOK { + h.TTL |= edns0DNSSECOK + } + return nil +} + +// DNSSECAllowed reports whether the DNSSEC OK bit is set. +func (h *ResourceHeader) DNSSECAllowed() bool { + return h.TTL&edns0DNSSECOKMask == edns0DNSSECOK // RFC 6891 section 6.1.3 +} + +// ExtendedRCode returns an extended RCode. +// +// The provided rcode must be the RCode in DNS message header. +func (h *ResourceHeader) ExtendedRCode(rcode RCode) RCode { + if h.TTL&ednsVersionMask == edns0Version { // RFC 6891 section 6.1.3 + return RCode(h.TTL>>24<<4) | rcode + } + return rcode +} + +func skipResource(msg []byte, off int) (int, error) { + newOff, err := skipName(msg, off) + if err != nil { + return off, &nestedError{"Name", err} + } + if newOff, err = skipType(msg, newOff); err != nil { + return off, &nestedError{"Type", err} + } + if newOff, err = skipClass(msg, newOff); err != nil { + return off, &nestedError{"Class", err} + } + if newOff, err = skipUint32(msg, newOff); err != nil { + return off, &nestedError{"TTL", err} + } + length, newOff, err := unpackUint16(msg, newOff) + if err != nil { + return off, &nestedError{"Length", err} + } + if newOff += int(length); newOff > len(msg) { + return off, errResourceLen + } + return newOff, nil +} + +// packUint16 appends the wire format of field to msg. +func packUint16(msg []byte, field uint16) []byte { + return append(msg, byte(field>>8), byte(field)) +} + +func unpackUint16(msg []byte, off int) (uint16, int, error) { + if off+uint16Len > len(msg) { + return 0, off, errBaseLen + } + return uint16(msg[off])<<8 | uint16(msg[off+1]), off + uint16Len, nil +} + +func skipUint16(msg []byte, off int) (int, error) { + if off+uint16Len > len(msg) { + return off, errBaseLen + } + return off + uint16Len, nil +} + +// packType appends the wire format of field to msg. +func packType(msg []byte, field Type) []byte { + return packUint16(msg, uint16(field)) +} + +func unpackType(msg []byte, off int) (Type, int, error) { + t, o, err := unpackUint16(msg, off) + return Type(t), o, err +} + +func skipType(msg []byte, off int) (int, error) { + return skipUint16(msg, off) +} + +// packClass appends the wire format of field to msg. +func packClass(msg []byte, field Class) []byte { + return packUint16(msg, uint16(field)) +} + +func unpackClass(msg []byte, off int) (Class, int, error) { + c, o, err := unpackUint16(msg, off) + return Class(c), o, err +} + +func skipClass(msg []byte, off int) (int, error) { + return skipUint16(msg, off) +} + +// packUint32 appends the wire format of field to msg. +func packUint32(msg []byte, field uint32) []byte { + return append( + msg, + byte(field>>24), + byte(field>>16), + byte(field>>8), + byte(field), + ) +} + +func unpackUint32(msg []byte, off int) (uint32, int, error) { + if off+uint32Len > len(msg) { + return 0, off, errBaseLen + } + v := uint32(msg[off])<<24 | uint32(msg[off+1])<<16 | uint32(msg[off+2])<<8 | uint32(msg[off+3]) + return v, off + uint32Len, nil +} + +func skipUint32(msg []byte, off int) (int, error) { + if off+uint32Len > len(msg) { + return off, errBaseLen + } + return off + uint32Len, nil +} + +// packText appends the wire format of field to msg. +func packText(msg []byte, field string) ([]byte, error) { + l := len(field) + if l > 255 { + return nil, errStringTooLong + } + msg = append(msg, byte(l)) + msg = append(msg, field...) + + return msg, nil +} + +func unpackText(msg []byte, off int) (string, int, error) { + if off >= len(msg) { + return "", off, errBaseLen + } + beginOff := off + 1 + endOff := beginOff + int(msg[off]) + if endOff > len(msg) { + return "", off, errCalcLen + } + return string(msg[beginOff:endOff]), endOff, nil +} + +// packBytes appends the wire format of field to msg. +func packBytes(msg []byte, field []byte) []byte { + return append(msg, field...) +} + +func unpackBytes(msg []byte, off int, field []byte) (int, error) { + newOff := off + len(field) + if newOff > len(msg) { + return off, errBaseLen + } + copy(field, msg[off:newOff]) + return newOff, nil +} + +const nonEncodedNameMax = 254 + +// A Name is a non-encoded and non-escaped domain name. It is used instead of strings to avoid +// allocations. +type Name struct { + Data [255]byte + Length uint8 +} + +// NewName creates a new Name from a string. +func NewName(name string) (Name, error) { + n := Name{Length: uint8(len(name))} + if len(name) > len(n.Data) { + return Name{}, errCalcLen + } + copy(n.Data[:], name) + return n, nil +} + +// MustNewName creates a new Name from a string and panics on error. +func MustNewName(name string) Name { + n, err := NewName(name) + if err != nil { + panic("creating name: " + err.Error()) + } + return n +} + +// String implements fmt.Stringer.String. +// +// Note: characters inside the labels are not escaped in any way. +func (n Name) String() string { + return string(n.Data[:n.Length]) +} + +// GoString implements fmt.GoStringer.GoString. +func (n *Name) GoString() string { + return `dnsmessage.MustNewName("` + printString(n.Data[:n.Length]) + `")` +} + +// pack appends the wire format of the Name to msg. +// +// Domain names are a sequence of counted strings split at the dots. They end +// with a zero-length string. Compression can be used to reuse domain suffixes. +// +// The compression map will be updated with new domain suffixes. If compression +// is nil, compression will not be used. +func (n *Name) pack(msg []byte, compression map[string]uint16, compressionOff int) ([]byte, error) { + oldMsg := msg + + if n.Length > nonEncodedNameMax { + return nil, errNameTooLong + } + + // Add a trailing dot to canonicalize name. + if n.Length == 0 || n.Data[n.Length-1] != '.' { + return oldMsg, errNonCanonicalName + } + + // Allow root domain. + if n.Data[0] == '.' && n.Length == 1 { + return append(msg, 0), nil + } + + var nameAsStr string + + // Emit sequence of counted strings, chopping at dots. + for i, begin := 0, 0; i < int(n.Length); i++ { + // Check for the end of the segment. + if n.Data[i] == '.' { + // The two most significant bits have special meaning. + // It isn't allowed for segments to be long enough to + // need them. + if i-begin >= 1<<6 { + return oldMsg, errSegTooLong + } + + // Segments must have a non-zero length. + if i-begin == 0 { + return oldMsg, errZeroSegLen + } + + msg = append(msg, byte(i-begin)) + + for j := begin; j < i; j++ { + msg = append(msg, n.Data[j]) + } + + begin = i + 1 + continue + } + + // We can only compress domain suffixes starting with a new + // segment. A pointer is two bytes with the two most significant + // bits set to 1 to indicate that it is a pointer. + if (i == 0 || n.Data[i-1] == '.') && compression != nil { + if ptr, ok := compression[string(n.Data[i:n.Length])]; ok { + // Hit. Emit a pointer instead of the rest of + // the domain. + return append(msg, byte(ptr>>8|0xC0), byte(ptr)), nil + } + + // Miss. Add the suffix to the compression table if the + // offset can be stored in the available 14 bits. + newPtr := len(msg) - compressionOff + if newPtr <= int(^uint16(0)>>2) { + if nameAsStr == "" { + // allocate n.Data on the heap once, to avoid allocating it + // multiple times (for next labels). + nameAsStr = string(n.Data[:n.Length]) + } + compression[nameAsStr[i:]] = uint16(newPtr) + } + } + } + return append(msg, 0), nil +} + +// unpack unpacks a domain name. +func (n *Name) unpack(msg []byte, off int) (int, error) { + // currOff is the current working offset. + currOff := off + + // newOff is the offset where the next record will start. Pointers lead + // to data that belongs to other names and thus doesn't count towards to + // the usage of this name. + newOff := off + + // ptr is the number of pointers followed. + var ptr int + + // Name is a slice representation of the name data. + name := n.Data[:0] + +Loop: + for { + if currOff >= len(msg) { + return off, errBaseLen + } + c := int(msg[currOff]) + currOff++ + switch c & 0xC0 { + case 0x00: // String segment + if c == 0x00 { + // A zero length signals the end of the name. + break Loop + } + endOff := currOff + c + if endOff > len(msg) { + return off, errCalcLen + } + + // Reject names containing dots. + // See issue golang/go#56246 + for _, v := range msg[currOff:endOff] { + if v == '.' { + return off, errInvalidName + } + } + // Reject names that are too long while unpacking + // See issue golang/go#77540 + if len(name)+(endOff-currOff) >= nonEncodedNameMax { + return off, errNameTooLong + } + name = append(name, msg[currOff:endOff]...) + name = append(name, '.') + currOff = endOff + case 0xC0: // Pointer + if currOff >= len(msg) { + return off, errInvalidPtr + } + c1 := msg[currOff] + currOff++ + if ptr == 0 { + newOff = currOff + } + // Don't follow too many pointers, maybe there's a loop. + if ptr++; ptr > 10 { + return off, errTooManyPtr + } + currOff = (c^0xC0)<<8 | int(c1) + default: + // Prefixes 0x80 and 0x40 are reserved. + return off, errReserved + } + } + if len(name) == 0 { + name = append(name, '.') + } + n.Length = uint8(len(name)) + if ptr == 0 { + newOff = currOff + } + return newOff, nil +} + +func skipName(msg []byte, off int) (int, error) { + // newOff is the offset where the next record will start. Pointers lead + // to data that belongs to other names and thus doesn't count towards to + // the usage of this name. + newOff := off + +Loop: + for { + if newOff >= len(msg) { + return off, errBaseLen + } + c := int(msg[newOff]) + newOff++ + switch c & 0xC0 { + case 0x00: + if c == 0x00 { + // A zero length signals the end of the name. + break Loop + } + // literal string + newOff += c + if newOff > len(msg) { + return off, errCalcLen + } + case 0xC0: + // Pointer to somewhere else in msg. + + // Pointers are two bytes. + newOff++ + + // Don't follow the pointer as the data here has ended. + break Loop + default: + // Prefixes 0x80 and 0x40 are reserved. + return off, errReserved + } + } + + return newOff, nil +} + +// A Question is a DNS query. +type Question struct { + Name Name + Type Type + Class Class +} + +// pack appends the wire format of the Question to msg. +func (q *Question) pack(msg []byte, compression map[string]uint16, compressionOff int) ([]byte, error) { + msg, err := q.Name.pack(msg, compression, compressionOff) + if err != nil { + return msg, &nestedError{"Name", err} + } + msg = packType(msg, q.Type) + return packClass(msg, q.Class), nil +} + +// GoString implements fmt.GoStringer.GoString. +func (q *Question) GoString() string { + return "dnsmessage.Question{" + + "Name: " + q.Name.GoString() + ", " + + "Type: " + q.Type.GoString() + ", " + + "Class: " + q.Class.GoString() + "}" +} + +func unpackResourceBody(msg []byte, off int, hdr ResourceHeader) (ResourceBody, int, error) { + var ( + r ResourceBody + err error + name string + ) + switch hdr.Type { + case TypeA: + var rb AResource + rb, err = unpackAResource(msg, off) + r = &rb + name = "A" + case TypeNS: + var rb NSResource + rb, err = unpackNSResource(msg, off) + r = &rb + name = "NS" + case TypeCNAME: + var rb CNAMEResource + rb, err = unpackCNAMEResource(msg, off) + r = &rb + name = "CNAME" + case TypeSOA: + var rb SOAResource + rb, err = unpackSOAResource(msg, off) + r = &rb + name = "SOA" + case TypePTR: + var rb PTRResource + rb, err = unpackPTRResource(msg, off) + r = &rb + name = "PTR" + case TypeMX: + var rb MXResource + rb, err = unpackMXResource(msg, off) + r = &rb + name = "MX" + case TypeTXT: + var rb TXTResource + rb, err = unpackTXTResource(msg, off, hdr.Length) + r = &rb + name = "TXT" + case TypeAAAA: + var rb AAAAResource + rb, err = unpackAAAAResource(msg, off) + r = &rb + name = "AAAA" + case TypeSRV: + var rb SRVResource + rb, err = unpackSRVResource(msg, off) + r = &rb + name = "SRV" + case TypeSVCB: + var rb SVCBResource + rb, err = unpackSVCBResource(msg, off, hdr.Length) + r = &rb + name = "SVCB" + case TypeHTTPS: + var rb HTTPSResource + rb.SVCBResource, err = unpackSVCBResource(msg, off, hdr.Length) + r = &rb + name = "HTTPS" + case TypeOPT: + var rb OPTResource + rb, err = unpackOPTResource(msg, off, hdr.Length) + r = &rb + name = "OPT" + default: + var rb UnknownResource + rb, err = unpackUnknownResource(hdr.Type, msg, off, hdr.Length) + r = &rb + name = "Unknown" + } + if err != nil { + return nil, off, &nestedError{name + " record", err} + } + return r, off + int(hdr.Length), nil +} + +// A CNAMEResource is a CNAME Resource record. +type CNAMEResource struct { + CNAME Name +} + +func (r *CNAMEResource) realType() Type { + return TypeCNAME +} + +// pack appends the wire format of the CNAMEResource to msg. +func (r *CNAMEResource) pack(msg []byte, compression map[string]uint16, compressionOff int) ([]byte, error) { + return r.CNAME.pack(msg, compression, compressionOff) +} + +// GoString implements fmt.GoStringer.GoString. +func (r *CNAMEResource) GoString() string { + return "dnsmessage.CNAMEResource{CNAME: " + r.CNAME.GoString() + "}" +} + +func unpackCNAMEResource(msg []byte, off int) (CNAMEResource, error) { + var cname Name + if _, err := cname.unpack(msg, off); err != nil { + return CNAMEResource{}, err + } + return CNAMEResource{cname}, nil +} + +// An MXResource is an MX Resource record. +type MXResource struct { + Pref uint16 + MX Name +} + +func (r *MXResource) realType() Type { + return TypeMX +} + +// pack appends the wire format of the MXResource to msg. +func (r *MXResource) pack(msg []byte, compression map[string]uint16, compressionOff int) ([]byte, error) { + oldMsg := msg + msg = packUint16(msg, r.Pref) + msg, err := r.MX.pack(msg, compression, compressionOff) + if err != nil { + return oldMsg, &nestedError{"MXResource.MX", err} + } + return msg, nil +} + +// GoString implements fmt.GoStringer.GoString. +func (r *MXResource) GoString() string { + return "dnsmessage.MXResource{" + + "Pref: " + printUint16(r.Pref) + ", " + + "MX: " + r.MX.GoString() + "}" +} + +func unpackMXResource(msg []byte, off int) (MXResource, error) { + pref, off, err := unpackUint16(msg, off) + if err != nil { + return MXResource{}, &nestedError{"Pref", err} + } + var mx Name + if _, err := mx.unpack(msg, off); err != nil { + return MXResource{}, &nestedError{"MX", err} + } + return MXResource{pref, mx}, nil +} + +// An NSResource is an NS Resource record. +type NSResource struct { + NS Name +} + +func (r *NSResource) realType() Type { + return TypeNS +} + +// pack appends the wire format of the NSResource to msg. +func (r *NSResource) pack(msg []byte, compression map[string]uint16, compressionOff int) ([]byte, error) { + return r.NS.pack(msg, compression, compressionOff) +} + +// GoString implements fmt.GoStringer.GoString. +func (r *NSResource) GoString() string { + return "dnsmessage.NSResource{NS: " + r.NS.GoString() + "}" +} + +func unpackNSResource(msg []byte, off int) (NSResource, error) { + var ns Name + if _, err := ns.unpack(msg, off); err != nil { + return NSResource{}, err + } + return NSResource{ns}, nil +} + +// A PTRResource is a PTR Resource record. +type PTRResource struct { + PTR Name +} + +func (r *PTRResource) realType() Type { + return TypePTR +} + +// pack appends the wire format of the PTRResource to msg. +func (r *PTRResource) pack(msg []byte, compression map[string]uint16, compressionOff int) ([]byte, error) { + return r.PTR.pack(msg, compression, compressionOff) +} + +// GoString implements fmt.GoStringer.GoString. +func (r *PTRResource) GoString() string { + return "dnsmessage.PTRResource{PTR: " + r.PTR.GoString() + "}" +} + +func unpackPTRResource(msg []byte, off int) (PTRResource, error) { + var ptr Name + if _, err := ptr.unpack(msg, off); err != nil { + return PTRResource{}, err + } + return PTRResource{ptr}, nil +} + +// An SOAResource is an SOA Resource record. +type SOAResource struct { + NS Name + MBox Name + Serial uint32 + Refresh uint32 + Retry uint32 + Expire uint32 + + // MinTTL the is the default TTL of Resources records which did not + // contain a TTL value and the TTL of negative responses. (RFC 2308 + // Section 4) + MinTTL uint32 +} + +func (r *SOAResource) realType() Type { + return TypeSOA +} + +// pack appends the wire format of the SOAResource to msg. +func (r *SOAResource) pack(msg []byte, compression map[string]uint16, compressionOff int) ([]byte, error) { + oldMsg := msg + msg, err := r.NS.pack(msg, compression, compressionOff) + if err != nil { + return oldMsg, &nestedError{"SOAResource.NS", err} + } + msg, err = r.MBox.pack(msg, compression, compressionOff) + if err != nil { + return oldMsg, &nestedError{"SOAResource.MBox", err} + } + msg = packUint32(msg, r.Serial) + msg = packUint32(msg, r.Refresh) + msg = packUint32(msg, r.Retry) + msg = packUint32(msg, r.Expire) + return packUint32(msg, r.MinTTL), nil +} + +// GoString implements fmt.GoStringer.GoString. +func (r *SOAResource) GoString() string { + return "dnsmessage.SOAResource{" + + "NS: " + r.NS.GoString() + ", " + + "MBox: " + r.MBox.GoString() + ", " + + "Serial: " + printUint32(r.Serial) + ", " + + "Refresh: " + printUint32(r.Refresh) + ", " + + "Retry: " + printUint32(r.Retry) + ", " + + "Expire: " + printUint32(r.Expire) + ", " + + "MinTTL: " + printUint32(r.MinTTL) + "}" +} + +func unpackSOAResource(msg []byte, off int) (SOAResource, error) { + var ns Name + off, err := ns.unpack(msg, off) + if err != nil { + return SOAResource{}, &nestedError{"NS", err} + } + var mbox Name + if off, err = mbox.unpack(msg, off); err != nil { + return SOAResource{}, &nestedError{"MBox", err} + } + serial, off, err := unpackUint32(msg, off) + if err != nil { + return SOAResource{}, &nestedError{"Serial", err} + } + refresh, off, err := unpackUint32(msg, off) + if err != nil { + return SOAResource{}, &nestedError{"Refresh", err} + } + retry, off, err := unpackUint32(msg, off) + if err != nil { + return SOAResource{}, &nestedError{"Retry", err} + } + expire, off, err := unpackUint32(msg, off) + if err != nil { + return SOAResource{}, &nestedError{"Expire", err} + } + minTTL, _, err := unpackUint32(msg, off) + if err != nil { + return SOAResource{}, &nestedError{"MinTTL", err} + } + return SOAResource{ns, mbox, serial, refresh, retry, expire, minTTL}, nil +} + +// A TXTResource is a TXT Resource record. +type TXTResource struct { + TXT []string +} + +func (r *TXTResource) realType() Type { + return TypeTXT +} + +// pack appends the wire format of the TXTResource to msg. +func (r *TXTResource) pack(msg []byte, compression map[string]uint16, compressionOff int) ([]byte, error) { + oldMsg := msg + for _, s := range r.TXT { + var err error + msg, err = packText(msg, s) + if err != nil { + return oldMsg, err + } + } + return msg, nil +} + +// GoString implements fmt.GoStringer.GoString. +func (r *TXTResource) GoString() string { + s := "dnsmessage.TXTResource{TXT: []string{" + if len(r.TXT) == 0 { + return s + "}}" + } + s += `"` + printString([]byte(r.TXT[0])) + for _, t := range r.TXT[1:] { + s += `", "` + printString([]byte(t)) + } + return s + `"}}` +} + +func unpackTXTResource(msg []byte, off int, length uint16) (TXTResource, error) { + txts := make([]string, 0, 1) + for n := uint16(0); n < length; { + var t string + var err error + if t, off, err = unpackText(msg, off); err != nil { + return TXTResource{}, &nestedError{"text", err} + } + // Check if we got too many bytes. + if length-n < uint16(len(t))+1 { + return TXTResource{}, errCalcLen + } + n += uint16(len(t)) + 1 + txts = append(txts, t) + } + return TXTResource{txts}, nil +} + +// An SRVResource is an SRV Resource record. +type SRVResource struct { + Priority uint16 + Weight uint16 + Port uint16 + Target Name // Not compressed as per RFC 2782. +} + +func (r *SRVResource) realType() Type { + return TypeSRV +} + +// pack appends the wire format of the SRVResource to msg. +func (r *SRVResource) pack(msg []byte, compression map[string]uint16, compressionOff int) ([]byte, error) { + oldMsg := msg + msg = packUint16(msg, r.Priority) + msg = packUint16(msg, r.Weight) + msg = packUint16(msg, r.Port) + msg, err := r.Target.pack(msg, nil, compressionOff) + if err != nil { + return oldMsg, &nestedError{"SRVResource.Target", err} + } + return msg, nil +} + +// GoString implements fmt.GoStringer.GoString. +func (r *SRVResource) GoString() string { + return "dnsmessage.SRVResource{" + + "Priority: " + printUint16(r.Priority) + ", " + + "Weight: " + printUint16(r.Weight) + ", " + + "Port: " + printUint16(r.Port) + ", " + + "Target: " + r.Target.GoString() + "}" +} + +func unpackSRVResource(msg []byte, off int) (SRVResource, error) { + priority, off, err := unpackUint16(msg, off) + if err != nil { + return SRVResource{}, &nestedError{"Priority", err} + } + weight, off, err := unpackUint16(msg, off) + if err != nil { + return SRVResource{}, &nestedError{"Weight", err} + } + port, off, err := unpackUint16(msg, off) + if err != nil { + return SRVResource{}, &nestedError{"Port", err} + } + var target Name + if _, err := target.unpack(msg, off); err != nil { + return SRVResource{}, &nestedError{"Target", err} + } + return SRVResource{priority, weight, port, target}, nil +} + +// An AResource is an A Resource record. +type AResource struct { + A [4]byte +} + +func (r *AResource) realType() Type { + return TypeA +} + +// pack appends the wire format of the AResource to msg. +func (r *AResource) pack(msg []byte, compression map[string]uint16, compressionOff int) ([]byte, error) { + return packBytes(msg, r.A[:]), nil +} + +// GoString implements fmt.GoStringer.GoString. +func (r *AResource) GoString() string { + return "dnsmessage.AResource{" + + "A: [4]byte{" + printByteSlice(r.A[:]) + "}}" +} + +func unpackAResource(msg []byte, off int) (AResource, error) { + var a [4]byte + if _, err := unpackBytes(msg, off, a[:]); err != nil { + return AResource{}, err + } + return AResource{a}, nil +} + +// An AAAAResource is an AAAA Resource record. +type AAAAResource struct { + AAAA [16]byte +} + +func (r *AAAAResource) realType() Type { + return TypeAAAA +} + +// GoString implements fmt.GoStringer.GoString. +func (r *AAAAResource) GoString() string { + return "dnsmessage.AAAAResource{" + + "AAAA: [16]byte{" + printByteSlice(r.AAAA[:]) + "}}" +} + +// pack appends the wire format of the AAAAResource to msg. +func (r *AAAAResource) pack(msg []byte, compression map[string]uint16, compressionOff int) ([]byte, error) { + return packBytes(msg, r.AAAA[:]), nil +} + +func unpackAAAAResource(msg []byte, off int) (AAAAResource, error) { + var aaaa [16]byte + if _, err := unpackBytes(msg, off, aaaa[:]); err != nil { + return AAAAResource{}, err + } + return AAAAResource{aaaa}, nil +} + +// An OPTResource is an OPT pseudo Resource record. +// +// The pseudo resource record is part of the extension mechanisms for DNS +// as defined in RFC 6891. +type OPTResource struct { + Options []Option +} + +// An Option represents a DNS message option within OPTResource. +// +// The message option is part of the extension mechanisms for DNS as +// defined in RFC 6891. +type Option struct { + Code uint16 // option code + Data []byte +} + +// GoString implements fmt.GoStringer.GoString. +func (o *Option) GoString() string { + return "dnsmessage.Option{" + + "Code: " + printUint16(o.Code) + ", " + + "Data: []byte{" + printByteSlice(o.Data) + "}}" +} + +func (r *OPTResource) realType() Type { + return TypeOPT +} + +func (r *OPTResource) pack(msg []byte, compression map[string]uint16, compressionOff int) ([]byte, error) { + for _, opt := range r.Options { + msg = packUint16(msg, opt.Code) + l := uint16(len(opt.Data)) + msg = packUint16(msg, l) + msg = packBytes(msg, opt.Data) + } + return msg, nil +} + +// GoString implements fmt.GoStringer.GoString. +func (r *OPTResource) GoString() string { + s := "dnsmessage.OPTResource{Options: []dnsmessage.Option{" + if len(r.Options) == 0 { + return s + "}}" + } + s += r.Options[0].GoString() + for _, o := range r.Options[1:] { + s += ", " + o.GoString() + } + return s + "}}" +} + +func unpackOPTResource(msg []byte, off int, length uint16) (OPTResource, error) { + var opts []Option + for oldOff := off; off < oldOff+int(length); { + var err error + var o Option + o.Code, off, err = unpackUint16(msg, off) + if err != nil { + return OPTResource{}, &nestedError{"Code", err} + } + var l uint16 + l, off, err = unpackUint16(msg, off) + if err != nil { + return OPTResource{}, &nestedError{"Data", err} + } + o.Data = make([]byte, l) + if copy(o.Data, msg[off:]) != int(l) { + return OPTResource{}, &nestedError{"Data", errCalcLen} + } + off += int(l) + opts = append(opts, o) + } + return OPTResource{opts}, nil +} + +// An UnknownResource is a catch-all container for unknown record types. +type UnknownResource struct { + Type Type + Data []byte +} + +func (r *UnknownResource) realType() Type { + return r.Type +} + +// pack appends the wire format of the UnknownResource to msg. +func (r *UnknownResource) pack(msg []byte, compression map[string]uint16, compressionOff int) ([]byte, error) { + return packBytes(msg, r.Data[:]), nil +} + +// GoString implements fmt.GoStringer.GoString. +func (r *UnknownResource) GoString() string { + return "dnsmessage.UnknownResource{" + + "Type: " + r.Type.GoString() + ", " + + "Data: []byte{" + printByteSlice(r.Data) + "}}" +} + +func unpackUnknownResource(recordType Type, msg []byte, off int, length uint16) (UnknownResource, error) { + parsed := UnknownResource{ + Type: recordType, + Data: make([]byte, length), + } + if _, err := unpackBytes(msg, off, parsed.Data); err != nil { + return UnknownResource{}, err + } + return parsed, nil +} diff --git a/vendor/golang.org/x/net/dns/dnsmessage/svcb.go b/vendor/golang.org/x/net/dns/dnsmessage/svcb.go new file mode 100644 index 0000000000..977d91eadc --- /dev/null +++ b/vendor/golang.org/x/net/dns/dnsmessage/svcb.go @@ -0,0 +1,333 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package dnsmessage + +import ( + "slices" +) + +// An SVCBResource is an SVCB Resource record. +type SVCBResource struct { + Priority uint16 + Target Name + Params []SVCParam // Must be in strict increasing order by Key. +} + +func (r *SVCBResource) realType() Type { + return TypeSVCB +} + +// GoString implements fmt.GoStringer.GoString. +func (r *SVCBResource) GoString() string { + b := []byte("dnsmessage.SVCBResource{" + + "Priority: " + printUint16(r.Priority) + ", " + + "Target: " + r.Target.GoString() + ", " + + "Params: []dnsmessage.SVCParam{") + if len(r.Params) > 0 { + b = append(b, r.Params[0].GoString()...) + for _, p := range r.Params[1:] { + b = append(b, ", "+p.GoString()...) + } + } + b = append(b, "}}"...) + return string(b) +} + +// An HTTPSResource is an HTTPS Resource record. +// It has the same format as the SVCB record. +type HTTPSResource struct { + // Alias for SVCB resource record. + SVCBResource +} + +func (r *HTTPSResource) realType() Type { + return TypeHTTPS +} + +// GoString implements fmt.GoStringer.GoString. +func (r *HTTPSResource) GoString() string { + return "dnsmessage.HTTPSResource{SVCBResource: " + r.SVCBResource.GoString() + "}" +} + +// GetParam returns a parameter value by key. +func (r *SVCBResource) GetParam(key SVCParamKey) (value []byte, ok bool) { + for i := range r.Params { + if r.Params[i].Key == key { + return r.Params[i].Value, true + } + if r.Params[i].Key > key { + break + } + } + return nil, false +} + +// SetParam sets a parameter value by key. +// The Params list is kept sorted by key. +func (r *SVCBResource) SetParam(key SVCParamKey, value []byte) { + i := 0 + for i < len(r.Params) { + if r.Params[i].Key >= key { + break + } + i++ + } + + if i < len(r.Params) && r.Params[i].Key == key { + r.Params[i].Value = value + return + } + + r.Params = slices.Insert(r.Params, i, SVCParam{Key: key, Value: value}) +} + +// DeleteParam deletes a parameter by key. +// It returns true if the parameter was present. +func (r *SVCBResource) DeleteParam(key SVCParamKey) bool { + for i := range r.Params { + if r.Params[i].Key == key { + r.Params = slices.Delete(r.Params, i, i+1) + return true + } + if r.Params[i].Key > key { + break + } + } + return false +} + +// A SVCParam is a service parameter. +type SVCParam struct { + Key SVCParamKey + Value []byte +} + +// GoString implements fmt.GoStringer.GoString. +func (p SVCParam) GoString() string { + return "dnsmessage.SVCParam{" + + "Key: " + p.Key.GoString() + ", " + + "Value: []byte{" + printByteSlice(p.Value) + "}}" +} + +// A SVCParamKey is a key for a service parameter. +type SVCParamKey uint16 + +// Values defined at https://www.iana.org/assignments/dns-svcb/dns-svcb.xhtml#dns-svcparamkeys. +const ( + SVCParamMandatory SVCParamKey = 0 + SVCParamALPN SVCParamKey = 1 + SVCParamNoDefaultALPN SVCParamKey = 2 + SVCParamPort SVCParamKey = 3 + SVCParamIPv4Hint SVCParamKey = 4 + SVCParamECH SVCParamKey = 5 + SVCParamIPv6Hint SVCParamKey = 6 + SVCParamDOHPath SVCParamKey = 7 + SVCParamOHTTP SVCParamKey = 8 + SVCParamTLSSupportedGroups SVCParamKey = 9 +) + +var svcParamKeyNames = map[SVCParamKey]string{ + SVCParamMandatory: "Mandatory", + SVCParamALPN: "ALPN", + SVCParamNoDefaultALPN: "NoDefaultALPN", + SVCParamPort: "Port", + SVCParamIPv4Hint: "IPv4Hint", + SVCParamECH: "ECH", + SVCParamIPv6Hint: "IPv6Hint", + SVCParamDOHPath: "DOHPath", + SVCParamOHTTP: "OHTTP", + SVCParamTLSSupportedGroups: "TLSSupportedGroups", +} + +// String implements fmt.Stringer.String. +func (k SVCParamKey) String() string { + if n, ok := svcParamKeyNames[k]; ok { + return n + } + return printUint16(uint16(k)) +} + +// GoString implements fmt.GoStringer.GoString. +func (k SVCParamKey) GoString() string { + if n, ok := svcParamKeyNames[k]; ok { + return "dnsmessage.SVCParam" + n + } + return printUint16(uint16(k)) +} + +func (r *SVCBResource) pack(msg []byte, _ map[string]uint16, _ int) ([]byte, error) { + oldMsg := msg + msg = packUint16(msg, r.Priority) + // https://datatracker.ietf.org/doc/html/rfc3597#section-4 prohibits name + // compression for RR types that are not "well-known". + // https://datatracker.ietf.org/doc/html/rfc9460#section-2.2 explicitly states that + // compression of the Target is prohibited, following RFC 3597. + msg, err := r.Target.pack(msg, nil, 0) + if err != nil { + return oldMsg, &nestedError{"SVCBResource.Target", err} + } + for i, param := range r.Params { + if i > 0 && param.Key <= r.Params[i-1].Key { + return oldMsg, &nestedError{"SVCBResource.Params", errParamOutOfOrder} + } + if len(param.Value) > (1<<16)-1 { + return oldMsg, &nestedError{"SVCBResource.Params", errTooLongSVCBValue} + } + msg = packUint16(msg, uint16(param.Key)) + msg = packUint16(msg, uint16(len(param.Value))) + msg = append(msg, param.Value...) + } + return msg, nil +} + +func unpackSVCBResource(msg []byte, off int, length uint16) (SVCBResource, error) { + // Wire format reference: https://www.rfc-editor.org/rfc/rfc9460.html#section-2.2. + r := SVCBResource{} + paramsOff := off + bodyEnd := off + int(length) + + if bodyEnd > len(msg) { + return SVCBResource{}, errResourceLen + } + + var err error + if r.Priority, paramsOff, err = unpackUint16(msg, paramsOff); err != nil { + return SVCBResource{}, &nestedError{"Priority", err} + } + + if paramsOff, err = r.Target.unpack(msg, paramsOff); err != nil { + return SVCBResource{}, &nestedError{"Target", err} + } + + // Two-pass parsing to avoid allocations. + // First, count the number of params. + n := 0 + var totalValueLen uint16 + off = paramsOff + var previousKey uint16 + for off < bodyEnd { + var key, size uint16 + if key, off, err = unpackUint16(msg, off); err != nil { + return SVCBResource{}, &nestedError{"Params key", err} + } + if n > 0 && key <= previousKey { + // As per https://www.rfc-editor.org/rfc/rfc9460.html#section-2.2, clients MUST + // consider the RR malformed if the SvcParamKeys are not in strictly increasing numeric order + return SVCBResource{}, &nestedError{"Params", errParamOutOfOrder} + } + if size, off, err = unpackUint16(msg, off); err != nil { + return SVCBResource{}, &nestedError{"Params value length", err} + } + if off+int(size) > bodyEnd { + return SVCBResource{}, errResourceLen + } + previousKey = key + totalValueLen += size + off += int(size) + n++ + } + if off != bodyEnd { + return SVCBResource{}, errResourceLen + } + + // Second, fill in the params. + r.Params = make([]SVCParam, n) + // valuesBuf is used to hold all param values to reduce allocations. + // Each param's Value slice will point into this buffer. + valuesBuf := make([]byte, totalValueLen) + off = paramsOff + for i := 0; i < n; i++ { + p := &r.Params[i] + var key, size uint16 + if key, off, err = unpackUint16(msg, off); err != nil { + return SVCBResource{}, &nestedError{"param key", err} + } + p.Key = SVCParamKey(key) + if size, off, err = unpackUint16(msg, off); err != nil { + return SVCBResource{}, &nestedError{"param length", err} + } + if len(msg[off:]) < int(size) { + return SVCBResource{}, &nestedError{"param value", errCalcLen} + } + if copy(valuesBuf, msg[off:][:int(size)]) != int(size) { + return SVCBResource{}, &nestedError{"param value", errCalcLen} + } + p.Value = valuesBuf[:size:size] + valuesBuf = valuesBuf[size:] + off += int(size) + } + + return r, nil +} + +// genericSVCBResource parses a single Resource Record compatible with SVCB. +func (p *Parser) genericSVCBResource(svcbType Type) (SVCBResource, error) { + if !p.resHeaderValid || p.resHeaderType != svcbType { + return SVCBResource{}, ErrNotStarted + } + r, err := unpackSVCBResource(p.msg, p.off, p.resHeaderLength) + if err != nil { + return SVCBResource{}, err + } + p.off += int(p.resHeaderLength) + p.resHeaderValid = false + p.index++ + return r, nil +} + +// SVCBResource parses a single SVCBResource. +// +// One of the XXXHeader methods must have been called before calling this +// method. +func (p *Parser) SVCBResource() (SVCBResource, error) { + return p.genericSVCBResource(TypeSVCB) +} + +// HTTPSResource parses a single HTTPSResource. +// +// One of the XXXHeader methods must have been called before calling this +// method. +func (p *Parser) HTTPSResource() (HTTPSResource, error) { + svcb, err := p.genericSVCBResource(TypeHTTPS) + if err != nil { + return HTTPSResource{}, err + } + return HTTPSResource{svcb}, nil +} + +// genericSVCBResource is the generic implementation for adding SVCB-like resources. +func (b *Builder) genericSVCBResource(h ResourceHeader, r SVCBResource) error { + if err := b.checkResourceSection(); err != nil { + return err + } + msg, lenOff, err := h.pack(b.msg, b.compression, b.start) + if err != nil { + return &nestedError{"ResourceHeader", err} + } + preLen := len(msg) + if msg, err = r.pack(msg, b.compression, b.start); err != nil { + return &nestedError{"ResourceBody", err} + } + if err := h.fixLen(msg, lenOff, preLen); err != nil { + return err + } + if err := b.incrementSectionCount(); err != nil { + return err + } + b.msg = msg + return nil +} + +// SVCBResource adds a single SVCBResource. +func (b *Builder) SVCBResource(h ResourceHeader, r SVCBResource) error { + h.Type = r.realType() + return b.genericSVCBResource(h, r) +} + +// HTTPSResource adds a single HTTPSResource. +func (b *Builder) HTTPSResource(h ResourceHeader, r HTTPSResource) error { + h.Type = r.realType() + return b.genericSVCBResource(h, r.SVCBResource) +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 0a9821bfd8..2af767a13b 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1271,6 +1271,7 @@ golang.org/x/exp/slices # golang.org/x/net v0.58.0 ## explicit; go 1.25.0 golang.org/x/net/bpf +golang.org/x/net/dns/dnsmessage golang.org/x/net/http/httpguts golang.org/x/net/http2 golang.org/x/net/http2/hpack From 190b956b93b0b5d230822009606d2ffea67cfed6 Mon Sep 17 00:00:00 2001 From: Max Thompson Date: Fri, 9 Oct 2026 18:16:19 +0000 Subject: [PATCH 34/44] ate-setup sets the actor JWT issuer (ACTOR_JWT_ISSUER), and ate-api-server requires it (#2257) Part of #1756. - ate-api-server's `--actor-jwt-issuer` is now required instead of defaulting to `https://idp..svc`, so the issuer comes from one setting that ate-idp-server can share rather than from two binaries' defaults agreeing. - ate-setup takes the issuer from a new `actorJWT.issuer` setting (`ACTOR_JWT_ISSUER`, `--actor-jwt-issuer`, or the config file), defaulting to `https://idp..svc`, and validates it before touching the cluster. - ate-setup writes it to the `ate-api-server-envvars` ConfigMap as `ATE_API_ACTOR_JWT_ISSUER`, and the manifest passes `--actor-jwt-issuer=@env`, the same path the PostgreSQL settings take. Changing it rolls ate-api-server through the existing env-hash annotation. Anything that applies `ate-api-server.yaml` without ate-setup has to add `ATE_API_ACTOR_JWT_ISSUER` to that ConfigMap, or ate-api-server exits at startup. Testing: unit tests for the required flag, the `@env` lookup, ate-setup's default and override, and the ConfigMap write. I didn't deploy it by hand; the e2e install in CI is its first real run. - [ ] Tests pass - [ ] Appropriate changes to documentation are included in the PR --- cmd/ate-setup/internal/config/config.go | 4 ++ cmd/ate-setup/internal/config/config_test.go | 42 +++++++++++++++ .../internal/config/coverage_test.go | 2 + cmd/ate-setup/internal/config/load.go | 11 ++++ cmd/ate-setup/internal/config/setting.go | 6 +++ cmd/ate-setup/internal/steps/apienv.go | 4 +- cmd/ate-setup/internal/steps/apienv_test.go | 22 ++++++++ cmd/ateapi/main.go | 21 +++----- cmd/ateapi/main_test.go | 53 +++++++++---------- docs/operator-install.md | 1 + hack/install-ate.sh | 2 + manifests/ate-install/ate-api-server.yaml | 1 + 12 files changed, 127 insertions(+), 42 deletions(-) diff --git a/cmd/ate-setup/internal/config/config.go b/cmd/ate-setup/internal/config/config.go index 0042ac184d..8050a58aea 100644 --- a/cmd/ate-setup/internal/config/config.go +++ b/cmd/ate-setup/internal/config/config.go @@ -121,6 +121,10 @@ type Config struct { // pool (ACTOR_JWT_ALGORITHM): ES256 or RS256. ActorJWTAlgorithm string + // ActorJWTIssuer is the iss claim of actor JWTs (ACTOR_JWT_ISSUER). It + // defaults to the in-cluster URL of the install's ate-idp-server Service. + ActorJWTIssuer string + // BucketName is the snapshot bucket demos are templated with. BucketName string diff --git a/cmd/ate-setup/internal/config/config_test.go b/cmd/ate-setup/internal/config/config_test.go index edba7d2de8..240446ecfe 100644 --- a/cmd/ate-setup/internal/config/config_test.go +++ b/cmd/ate-setup/internal/config/config_test.go @@ -39,6 +39,7 @@ func loadEnv(t *testing.T) { t.Setenv("NO_DEV_ENV", "1") for _, name := range []string{ "ACTOR_JWT_ALGORITHM", + "ACTOR_JWT_ISSUER", "ANTHROPIC_API_KEY", "ATE_ADDITIONAL_EGRESS_EXTPROC_SERVICE", "ATE_API_POSTGRES_CLOUDSQL_GSA", @@ -435,6 +436,47 @@ func TestLoadActorJWTAlgorithm(t *testing.T) { } } +func TestLoadActorJWTIssuer(t *testing.T) { + for _, tt := range []struct { + name string + env string + namespace string + want string + wantErr bool + }{ + {name: "unset", want: "https://idp.ate-system.svc"}, + {name: "unset in a relocated install", namespace: "team-a", want: "https://idp.team-a.svc"}, + {name: "set is used as given", env: "https://idp.example.com/prod/", want: "https://idp.example.com/prod/"}, + {name: "not https", env: "http://idp.example.com", wantErr: true}, + } { + t.Run(tt.name, func(t *testing.T) { + loadEnv(t) + if tt.env != "" { + t.Setenv("ACTOR_JWT_ISSUER", tt.env) + } + if tt.namespace != "" { + t.Setenv("ATE_NAMESPACE", tt.namespace) + } else { + unsetEnv(t, "ATE_NAMESPACE") + } + + cfg, err := Load(Options{}) + if tt.wantErr { + if err == nil { + t.Fatalf("Load() with ACTOR_JWT_ISSUER=%q returned nil error", tt.env) + } + return + } + if err != nil { + t.Fatalf("Load() error = %v", err) + } + if cfg.ActorJWTIssuer != tt.want { + t.Errorf("ActorJWTIssuer = %q, want %q", cfg.ActorJWTIssuer, tt.want) + } + }) + } +} + // The endpoint has to reach both the Go steps and the shell scripts ate-setup // still delegates to, or the two halves of an install export different // collectors. diff --git a/cmd/ate-setup/internal/config/coverage_test.go b/cmd/ate-setup/internal/config/coverage_test.go index 9b89535f7a..287c192027 100644 --- a/cmd/ate-setup/internal/config/coverage_test.go +++ b/cmd/ate-setup/internal/config/coverage_test.go @@ -98,6 +98,8 @@ func valueFor(key string) string { return `{"name":"k8s.io"}` case "actorJWT.algorithm": return "ES256" + case "actorJWT.issuer": + return "https://idp.example.com" case "csi.setup": return "none" } diff --git a/cmd/ate-setup/internal/config/load.go b/cmd/ate-setup/internal/config/load.go index 0fee8fb34e..81fe2aa2dc 100644 --- a/cmd/ate-setup/internal/config/load.go +++ b/cmd/ate-setup/internal/config/load.go @@ -24,6 +24,8 @@ import ( "github.com/spf13/pflag" "github.com/agent-substrate/substrate/cmd/ate-setup/internal/images" + "github.com/agent-substrate/substrate/internal/installdefaults" + "github.com/agent-substrate/substrate/internal/oidcdiscovery" ) // ConfigPathEnv names the configuration document when --config is not given. @@ -196,6 +198,10 @@ func buildConfig(root string, env map[string]string, r *Resolved) (*Config, erro ownerConnectionString = readWriteConnectionString } instance, _ := r.Value("ateapi.postgres.cloudsql.instance") + actorJWTIssuer := r.String("actorJWT.issuer") + if actorJWTIssuer == "" { + actorJWTIssuer = installdefaults.ActorJWTIssuer(r.String("namespace")) + } cfg := &Config{ Root: root, @@ -211,6 +217,7 @@ func buildConfig(root string, env map[string]string, r *Resolved) (*Config, erro KODockerRepo: r.String("ko.dockerRepo"), KODefaultPlatforms: r.String("ko.defaultPlatforms"), ActorJWTAlgorithm: r.String("actorJWT.algorithm"), + ActorJWTIssuer: actorJWTIssuer, DockerBuildFlags: strings.Fields(r.String("docker.buildFlags")), Images: images.Source{ Repo: strings.TrimSuffix(r.String("images.repo"), "/"), @@ -367,6 +374,10 @@ func validateResolved(cfg *Config, r *Resolved) error { alg, _ := r.Value("actorJWT.algorithm") return &InvalidError{Value: alg, Want: "ES256 or RS256"} } + if err := oidcdiscovery.ValidateIssuer(cfg.ActorJWTIssuer); err != nil { + issuer, _ := r.Value("actorJWT.issuer") + return fmt.Errorf("%s: %w (from %s)", issuer.Setting.Key, err, issuer.From.Describe(issuer.Setting)) + } if cfg.AdditionalEgressExtprocService != "" { extproc, _ := r.Value("atenet.egress.additionalExtprocService") if err := validateExtprocService(extproc); err != nil { diff --git a/cmd/ate-setup/internal/config/setting.go b/cmd/ate-setup/internal/config/setting.go index 6f102b6c01..0f803f2a11 100644 --- a/cmd/ate-setup/internal/config/setting.go +++ b/cmd/ate-setup/internal/config/setting.go @@ -122,6 +122,12 @@ var Registry = []Setting{ Kind: KindString, Default: "ES256", Usage: "Signing algorithm of the key in a new actor JWT pool: ES256 or RS256", }, + { + // The default depends on the namespace, so buildConfig applies it. + Key: "actorJWT.issuer", Env: "ACTOR_JWT_ISSUER", Flag: "actor-jwt-issuer", Kind: KindString, + Usage: "iss claim of actor JWTs, verbatim. Relying parties fetch /.well-known/openid-configuration " + + "(default https://idp..svc)", + }, { // Whitespace-separated, as the environment variable is. The registry // has no list kind, so the value is carried as written and split diff --git a/cmd/ate-setup/internal/steps/apienv.go b/cmd/ate-setup/internal/steps/apienv.go index 209f2af5c9..2669a533d2 100644 --- a/cmd/ate-setup/internal/steps/apienv.go +++ b/cmd/ate-setup/internal/steps/apienv.go @@ -34,7 +34,8 @@ const envHashAnnotation = "ate.dev/env-hash" // CreateAPIServerEnvVars reconciles how ate-api-server reaches its PostgreSQL // store: both DSNs and the schema into the ate-api-server-secret-envvars Secret, // stable roles and Cloud SQL settings into the ConfigMap, -// and an external server CA into postgres-server-ca. +// and an external server CA into postgres-server-ca. The ConfigMap also +// carries the actor JWT issuer. // // ate-api-server.yaml pulls both in through optional envFrom sources and // resolves the PostgreSQL connection, role, and schema flags from @@ -112,6 +113,7 @@ func (e *Env) CreateAPIServerEnvVars(ctx context.Context) error { if poolMaxConns != "" { configVars["ATE_API_POSTGRES_POOL_MAX_CONNS"] = poolMaxConns } + configVars["ATE_API_ACTOR_JWT_ISSUER"] = e.Cfg.ActorJWTIssuer if err := e.Kube.ApplyConfigMap(ctx, e.Namespace(), ConfigMapAPIEnvVars, configVars); err != nil { return err } diff --git a/cmd/ate-setup/internal/steps/apienv_test.go b/cmd/ate-setup/internal/steps/apienv_test.go index 6d51453da5..f47616e465 100644 --- a/cmd/ate-setup/internal/steps/apienv_test.go +++ b/cmd/ate-setup/internal/steps/apienv_test.go @@ -177,6 +177,28 @@ func TestCreateAPIServerEnvVarsPoolSize(t *testing.T) { } } +func TestCreateAPIServerEnvVarsActorJWTIssuer(t *testing.T) { + cfg := config.Config{ + PostgresReadWriteConnectionString: "postgres://runtime@postgres/atepg", + ActorJWTIssuer: "https://idp.example.com/prod", + } + e := &Env{Cfg: &cfg, Kube: fakeKube(t, + &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: NamespaceAteSystem}}, + &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: ConfigMapAPIEnvVars, Namespace: NamespaceAteSystem}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: SecretAPIEnvVars, Namespace: NamespaceAteSystem}}, + )} + if err := e.CreateAPIServerEnvVars(t.Context()); err != nil { + t.Fatal(err) + } + cm, err := e.Kube.GetConfigMap(t.Context(), NamespaceAteSystem, ConfigMapAPIEnvVars) + if err != nil { + t.Fatal(err) + } + if got := cm.Data["ATE_API_ACTOR_JWT_ISSUER"]; got != cfg.ActorJWTIssuer { + t.Errorf("ATE_API_ACTOR_JWT_ISSUER = %q, want %q", got, cfg.ActorJWTIssuer) + } +} + func TestCreateAPIServerEnvVarsAdoptsPostgresIdentity(t *testing.T) { const recordedDSN = "user=svc@p.iam host=127.0.0.1 dbname=atepg" const explicitOwnerDSN = "user=new-owner@p.iam host=127.0.0.1 dbname=atepg" diff --git a/cmd/ateapi/main.go b/cmd/ateapi/main.go index 3b3e383d66..ebc93e65d8 100644 --- a/cmd/ateapi/main.go +++ b/cmd/ateapi/main.go @@ -85,7 +85,7 @@ var ( authzBootstrapOwners = pflag.StringSlice("authz-bootstrap-owners", nil, "Principal IDs that are always global owners while listed, independent of the stored global AccessPolicy. Removing an ID revokes its access on restart. At least one is required when --experimental-enable-authz is set.") actorIDJWTPoolFile = pflag.String("actor-id-jwt-pool", "", "The file that contains the serialized JWT authority pool for signing actor JWTs") - actorJWTIssuer = pflag.String("actor-jwt-issuer", "", "Issuer URL placed in the iss claim of actor JWTs. Relying parties fetch /.well-known/openid-configuration to verify them. Must be https with no query or fragment. Empty means https://"+installdefaults.IDPServiceName+"..svc.") + actorJWTIssuer = pflag.String("actor-jwt-issuer", "", "Issuer URL placed in the iss claim of actor JWTs. Relying parties fetch /.well-known/openid-configuration to verify them. Must be https with no query or fragment. Required.") defaultEgressGatewayAddress = pflag.String("default-egress-gateway-address", "", "Default address (host:port) of the egress PEP that each actor's atunnel dials. Sent on every atelet Run and Restore, so it takes effect at the actor's next activation. Empty leaves actors with no TCP egress.") actorIDCAPoolFile = pflag.String("actor-id-ca-pool", "", "The file that contains the CA pool for signing actor JWTs") @@ -125,11 +125,9 @@ func main() { if *templateResyncInterval < minResyncInterval { serverboot.Fatal(ctx, "Invalid --template-resync-interval", fmt.Errorf("must be at least %s", minResyncInterval)) } - resolvedActorJWTIssuer, err := resolveActorJWTIssuer(*actorJWTIssuer, installdefaults.NamespaceFromPodEnv()) - if err != nil { + if err := validateActorJWTIssuer(*actorJWTIssuer); err != nil { serverboot.Fatal(ctx, "Invalid --actor-jwt-issuer", err) } - slog.InfoContext(ctx, "Resolved actor JWT issuer", slog.String("actor-jwt-issuer", resolvedActorJWTIssuer)) // Kept separate from ctx so that in-progress work (clients, informers) is // not cancelled the moment SIGTERM arrives. The drainOnShutdown @@ -299,7 +297,7 @@ func main() { *defaultEgressGatewayAddress, volPlugins, objectstoresnapshotv1.NewControlProviderClient(snapshotPluginConn), - resolvedActorJWTIssuer, + *actorJWTIssuer, actorIDJWTAuthorityPool, actorIDCAPool, ) @@ -404,6 +402,7 @@ func loadFlagsFromEnv() error { {postgresReadWriteRole, "ATE_API_POSTGRES_READ_WRITE_ROLE"}, {postgresOwnerRole, "ATE_API_POSTGRES_OWNER_ROLE"}, {postgresSchema, "ATE_API_POSTGRES_SCHEMA"}, + {actorJWTIssuer, "ATE_API_ACTOR_JWT_ISSUER"}, } for _, o := range overrides { if *o.flag == "@env" { @@ -630,15 +629,9 @@ func buildJWTProviders(ctx context.Context, cfg *apiauthn.AuthenticationConfig) return serverCfg, nil } -// resolveActorJWTIssuer applies the install default to an empty -// --actor-jwt-issuer and validates the result. -func resolveActorJWTIssuer(flagValue, namespace string) (string, error) { - issuer := flagValue +func validateActorJWTIssuer(issuer string) error { if issuer == "" { - issuer = installdefaults.ActorJWTIssuer(namespace) - } - if err := oidcdiscovery.ValidateIssuer(issuer); err != nil { - return "", err + return errors.New("required") } - return issuer, nil + return oidcdiscovery.ValidateIssuer(issuer) } diff --git a/cmd/ateapi/main_test.go b/cmd/ateapi/main_test.go index 58522fc209..9c03aab650 100644 --- a/cmd/ateapi/main_test.go +++ b/cmd/ateapi/main_test.go @@ -104,35 +104,34 @@ func TestLoadFlagsFromEnvPoolMaxConns(t *testing.T) { } } -func TestResolveActorJWTIssuer(t *testing.T) { - tests := []struct { - name string - flagValue string - namespace string - want string - wantErr bool +func TestValidateActorJWTIssuer(t *testing.T) { + for _, tt := range []struct { + issuer string + wantErr bool }{ - {name: "unset uses the namespace's idp Service", namespace: "ate-system", want: "https://idp.ate-system.svc"}, - {name: "unset in a relocated install", namespace: "team-a", want: "https://idp.team-a.svc"}, - {name: "set is used as given", flagValue: "https://idp.example.com/prod/", namespace: "ate-system", want: "https://idp.example.com/prod/"}, - {name: "set but invalid", flagValue: "http://idp.example.com", namespace: "ate-system", wantErr: true}, + {issuer: "", wantErr: true}, + {issuer: "http://idp.example.com", wantErr: true}, + {issuer: "https://idp.ate-system.svc"}, + {issuer: "https://idp.example.com/prod/"}, + } { + err := validateActorJWTIssuer(tt.issuer) + if gotErr := err != nil; gotErr != tt.wantErr { + t.Errorf("validateActorJWTIssuer(%q) = %v, want error %t", tt.issuer, err, tt.wantErr) + } } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := resolveActorJWTIssuer(tt.flagValue, tt.namespace) - if tt.wantErr { - if err == nil { - t.Fatalf("resolveActorJWTIssuer(%q, %q) = %q, want error", tt.flagValue, tt.namespace, got) - } - return - } - if err != nil { - t.Fatalf("resolveActorJWTIssuer(%q, %q) returned error: %v", tt.flagValue, tt.namespace, err) - } - if got != tt.want { - t.Errorf("resolveActorJWTIssuer(%q, %q) = %q, want %q", tt.flagValue, tt.namespace, got, tt.want) - } - }) +} + +func TestLoadFlagsFromEnvActorJWTIssuer(t *testing.T) { + old := *actorJWTIssuer + t.Cleanup(func() { *actorJWTIssuer = old }) + *actorJWTIssuer = "@env" + t.Setenv("ATE_API_ACTOR_JWT_ISSUER", "https://idp.example.com") + + if err := loadFlagsFromEnv(); err != nil { + t.Fatal(err) + } + if *actorJWTIssuer != "https://idp.example.com" { + t.Errorf("--actor-jwt-issuer = %q, want %q", *actorJWTIssuer, "https://idp.example.com") } } diff --git a/docs/operator-install.md b/docs/operator-install.md index da9b2d7fe6..531dbecf67 100644 --- a/docs/operator-install.md +++ b/docs/operator-install.md @@ -267,6 +267,7 @@ command. A setting with no default is unset unless something supplies it. | Key | Environment variable | Flag | Default | Applies to | |----------------------------------------------|-------------------------------------------------|----------------------------------------------------|-----------------------|------------------------------------------| | `actorJWT.algorithm` | `ACTOR_JWT_ALGORITHM` | `--actor-jwt-algorithm` | `ES256` | (global) | +| `actorJWT.issuer` | `ACTOR_JWT_ISSUER` | `--actor-jwt-issuer` | — | (global) | | `ateapi.expectedJWTIssuer` | `EXPECTED_JWT_ISSUER` | `--ateapi-expected-jwt-issuer` | — | (global) | | `ateapi.postgres.cloudsql.gsa` | `ATE_API_POSTGRES_CLOUDSQL_GSA` | `--ateapi-postgres-cloudsql-gsa` | — | (global) | | `ateapi.postgres.cloudsql.iamAuth` | `ATE_API_POSTGRES_CLOUDSQL_IAM_AUTH` | `--ateapi-postgres-cloudsql-iam-auth` | — | (global) | diff --git a/hack/install-ate.sh b/hack/install-ate.sh index 4036be08cb..529464f6ed 100755 --- a/hack/install-ate.sh +++ b/hack/install-ate.sh @@ -165,6 +165,8 @@ usage() { echo " else the cluster's OIDC discovery document" echo " ACTOR_JWT_ALGORITHM Signing algorithm of a newly created actor JWT pool: ES256 (default) | RS256." echo " Use RS256 (a 4096-bit RSA key) for relying parties that don't support ES256. Has no effect once the pool exists." + echo " ACTOR_JWT_ISSUER iss claim of actor JWTs, verbatim. Relying parties fetch /.well-known/openid-configuration." + echo " Default: https://idp..svc" echo "" echo "Benchmarks (see benchmarking/README.md for details and customization):" echo "" diff --git a/manifests/ate-install/ate-api-server.yaml b/manifests/ate-install/ate-api-server.yaml index 5718480cec..2e325ff40c 100644 --- a/manifests/ate-install/ate-api-server.yaml +++ b/manifests/ate-install/ate-api-server.yaml @@ -136,6 +136,7 @@ spec: - --postgres-owner-role=@env - --postgres-schema=@env - --actor-id-jwt-pool=/run/actor-id-jwt-pool/pool.json + - --actor-jwt-issuer=@env - --actor-id-ca-pool=/run/actor-id-ca-pool/pool.json - --atelet-client-cred-bundle=/run/podidentity.podcert.ate.dev/credential-bundle.pem - --pod-identity-ca-certs=/run/podidentity.podcert.ate.dev/trust-bundle.pem From 9b2ecb94395c8c024effad0f131c6a87c2997b67 Mon Sep 17 00:00:00 2001 From: Luiz Oliveira Date: Fri, 9 Oct 2026 19:49:14 +0000 Subject: [PATCH 35/44] Rename target_ateom_uid to worker_pod_uid (#2423) Fixes #2416 --- .../controlapi/functionaltest/actor_test.go | 12 ++--- .../internal/controlapi/workflow_delete.go | 8 ++-- .../internal/controlapi/workflow_pause.go | 2 +- .../internal/controlapi/workflow_resume.go | 6 +-- .../internal/controlapi/workflow_suspend.go | 2 +- cmd/atelet/lifecycle_test.go | 48 +++++++++---------- cmd/atelet/main.go | 31 ++++++------ cmd/atelet/main_test.go | 28 +++++------ cmd/ateom-gvisor/main.go | 4 +- cmd/ateom-microvm/main.go | 4 +- internal/proto/ateletpb/atelet.pb.go | 48 +++++++++---------- internal/proto/ateletpb/atelet.proto | 8 ++-- internal/resources/validate.go | 8 ++-- internal/resources/validate_test.go | 6 +-- 14 files changed, 107 insertions(+), 108 deletions(-) diff --git a/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go b/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go index 25cb9e1f41..a77716e968 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go @@ -1241,8 +1241,8 @@ func TestDeleteActor_Crashed(t *testing.T) { } if !tc.fakeAtelet.TerminateCalled { t.Errorf("expected Terminate call to clean up assigned node for crashed actor") - } else if gotUID := tc.fakeAtelet.TerminateRequest.GetTargetAteomUid(); gotUID != "" { - t.Errorf("TerminateRequest.TargetAteomUid = %q, want empty for crashed actor", gotUID) + } else if gotUID := tc.fakeAtelet.TerminateRequest.GetWorkerPodUid(); gotUID != "" { + t.Errorf("TerminateRequest.WorkerPodUid = %q, want empty for crashed actor", gotUID) } _, err = tc.client.GetActor(ctx, &ateapipb.GetActorRequest{ @@ -5405,8 +5405,8 @@ func TestRevertActor_FromPaused(t *testing.T) { } if !tc.fakeAtelet.TerminateCalled { t.Errorf("expected Terminate call to clean up assigned node for paused actor") - } else if gotUID := tc.fakeAtelet.TerminateRequest.GetTargetAteomUid(); gotUID != "" { - t.Errorf("TerminateRequest.TargetAteomUid = %q, want empty for paused actor", gotUID) + } else if gotUID := tc.fakeAtelet.TerminateRequest.GetWorkerPodUid(); gotUID != "" { + t.Errorf("TerminateRequest.WorkerPodUid = %q, want empty for paused actor", gotUID) } if tc.fakeAtelet.CheckpointCalled { t.Errorf("RevertActor checkpointed the workload, want the execution discarded") @@ -5486,8 +5486,8 @@ func TestRevertActor_FromCrashed(t *testing.T) { } if !tc.fakeAtelet.TerminateCalled { t.Errorf("expected Terminate call to clean up assigned node for crashed actor") - } else if gotUID := tc.fakeAtelet.TerminateRequest.GetTargetAteomUid(); gotUID != "" { - t.Errorf("TerminateRequest.TargetAteomUid = %q, want empty for crashed actor", gotUID) + } else if gotUID := tc.fakeAtelet.TerminateRequest.GetWorkerPodUid(); gotUID != "" { + t.Errorf("TerminateRequest.WorkerPodUid = %q, want empty for crashed actor", gotUID) } if tc.fakeAtelet.CheckpointCalled { t.Errorf("RevertActor checkpointed the workload, want the execution discarded") diff --git a/cmd/ateapi/internal/controlapi/workflow_delete.go b/cmd/ateapi/internal/controlapi/workflow_delete.go index 759facf0c4..5a4337fdc5 100644 --- a/cmd/ateapi/internal/controlapi/workflow_delete.go +++ b/cmd/ateapi/internal/controlapi/workflow_delete.go @@ -131,9 +131,9 @@ func (w *ActorWorkflow) ensureAteletTerminated(ctx context.Context, actorRef res return nil } - targetAteomUID := "" + workerPodUID := "" if assignment := actor.GetStatus().GetWorkerAssignment(); assignment != nil { - targetAteomUID = assignment.GetWorkerPodUid() + workerPodUID = assignment.GetWorkerPodUid() if workerName := assignment.GetWorker().GetName(); workerName != "" { // Ask whether the worker still HOSTS this actor, not whether its one // assignment happens to be this actor: a worker hosting several is the @@ -146,7 +146,7 @@ func (w *ActorWorkflow) ensureAteletTerminated(ctx context.Context, actorRef res slog.InfoContext(ctx, "worker is no longer assigned to this actor, skipping ateom workload termination", slog.String("worker", workerName), slog.Any("actor", actorRef)) - targetAteomUID = "" + workerPodUID = "" } } } @@ -192,7 +192,7 @@ func (w *ActorWorkflow) ensureAteletTerminated(ctx context.Context, actorRef res } req := &ateletpb.TerminateRequest{ - TargetAteomUid: targetAteomUID, + WorkerPodUid: workerPodUID, Atespace: actor.GetMetadata().GetAtespace(), ActorName: actor.GetMetadata().GetName(), ActorUid: actor.GetMetadata().GetUid(), diff --git a/cmd/ateapi/internal/controlapi/workflow_pause.go b/cmd/ateapi/internal/controlapi/workflow_pause.go index b00ae7a8cb..bac9a6e5ce 100644 --- a/cmd/ateapi/internal/controlapi/workflow_pause.go +++ b/cmd/ateapi/internal/controlapi/workflow_pause.go @@ -191,7 +191,7 @@ func (w *ActorWorkflow) ensureAteletPaused(ctx context.Context, actorRef resourc // into the snapshot manifest. _, inProgressLocal := findLatestSnapshotStorage(actor.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS) req := &ateletpb.CheckpointRequest{ - TargetAteomUid: assignment.GetWorkerPodUid(), + WorkerPodUid: assignment.GetWorkerPodUid(), Atespace: actor.GetMetadata().GetAtespace(), ActorName: actor.GetMetadata().GetName(), ActorTemplateAtespace: actor.GetActorTemplate().GetAtespace(), diff --git a/cmd/ateapi/internal/controlapi/workflow_resume.go b/cmd/ateapi/internal/controlapi/workflow_resume.go index f9bd159582..b9bfbff09f 100644 --- a/cmd/ateapi/internal/controlapi/workflow_resume.go +++ b/cmd/ateapi/internal/controlapi/workflow_resume.go @@ -700,7 +700,7 @@ func (w *ActorWorkflow) ensureAteletRestored(ctx context.Context, actorRef resou tele.SnapshotKind = ateattr.SnapshotKindLocal req := &ateletpb.RestoreRequest{ - TargetAteomUid: assignment.GetWorkerPodUid(), + WorkerPodUid: assignment.GetWorkerPodUid(), Atespace: actor.GetMetadata().GetAtespace(), ActorName: actor.GetMetadata().GetName(), ActorTemplateAtespace: actor.GetActorTemplate().GetAtespace(), @@ -732,7 +732,7 @@ func (w *ActorWorkflow) ensureAteletRestored(ctx context.Context, actorRef resou } tele.WireFidelity = ateattr.SnapshotFidelityValue(scope) req := &ateletpb.RestoreRequest{ - TargetAteomUid: assignment.GetWorkerPodUid(), + WorkerPodUid: assignment.GetWorkerPodUid(), Atespace: actor.GetMetadata().GetAtespace(), ActorName: actor.GetMetadata().GetName(), ActorTemplateAtespace: actor.GetActorTemplate().GetAtespace(), @@ -760,7 +760,7 @@ func (w *ActorWorkflow) ensureAteletRestored(ctx context.Context, actorRef resou tele.SnapshotKind = ateattr.SnapshotKindBoot req := &ateletpb.RunRequest{ - TargetAteomUid: assignment.GetWorkerPodUid(), + WorkerPodUid: assignment.GetWorkerPodUid(), Atespace: actor.GetMetadata().GetAtespace(), ActorName: actor.GetMetadata().GetName(), ActorTemplateAtespace: actor.GetActorTemplate().GetAtespace(), diff --git a/cmd/ateapi/internal/controlapi/workflow_suspend.go b/cmd/ateapi/internal/controlapi/workflow_suspend.go index e6d462ddbb..28f3893cd7 100644 --- a/cmd/ateapi/internal/controlapi/workflow_suspend.go +++ b/cmd/ateapi/internal/controlapi/workflow_suspend.go @@ -255,7 +255,7 @@ func (w *ActorWorkflow) ensureAteletSuspended(ctx context.Context, actorRef reso // into the snapshot manifest. _, inProgressDurable := findLatestSnapshotStorage(actor.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS) req := &ateletpb.CheckpointRequest{ - TargetAteomUid: assignment.GetWorkerPodUid(), + WorkerPodUid: assignment.GetWorkerPodUid(), Atespace: actor.GetMetadata().GetAtespace(), ActorName: actor.GetMetadata().GetName(), ActorTemplateAtespace: actor.GetActorTemplate().GetAtespace(), diff --git a/cmd/atelet/lifecycle_test.go b/cmd/atelet/lifecycle_test.go index b80a54d6ae..80c332c0fe 100644 --- a/cmd/atelet/lifecycle_test.go +++ b/cmd/atelet/lifecycle_test.go @@ -149,7 +149,7 @@ func TestLocalSnapshotGC(t *testing.T) { atespace = "ate-demo" actorName = "counter" actorUID = "actor-uid-1" - ateomUID = "ateom-uid-1" + workerPodUID = "worker-pod-uid-1" snapshotName = "pause-snap-1" ) @@ -191,7 +191,7 @@ func TestLocalSnapshotGC(t *testing.T) { ActorUid: actorUID, ActorTemplateAtespace: "default", ActorTemplateName: "counter", - TargetAteomUid: ateomUID, + WorkerPodUid: workerPodUID, SandboxAssets: sandboxAssets, Spec: spec, }); err != nil { @@ -205,7 +205,7 @@ func TestLocalSnapshotGC(t *testing.T) { ActorUid: actorUID, ActorTemplateAtespace: "default", ActorTemplateName: "counter", - TargetAteomUid: ateomUID, + WorkerPodUid: workerPodUID, Spec: spec, Fidelity: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, Type: ateletpb.CheckpointType_CHECKPOINT_TYPE_LOCAL, @@ -227,7 +227,7 @@ func TestLocalSnapshotGC(t *testing.T) { ActorUid: actorUID, ActorTemplateAtespace: "default", ActorTemplateName: "counter", - TargetAteomUid: ateomUID, + WorkerPodUid: workerPodUID, SandboxAssets: sandboxAssets, Spec: spec, Fidelity: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, @@ -255,7 +255,7 @@ func TestLocalSnapshotGC(t *testing.T) { ActorUid: actorUID, ActorTemplateAtespace: "default", ActorTemplateName: "counter", - TargetAteomUid: ateomUID, + WorkerPodUid: workerPodUID, Spec: spec, }); err != nil { t.Fatalf("Terminate: %v", err) @@ -305,7 +305,7 @@ func TestRestoreUsesRequestSandboxAssets(t *testing.T) { atespace = "ate-demo" actorName = "counter" actorUID = "actor-uid-1" - ateomUID = "ateom-uid-1" + workerPodUID = "worker-pod-uid-1" snapshotName = "pause-snap-1" ) @@ -351,7 +351,7 @@ func TestRestoreUsesRequestSandboxAssets(t *testing.T) { ActorUid: actorUID, ActorTemplateAtespace: "default", ActorTemplateName: "counter", - TargetAteomUid: ateomUID, + WorkerPodUid: workerPodUID, SandboxAssets: assetsWithPause(checkpointPause), Spec: spec, }); err != nil { @@ -364,7 +364,7 @@ func TestRestoreUsesRequestSandboxAssets(t *testing.T) { ActorUid: actorUID, ActorTemplateAtespace: "default", ActorTemplateName: "counter", - TargetAteomUid: ateomUID, + WorkerPodUid: workerPodUID, Spec: spec, Fidelity: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, Type: ateletpb.CheckpointType_CHECKPOINT_TYPE_LOCAL, @@ -393,7 +393,7 @@ func TestRestoreUsesRequestSandboxAssets(t *testing.T) { ActorUid: actorUID, ActorTemplateAtespace: "default", ActorTemplateName: "counter", - TargetAteomUid: ateomUID, + WorkerPodUid: workerPodUID, SandboxAssets: assetsWithPause(restorePause), Spec: spec, Fidelity: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, @@ -490,14 +490,14 @@ func TestActivationFailureBeforeRegistration(t *testing.T) { } if tc.restore { _, err = s.Restore(ctx, &ateletpb.RestoreRequest{ - Atespace: "team-a", ActorName: "actor-1", ActorUid: actorUID, TargetAteomUid: "ateom-uid-1", + Atespace: "team-a", ActorName: "actor-1", ActorUid: actorUID, WorkerPodUid: "worker-pod-uid-1", SandboxAssets: assets, Spec: spec, Fidelity: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, Type: ateletpb.CheckpointType_CHECKPOINT_TYPE_LOCAL, Config: &ateletpb.RestoreRequest_LocalConfig{LocalConfig: &ateletpb.LocalCheckpointConfiguration{SnapshotName: snapshotName}}, }) } else { _, err = s.Run(ctx, &ateletpb.RunRequest{ - Atespace: "team-a", ActorName: "actor-1", ActorUid: actorUID, TargetAteomUid: "ateom-uid-1", + Atespace: "team-a", ActorName: "actor-1", ActorUid: actorUID, WorkerPodUid: "worker-pod-uid-1", SandboxAssets: assets, Spec: spec, }) } @@ -560,10 +560,10 @@ func TestRunFailureAfterRegistrationRemovesOwnRegistration(t *testing.T) { }}, } _, err := s.Run(t.Context(), &ateletpb.RunRequest{ - Atespace: "team-a", - ActorName: "actor-run", - ActorUid: "actor-uid-run", - TargetAteomUid: "ateom-uid-1", + Atespace: "team-a", + ActorName: "actor-run", + ActorUid: "actor-uid-run", + WorkerPodUid: "worker-pod-uid-1", SandboxAssets: &ateletpb.SandboxAssets{ SandboxClass: "gvisor", PauseImage: "://invalid-image", @@ -612,10 +612,10 @@ func TestRestoreFailureAfterRegistrationRemovesOwnRegistration(t *testing.T) { systemInfoVolumes: refresher, } _, err := s.Restore(t.Context(), &ateletpb.RestoreRequest{ - Atespace: "team-a", - ActorName: "actor-restore-after", - ActorUid: actorUID, - TargetAteomUid: "ateom-uid-1", + Atespace: "team-a", + ActorName: "actor-restore-after", + ActorUid: actorUID, + WorkerPodUid: "worker-pod-uid-1", SandboxAssets: &ateletpb.SandboxAssets{ SandboxClass: "gvisor", PauseImage: "://invalid-image", @@ -641,11 +641,11 @@ func TestRestoreFailureAfterRegistrationRemovesOwnRegistration(t *testing.T) { } } -// TestTerminateWithoutTargetAteomUID verifies that Terminate with an empty -// TargetAteomUid (used when cleaning up the assigned node for a paused or +// TestTerminateWithoutWorkerPodUID verifies that Terminate with an empty +// WorkerPodUid (used when cleaning up the assigned node for a paused or // crashed actor that no longer has a worker pod) skips dialing ateom while // still pruning local checkpoints and removing actor directories on the node. -func TestTerminateWithoutTargetAteomUID(t *testing.T) { +func TestTerminateWithoutWorkerPodUID(t *testing.T) { useTempNodeDirs(t) ctx := t.Context() @@ -685,14 +685,14 @@ func TestTerminateWithoutTargetAteomUID(t *testing.T) { ActorUid: actorUID, ActorTemplateAtespace: "default", ActorTemplateName: "counter", - TargetAteomUid: "", + WorkerPodUid: "", Spec: spec, }); err != nil { t.Fatalf("Terminate: %v", err) } if got := ateom.actorDirs["TerminateWorkload"]; got != nil { - t.Errorf("TerminateWorkload was called on ateom (%v), want skipped when TargetAteomUid is empty", got) + t.Errorf("TerminateWorkload was called on ateom (%v), want skipped when WorkerPodUid is empty", got) } if _, err := os.Stat(ateletpath.LocalCheckpointsDir(actorUID)); !os.IsNotExist(err) { t.Errorf("local checkpoint dir survived terminate: %v", err) diff --git a/cmd/atelet/main.go b/cmd/atelet/main.go index 085ad698ce..99f4c78d91 100644 --- a/cmd/atelet/main.go +++ b/cmd/atelet/main.go @@ -542,13 +542,13 @@ func (s *AteomHerder) Run(ctx context.Context, req *ateletpb.RunRequest) (resp * return nil, err } containerSpecs, err := s.prepareOCIBundles(ctx, actorUID, actorRef, - req.GetSpec(), sandboxRec.PauseImage, req.GetTargetAteomUid(), + req.GetSpec(), sandboxRec.PauseImage, ) if err != nil { return nil, err } - client, err := s.dialAteom(ctx, req.GetTargetAteomUid()) + client, err := s.dialAteom(ctx, req.GetWorkerPodUid()) if err != nil { return nil, err } @@ -678,7 +678,7 @@ func (s *AteomHerder) Checkpoint(ctx context.Context, req *ateletpb.CheckpointRe checkpointDir := ateletpath.CheckpointStateDir(actorUID) - client, err := s.dialAteom(ctx, req.GetTargetAteomUid()) + client, err := s.dialAteom(ctx, req.GetWorkerPodUid()) if err != nil { return nil, err } @@ -1211,7 +1211,7 @@ func (s *AteomHerder) Restore(ctx context.Context, req *ateletpb.RestoreRequest) return err } t := time.Now() - containerSpecs, err = s.prepareOCIBundles(gctx, actorUID, actorRef, req.GetSpec(), runtimeRec.PauseImage, req.GetTargetAteomUid()) + containerSpecs, err = s.prepareOCIBundles(gctx, actorUID, actorRef, req.GetSpec(), runtimeRec.PauseImage) dBundles = time.Since(t) if err != nil { prepFailedPhase = ateattr.SnapshotPhaseOCIUnpack @@ -1229,7 +1229,7 @@ func (s *AteomHerder) Restore(ctx context.Context, req *ateletpb.RestoreRequest) return nil, err } - client, err := s.dialAteom(ctx, req.GetTargetAteomUid()) + client, err := s.dialAteom(ctx, req.GetWorkerPodUid()) if err != nil { return nil, err } @@ -1290,7 +1290,7 @@ func (s *AteomHerder) Terminate(ctx context.Context, req *ateletpb.TerminateRequ actorRef := resources.ActorRef{Atespace: req.GetAtespace(), Name: req.GetActorName()} actorUID := req.GetActorUid() - if req.GetTargetAteomUid() != "" { + if req.GetWorkerPodUid() != "" { var assetPaths map[string]string sandboxRec, err := readSandboxRecord(actorUID) if err != nil { @@ -1302,7 +1302,7 @@ func (s *AteomHerder) Terminate(ctx context.Context, req *ateletpb.TerminateRequ } assetPaths = paths - client, err := s.dialAteom(ctx, req.GetTargetAteomUid()) + client, err := s.dialAteom(ctx, req.GetWorkerPodUid()) if err != nil { return nil, fmt.Errorf("failed to dial ateom for terminate (actor: %s, actorUID: %s): %w", actorRef, actorUID, err) } @@ -1474,7 +1474,6 @@ func (s *AteomHerder) prepareOCIBundles( actorRef resources.ActorRef, spec *ateletpb.WorkloadSpec, pauseImage string, - targetAteomUid string, ) ([]*ateompb.ContainerSpec, error) { // Prepare host folders for volume types that need them. for _, vol := range spec.GetVolumes() { @@ -1532,10 +1531,10 @@ func (s *AteomHerder) prepareOCIBundles( // dialAteom opens (or reuses) the gRPC connection to the target ateom // pod and returns an ateom client. -func (s *AteomHerder) dialAteom(ctx context.Context, targetAteomUid string) (ateompb.AteomClient, error) { - conn, err := s.ateomDialer.DialAteomPod(ctx, targetAteomUid) +func (s *AteomHerder) dialAteom(ctx context.Context, workerPodUID string) (ateompb.AteomClient, error) { + conn, err := s.ateomDialer.DialAteomPod(ctx, workerPodUID) if err != nil { - return nil, fmt.Errorf("while getting ateom conn for %s: %w", targetAteomUid, err) + return nil, fmt.Errorf("while getting ateom conn for %s: %w", workerPodUID, err) } return ateompb.NewAteomClient(conn), nil } @@ -1708,7 +1707,7 @@ func validateRunRequest(req *ateletpb.RunRequest) error { return errs.ToAggregate() } // TODO: Migrate all validations below to the validation framework. - if err := resources.ValidateAteomUID(req.GetTargetAteomUid()); err != nil { + if err := resources.ValidateWorkerPodUID(req.GetWorkerPodUid()); err != nil { return err } names := make([]string, 0, len(req.GetSpec().GetContainers())) @@ -1727,7 +1726,7 @@ func validateCheckpointRequest(req *ateletpb.CheckpointRequest) error { return errs.ToAggregate() } // TODO: Migrate all validations below to the validation framework. - if err := resources.ValidateAteomUID(req.GetTargetAteomUid()); err != nil { + if err := resources.ValidateWorkerPodUID(req.GetWorkerPodUid()); err != nil { return err } names := make([]string, 0, len(req.GetSpec().GetContainers())) @@ -1766,7 +1765,7 @@ func validateRestoreRequest(req *ateletpb.RestoreRequest) error { return errs.ToAggregate() } // TODO: Migrate all validations below to the validation framework. - if err := resources.ValidateAteomUID(req.GetTargetAteomUid()); err != nil { + if err := resources.ValidateWorkerPodUID(req.GetWorkerPodUid()); err != nil { return err } names := make([]string, 0, len(req.GetSpec().GetContainers())) @@ -1808,8 +1807,8 @@ func validateTerminateRequest(req *ateletpb.TerminateRequest) error { if len(errs) > 0 { return errs.ToAggregate() } - if req.GetTargetAteomUid() != "" { - if err := resources.ValidateAteomUID(req.GetTargetAteomUid()); err != nil { + if req.GetWorkerPodUid() != "" { + if err := resources.ValidateWorkerPodUID(req.GetWorkerPodUid()); err != nil { return err } } diff --git a/cmd/atelet/main_test.go b/cmd/atelet/main_test.go index 14f6f39b26..921267e907 100644 --- a/cmd/atelet/main_test.go +++ b/cmd/atelet/main_test.go @@ -259,7 +259,7 @@ func TestPrepareOCIBundlesPause(t *testing.T) { useTempNodeDirs(t) const actorUID = "actor-uid-1" s := &AteomHerder{imageCache: newImageVolumeStore(t)} - if _, err := s.prepareOCIBundles(t.Context(), actorUID, resources.ActorRef{}, spec, tc.pauseImage, "ateom-uid-1"); err != nil { + if _, err := s.prepareOCIBundles(t.Context(), actorUID, resources.ActorRef{}, spec, tc.pauseImage); err != nil { t.Fatalf("prepareOCIBundles: %v", err) } if _, err := os.Stat(filepath.Join(ateletpath.OCIBundlePath(actorUID, "app"), imagecache.OverlaySpecFileName)); err != nil { @@ -295,7 +295,7 @@ func TestPrepareOCIBundlesFillsContainerSpec(t *testing.T) { } s := &AteomHerder{imageCache: newImageVolumeStore(t)} - got, err := s.prepareOCIBundles(t.Context(), "actor-uid-1", resources.ActorRef{}, spec, "", "ateom-uid-1") + got, err := s.prepareOCIBundles(t.Context(), "actor-uid-1", resources.ActorRef{}, spec, "") if err != nil { t.Fatalf("prepareOCIBundles: %v", err) } @@ -546,7 +546,7 @@ func validRunRequest() *ateletpb.RunRequest { ActorName: "counter-1", ActorTemplateAtespace: "ate-demo", ActorTemplateName: "counter", - TargetAteomUid: "422938ba-8860-4983-a25d-d6bcb0a69d4e", + WorkerPodUid: "422938ba-8860-4983-a25d-d6bcb0a69d4e", ActorUid: "123e4567-e89b-12d3-a456-426614174000", Spec: &ateletpb.WorkloadSpec{Containers: []*ateletpb.Container{{Name: "worker"}}}, } @@ -558,7 +558,7 @@ func validCheckpointRequest() *ateletpb.CheckpointRequest { ActorName: "counter-1", ActorTemplateAtespace: "ate-demo", ActorTemplateName: "counter", - TargetAteomUid: "422938ba-8860-4983-a25d-d6bcb0a69d4e", + WorkerPodUid: "422938ba-8860-4983-a25d-d6bcb0a69d4e", ActorUid: "123e4567-e89b-12d3-a456-426614174000", Spec: &ateletpb.WorkloadSpec{Containers: []*ateletpb.Container{{Name: "worker"}}}, Type: ateletpb.CheckpointType_CHECKPOINT_TYPE_EXTERNAL, @@ -577,7 +577,7 @@ func validRestoreRequest() *ateletpb.RestoreRequest { ActorName: "counter-1", ActorTemplateAtespace: "ate-demo", ActorTemplateName: "counter", - TargetAteomUid: "422938ba-8860-4983-a25d-d6bcb0a69d4e", + WorkerPodUid: "422938ba-8860-4983-a25d-d6bcb0a69d4e", ActorUid: "123e4567-e89b-12d3-a456-426614174000", Spec: &ateletpb.WorkloadSpec{Containers: []*ateletpb.Container{{Name: "worker"}}}, Type: ateletpb.CheckpointType_CHECKPOINT_TYPE_EXTERNAL, @@ -601,7 +601,7 @@ func TestValidateRunRequest(t *testing.T) { wantErr bool }{ {"valid", func(*ateletpb.RunRequest) {}, false}, - {"invalid ateom uid", func(r *ateletpb.RunRequest) { r.TargetAteomUid = "../escape" }, true}, + {"invalid worker pod uid", func(r *ateletpb.RunRequest) { r.WorkerPodUid = "../escape" }, true}, {"invalid atespace", func(r *ateletpb.RunRequest) { r.Atespace = "../escape" }, true}, {"invalid actor name", func(r *ateletpb.RunRequest) { r.ActorName = "../escape" }, true}, {"invalid actor uid", func(r *ateletpb.RunRequest) { r.ActorUid = "../escape" }, true}, @@ -641,7 +641,7 @@ func TestValidateCheckpointRequest(t *testing.T) { {"valid", makeReq(), false}, {"empty snapshot uri", makeReq(func(r *ateletpb.CheckpointRequest) { r.GetExternalConfig().SnapshotUri = "" }), true}, {"bucketless snapshot uri", makeReq(func(r *ateletpb.CheckpointRequest) { r.GetExternalConfig().SnapshotUri = "relative/path" }), true}, - {"invalid ateom uid", makeReq(func(r *ateletpb.CheckpointRequest) { r.TargetAteomUid = "../escape" }), true}, + {"invalid worker pod uid", makeReq(func(r *ateletpb.CheckpointRequest) { r.WorkerPodUid = "../escape" }), true}, {"invalid atespace", makeReq(func(r *ateletpb.CheckpointRequest) { r.Atespace = "../escape" }), true}, {"invalid actor name", makeReq(func(r *ateletpb.CheckpointRequest) { r.ActorName = "../escape" }), true}, {"invalid actor uid", makeReq(func(r *ateletpb.CheckpointRequest) { r.ActorUid = "../escape" }), true}, @@ -704,7 +704,7 @@ func TestValidateRestoreRequest(t *testing.T) { {"missing sandbox assets", makeReq(func(r *ateletpb.RestoreRequest) { r.SandboxAssets = nil }), true}, {"empty snapshot uri", makeReq(func(r *ateletpb.RestoreRequest) { r.GetExternalConfig().SnapshotUri = "" }), true}, {"bucketless snapshot uri", makeReq(func(r *ateletpb.RestoreRequest) { r.GetExternalConfig().SnapshotUri = "relative/path" }), true}, - {"invalid ateom uid", makeReq(func(r *ateletpb.RestoreRequest) { r.TargetAteomUid = "../escape" }), true}, + {"invalid worker pod uid", makeReq(func(r *ateletpb.RestoreRequest) { r.WorkerPodUid = "../escape" }), true}, {"invalid atespace", makeReq(func(r *ateletpb.RestoreRequest) { r.Atespace = "../escape" }), true}, {"invalid actor name", makeReq(func(r *ateletpb.RestoreRequest) { r.ActorName = "../escape" }), true}, {"invalid actor uid", makeReq(func(r *ateletpb.RestoreRequest) { r.ActorUid = "../escape" }), true}, @@ -910,14 +910,14 @@ func TestFetchAssetStreaming(t *testing.T) { func TestRPCBoundariesReject(t *testing.T) { s := &AteomHerder{} ctx := context.Background() - badUID := "../escape" // valid actor ref, invalid ateom UID + badUID := "../escape" // valid actor ref, invalid worker pod UID const okAtespace, okID, okActorUID = "ate-demo", "counter-1", "123e4567-e89b-12d3-a456-426614174000" okSpec := &ateletpb.WorkloadSpec{Containers: []*ateletpb.Container{{Name: "worker"}}} wantInvalidArgument := func(t *testing.T, rpc string, err error) { t.Helper() if err == nil { - t.Errorf("%s accepted an invalid target ateom UID", rpc) + t.Errorf("%s accepted an invalid worker pod UID", rpc) return } if code := apierror.Code(err); code != codes.InvalidArgument { @@ -928,21 +928,21 @@ func TestRPCBoundariesReject(t *testing.T) { t.Run("Run", func(t *testing.T) { _, err := s.Run(ctx, &ateletpb.RunRequest{ Atespace: okAtespace, ActorName: okID, - ActorUid: okActorUID, TargetAteomUid: badUID, Spec: okSpec, + ActorUid: okActorUID, WorkerPodUid: badUID, Spec: okSpec, }) wantInvalidArgument(t, "Run", err) }) t.Run("Checkpoint", func(t *testing.T) { _, err := s.Checkpoint(ctx, &ateletpb.CheckpointRequest{ Atespace: okAtespace, ActorName: okID, - ActorUid: okActorUID, TargetAteomUid: badUID, Spec: okSpec, + ActorUid: okActorUID, WorkerPodUid: badUID, Spec: okSpec, }) wantInvalidArgument(t, "Checkpoint", err) }) t.Run("Restore", func(t *testing.T) { _, err := s.Restore(ctx, &ateletpb.RestoreRequest{ Atespace: okAtespace, ActorName: okID, - ActorUid: okActorUID, TargetAteomUid: badUID, Spec: okSpec, + ActorUid: okActorUID, WorkerPodUid: badUID, Spec: okSpec, }) wantInvalidArgument(t, "Restore", err) }) @@ -950,7 +950,7 @@ func TestRPCBoundariesReject(t *testing.T) { _, err := s.Terminate(ctx, &ateletpb.TerminateRequest{ Atespace: okAtespace, ActorName: okID, ActorUid: okActorUID, ActorTemplateAtespace: "default", ActorTemplateName: "template", - TargetAteomUid: badUID, Spec: okSpec, + WorkerPodUid: badUID, Spec: okSpec, }) wantInvalidArgument(t, "Terminate", err) }) diff --git a/cmd/ateom-gvisor/main.go b/cmd/ateom-gvisor/main.go index 663bac2c6f..a54f4a6539 100644 --- a/cmd/ateom-gvisor/main.go +++ b/cmd/ateom-gvisor/main.go @@ -181,8 +181,8 @@ func do(ctx context.Context) error { // Create ateom dir ateomDir := nodepath.AteomPath(*podUID) - if err := resources.ValidateAteomUID(*podUID); err != nil { - return fmt.Errorf("in resources.ValidateAteomUID: %w", err) + if err := resources.ValidateWorkerPodUID(*podUID); err != nil { + return fmt.Errorf("in resources.ValidateWorkerPodUID: %w", err) } if err := os.MkdirAll(ateomDir, 0o700); err != nil { return fmt.Errorf("in os.MkdirAll(%q): %w", ateomDir, err) diff --git a/cmd/ateom-microvm/main.go b/cmd/ateom-microvm/main.go index bf5c3b2fae..501d907001 100644 --- a/cmd/ateom-microvm/main.go +++ b/cmd/ateom-microvm/main.go @@ -171,8 +171,8 @@ func do(ctx context.Context) error { // Create ateom dir. ateomDir := nodepath.AteomPath(*podUID) - if err := resources.ValidateAteomUID(*podUID); err != nil { - return fmt.Errorf("in resources.ValidateAteomUID: %w", err) + if err := resources.ValidateWorkerPodUID(*podUID); err != nil { + return fmt.Errorf("in resources.ValidateWorkerPodUID: %w", err) } if err := os.MkdirAll(ateomDir, 0o700); err != nil { return fmt.Errorf("in os.MkdirAll(%q): %w", ateomDir, err) diff --git a/internal/proto/ateletpb/atelet.pb.go b/internal/proto/ateletpb/atelet.pb.go index 90b451c20d..5d7f8cb916 100644 --- a/internal/proto/ateletpb/atelet.pb.go +++ b/internal/proto/ateletpb/atelet.pb.go @@ -767,7 +767,7 @@ func (x *MintActorCertificateResponse) GetActorCertificates() [][]byte { type TerminateRequest struct { state protoimpl.MessageState `protogen:"open.v1"` // If unset, the atelet will just cleanup node resources for the actor. - TargetAteomUid string `protobuf:"bytes,1,opt,name=target_ateom_uid,json=targetAteomUid,proto3" json:"target_ateom_uid,omitempty"` + WorkerPodUid string `protobuf:"bytes,1,opt,name=worker_pod_uid,json=workerPodUid,proto3" json:"worker_pod_uid,omitempty"` Atespace string `protobuf:"bytes,2,opt,name=atespace,proto3" json:"atespace,omitempty"` ActorName string `protobuf:"bytes,3,opt,name=actor_name,json=actorName,proto3" json:"actor_name,omitempty"` ActorUid string `protobuf:"bytes,4,opt,name=actor_uid,json=actorUid,proto3" json:"actor_uid,omitempty"` @@ -808,9 +808,9 @@ func (*TerminateRequest) Descriptor() ([]byte, []int) { return file_atelet_proto_rawDescGZIP(), []int{10} } -func (x *TerminateRequest) GetTargetAteomUid() string { +func (x *TerminateRequest) GetWorkerPodUid() string { if x != nil { - return x.TargetAteomUid + return x.WorkerPodUid } return "" } @@ -895,7 +895,7 @@ func (*TerminateResponse) Descriptor() ([]byte, []int) { type RunRequest struct { state protoimpl.MessageState `protogen:"open.v1"` - TargetAteomUid string `protobuf:"bytes,1,opt,name=target_ateom_uid,json=targetAteomUid,proto3" json:"target_ateom_uid,omitempty"` + WorkerPodUid string `protobuf:"bytes,1,opt,name=worker_pod_uid,json=workerPodUid,proto3" json:"worker_pod_uid,omitempty"` Atespace string `protobuf:"bytes,2,opt,name=atespace,proto3" json:"atespace,omitempty"` ActorName string `protobuf:"bytes,3,opt,name=actor_name,json=actorName,proto3" json:"actor_name,omitempty"` ActorUid string `protobuf:"bytes,4,opt,name=actor_uid,json=actorUid,proto3" json:"actor_uid,omitempty"` @@ -947,9 +947,9 @@ func (*RunRequest) Descriptor() ([]byte, []int) { return file_atelet_proto_rawDescGZIP(), []int{12} } -func (x *RunRequest) GetTargetAteomUid() string { +func (x *RunRequest) GetWorkerPodUid() string { if x != nil { - return x.TargetAteomUid + return x.WorkerPodUid } return "" } @@ -2511,7 +2511,7 @@ func (x *ExternalRestoreConfiguration) GetSnapshotUri() string { type CheckpointRequest struct { state protoimpl.MessageState `protogen:"open.v1"` - TargetAteomUid string `protobuf:"bytes,1,opt,name=target_ateom_uid,json=targetAteomUid,proto3" json:"target_ateom_uid,omitempty"` + WorkerPodUid string `protobuf:"bytes,1,opt,name=worker_pod_uid,json=workerPodUid,proto3" json:"worker_pod_uid,omitempty"` Atespace string `protobuf:"bytes,2,opt,name=atespace,proto3" json:"atespace,omitempty"` ActorName string `protobuf:"bytes,3,opt,name=actor_name,json=actorName,proto3" json:"actor_name,omitempty"` ActorUid string `protobuf:"bytes,4,opt,name=actor_uid,json=actorUid,proto3" json:"actor_uid,omitempty"` @@ -2565,9 +2565,9 @@ func (*CheckpointRequest) Descriptor() ([]byte, []int) { return file_atelet_proto_rawDescGZIP(), []int{39} } -func (x *CheckpointRequest) GetTargetAteomUid() string { +func (x *CheckpointRequest) GetWorkerPodUid() string { if x != nil { - return x.TargetAteomUid + return x.WorkerPodUid } return "" } @@ -2852,7 +2852,7 @@ func (*UploadPausedCheckpointResponse) Descriptor() ([]byte, []int) { type RestoreRequest struct { state protoimpl.MessageState `protogen:"open.v1"` - TargetAteomUid string `protobuf:"bytes,1,opt,name=target_ateom_uid,json=targetAteomUid,proto3" json:"target_ateom_uid,omitempty"` + WorkerPodUid string `protobuf:"bytes,1,opt,name=worker_pod_uid,json=workerPodUid,proto3" json:"worker_pod_uid,omitempty"` Atespace string `protobuf:"bytes,2,opt,name=atespace,proto3" json:"atespace,omitempty"` ActorName string `protobuf:"bytes,3,opt,name=actor_name,json=actorName,proto3" json:"actor_name,omitempty"` ActorUid string `protobuf:"bytes,4,opt,name=actor_uid,json=actorUid,proto3" json:"actor_uid,omitempty"` @@ -2914,9 +2914,9 @@ func (*RestoreRequest) Descriptor() ([]byte, []int) { return file_atelet_proto_rawDescGZIP(), []int{43} } -func (x *RestoreRequest) GetTargetAteomUid() string { +func (x *RestoreRequest) GetWorkerPodUid() string { if x != nil { - return x.TargetAteomUid + return x.WorkerPodUid } return "" } @@ -3119,9 +3119,9 @@ const file_atelet_proto_rawDesc = "" + "\tactor_uid\x18\x05 \x01(\tR\bactorUid\x12>\n" + "\x1bcertificate_signing_request\x18\x01 \x01(\fR\x19certificateSigningRequest\"M\n" + "\x1cMintActorCertificateResponse\x12-\n" + - "\x12actor_certificates\x18\x01 \x03(\fR\x11actorCertificates\"\xa6\x02\n" + - "\x10TerminateRequest\x12(\n" + - "\x10target_ateom_uid\x18\x01 \x01(\tR\x0etargetAteomUid\x12\x1a\n" + + "\x12actor_certificates\x18\x01 \x03(\fR\x11actorCertificates\"\xa2\x02\n" + + "\x10TerminateRequest\x12$\n" + + "\x0eworker_pod_uid\x18\x01 \x01(\tR\fworkerPodUid\x12\x1a\n" + "\batespace\x18\x02 \x01(\tR\batespace\x12\x1d\n" + "\n" + "actor_name\x18\x03 \x01(\tR\tactorName\x12\x1b\n" + @@ -3129,10 +3129,10 @@ const file_atelet_proto_rawDesc = "" + "\x17actor_template_atespace\x18\x05 \x01(\tR\x15actorTemplateAtespace\x12.\n" + "\x13actor_template_name\x18\x06 \x01(\tR\x11actorTemplateName\x12(\n" + "\x04spec\x18\a \x01(\v2\x14.atelet.WorkloadSpecR\x04spec\"\x13\n" + - "\x11TerminateResponse\"\xf4\x03\n" + + "\x11TerminateResponse\"\xf0\x03\n" + "\n" + - "RunRequest\x12(\n" + - "\x10target_ateom_uid\x18\x01 \x01(\tR\x0etargetAteomUid\x12\x1a\n" + + "RunRequest\x12$\n" + + "\x0eworker_pod_uid\x18\x01 \x01(\tR\fworkerPodUid\x12\x1a\n" + "\batespace\x18\x02 \x01(\tR\batespace\x12\x1d\n" + "\n" + "actor_name\x18\x03 \x01(\tR\tactorName\x12\x1b\n" + @@ -3247,9 +3247,9 @@ const file_atelet_proto_rawDesc = "" + "\x1fExternalCheckpointConfiguration\x12!\n" + "\fsnapshot_uri\x18\x01 \x01(\tR\vsnapshotUri\"A\n" + "\x1cExternalRestoreConfiguration\x12!\n" + - "\fsnapshot_uri\x18\x01 \x01(\tR\vsnapshotUri\"\xb2\x04\n" + - "\x11CheckpointRequest\x12(\n" + - "\x10target_ateom_uid\x18\x01 \x01(\tR\x0etargetAteomUid\x12\x1a\n" + + "\fsnapshot_uri\x18\x01 \x01(\tR\vsnapshotUri\"\xae\x04\n" + + "\x11CheckpointRequest\x12$\n" + + "\x0eworker_pod_uid\x18\x01 \x01(\tR\fworkerPodUid\x12\x1a\n" + "\batespace\x18\x02 \x01(\tR\batespace\x12\x1d\n" + "\n" + "actor_name\x18\x03 \x01(\tR\tactorName\x12\x1b\n" + @@ -3274,9 +3274,9 @@ const file_atelet_proto_rawDesc = "" + "\x13local_snapshot_name\x18\x06 \x01(\tR\x11localSnapshotName\x128\n" + "\x18destination_snapshot_uri\x18\a \x01(\tR\x16destinationSnapshotUri\x12C\n" + "\x10desired_fidelity\x18\b \x01(\x0e2\x18.atelet.SnapshotFidelityR\x0fdesiredFidelity\" \n" + - "\x1eUploadPausedCheckpointResponse\"\x80\x06\n" + - "\x0eRestoreRequest\x12(\n" + - "\x10target_ateom_uid\x18\x01 \x01(\tR\x0etargetAteomUid\x12\x1a\n" + + "\x1eUploadPausedCheckpointResponse\"\xfc\x05\n" + + "\x0eRestoreRequest\x12$\n" + + "\x0eworker_pod_uid\x18\x01 \x01(\tR\fworkerPodUid\x12\x1a\n" + "\batespace\x18\x02 \x01(\tR\batespace\x12\x1d\n" + "\n" + "actor_name\x18\x03 \x01(\tR\tactorName\x12\x1b\n" + diff --git a/internal/proto/ateletpb/atelet.proto b/internal/proto/ateletpb/atelet.proto index 5f6f644848..4715da5d57 100644 --- a/internal/proto/ateletpb/atelet.proto +++ b/internal/proto/ateletpb/atelet.proto @@ -203,7 +203,7 @@ service AteomHerder { message TerminateRequest { // If unset, the atelet will just cleanup node resources for the actor. - string target_ateom_uid = 1; + string worker_pod_uid = 1; string atespace = 2; string actor_name = 3; @@ -219,7 +219,7 @@ message TerminateResponse { } message RunRequest { - string target_ateom_uid = 1; + string worker_pod_uid = 1; string atespace = 2; string actor_name = 3; @@ -477,7 +477,7 @@ message ExternalRestoreConfiguration { } message CheckpointRequest { - string target_ateom_uid = 1; + string worker_pod_uid = 1; string atespace = 2; string actor_name = 3; @@ -534,7 +534,7 @@ message UploadPausedCheckpointResponse { } message RestoreRequest { - string target_ateom_uid = 1; + string worker_pod_uid = 1; string atespace = 2; string actor_name = 3; diff --git a/internal/resources/validate.go b/internal/resources/validate.go index 855772d7b9..104f043435 100644 --- a/internal/resources/validate.go +++ b/internal/resources/validate.go @@ -88,13 +88,13 @@ func IsValidResourceName(name string) bool { return len(content.IsDNS1123Label(name)) == 0 } -// ValidateAteomUID rejects a target ateom pod UID that could escape the host +// ValidateWorkerPodUID rejects a worker pod UID that could escape the host // path built from it: the ateom control socket (.../ateoms//ateom.sock). // Kubernetes pod UIDs are UUIDs, which are valid DNS-1123 labels, so a label // check accepts every legitimate value while rejecting separators and "..". -func ValidateAteomUID(targetAteomUID string) error { - if errs := content.IsDNS1123Label(targetAteomUID); len(errs) > 0 { - return fmt.Errorf("invalid target ateom UID %q: %s", targetAteomUID, strings.Join(errs, "; ")) +func ValidateWorkerPodUID(workerPodUID string) error { + if errs := content.IsDNS1123Label(workerPodUID); len(errs) > 0 { + return fmt.Errorf("invalid worker pod UID %q: %s", workerPodUID, strings.Join(errs, "; ")) } return nil } diff --git a/internal/resources/validate_test.go b/internal/resources/validate_test.go index aa14889dfa..cec0e0864a 100644 --- a/internal/resources/validate_test.go +++ b/internal/resources/validate_test.go @@ -101,7 +101,7 @@ func TestIsValidResourceName(t *testing.T) { } } -func TestValidateAteomUID(t *testing.T) { +func TestValidateWorkerPodUID(t *testing.T) { tests := []struct { name string uid string @@ -116,8 +116,8 @@ func TestValidateAteomUID(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if err := ValidateAteomUID(tt.uid); (err != nil) != tt.wantErr { - t.Errorf("ValidateAteomUID(%q) err = %v, wantErr %v", tt.uid, err, tt.wantErr) + if err := ValidateWorkerPodUID(tt.uid); (err != nil) != tt.wantErr { + t.Errorf("ValidateWorkerPodUID(%q) err = %v, wantErr %v", tt.uid, err, tt.wantErr) } }) } From 55b6e00e5585f63c1917eb8012655c9b8fecb8fa Mon Sep 17 00:00:00 2001 From: Julian Gutierrez Oschmann Date: Fri, 9 Oct 2026 20:35:05 +0000 Subject: [PATCH 36/44] Rename `MintActorJWTRequest.audience` to `audiences`. (#2421) The field is repeated (so it should be plural), and it's also not aligned with other messages that call it `audiences`. Fixes #2408 --- cmd/ateapi/internal/apivalidation/actor_test.go | 2 +- .../apivalidation/zz_generated.validation.go | 6 +++--- cmd/ateapi/internal/controlapi/actor.go | 4 ++-- .../internal/controlapi/functionaltest/actor_test.go | 2 +- .../internal/router/egress/actorjwt/actorjwt.go | 2 +- .../internal/router/egress/actorjwt/actorjwt_test.go | 2 +- .../internal/router/egress/credentials_test.go | 2 +- pkg/proto/ateapipb/ateapi.pb.go | 12 ++++++------ pkg/proto/ateapipb/ateapi.proto | 2 +- 9 files changed, 17 insertions(+), 17 deletions(-) diff --git a/cmd/ateapi/internal/apivalidation/actor_test.go b/cmd/ateapi/internal/apivalidation/actor_test.go index 7b2a2cafbd..6b4cfcc627 100644 --- a/cmd/ateapi/internal/apivalidation/actor_test.go +++ b/cmd/ateapi/internal/apivalidation/actor_test.go @@ -1133,7 +1133,7 @@ func TestValidateMintActorJWTRequest(t *testing.T) { withExpiration := func(seconds int64) *ateapipb.MintActorJWTRequest { return &ateapipb.MintActorJWTRequest{ Actor: &ateapipb.ObjectRef{Atespace: "ns1", Name: "id1"}, - Audience: []string{"https://example.com"}, + Audiences: []string{"https://example.com"}, ExpirationSeconds: seconds, } } diff --git a/cmd/ateapi/internal/apivalidation/zz_generated.validation.go b/cmd/ateapi/internal/apivalidation/zz_generated.validation.go index 69c505dd3c..e083f0d18b 100644 --- a/cmd/ateapi/internal/apivalidation/zz_generated.validation.go +++ b/cmd/ateapi/internal/apivalidation/zz_generated.validation.go @@ -5629,7 +5629,7 @@ func Validate_MintActorJWTRequest( errs = append(errs, fn(fldPath.Child("actor"), obj.Actor, oldVal, oldObj != nil)...) } - { // field ateapipb.MintActorJWTRequest.Audience + { // field ateapipb.MintActorJWTRequest.Audiences fn := func( fldPath *field.Path, obj, oldObj []string, @@ -5667,9 +5667,9 @@ func Validate_MintActorJWTRequest( } oldVal := safe.Field(oldObj, func(oldObj *ateapipb.MintActorJWTRequest) []string { - return oldObj.Audience + return oldObj.Audiences }) - errs = append(errs, fn(fldPath.Child("audience"), obj.Audience, oldVal, oldObj != nil)...) + errs = append(errs, fn(fldPath.Child("audiences"), obj.Audiences, oldVal, oldObj != nil)...) } { // field ateapipb.MintActorJWTRequest.ExpirationSeconds diff --git a/cmd/ateapi/internal/controlapi/actor.go b/cmd/ateapi/internal/controlapi/actor.go index 8ced1cb406..c6ca082f38 100644 --- a/cmd/ateapi/internal/controlapi/actor.go +++ b/cmd/ateapi/internal/controlapi/actor.go @@ -562,7 +562,7 @@ func (s *RPCService) MintActorJWT(ctx context.Context, req *ateapipb.MintActorJW } // We only issue tokens with audience bindings. - if len(req.GetAudience()) == 0 { + if len(req.GetAudiences()) == 0 { return nil, fmt.Errorf("at least one audience must be requested") } @@ -573,7 +573,7 @@ func (s *RPCService) MintActorJWT(ctx context.Context, req *ateapipb.MintActorJW actorClaims := &actoridjwt.Claims{ Issuer: s.actorJWTIssuer, Subject: fmt.Sprintf("actor/%s/%s", dbActor.GetMetadata().GetAtespace(), dbActor.GetMetadata().GetName()), - Audiences: req.GetAudience(), + Audiences: req.GetAudiences(), Expiration: expiresAt, NotBefore: now.Add(-5 * time.Minute), IssuedAt: now, diff --git a/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go b/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go index a77716e968..4e56ab6816 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go @@ -5210,7 +5210,7 @@ func TestMintActorJWT_Success(t *testing.T) { Atespace: createResp.GetMetadata().GetAtespace(), Name: createResp.GetMetadata().GetName(), }, - Audience: []string{"foo"}, + Audiences: []string{"foo"}, ExpirationSeconds: 1800, }) if err != nil { diff --git a/cmd/atenet/internal/router/egress/actorjwt/actorjwt.go b/cmd/atenet/internal/router/egress/actorjwt/actorjwt.go index 1115b80520..3c65c0783e 100644 --- a/cmd/atenet/internal/router/egress/actorjwt/actorjwt.go +++ b/cmd/atenet/internal/router/egress/actorjwt/actorjwt.go @@ -50,7 +50,7 @@ func New(client ateapipb.ControlClient) *Minter { func (m *Minter) Token(ctx context.Context, ref resources.ActorRef, src *ateapipb.ActorJWTSource) (string, error) { req := &ateapipb.MintActorJWTRequest{ Actor: ref.ToObjectRef(), - Audience: slices.Sorted(slices.Values(src.GetAudiences())), + Audiences: slices.Sorted(slices.Values(src.GetAudiences())), ExpirationSeconds: src.GetExpirationSeconds(), } key, err := tokenKey(req) diff --git a/cmd/atenet/internal/router/egress/actorjwt/actorjwt_test.go b/cmd/atenet/internal/router/egress/actorjwt/actorjwt_test.go index 7fa35d5fe1..71a79dfc73 100644 --- a/cmd/atenet/internal/router/egress/actorjwt/actorjwt_test.go +++ b/cmd/atenet/internal/router/egress/actorjwt/actorjwt_test.go @@ -114,7 +114,7 @@ func TestReuseTokenUntilAThirdOfItsLifetimeRemains(t *testing.T) { wantToken(t, m, testActor, src, "jwt-1") want := &ateapipb.MintActorJWTRequest{ Actor: &ateapipb.ObjectRef{Atespace: "default", Name: "my-actor"}, - Audience: []string{"https://a.example", "https://b.example"}, + Audiences: []string{"https://a.example", "https://b.example"}, ExpirationSeconds: 900, } if got := ctl.last.Load(); !proto.Equal(got, want) { diff --git a/cmd/atenet/internal/router/egress/credentials_test.go b/cmd/atenet/internal/router/egress/credentials_test.go index 4cc9656411..49630191c9 100644 --- a/cmd/atenet/internal/router/egress/credentials_test.go +++ b/cmd/atenet/internal/router/egress/credentials_test.go @@ -310,7 +310,7 @@ func TestActorJWTInjection(t *testing.T) { } want := &ateapipb.MintActorJWTRequest{ Actor: &ateapipb.ObjectRef{Atespace: testEgressAtespace, Name: testEgressActor}, - Audience: []string{"https://api.example.com"}, + Audiences: []string{"https://api.example.com"}, ExpirationSeconds: 900, } if got := client.lastMint.Load(); !proto.Equal(got, want) { diff --git a/pkg/proto/ateapipb/ateapi.pb.go b/pkg/proto/ateapipb/ateapi.pb.go index 02e7410462..7e063939be 100644 --- a/pkg/proto/ateapipb/ateapi.pb.go +++ b/pkg/proto/ateapipb/ateapi.pb.go @@ -5671,7 +5671,7 @@ type MintActorJWTRequest struct { // +k8s:maxItems=16 # guardrail; tokens realistically bind a handful of audiences // +k8s:listType=set // +k8s:eachVal=+k8s:maxLength=512 # audiences are caller-defined URIs; bound only - Audience []string `protobuf:"bytes,1,rep,name=audience,proto3" json:"audience,omitempty"` + Audiences []string `protobuf:"bytes,1,rep,name=audiences,proto3" json:"audiences,omitempty"` // Lifetime of the JWT, in seconds. // // +k8s:required @@ -5719,9 +5719,9 @@ func (x *MintActorJWTRequest) GetActor() *ObjectRef { return nil } -func (x *MintActorJWTRequest) GetAudience() []string { +func (x *MintActorJWTRequest) GetAudiences() []string { if x != nil { - return x.Audience + return x.Audiences } return nil } @@ -8599,10 +8599,10 @@ const file_ateapi_proto_rawDesc = "" + "\x05actor\x18\x01 \x01(\v2\x11.ateapi.ObjectRefR\x05actor\x12/\n" + "\aoptions\x18\x02 \x01(\v2\x15.ateapi.DeleteOptionsR\aoptions\"4\n" + "\rGetTagRequest\x12#\n" + - "\x03tag\x18\x01 \x01(\v2\x11.ateapi.ObjectRefR\x03tag\"\x89\x01\n" + + "\x03tag\x18\x01 \x01(\v2\x11.ateapi.ObjectRefR\x03tag\"\x8b\x01\n" + "\x13MintActorJWTRequest\x12'\n" + - "\x05actor\x18\x05 \x01(\v2\x11.ateapi.ObjectRefR\x05actor\x12\x1a\n" + - "\baudience\x18\x01 \x03(\tR\baudience\x12-\n" + + "\x05actor\x18\x05 \x01(\v2\x11.ateapi.ObjectRefR\x05actor\x12\x1c\n" + + "\taudiences\x18\x01 \x03(\tR\taudiences\x12-\n" + "\x12expiration_seconds\x18\b \x01(\x03R\x11expirationSeconds\"s\n" + "\x14MintActorJWTResponse\x12 \n" + "\tactor_jwt\x18\x01 \x01(\tB\x03\x80\x01\x01R\bactorJwt\x129\n" + diff --git a/pkg/proto/ateapipb/ateapi.proto b/pkg/proto/ateapipb/ateapi.proto index ca0195c8d7..4541749380 100644 --- a/pkg/proto/ateapipb/ateapi.proto +++ b/pkg/proto/ateapipb/ateapi.proto @@ -1784,7 +1784,7 @@ message MintActorJWTRequest { // +k8s:maxItems=16 # guardrail; tokens realistically bind a handful of audiences // +k8s:listType=set // +k8s:eachVal=+k8s:maxLength=512 # audiences are caller-defined URIs; bound only - repeated string audience = 1; + repeated string audiences = 1; // Lifetime of the JWT, in seconds. // From ea273bdabc04ebb9060ed486148199b2922c43b6 Mon Sep 17 00:00:00 2001 From: Taahir Ahmed Date: Fri, 9 Oct 2026 20:35:20 +0000 Subject: [PATCH 37/44] Remove makefile (#2374) Our entry points for building images and installing are the scripts under hack/ and cmd/ate-setup. Removing the Makefile cleans up a dev entrypoint we are no longer using. --- .github/workflows/pr-workflow.yaml | 10 +- .gitignore | 4 +- AGENTS.md | 23 ++- Makefile | 185 ------------------ benchmarking/automation/README.md | 2 +- cmd/ate-setup/README.md | 2 +- cmd/ate-setup/cli-diff.md | 2 +- cmd/ate-setup/commands.md | 16 +- cmd/ate-setup/differences.md | 4 +- cmd/ate-setup/internal/cmd/publish.go | 17 ++ cmd/ate-setup/internal/images/dockerfile.go | 61 +++++- .../internal/images/dockerfile_test.go | 46 +++++ cmd/ate-setup/internal/images/images.go | 3 +- cmd/ate-setup/internal/images/images_test.go | 33 ---- cmd/ate-setup/internal/ko/ko.go | 32 ++- cmd/ate-setup/internal/ko/ko_test.go | 59 +++++- cmd/ate-setup/internal/steps/env.go | 6 +- cmd/ate-setup/internal/steps/overlay_test.go | 4 +- cmd/ate-setup/internal/steps/publish.go | 44 +++++ cmd/ate-setup/internal/steps/publish_test.go | 12 ++ cmd/dataplane/envoy/README.md | 4 +- docs/observability.md | 4 +- internal/e2e/suites/security/README.md | 6 +- internal/version/version.go | 1 + 24 files changed, 299 insertions(+), 281 deletions(-) delete mode 100644 Makefile diff --git a/.github/workflows/pr-workflow.yaml b/.github/workflows/pr-workflow.yaml index 199c874c60..da4889330b 100644 --- a/.github/workflows/pr-workflow.yaml +++ b/.github/workflows/pr-workflow.yaml @@ -97,7 +97,7 @@ jobs: run: | sudo apt-get update -qq sudo apt-get install -y -qq --no-install-recommends clang libclang-dev - make test-dynamic-modules + hack/test-dynamic-modules.sh # Root-gated tests (overlay mounts, whiteout mknod, trusted.* xattrs, ...) # skip for the unprivileged runner user above; rerun the packages that # contain them (any test importing internal/roottest) under sudo. @@ -109,9 +109,7 @@ jobs: # file that was never written fails too. - name: Require tests to have run if: always() - run: | - make build-junittool - bin/junittool verify -manifest "${ARTIFACTS}/expected-junit.txt" + run: go -C tools/junittool run . verify -manifest "${ARTIFACTS}/expected-junit.txt" - name: Upload test results if: always() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 @@ -278,9 +276,7 @@ jobs: # run-e2e.sh refuses to run in CI without E2E_JUNIT_FILE. - name: Require every lane to have run tests if: always() - run: | - make build-junittool - bin/junittool verify -manifest "${ARTIFACTS}/expected-junit.txt" + run: go -C tools/junittool run . verify -manifest "${ARTIFACTS}/expected-junit.txt" - name: Save Go build cache # Main only, so PRs restore the latest main build rather than each # storing its own copy. diff --git a/.gitignore b/.gitignore index d680cfcc50..50656545db 100644 --- a/.gitignore +++ b/.gitignore @@ -60,8 +60,8 @@ Thumbs.db /.worktrees/ # Stray outputs from `go build` without -o. Root-module tools land at the repo -# root; tools with their own go.mod land beside their source. Use -# `make build-`, which writes to /bin/ above. +# root; tools with their own go.mod land beside their source. Build with +# `-o bin/`, which is ignored above. /validate-image-cache /setup-gcp /tools/apitool/apitool diff --git a/AGENTS.md b/AGENTS.md index 7addcfdb4e..010fb62b70 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -40,22 +40,21 @@ See `docs/dev/code-layout.md` for the full rationale and per-directory details. ## Build and Test Commands -Agent Substrate uses a `Makefile` for its build and test tasks. +The dev entry points are the scripts under `hack/` and the `cmd/ate-setup` installer. -### Building -- **Binaries**: `make build` (builds images and `kubectl-ate`) or `make build-atectl` -- **Images**: `make build-images` (uses ko to build container images) -- **Demos**: `make build-demos` -- **Release images**: `make build-release-images KO_DOCKER_REPO=REPO VERSION=TAG` (every image a pre-built install needs, all tagged `TAG`, including the docker-built `envoy-dataplane`; `make build-envoy-dataplane` builds only that one) +### Building and Installing +- **Install from source**: `go run ./cmd/ate-setup deploy ate-system` (or the `hack/install-ate.sh` shim) builds every image with ko as part of the install; see `cmd/ate-setup/README.md` +- **Binaries**: `go build -o bin/kubectl-ate ./cmd/kubectl-ate`, `go build -o bin/ate-setup ./cmd/ate-setup` +- **Release images**: `VERSION=TAG go run ./cmd/ate-setup publish release-images --ko-docker-repo REPO` (every image a pre-built install needs, all tagged `TAG`, including the docker-built `envoy-dataplane`) ### Testing and Verification -- **Run Unit Tests**: `make test` -- **Run E2E Tests**: `make e2e` (Requires GCP cluster setup and built images) -- **Run Linters and Verifiers**: `make verify` (Includes `go vet` and checks for formatting, boilerplate headers, licenses, and go modules) +- **Run Unit Tests**: `go test -race ./...` +- **Run E2E Tests**: `hack/run-e2e.sh` (Requires a cluster with Agent Substrate installed; `hack/run-e2e-kind.sh` for kind) +- **Run Linters and Verifiers**: `hack/verify-all.sh` (formatting, golangci-lint, boilerplate headers, licenses, go modules, generated code, and the metrics registry) ## Code Style Guidelines -- **Go Formatting**: Code must be formatted with `gofmt`. Run `make fmt` to automatically format all files before submitting changes. +- **Go Formatting**: Code must be formatted with `gofmt`. Run `hack/update/gofmt.sh` to automatically format all files before submitting changes. - **Copyright Headers**: All files must contain appropriate copyright and license headers. See templates in `hack/boilerplate/`. - **Modularity**: Submit small, focused Pull Requests that touch a limited part of the codebase for easier reviews and rebasing. - **Go Modules**: Ensure `go.mod` is clean. Run `go mod tidy` if adding or removing dependencies. @@ -79,7 +78,7 @@ Do not add `reserved` statements to protos, deprecated aliases, fallbacks for ol `docs/metrics/registry/` is an [OpenTelemetry Weaver](https://github.com/open-telemetry/weaver) registry. `metrics.yaml` defines every metric instrument the ate system components emit, plus every attribute any signal uses and the permitted values of each. `events.yaml` defines the log events emitted over OTLP. Read them to find an instrument, an event, or their attributes. -If you add or rename an instrument or an event, or add an attribute, follow `docs/dev/best-practices/metrics.md`, update the registry, and run `hack/verify/metrics.sh`. `make verify` runs the same check. Weaver reads the whole directory, so a new file in it is checked straight away. +If you add or rename an instrument or an event, or add an attribute, follow `docs/dev/best-practices/metrics.md`, update the registry, and run `hack/verify/metrics.sh`. `hack/verify-all.sh` runs the same check. Weaver reads the whole directory, so a new file in it is checked straight away. Some attributes are defined in the registry but barred from metric labels: actor identity is the main one. The `note` on each says so, and `cardinality_rules` in `docs/metrics/substrate.yaml` is where the bar lives. @@ -91,7 +90,7 @@ See the [metric registry](docs/observability.md#the-metric-registry) section of 1. Write tests for all new code. We will not merge code that lacks tests. 2. Ensure changes do not break existing tests. -3. Run `make verify` locally before requesting a code review to catch common issues like missed copyright headers or formatting drift. +3. Run `go test -race ./...` and `hack/verify-all.sh` locally before requesting a code review to catch common issues like missed copyright headers or formatting drift. 4. For end-to-end tests involving the actual infrastructure, ensure you have a running cluster (setup via `hack/ate-dev-env.sh.example` and `go run ./tools/setup-gcp bootstrap`). 5. A test that skips when a precondition is missing — Docker, a cluster, an artifact directory — must **fail** on it in CI, because a skip and a pass are the same exit code. Resolve strictness through a named predicate rather than an inline `os.Getenv("CI")`; `cmd/ateapi/internal/store/dockerenv.Required()` is the reference implementation. Prove the strict branch red before merging: a guard only ever observed passing is not known to guard anything. See `docs/dev/best-practices/ci-fail-closed.md`. diff --git a/Makefile b/Makefile deleted file mode 100644 index f01760e765..0000000000 --- a/Makefile +++ /dev/null @@ -1,185 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Default project ID, can be overridden -PROJECT_ID ?= $(shell echo $${USER}-gke-dev) - -# Ko configuration -export KO_DOCKER_REPO := gcr.io/$(PROJECT_ID)/ate-images - -# Go commands -GO := go -KO := hack/run-tool.sh ko - -# Flags every ko image build gets, e.g. `make build-images KO_FLAGS=--push=false`. -# Empty by default, so ko runs on its own defaults and whatever .ko.yaml configures. -KO_FLAGS ?= - -# Image naming, kept out of KO_FLAGS so that overriding those does not drop it. -# Each image is published as /, which is -# the name `ate-setup deploy --image-repo` looks for. ko's default appends an -# md5 of the full import path instead, which nothing outside ko can predict. -# cmd/ate-setup/internal/ko passes the same flag. -KO_NAMING := --base-import-paths - -# Image tags, kept out of KO_FLAGS for the same reason. Empty by default, so ko -# tags `latest`; build-release-images sets it to $(VERSION). -KO_TAGS := - -# Binaries -BINDIR := bin/ -ATECTL := $(BINDIR)/kubectl-ate -ATESETUP := $(BINDIR)/ate-setup - -# Version stamping. Override on the make command line to pin -# (e.g. `make VERSION=v0.5.0 build`). -VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo dev) -VERSION_PKG := github.com/agent-substrate/substrate/internal/version -LDFLAGS := -X=$(VERSION_PKG).Version=$(VERSION) - -# Every image the installer can deploy, defined once. These two sets together -# have to cover images.Components in cmd/ate-setup/internal/images: a package -# missing here has no image for a build from source, and one missing there has -# none for an install from a release. envoy-dataplane is built from a -# Dockerfile, not with ko, so it is in neither; see build-envoy-dataplane. -CONTROL_PLANE_IMAGES := ./cmd/ateapi \ - ./cmd/atecontroller \ - ./cmd/atelet \ - ./cmd/atenet \ - ./cmd/credential-provider/kubernetes-secrets \ - ./cmd/podcertcontroller \ - ./cmd/snapshot-plugin -WORKER_IMAGES := ./cmd/ateom-gvisor \ - ./cmd/ateom-microvm -DEMO_IMAGES := ./demos/counter \ - ./demos/egress \ - ./demos/multi-template/fspersist \ - ./demos/sandbox -ALL_IMAGES := $(CONTROL_PLANE_IMAGES) $(WORKER_IMAGES) - -# Developer builds may leave components out, e.g. the microvm image, the one -# image built from a debian base rather than distroless static: -# make build-images SKIP_IMAGES=./cmd/ateom-microvm -# Overriding IMAGES or DEMOS on the command line builds exactly that set. -SKIP_IMAGES ?= -IMAGES := $(filter-out $(SKIP_IMAGES),$(ALL_IMAGES)) -DEMOS := $(filter-out $(SKIP_IMAGES),$(DEMO_IMAGES)) - -# Images ko cannot build: envoy-dataplane is Envoy plus a Rust dynamic module. -# It is pushed as $(KO_DOCKER_REPO)/envoy-dataplane:$(VERSION), beside the ko -# images, where `ate-setup deploy --image-repo --image-tag` looks for it. A -# build from source doesn't need this; ate-setup builds the image itself. -# -# Platforms default to KO_DEFAULTPLATFORMS, else linux/amd64, as in ate-setup. -# Another architecture compiles Rust under QEMU, which the build host must have -# registered with binfmt. Extra buildx flags go in DOCKER_BUILD_FLAGS. -DOCKERFILE_PLATFORMS ?= $(or $(KO_DEFAULTPLATFORMS),linux/amd64) -DOCKER_BUILD_FLAGS ?= - -.PHONY: all -all: build - -.PHONY: build -build: build-images build-atectl build-ate-setup - -.PHONY: build-images -build-images: - $(KO) build $(KO_NAMING) $(KO_TAGS) $(KO_FLAGS) \ - --ldflags="$(LDFLAGS)" \ - $(IMAGES) - -.PHONY: build-atectl -build-atectl: - $(GO) build -ldflags "$(LDFLAGS)" -o $(ATECTL) ./cmd/kubectl-ate - -# The cluster installer. hack/install-ate.sh is a shim over it; see -# cmd/ate-setup/commands.md for the flag-by-flag mapping between the two. -.PHONY: build-ate-setup -build-ate-setup: - $(GO) build -ldflags "$(LDFLAGS)" -o $(ATESETUP) ./cmd/ate-setup - -.PHONY: build-atenet -build-atenet: - $(GO) build -ldflags "$(LDFLAGS)" -o $(BINDIR)/atenet ./cmd/atenet - -# Registers the JUnit files a CI run will write, and verifies they hold tests. -# Separate module, so it builds by path. -o keeps the binary in $(BINDIR), -# which is gitignored and removed by `clean`. -.PHONY: build-junittool -build-junittool: - $(GO) -C tools/junittool build -o $(CURDIR)/$(BINDIR)/junittool . - -.PHONY: build-demos -build-demos: - $(KO) build $(KO_NAMING) $(KO_TAGS) $(KO_FLAGS) \ - --ldflags="$(LDFLAGS)" \ - $(DEMOS) - -.PHONY: build-envoy-dataplane -build-envoy-dataplane: - docker buildx build --push $(DOCKER_BUILD_FLAGS) \ - --platform=$(DOCKERFILE_PLATFORMS) \ - -t $(KO_DOCKER_REPO)/envoy-dataplane:$(VERSION) \ - cmd/dataplane/envoy - -# Every image a pre-built install needs, all tagged $(VERSION), which is what -# `ate-setup deploy --image-repo $(KO_DOCKER_REPO) --image-tag $(VERSION)` -# installs. Stage a release with e.g. -# make build-release-images KO_DOCKER_REPO=REPO VERSION=TAG -.PHONY: build-release-images -build-release-images: KO_TAGS = --tags=$(VERSION) -build-release-images: build-images build-demos build-envoy-dataplane - -.PHONY: test -test: - $(GO) test -race ./... - -# The Envoy dynamic modules are Rust. CI runs this target. -.PHONY: test-dynamic-modules -test-dynamic-modules: - hack/test-dynamic-modules.sh - -.PHONY: e2e -e2e: build build-demos - hack/run-e2e.sh - -.PHONY: fmt verify-fmt - -# Prints the Go ldflags (used for scripts to do version stamping). -ldflags: - @for flag in $(LDFLAGS); do \ - echo $$flag; \ - done - -# Formats all Go files in the project -fmt: - @./hack/update/gofmt.sh - -# Fails if any Go files are not formatted properly -verify-fmt: - @./hack/verify/gofmt.sh - -.PHONY: lint - -# Runs golangci-lint and fails on any reported issues. -lint: - @./hack/verify/golangci-lint.sh - -.PHONY: verify -verify: test - bash hack/verify-all.sh - -.PHONY: clean -clean: - rm -rf $(BINDIR) diff --git a/benchmarking/automation/README.md b/benchmarking/automation/README.md index 029c550cdc..a8f8aa6003 100644 --- a/benchmarking/automation/README.md +++ b/benchmarking/automation/README.md @@ -23,7 +23,7 @@ the router capacity benchmark — see `ateArgs`) + `benchmarking/workloads/deploy.sh --deploy --sandbox-class ` (these build & push substrate / workload images via `ko` as part of their deploy steps — there's no separate - `make build-images` step). For a `microvm` test the orchestrator also + image build step). For a `microvm` test the orchestrator also runs `hack/install-microvm-deps.sh --install` between the two, which stages kata + cloud-hypervisor + virtiofsd assets to the cluster's object store bucket and applies the cluster-wide `microvm` SandboxConfig. diff --git a/cmd/ate-setup/README.md b/cmd/ate-setup/README.md index 72316e675e..9d99f27b31 100644 --- a/cmd/ate-setup/README.md +++ b/cmd/ate-setup/README.md @@ -4,7 +4,7 @@ Installs and tears down Agent Substrate on a Kubernetes cluster. ``` go run ./cmd/ate-setup [global flags] [flags] -make build-ate-setup # builds bin/ate-setup +go build -o bin/ate-setup ./cmd/ate-setup ``` `ate-setup` is the installer. `hack/install-ate.sh` is a shim over it, kept so diff --git a/cmd/ate-setup/cli-diff.md b/cmd/ate-setup/cli-diff.md index 69eeffe46d..da775db851 100644 --- a/cmd/ate-setup/cli-diff.md +++ b/cmd/ate-setup/cli-diff.md @@ -41,7 +41,7 @@ no `KUBECTL_CONTEXT`, a `gcloud container clusters get-credentials` per action rather than one for the whole line. `go` is therefore required to run the installer. In exchange, `kubectl`, `jq`, -`openssl`, `sed`, `base64`, and `make` are not; see the external-binaries table +`openssl`, `sed`, and `base64` are not; see the external-binaries table in [`differences.md`](differences.md#external-binaries) for what is still needed. diff --git a/cmd/ate-setup/commands.md b/cmd/ate-setup/commands.md index d6a05c830e..5131b60c4c 100644 --- a/cmd/ate-setup/commands.md +++ b/cmd/ate-setup/commands.md @@ -63,9 +63,9 @@ toolchain, and write access to a registry. `REPO` has to hold every component image the manifests reference, all under the same tag, which is how a release publishes them. A release that adds a component has to publish it alongside the others before a pre-built install can use it. -`make build-release-images KO_DOCKER_REPO=REPO VERSION=TAG` publishes the full -set, including `envoy-dataplane`, which is built from a Dockerfile with `docker -buildx` rather than with `ko`. A build from source builds that image itself, so +`VERSION=TAG ate-setup publish release-images --ko-docker-repo REPO` publishes +the full set, including `envoy-dataplane`, which is built from a Dockerfile with +`docker buildx` rather than with `ko`. A build from source builds that image itself, so it needs `docker` as well as `ko`. Each reference is then pinned to the digest its tag names, which takes one HEAD request per image, so the installer needs read access to `REPO` and not only the @@ -105,9 +105,15 @@ a running cluster. | `ate-setup` | `hack/install-ate.sh` | |---|---| | `publish worker-images` | (no shell equivalent) | +| `publish release-images` | (no shell equivalent) | -Builds and pushes the ateom worker images for the checked-out build and prints -their refs; a WorkerPool points `spec.workerImage` to a build to use the ateom. +`publish worker-images` builds and pushes the ateom worker images for the +checked-out build and prints their refs; a WorkerPool points `spec.workerImage` +to a build to use the ateom. + +`publish release-images` builds and pushes every image a pre-built install +needs, all tagged with the build version; see [Installing a +release](#installing-a-release). ## Delete diff --git a/cmd/ate-setup/differences.md b/cmd/ate-setup/differences.md index 07104b6d6d..6e92de84d5 100644 --- a/cmd/ate-setup/differences.md +++ b/cmd/ate-setup/differences.md @@ -43,9 +43,9 @@ These were treated as contracts and reproduced exactly: - **Rendered bytes.** `authentication.yaml` is trimmed of its trailing newline because the shell built it inside `$(...)`, which strips them. Installing over a shell-installed cluster must not rewrite the ConfigMap. -- **ko's ldflags.** `make ldflags` emitted `-X=.Version=$(git +- **ko's ldflags.** The shell scripts stamped `-X=.Version=$(git describe --tags --always --dirty)`; `ko.Runner.ldflags` computes the identical - string without depending on make. + string. ## Execution model diff --git a/cmd/ate-setup/internal/cmd/publish.go b/cmd/ate-setup/internal/cmd/publish.go index 96525ace90..48d7718440 100644 --- a/cmd/ate-setup/internal/cmd/publish.go +++ b/cmd/ate-setup/internal/cmd/publish.go @@ -38,7 +38,24 @@ ateom ref.`, }, } +var publishReleaseImagesCmd = &cobra.Command{ + Use: "release-images", + Short: "Build and push every image a pre-built install needs, tagged with the build version", + Long: `Build and push every component image, including the Dockerfile-built +envoy-dataplane, to KO_DOCKER_REPO, all tagged with the build version (VERSION, +else git describe), and print their pushed references. + +The result is what "deploy --image-repo REPO --image-tag TAG" installs: + + VERSION=TAG ate-setup publish release-images --ko-docker-repo REPO`, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + return env.PublishReleaseImages(cmd.Context(), cmd.OutOrStdout()) + }, +} + func init() { rootCmd.AddCommand(publishCmd) publishCmd.AddCommand(publishWorkerImagesCmd) + publishCmd.AddCommand(publishReleaseImagesCmd) } diff --git a/cmd/ate-setup/internal/images/dockerfile.go b/cmd/ate-setup/internal/images/dockerfile.go index b59c933568..5b0549d126 100644 --- a/cmd/ate-setup/internal/images/dockerfile.go +++ b/cmd/ate-setup/internal/images/dockerfile.go @@ -18,21 +18,54 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" + "io/fs" "os" "os/exec" + "path/filepath" "strings" "time" + + "sigs.k8s.io/yaml" ) -// dockerfilePlatforms is the buildx --platform value: the install's -// KO_DEFAULTPLATFORMS, so the image lands on the same nodes as the ko images, -// or linux/amd64 when unset. -func dockerfilePlatforms(koDefaultPlatforms string) string { - if koDefaultPlatforms == "" { - return "linux/amd64" +// dockerfilePlatforms is the buildx --platform value, chosen the way ko +// chooses its own so the image runs on the same nodes as the ko images: +// KO_DEFAULTPLATFORMS, else defaultPlatforms from the ko config, else +// linux/amd64. +func dockerfilePlatforms(rootDir, koDefaultPlatforms string) (string, error) { + if koDefaultPlatforms != "" { + return koDefaultPlatforms, nil + } + path := filepath.Join(rootDir, ".ko.yaml") + // ko runs from rootDir, so a relative KO_CONFIG_PATH is relative to it. + if p := os.Getenv("KO_CONFIG_PATH"); p != "" { + if !filepath.IsAbs(p) { + p = filepath.Join(rootDir, p) + } + path = p + if info, err := os.Stat(p); err == nil && info.IsDir() { + path = filepath.Join(p, ".ko.yaml") + } + } + raw, err := os.ReadFile(path) + if errors.Is(err, fs.ErrNotExist) { + return "linux/amd64", nil + } + if err != nil { + return "", fmt.Errorf("while reading the ko config: %w", err) + } + var koConfig struct { + DefaultPlatforms []string `json:"defaultPlatforms"` } - return koDefaultPlatforms + if err := yaml.Unmarshal(raw, &koConfig); err != nil { + return "", fmt.Errorf("while parsing %s: %w", path, err) + } + if len(koConfig.DefaultPlatforms) == 0 { + return "linux/amd64", nil + } + return strings.Join(koConfig.DefaultPlatforms, ","), nil } // dockerBuildArgs returns the docker arguments that build contextPath for @@ -51,11 +84,21 @@ func dockerBuildArgs(platforms string, extraFlags []string, tag, contextPath str // push to; the returned reference always uses the digest, so a stale tag can // never be resolved by accident. func BuildDockerfileImage(ctx context.Context, rootDir, dockerRepo, imageName, contextPath, koDefaultPlatforms string, extraFlags []string) (string, error) { + return PublishDockerfileImage(ctx, rootDir, dockerRepo, imageName, fmt.Sprintf("build-%d", time.Now().Unix()), contextPath, koDefaultPlatforms, extraFlags) +} + +// PublishDockerfileImage is BuildDockerfileImage with the tag chosen by the +// caller, for a release that publishes every image under one tag. +func PublishDockerfileImage(ctx context.Context, rootDir, dockerRepo, imageName, tag, contextPath, koDefaultPlatforms string, extraFlags []string) (string, error) { repo := strings.TrimSuffix(dockerRepo, "/") + "/" + imageName - stageTag := fmt.Sprintf("%s:build-%d", repo, time.Now().Unix()) + stageTag := repo + ":" + tag + platforms, err := dockerfilePlatforms(rootDir, koDefaultPlatforms) + if err != nil { + return "", err + } build := exec.CommandContext(ctx, "docker", - dockerBuildArgs(dockerfilePlatforms(koDefaultPlatforms), extraFlags, stageTag, contextPath)...) + dockerBuildArgs(platforms, extraFlags, stageTag, contextPath)...) build.Dir = rootDir // The shell version sent build output to stderr so it could capture the // image reference on stdout; keeping that split makes the two behave the diff --git a/cmd/ate-setup/internal/images/dockerfile_test.go b/cmd/ate-setup/internal/images/dockerfile_test.go index c018c3dac7..97fea6f08f 100644 --- a/cmd/ate-setup/internal/images/dockerfile_test.go +++ b/cmd/ate-setup/internal/images/dockerfile_test.go @@ -15,6 +15,9 @@ package images import ( + "cmp" + "os" + "path/filepath" "slices" "testing" ) @@ -31,3 +34,46 @@ func TestDockerBuildArgs(t *testing.T) { t.Errorf("dockerBuildArgs() without flags = %q, want %q", got, want) } } + +// ko builds for .ko.yaml's defaultPlatforms unless KO_DEFAULTPLATFORMS says +// otherwise, and a Dockerfile image installed beside the ko images has to +// cover the same platforms or it cannot run on some of their nodes. +func TestDockerfilePlatforms(t *testing.T) { + const twoPlatforms = "defaultPlatforms:\n - linux/amd64\n - linux/arm64\n" + for _, tc := range []struct { + name string + koYAML string // empty: no ko config + koYAMLFile string // where it is written, relative to the root; default .ko.yaml + koConfigPath string + koPlatforms string + want string + }{ + {name: "explicit platforms win", koYAML: twoPlatforms, koPlatforms: "linux/s390x", want: "linux/s390x"}, + {name: "ko config", koYAML: twoPlatforms, want: "linux/amd64,linux/arm64"}, + {name: "ko config without platforms", koYAML: "defaultBaseImage: example.com/base\n", want: "linux/amd64"}, + {name: "no ko config", want: "linux/amd64"}, + {name: "KO_CONFIG_PATH file", koYAML: twoPlatforms, koYAMLFile: "conf/ko.yaml", koConfigPath: "conf/ko.yaml", want: "linux/amd64,linux/arm64"}, + {name: "KO_CONFIG_PATH directory", koYAML: twoPlatforms, koYAMLFile: "conf/.ko.yaml", koConfigPath: "conf", want: "linux/amd64,linux/arm64"}, + } { + t.Run(tc.name, func(t *testing.T) { + root := t.TempDir() + t.Setenv("KO_CONFIG_PATH", tc.koConfigPath) + if tc.koYAML != "" { + file := filepath.Join(root, cmp.Or(tc.koYAMLFile, ".ko.yaml")) + if err := os.MkdirAll(filepath.Dir(file), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(file, []byte(tc.koYAML), 0o644); err != nil { + t.Fatal(err) + } + } + got, err := dockerfilePlatforms(root, tc.koPlatforms) + if err != nil { + t.Fatalf("dockerfilePlatforms() error = %v", err) + } + if got != tc.want { + t.Errorf("dockerfilePlatforms() = %q, want %q", got, tc.want) + } + }) + } +} diff --git a/cmd/ate-setup/internal/images/images.go b/cmd/ate-setup/internal/images/images.go index f106359947..dd4e97f6eb 100644 --- a/cmd/ate-setup/internal/images/images.go +++ b/cmd/ate-setup/internal/images/images.go @@ -69,8 +69,7 @@ func KoReference(pkg string) string { return "ko://" + ModulePath + "/" + pkg } // well, which changes nothing while every package here already is. // // This only describes reality because every ko invocation asks for that -// naming — internal/ko passes the flag, and the Makefile passes it as -// KO_NAMING. ko's own default appends an md5 of the full import path, which +// naming: internal/ko passes the flag. ko's own default appends an md5 of the full import path, which // nothing here can reproduce, so a build that loses the flag publishes images // no --image-repo install can find. func ImageName(pkg string) string { return path.Base(pkg) } diff --git a/cmd/ate-setup/internal/images/images_test.go b/cmd/ate-setup/internal/images/images_test.go index 9006a03191..523e01950f 100644 --- a/cmd/ate-setup/internal/images/images_test.go +++ b/cmd/ate-setup/internal/images/images_test.go @@ -134,36 +134,3 @@ func TestImageNamesAreUnique(t *testing.T) { byName[name] = pkg } } - -// ImageName is only right if the images were published under that naming, and -// the Makefile is what publishes them. Losing the flag there breaks nothing -// visible -- an install from source never looks an image up by name -- and -// surfaces much later as an --image-repo install that cannot find a single -// component. -func TestMakefilePublishesWithBaseImportPaths(t *testing.T) { - root, err := config.RepoRoot() - if err != nil { - t.Fatalf("resolving repo root: %v", err) - } - makefile, err := os.ReadFile(filepath.Join(root, "Makefile")) - if err != nil { - t.Fatalf("reading the Makefile: %v", err) - } - - var koBuilds int - for line := range strings.Lines(string(makefile)) { - if !strings.Contains(line, "$(KO) build") { - continue - } - koBuilds++ - if !strings.Contains(line, "$(KO_NAMING)") { - t.Errorf("Makefile line %q builds images without $(KO_NAMING), so they publish under ko's md5 naming", strings.TrimSpace(line)) - } - } - if koBuilds == 0 { - t.Error("found no `$(KO) build` line in the Makefile; this test no longer checks anything") - } - if !strings.Contains(string(makefile), "KO_NAMING := --base-import-paths") { - t.Error("the Makefile no longer defines KO_NAMING as --base-import-paths") - } -} diff --git a/cmd/ate-setup/internal/ko/ko.go b/cmd/ate-setup/internal/ko/ko.go index a176226340..44d8d82852 100644 --- a/cmd/ate-setup/internal/ko/ko.go +++ b/cmd/ate-setup/internal/ko/ko.go @@ -37,7 +37,6 @@ import ( ) // versionPkg is the package whose Version variable receives the build stamp. -// It matches VERSION_PKG in the Makefile. const versionPkg = "github.com/agent-substrate/substrate/internal/version" // baseImportPaths publishes each image as the last element of its import path, @@ -46,8 +45,7 @@ const versionPkg = "github.com/agent-substrate/substrate/internal/version" // ko's default instead appends an md5 of the full import path, which nothing // outside ko can predict. images.ImageName is the other half of this: it is // how `--image-repo` addresses a published image, and it can only be right if -// the images were published under this naming. The Makefile passes the same -// flag, as KO_NAMING. +// the images were published under this naming. const baseImportPaths = "--base-import-paths" // Runner invokes ko with a fixed repository root and environment. @@ -139,6 +137,26 @@ func (r *Runner) Build(ctx context.Context, importPath string) (string, error) { return lines[len(lines)-1], nil } +// BuildTagged builds and publishes the images for importPaths in one ko +// invocation, tagging each with tag, and returns their pushed references. +func (r *Runner) BuildTagged(ctx context.Context, tag string, importPaths ...string) ([]string, error) { + defer log.Elapsed(time.Now(), "ko build --tags="+tag) + cmd := exec.CommandContext(ctx, r.binary, append(r.args("build", importPaths...), "--tags="+tag)...) + cmd.Dir = r.Root + cmd.Env = append(os.Environ(), r.Env...) + var stdout bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = r.Stderr + if err := cmd.Run(); err != nil { + return nil, fmt.Errorf("while running ko build --tags=%s: %w", tag, err) + } + refs := strings.Fields(stdout.String()) + if len(refs) != len(importPaths) { + return nil, fmt.Errorf("ko build printed %d image references for %d packages", len(refs), len(importPaths)) + } + return refs, nil +} + // ResolvePath resolves a manifest file or directory. func (r *Runner) ResolvePath(ctx context.Context, path string) ([]byte, error) { return r.Resolve(ctx, path, nil) @@ -160,15 +178,13 @@ func (r *Runner) args(subcommand string, target ...string) []string { return args } -// ldflags returns the version stamp ko should bake into binaries. The shell -// scripts shelled out to `make ldflags` for this; computing it here keeps the -// value identical without depending on make. +// ldflags returns the version stamp ko should bake into binaries. func (r *Runner) ldflags() []string { return []string{fmt.Sprintf("-X=%s.Version=%s", versionPkg, BuildVersion(r.Root))} } -// BuildVersion mirrors the Makefile's VERSION: `git describe`, or "dev" when -// git has nothing to say. It is what ko stamps into the binaries. +// BuildVersion is the VERSION environment variable, else `git describe`, or +// "dev" when git has nothing to say. It is what ko stamps into the binaries. func BuildVersion(root string) string { if v := os.Getenv("VERSION"); v != "" { return v diff --git a/cmd/ate-setup/internal/ko/ko_test.go b/cmd/ate-setup/internal/ko/ko_test.go index 55e427d451..a1bbfa9e6a 100644 --- a/cmd/ate-setup/internal/ko/ko_test.go +++ b/cmd/ate-setup/internal/ko/ko_test.go @@ -15,7 +15,10 @@ package ko import ( + "os" + "path/filepath" "slices" + "strings" "testing" ) @@ -66,7 +69,7 @@ func TestBuildVersionPrefersTheEnvironment(t *testing.T) { } } -// A source tarball has no git metadata, and the Makefile falls back to "dev" +// A source tarball has no git metadata, so BuildVersion falls back to "dev" // there rather than stamping an empty version. func TestBuildVersionFallsBackToDev(t *testing.T) { t.Setenv("VERSION", "") @@ -74,3 +77,57 @@ func TestBuildVersionFallsBackToDev(t *testing.T) { t.Errorf("BuildVersion() = %q, want dev", got) } } + +// fakeKo writes a ko stand-in that records its arguments, one per line, and +// prints stdout. +func fakeKo(t *testing.T, stdout string) (binary, argsFile string) { + t.Helper() + dir := t.TempDir() + argsFile = filepath.Join(dir, "args") + binary = filepath.Join(dir, "ko") + script := "#!/bin/sh\nprintf '%s\\n' \"$@\" > " + argsFile + "\nprintf '" + stdout + "'\n" + if err := os.WriteFile(binary, []byte(script), 0o755); err != nil { + t.Fatal(err) + } + return binary, argsFile +} + +// A release is only installable if every image carries the tag and the +// --base-import-paths name that --image-repo looks it up by. +func TestBuildTagged(t *testing.T) { + t.Setenv("VERSION", "v1.2.3") + binary, argsFile := fakeKo(t, "repo/ateapi:v1.2.3@sha256:a\\nrepo/atelet:v1.2.3@sha256:b\\n") + r := &Runner{Root: t.TempDir(), Stderr: os.Stderr, binary: binary} + + refs, err := r.BuildTagged(t.Context(), "v1.2.3", "./cmd/ateapi", "./cmd/atelet") + if err != nil { + t.Fatalf("BuildTagged() error = %v", err) + } + if want := []string{"repo/ateapi:v1.2.3@sha256:a", "repo/atelet:v1.2.3@sha256:b"}; !slices.Equal(refs, want) { + t.Errorf("BuildTagged() = %q, want %q", refs, want) + } + raw, err := os.ReadFile(argsFile) + if err != nil { + t.Fatal(err) + } + args := strings.Fields(string(raw)) + if got := args[:3]; !slices.Equal(got, []string{"build", "./cmd/ateapi", "./cmd/atelet"}) { + t.Errorf("args[:3] = %v, want [build ./cmd/ateapi ./cmd/atelet]", got) + } + for _, want := range []string{"--tags=v1.2.3", "--base-import-paths"} { + if !slices.Contains(args, want) { + t.Errorf("args = %v, want %s", args, want) + } + } +} + +// A ref missing from ko's output would leave a release short an image, so it +// fails the build rather than printing an incomplete list. +func TestBuildTaggedRejectsMissingRefs(t *testing.T) { + binary, _ := fakeKo(t, "repo/ateapi:v1@sha256:a\\n") + r := &Runner{Root: t.TempDir(), Stderr: os.Stderr, binary: binary} + + if _, err := r.BuildTagged(t.Context(), "v1", "./cmd/ateapi", "./cmd/atelet"); err == nil { + t.Error("BuildTagged() error = nil, want one for 1 ref from 2 packages") + } +} diff --git a/cmd/ate-setup/internal/steps/env.go b/cmd/ate-setup/internal/steps/env.go index ad569874c7..1e22e5b889 100644 --- a/cmd/ate-setup/internal/steps/env.go +++ b/cmd/ate-setup/internal/steps/env.go @@ -166,8 +166,8 @@ func (e *Env) koRunner() (*ko.Runner, error) { // KO_DOCKER_REPO. A pre-built install builds nothing: --image-repo installs // images someone else published, and needs neither docker nor a registry to // push to, so the image is pinned from REPO/name:TAG like every ko image. -// That only works if the release published it there, which the Makefile's -// build- target does. +// That only works if the release published it there, which +// `ate-setup publish release-images` does. func (e *Env) dockerfileImage(ctx context.Context, name, contextDir string) (string, error) { if !e.Cfg.Images.IsPrebuilt() { return images.BuildDockerfileImage(ctx, e.Cfg.Root, e.Cfg.KODockerRepo, name, e.Cfg.Path(contextDir), e.Cfg.KODefaultPlatforms, e.Cfg.DockerBuildFlags) @@ -183,7 +183,7 @@ func (e *Env) dockerfileImage(ctx context.Context, name, contextDir string) (str ref, err := prebuilt.Pin(ctx, name) if err != nil { return "", fmt.Errorf("%s is built from a Dockerfile, not with ko, so a release has to publish it "+ - "alongside the other images (make build-%s): %w", name, name, err) + "alongside the other images (ate-setup publish release-images): %w", name, err) } return ref, nil } diff --git a/cmd/ate-setup/internal/steps/overlay_test.go b/cmd/ate-setup/internal/steps/overlay_test.go index 440febd241..c047a029c3 100644 --- a/cmd/ate-setup/internal/steps/overlay_test.go +++ b/cmd/ate-setup/internal/steps/overlay_test.go @@ -828,7 +828,7 @@ func TestRenderAtenetEgressManifestPrebuilt(t *testing.T) { } // A release that did not publish envoy-dataplane fails the install with a -// message naming the image and the target that publishes it, rather than a bare +// message naming the image and the command that publishes it, rather than a bare // registry error. func TestDockerfileImagePrebuiltNotPublished(t *testing.T) { src := images.Source{Repo: "example.com/substrate", Tag: "v1.2.3"} @@ -844,7 +844,7 @@ func TestDockerfileImagePrebuiltNotPublished(t *testing.T) { t.Fatal("dockerfileImage() error = nil, want one") } for _, want := range []string{ - "make build-envoy-dataplane", + "ate-setup publish release-images", "resolving example.com/substrate/envoy-dataplane:v1.2.3 to a digest: MANIFEST_UNKNOWN", } { if !strings.Contains(err.Error(), want) { diff --git a/cmd/ate-setup/internal/steps/publish.go b/cmd/ate-setup/internal/steps/publish.go index 1760c54697..e43f207a6f 100644 --- a/cmd/ate-setup/internal/steps/publish.go +++ b/cmd/ate-setup/internal/steps/publish.go @@ -19,6 +19,7 @@ import ( "fmt" "io" + "github.com/agent-substrate/substrate/cmd/ate-setup/internal/images" "github.com/agent-substrate/substrate/cmd/ate-setup/internal/log" ) @@ -60,3 +61,46 @@ func (e *Env) PublishWorkerImages(ctx context.Context, w io.Writer) error { } return nil } + +// PublishReleaseImages builds and pushes every image a pre-built install +// needs, all tagged with the build version, and writes their pushed references +// to w. That is what `deploy --image-repo KO_DOCKER_REPO --image-tag VERSION` +// installs: the ko images in images.Components, plus envoy-dataplane, which is +// built from a Dockerfile. +func (e *Env) PublishReleaseImages(ctx context.Context, w io.Writer) error { + if e.Cfg.KODockerRepo == "" { + return fmt.Errorf("publishing release images needs a registry to push to; set KO_DOCKER_REPO or --ko-docker-repo") + } + tag, _, err := e.SubstrateVersion() + if err != nil { + return err + } + log.Stepf("publish_release_images (%s)", tag) + runner, err := e.koRunner() + if err != nil { + return err + } + pkgs := make([]string, 0, len(images.Components)) + for _, pkg := range images.Components { + pkgs = append(pkgs, "./"+pkg) + } + refs, err := runner.BuildTagged(ctx, tag, pkgs...) + if err != nil { + return err + } + envoy, err := images.PublishDockerfileImage(ctx, e.Cfg.Root, e.Cfg.KODockerRepo, envoyDataplaneImage, tag, + e.Cfg.Path(envoyDataplaneDockefile), e.Cfg.KODefaultPlatforms, e.Cfg.DockerBuildFlags) + if err != nil { + return err + } + refs = append(refs, envoy) + if _, err := fmt.Fprintf(w, "\nRelease images for %s:\n", tag); err != nil { + return err + } + for _, ref := range refs { + if _, err := fmt.Fprintln(w, ref); err != nil { + return err + } + } + return nil +} diff --git a/cmd/ate-setup/internal/steps/publish_test.go b/cmd/ate-setup/internal/steps/publish_test.go index 9cd51f6dde..99f4add104 100644 --- a/cmd/ate-setup/internal/steps/publish_test.go +++ b/cmd/ate-setup/internal/steps/publish_test.go @@ -15,6 +15,7 @@ package steps import ( + "io" "strings" "testing" @@ -41,3 +42,14 @@ func TestKoRunnerPrebuilt(t *testing.T) { t.Errorf("koRunner() error = %q; want it to name --image-repo", err) } } + +// Without a registry ko would fall back to its own default, so the release +// command refuses before building anything. +func TestPublishReleaseImagesNeedsARegistry(t *testing.T) { + e := &Env{Cfg: &config.Config{Root: t.TempDir()}} + + err := e.PublishReleaseImages(t.Context(), io.Discard) + if err == nil || !strings.Contains(err.Error(), "KO_DOCKER_REPO") { + t.Errorf("PublishReleaseImages() error = %v, want one naming KO_DOCKER_REPO", err) + } +} diff --git a/cmd/dataplane/envoy/README.md b/cmd/dataplane/envoy/README.md index 6c7dfa646d..23727118fc 100644 --- a/cmd/dataplane/envoy/README.md +++ b/cmd/dataplane/envoy/README.md @@ -23,10 +23,10 @@ During a build from source, `ate-setup` builds the image from this directory via A pre-built install (`ate-setup deploy --image-repo REPO --image-tag TAG`) builds nothing and pins `REPO/envoy-dataplane:TAG` instead, so a release publishes it with the other images: ```bash -make build-release-images KO_DOCKER_REPO=REPO VERSION=TAG +VERSION=TAG go run ./cmd/ate-setup publish release-images --ko-docker-repo REPO ``` -`make build-envoy-dataplane` builds just this image. It targets `KO_DEFAULTPLATFORMS`, or `linux/amd64` when unset; set `DOCKERFILE_PLATFORMS` to override. Building another architecture compiles Rust under QEMU, which must be registered with binfmt (e.g. `docker run --privileged --rm tonistiigi/binfmt --install arm64`). +The image targets the same platforms as the ko images: `KO_DEFAULTPLATFORMS`, else `defaultPlatforms` in `.ko.yaml` (linux/amd64 and linux/arm64), else `linux/amd64`. Building another architecture compiles Rust under QEMU, which must be registered with binfmt (e.g. `docker run --privileged --rm tonistiigi/binfmt --install arm64`). To build the image locally without pushing it: diff --git a/docs/observability.md b/docs/observability.md index ae2fd83e51..a69839434b 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -214,7 +214,7 @@ Three `event.name` values, which is the OTLP LogRecord's own field rather than a `ate.actor.usage_sampled` is the ateoms' record: one per actor per sampling interval, plus an `initial` record per activation and a `final` one when the ateom tears it down, told apart by `ate.stats.kind`. Its timestamp is when the measurement was read. Its measurements are named after the `ate.actor.stats.*` instruments and share their units, so `ate.stats.cpu.time` is seconds. They are absent, not zero, while the actor is not measurable, which the record says with `ate.stats.source` unspecified. `ate.stats.cpu.time` restarts at zero with each `ate.actor.epoch`, the unix-nano time the activation began, so a lifetime figure is the sum over epochs of each epoch's highest value; `ate.stats.memory.usage` and `ate.stats.memory.working_set` are absolute, and `ate.stats.memory.peak` is as the source reports it. The same measurements ride `WorkloadStatsSample` on the stats RPCs, with the epoch beside them. -A crash is its own name because an event name promises a set of attributes and a crash has a different severity and shape. There is no name per state: `ate.actor.state` already says which transition happened, so a consumer still selects on that one attribute and needs no map from a name to a state. All three names are in [`docs/metrics/registry/events.yaml`](metrics/registry/events.yaml), which `make verify` checks. +A crash is its own name because an event name promises a set of attributes and a crash has a different severity and shape. There is no name per state: `ate.actor.state` already says which transition happened, so a consumer still selects on that one attribute and needs no map from a name to a state. All three names are in [`docs/metrics/registry/events.yaml`](metrics/registry/events.yaml), which `hack/verify-all.sh` checks. The attributes are the same flat `ate.*` keys as the stdout record, so they arrive as real log attributes with no transform in front of them. Trace context is not among them: it goes on the record's own `TraceId` and `SpanId` fields, where the stdout record's top-level `trace_id`/`span_id` would be mapped to anyway. The instrumentation scope is `github.com/agent-substrate/substrate/internal/actorevent` for every actor event. Select or drop a stream by `event.name`: the lifecycle events must never be sampled, the usage samples may be. @@ -320,7 +320,7 @@ hack/verify/metrics.sh # the command that CI uses weaver registry check -r docs/metrics/registry # the same command, direct ``` -`make verify` runs the script. It uses a local `weaver` binary if there is one, and the official image if there is none. +`hack/verify-all.sh` runs the script. It uses a local `weaver` binary if there is one, and the official image if there is none. **To add or change an instrument:** follow [Metrics Best Practices](dev/best-practices/metrics.md), which covers the instrument choice, the labels, the Go, the test, and the registry entry; then change `metrics.yaml` and run the script. diff --git a/internal/e2e/suites/security/README.md b/internal/e2e/suites/security/README.md index 382f9e2bc0..0c06735843 100644 --- a/internal/e2e/suites/security/README.md +++ b/internal/e2e/suites/security/README.md @@ -57,11 +57,11 @@ Fast, no cluster needed. This is what CI's `run-tests` job does, minus the JUnit wrapping, and it also type-checks every E2E suite: ``` -go vet ./... && make verify && (cd tools/apitool && go test -race ./...) +go vet ./... && go test -race ./... && hack/verify-all.sh && (cd tools/apitool && go test -race ./...) ``` -`make verify` runs `go test -race ./...` and then `hack/verify-all.sh` -(gofmt, boilerplate, licenses, go.mod, metrics registry, generated code). +`hack/verify-all.sh` runs gofmt, boilerplate, licenses, go.mod, metrics +registry, and generated-code checks. Commit or stash first: the generated-code check refuses a dirty tree. The metrics check needs Docker or a local `weaver` at the version pinned in `hack/verify/metrics.sh`. The root-gated tests CI runs under sudo need Linux: diff --git a/internal/version/version.go b/internal/version/version.go index ba2db0d74e..dea37669e0 100644 --- a/internal/version/version.go +++ b/internal/version/version.go @@ -50,6 +50,7 @@ func init() { } } } + Version = info.Main.Path + ".Version=" + info.Main.Version } // String returns a human-readable single-line version summary, From 85b128a345b67c2cd4a4513aebd83dd3b186f365 Mon Sep 17 00:00:00 2001 From: Anish Ramasekar Date: Fri, 9 Oct 2026 22:42:25 +0000 Subject: [PATCH 38/44] Rename the object-store plugin proto package to objectstore.v1 (#2432) Renames the object-store plugin proto package from `objectstoresnapshot.v1` to `objectstore.v1`. It is split out of #2385 as its own refactor, since that PR adds a non-snapshot service to the same API. - Moves `pkg/proto/objectstoresnapshotpb/v1` to `pkg/proto/objectstorepb/v1` and `objectstoresnapshot.proto` to `objectstore.proto`, regenerated with `hack/update/codegen.sh`. - Updates every in-tree caller, the protoredact pinned field names, and `docs/architecture.md`. - Services, messages and fields keep their names. The `cmd/snapshot-plugin` binary, image and manifests are unchanged. Open #1907 will need a rebase. It adds no references to the old package, so only its existing conflicts with main apply. #### Breaking change gRPC method names change from `/objectstoresnapshot.v1./` to `/objectstore.v1./`. Out-of-tree plugins must regenerate from `objectstore.proto` and roll out with atelet and ate-api-server. Please add the `breaking-change` label. - [x] Tests pass - [x] Appropriate changes to documentation are included in the PR --- cmd/ateapi/internal/controlapi/service.go | 6 +- cmd/ateapi/internal/controlapi/workflow.go | 10 +- .../workflow_snapshot_plugin_test.go | 6 +- cmd/ateapi/main.go | 4 +- cmd/atelet/main.go | 8 +- cmd/atelet/snapshot_transfer.go | 6 +- cmd/atelet/snapshot_transfer_test.go | 10 +- cmd/snapshot-plugin/main.go | 6 +- docs/architecture.md | 2 +- internal/objectstoreplugin/control.go | 12 +- internal/objectstoreplugin/node.go | 12 +- .../objectstoreplugintest.go | 14 +- internal/objectstoreplugin/plugin_test.go | 54 +++---- internal/objectstoreplugin/socket_test.go | 20 +-- internal/protoredact/protoredact_test.go | 18 +-- .../v1/gen.go | 4 +- .../v1/objectstore.pb.go} | 148 +++++++++--------- .../v1/objectstore.proto} | 4 +- .../v1/objectstore_grpc.pb.go} | 20 +-- 19 files changed, 182 insertions(+), 182 deletions(-) rename pkg/proto/{objectstoresnapshotpb => objectstorepb}/v1/gen.go (94%) rename pkg/proto/{objectstoresnapshotpb/v1/objectstoresnapshot.pb.go => objectstorepb/v1/objectstore.pb.go} (71%) rename pkg/proto/{objectstoresnapshotpb/v1/objectstoresnapshot.proto => objectstorepb/v1/objectstore.proto} (98%) rename pkg/proto/{objectstoresnapshotpb/v1/objectstoresnapshot_grpc.pb.go => objectstorepb/v1/objectstore_grpc.pb.go} (96%) diff --git a/cmd/ateapi/internal/controlapi/service.go b/cmd/ateapi/internal/controlapi/service.go index f40caa3bb1..4e5db34875 100644 --- a/cmd/ateapi/internal/controlapi/service.go +++ b/cmd/ateapi/internal/controlapi/service.go @@ -27,7 +27,7 @@ import ( "github.com/agent-substrate/substrate/internal/volume/csi" listersv1alpha1 "github.com/agent-substrate/substrate/pkg/client/listers/api/v1alpha1" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" - objectstoresnapshotv1 "github.com/agent-substrate/substrate/pkg/proto/objectstoresnapshotpb/v1" + objectstorev1 "github.com/agent-substrate/substrate/pkg/proto/objectstorepb/v1" storagev1listers "k8s.io/client-go/listers/storage/v1" ) @@ -49,7 +49,7 @@ type RPCService struct { instruments *Instruments mu sync.RWMutex volumePlugins map[string]volume.VolumePluginControlPlane - snapshotPlugin objectstoresnapshotv1.ControlProviderClient + snapshotPlugin objectstorev1.ControlProviderClient actorJWTIssuer string actorIDJWTPool localjwtauthority.Pool @@ -83,7 +83,7 @@ func NewRPCService( instruments *Instruments, egressGatewayAddress string, volumePlugins map[string]volume.VolumePluginControlPlane, - snapshotPlugin objectstoresnapshotv1.ControlProviderClient, + snapshotPlugin objectstorev1.ControlProviderClient, actorJWTIssuer string, actorIDJWTPool localjwtauthority.Pool, actorIDCAPool localca.Pool, diff --git a/cmd/ateapi/internal/controlapi/workflow.go b/cmd/ateapi/internal/controlapi/workflow.go index 552b97c7a0..c15605e867 100644 --- a/cmd/ateapi/internal/controlapi/workflow.go +++ b/cmd/ateapi/internal/controlapi/workflow.go @@ -30,7 +30,7 @@ import ( "github.com/agent-substrate/substrate/internal/resources" listersv1alpha1 "github.com/agent-substrate/substrate/pkg/client/listers/api/v1alpha1" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" - objectstoresnapshotv1 "github.com/agent-substrate/substrate/pkg/proto/objectstoresnapshotpb/v1" + objectstorev1 "github.com/agent-substrate/substrate/pkg/proto/objectstorepb/v1" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/codes" @@ -110,7 +110,7 @@ type ActorWorkflow struct { instruments *Instruments egressGatewayAddress string pluginRegistry VolumePluginRegistry - snapshotPlugin objectstoresnapshotv1.ControlProviderClient + snapshotPlugin objectstorev1.ControlProviderClient } // NewActorWorkflow creates a new ActorWorkflow. instruments may be nil. @@ -127,7 +127,7 @@ func NewActorWorkflow( instruments *Instruments, egressGatewayAddress string, pluginRegistry VolumePluginRegistry, - snapshotPlugin objectstoresnapshotv1.ControlProviderClient, + snapshotPlugin objectstorev1.ControlProviderClient, ) *ActorWorkflow { return &ActorWorkflow{ store: store, @@ -146,14 +146,14 @@ func NewActorWorkflow( // cleanupSnapshot deletes every object under prefix through the control // snapshot plugin. func (w *ActorWorkflow) cleanupSnapshot(ctx context.Context, prefix resources.StoragePrefix) error { - _, err := w.snapshotPlugin.CleanupSnapshot(ctx, &objectstoresnapshotv1.CleanupSnapshotRequest{SnapshotUri: prefix.String()}) + _, err := w.snapshotPlugin.CleanupSnapshot(ctx, &objectstorev1.CleanupSnapshotRequest{SnapshotUri: prefix.String()}) return objectstoreplugin.CallError(err) } // copySnapshot copies every object under src to dst through the control // snapshot plugin. func (w *ActorWorkflow) copySnapshot(ctx context.Context, src, dst resources.StoragePrefix) error { - _, err := w.snapshotPlugin.CopySnapshot(ctx, &objectstoresnapshotv1.CopySnapshotRequest{SrcUri: src.String(), DstUri: dst.String()}) + _, err := w.snapshotPlugin.CopySnapshot(ctx, &objectstorev1.CopySnapshotRequest{SrcUri: src.String(), DstUri: dst.String()}) return objectstoreplugin.CallError(err) } diff --git a/cmd/ateapi/internal/controlapi/workflow_snapshot_plugin_test.go b/cmd/ateapi/internal/controlapi/workflow_snapshot_plugin_test.go index a70602307c..3344061492 100644 --- a/cmd/ateapi/internal/controlapi/workflow_snapshot_plugin_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_snapshot_plugin_test.go @@ -20,7 +20,7 @@ import ( "github.com/agent-substrate/substrate/internal/ateinterceptors" "github.com/agent-substrate/substrate/internal/resources" - objectstoresnapshotv1 "github.com/agent-substrate/substrate/pkg/proto/objectstoresnapshotpb/v1" + objectstorev1 "github.com/agent-substrate/substrate/pkg/proto/objectstorepb/v1" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -31,11 +31,11 @@ type failingControlPlugin struct { err error } -func (p failingControlPlugin) CleanupSnapshot(context.Context, *objectstoresnapshotv1.CleanupSnapshotRequest, ...grpc.CallOption) (*objectstoresnapshotv1.CleanupSnapshotResponse, error) { +func (p failingControlPlugin) CleanupSnapshot(context.Context, *objectstorev1.CleanupSnapshotRequest, ...grpc.CallOption) (*objectstorev1.CleanupSnapshotResponse, error) { return nil, p.err } -func (p failingControlPlugin) CopySnapshot(context.Context, *objectstoresnapshotv1.CopySnapshotRequest, ...grpc.CallOption) (*objectstoresnapshotv1.CopySnapshotResponse, error) { +func (p failingControlPlugin) CopySnapshot(context.Context, *objectstorev1.CopySnapshotRequest, ...grpc.CallOption) (*objectstorev1.CopySnapshotResponse, error) { return nil, p.err } diff --git a/cmd/ateapi/main.go b/cmd/ateapi/main.go index ebc93e65d8..e58e23cc36 100644 --- a/cmd/ateapi/main.go +++ b/cmd/ateapi/main.go @@ -47,7 +47,7 @@ import ( "github.com/agent-substrate/substrate/pkg/client/clientset/versioned" "github.com/agent-substrate/substrate/pkg/client/informers/externalversions" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" - objectstoresnapshotv1 "github.com/agent-substrate/substrate/pkg/proto/objectstoresnapshotpb/v1" + objectstorev1 "github.com/agent-substrate/substrate/pkg/proto/objectstorepb/v1" "github.com/jackc/pgx/v5/pgconn" "github.com/spf13/pflag" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" @@ -296,7 +296,7 @@ func main() { instruments, *defaultEgressGatewayAddress, volPlugins, - objectstoresnapshotv1.NewControlProviderClient(snapshotPluginConn), + objectstorev1.NewControlProviderClient(snapshotPluginConn), *actorJWTIssuer, actorIDJWTAuthorityPool, actorIDCAPool, diff --git a/cmd/atelet/main.go b/cmd/atelet/main.go index 99f4c78d91..244c3cc595 100644 --- a/cmd/atelet/main.go +++ b/cmd/atelet/main.go @@ -59,7 +59,7 @@ import ( "github.com/agent-substrate/substrate/pkg/client/informers/externalversions" "github.com/agent-substrate/substrate/pkg/objectstorage" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" - objectstoresnapshotv1 "github.com/agent-substrate/substrate/pkg/proto/objectstoresnapshotpb/v1" + objectstorev1 "github.com/agent-substrate/substrate/pkg/proto/objectstorepb/v1" "github.com/google/go-containerregistry/pkg/authn" "github.com/spf13/pflag" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" @@ -302,7 +302,7 @@ func main() { ateomDialer, wrappedAnonGCS, wrappedGCS, - objectstoresnapshotv1.NewNodeProviderClient(snapshotPluginConn), + objectstorev1.NewNodeProviderClient(snapshotPluginConn), imageCache, instruments, volPlugins, @@ -458,7 +458,7 @@ type AteomHerder struct { // through snapshotPlugin. gcsClient objectstorage.ObjectStorage // snapshotPlugin moves external snapshot files between the node and storage. - snapshotPlugin objectstoresnapshotv1.NodeProviderClient + snapshotPlugin objectstorev1.NodeProviderClient // snapshotScratchDir holds short-lived manifest directories; it must be // inside the node plugin's root. snapshotScratchDir string @@ -477,7 +477,7 @@ func NewService( ateomDialer *AteomDialer, anonGCSClient objectstorage.ObjectStorage, gcsClient objectstorage.ObjectStorage, - snapshotPlugin objectstoresnapshotv1.NodeProviderClient, + snapshotPlugin objectstorev1.NodeProviderClient, imageCache *imagecache.Store, instruments *Instruments, volumePlugins map[string]volume.VolumePluginWorkerPlane, diff --git a/cmd/atelet/snapshot_transfer.go b/cmd/atelet/snapshot_transfer.go index 0cbc579c49..485490d1b5 100644 --- a/cmd/atelet/snapshot_transfer.go +++ b/cmd/atelet/snapshot_transfer.go @@ -21,7 +21,7 @@ import ( "path/filepath" "github.com/agent-substrate/substrate/internal/objectstoreplugin" - objectstoresnapshotv1 "github.com/agent-substrate/substrate/pkg/proto/objectstoresnapshotpb/v1" + objectstorev1 "github.com/agent-substrate/substrate/pkg/proto/objectstorepb/v1" "google.golang.org/grpc" ) @@ -34,7 +34,7 @@ func (s *AteomHerder) fetchSnapshotFiles(ctx context.Context, snapshotURI, dstDi if len(files) == 0 { return nil } - _, err := s.snapshotPlugin.FetchSnapshot(ctx, &objectstoresnapshotv1.FetchSnapshotRequest{ + _, err := s.snapshotPlugin.FetchSnapshot(ctx, &objectstorev1.FetchSnapshotRequest{ SnapshotUri: snapshotURI, WritePath: dstDir, Files: files, @@ -50,7 +50,7 @@ func (s *AteomHerder) uploadSnapshotFiles(ctx context.Context, snapshotURI, srcD if len(files) == 0 { return nil } - _, err := s.snapshotPlugin.UploadSnapshot(ctx, &objectstoresnapshotv1.UploadSnapshotRequest{ + _, err := s.snapshotPlugin.UploadSnapshot(ctx, &objectstorev1.UploadSnapshotRequest{ SnapshotUri: snapshotURI, LocalPath: srcDir, Files: files, diff --git a/cmd/atelet/snapshot_transfer_test.go b/cmd/atelet/snapshot_transfer_test.go index a13873e789..82d7c3ca94 100644 --- a/cmd/atelet/snapshot_transfer_test.go +++ b/cmd/atelet/snapshot_transfer_test.go @@ -34,7 +34,7 @@ import ( "github.com/agent-substrate/substrate/internal/objectstoreplugin/objectstoreplugintest" "github.com/agent-substrate/substrate/internal/resources" "github.com/agent-substrate/substrate/pkg/objectstorage" - objectstoresnapshotv1 "github.com/agent-substrate/substrate/pkg/proto/objectstoresnapshotpb/v1" + objectstorev1 "github.com/agent-substrate/substrate/pkg/proto/objectstorepb/v1" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/connectivity" @@ -154,11 +154,11 @@ type failingNodePlugin struct { err error } -func (p failingNodePlugin) FetchSnapshot(context.Context, *objectstoresnapshotv1.FetchSnapshotRequest, ...grpc.CallOption) (*objectstoresnapshotv1.FetchSnapshotResponse, error) { +func (p failingNodePlugin) FetchSnapshot(context.Context, *objectstorev1.FetchSnapshotRequest, ...grpc.CallOption) (*objectstorev1.FetchSnapshotResponse, error) { return nil, p.err } -func (p failingNodePlugin) UploadSnapshot(context.Context, *objectstoresnapshotv1.UploadSnapshotRequest, ...grpc.CallOption) (*objectstoresnapshotv1.UploadSnapshotResponse, error) { +func (p failingNodePlugin) UploadSnapshot(context.Context, *objectstorev1.UploadSnapshotRequest, ...grpc.CallOption) (*objectstorev1.UploadSnapshotResponse, error) { return nil, p.err } @@ -258,7 +258,7 @@ func (s *pluginSidecar) start() { return } srv := grpc.NewServer() - objectstoresnapshotv1.RegisterNodeProviderServer(srv, s.plugin) + objectstorev1.RegisterNodeProviderServer(srv, s.plugin) healthpb.RegisterHealthServer(srv, health.NewServer()) go srv.Serve(lis) s.mu.Lock() @@ -297,7 +297,7 @@ func TestCheckpointUploadWaitsOutPluginOutage(t *testing.T) { t.Fatalf("WaitReady = %v", err) } s := &AteomHerder{ - snapshotPlugin: objectstoresnapshotv1.NewNodeProviderClient(conn), + snapshotPlugin: objectstorev1.NewNodeProviderClient(conn), snapshotScratchDir: t.TempDir(), } rec := sandboxAssetsRecord{ diff --git a/cmd/snapshot-plugin/main.go b/cmd/snapshot-plugin/main.go index d62391e416..1d95e9e394 100644 --- a/cmd/snapshot-plugin/main.go +++ b/cmd/snapshot-plugin/main.go @@ -35,7 +35,7 @@ import ( "github.com/agent-substrate/substrate/internal/objectstoreplugin" "github.com/agent-substrate/substrate/internal/serverboot" "github.com/agent-substrate/substrate/pkg/objectstorage" - objectstoresnapshotv1 "github.com/agent-substrate/substrate/pkg/proto/objectstoresnapshotpb/v1" + objectstorev1 "github.com/agent-substrate/substrate/pkg/proto/objectstorepb/v1" "github.com/spf13/pflag" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "google.golang.org/grpc" @@ -92,13 +92,13 @@ func main() { if err != nil { serverboot.Fatal(ctx, "Invalid --root", err) } - objectstoresnapshotv1.RegisterNodeProviderServer(srv, plugin) + objectstorev1.RegisterNodeProviderServer(srv, plugin) case "control": store, err := newObjectStore(ctx) if err != nil { serverboot.Fatal(ctx, "Failed to set up the object storage backend", err) } - objectstoresnapshotv1.RegisterControlProviderServer(srv, objectstoreplugin.NewControlPlugin(store)) + objectstorev1.RegisterControlProviderServer(srv, objectstoreplugin.NewControlPlugin(store)) default: fmt.Fprintln(os.Stderr, usage) os.Exit(2) diff --git a/docs/architecture.md b/docs/architecture.md index a09707ade9..45593ab924 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -327,7 +327,7 @@ The node-level subsystem manages the physical execution of sandboxes and the mov * **Lifecycle Management**: The `ateom` process invokes the sandbox runtime to checkpoint or restore processes within the physical pod boundaries — `runsc` for gVisor, or the Kata + Cloud Hypervisor stack for micro-VMs. (Note: the gVisor backend currently requires a `runsc` version with the `--allow-connected-on-save` flag to work around a bug in networking resumption during checkpointing.) - * **Storage Mover**: External snapshots move through a *snapshot plugin*, a sidecar that serves the `objectstoresnapshot.v1` gRPC API over a Unix socket in its pod. `atelet`'s sidecar (`NodeProvider`) uploads and fetches a snapshot's files; `ate-api-server`'s sidecar (`ControlProvider`) copies and deletes whole snapshots. Neither main process talks to object storage for snapshots. The in-tree plugin, `cmd/snapshot-plugin`, backs the API with GCS or S3; `ATE_STORAGE_BACKEND` on the sidecar (unset or `gcs`, or `s3` with the `AWS_*` variables) selects which, see `manifests/ate-install/atelet.yaml`, `ate-api-server.yaml`, and the kind overlay for an S3 example. `atelet` keeps a storage client of its own only for sandbox asset downloads. + * **Storage Mover**: External snapshots move through a *snapshot plugin*, a sidecar that serves the `objectstore.v1` gRPC API over a Unix socket in its pod. `atelet`'s sidecar (`NodeProvider`) uploads and fetches a snapshot's files; `ate-api-server`'s sidecar (`ControlProvider`) copies and deletes whole snapshots. Neither main process talks to object storage for snapshots. The in-tree plugin, `cmd/snapshot-plugin`, backs the API with GCS or S3; `ATE_STORAGE_BACKEND` on the sidecar (unset or `gcs`, or `s3` with the `AWS_*` variables) selects which, see `manifests/ate-install/atelet.yaml`, `ate-api-server.yaml`, and the kind overlay for an S3 example. `atelet` keeps a storage client of its own only for sandbox asset downloads. ### Sandbox Classes diff --git a/internal/objectstoreplugin/control.go b/internal/objectstoreplugin/control.go index 647226fd2e..69b46787f4 100644 --- a/internal/objectstoreplugin/control.go +++ b/internal/objectstoreplugin/control.go @@ -19,14 +19,14 @@ import ( "github.com/agent-substrate/substrate/internal/objectstore" "github.com/agent-substrate/substrate/internal/resources" - objectstoresnapshotv1 "github.com/agent-substrate/substrate/pkg/proto/objectstoresnapshotpb/v1" + objectstorev1 "github.com/agent-substrate/substrate/pkg/proto/objectstorepb/v1" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) // ControlPlugin serves ControlProvider on an object store. type ControlPlugin struct { - objectstoresnapshotv1.UnimplementedControlProviderServer + objectstorev1.UnimplementedControlProviderServer store objectstore.Store } @@ -37,7 +37,7 @@ func NewControlPlugin(store objectstore.Store) *ControlPlugin { } // CleanupSnapshot deletes every object under the given prefix. -func (p *ControlPlugin) CleanupSnapshot(ctx context.Context, req *objectstoresnapshotv1.CleanupSnapshotRequest) (*objectstoresnapshotv1.CleanupSnapshotResponse, error) { +func (p *ControlPlugin) CleanupSnapshot(ctx context.Context, req *objectstorev1.CleanupSnapshotRequest) (*objectstorev1.CleanupSnapshotResponse, error) { prefix, err := resources.ParseStoragePrefix(req.GetSnapshotUri()) if err != nil { return nil, status.Error(codes.InvalidArgument, err.Error()) @@ -45,11 +45,11 @@ func (p *ControlPlugin) CleanupSnapshot(ctx context.Context, req *objectstoresna if err := objectstore.DeletePrefix(ctx, p.store, prefix); err != nil { return nil, toStatus(err) } - return &objectstoresnapshotv1.CleanupSnapshotResponse{}, nil + return &objectstorev1.CleanupSnapshotResponse{}, nil } // CopySnapshot copies every object of src_uri to dst_uri. -func (p *ControlPlugin) CopySnapshot(ctx context.Context, req *objectstoresnapshotv1.CopySnapshotRequest) (*objectstoresnapshotv1.CopySnapshotResponse, error) { +func (p *ControlPlugin) CopySnapshot(ctx context.Context, req *objectstorev1.CopySnapshotRequest) (*objectstorev1.CopySnapshotResponse, error) { src, err := resources.ParseStoragePrefix(req.GetSrcUri()) if err != nil { return nil, status.Error(codes.InvalidArgument, err.Error()) @@ -61,5 +61,5 @@ func (p *ControlPlugin) CopySnapshot(ctx context.Context, req *objectstoresnapsh if err := objectstore.CopyPrefix(ctx, p.store, src, dst); err != nil { return nil, toStatus(err) } - return &objectstoresnapshotv1.CopySnapshotResponse{}, nil + return &objectstorev1.CopySnapshotResponse{}, nil } diff --git a/internal/objectstoreplugin/node.go b/internal/objectstoreplugin/node.go index 8dd9759753..6c7f8fa35d 100644 --- a/internal/objectstoreplugin/node.go +++ b/internal/objectstoreplugin/node.go @@ -31,7 +31,7 @@ import ( "github.com/agent-substrate/substrate/internal/resources" "github.com/agent-substrate/substrate/pkg/objectstorage" - objectstoresnapshotv1 "github.com/agent-substrate/substrate/pkg/proto/objectstoresnapshotpb/v1" + objectstorev1 "github.com/agent-substrate/substrate/pkg/proto/objectstorepb/v1" "golang.org/x/sync/errgroup" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -44,7 +44,7 @@ const manifestFile = "manifest.json" // NodePlugin serves NodeProvider on an object storage client. type NodePlugin struct { - objectstoresnapshotv1.UnimplementedNodeProviderServer + objectstorev1.UnimplementedNodeProviderServer gcsClient objectstorage.ObjectStorage // root confines every local path a caller names. @@ -61,7 +61,7 @@ func NewNodePlugin(gcsClient objectstorage.ObjectStorage, root string) (*NodePlu } // FetchSnapshot downloads the requested snapshot files into write_path. -func (p *NodePlugin) FetchSnapshot(ctx context.Context, req *objectstoresnapshotv1.FetchSnapshotRequest) (*objectstoresnapshotv1.FetchSnapshotResponse, error) { +func (p *NodePlugin) FetchSnapshot(ctx context.Context, req *objectstorev1.FetchSnapshotRequest) (*objectstorev1.FetchSnapshotResponse, error) { uri, dstDir, err := p.validate(req.GetSnapshotUri(), req.GetWritePath(), req.GetFiles()) if err != nil { return nil, err @@ -98,12 +98,12 @@ func (p *NodePlugin) FetchSnapshot(ctx context.Context, req *objectstoresnapshot if err := g.Wait(); err != nil { return nil, toStatus(err) } - return &objectstoresnapshotv1.FetchSnapshotResponse{}, nil + return &objectstorev1.FetchSnapshotResponse{}, nil } // UploadSnapshot uploads the requested files from local_path into the // snapshot. -func (p *NodePlugin) UploadSnapshot(ctx context.Context, req *objectstoresnapshotv1.UploadSnapshotRequest) (*objectstoresnapshotv1.UploadSnapshotResponse, error) { +func (p *NodePlugin) UploadSnapshot(ctx context.Context, req *objectstorev1.UploadSnapshotRequest) (*objectstorev1.UploadSnapshotResponse, error) { uri, srcDir, err := p.validate(req.GetSnapshotUri(), req.GetLocalPath(), req.GetFiles()) if err != nil { return nil, err @@ -146,7 +146,7 @@ func (p *NodePlugin) UploadSnapshot(ctx context.Context, req *objectstoresnapsho if err := g.Wait(); err != nil { return nil, toStatus(err) } - return &objectstoresnapshotv1.UploadSnapshotResponse{}, nil + return &objectstorev1.UploadSnapshotResponse{}, nil } // fetchManifest downloads the uncompressed manifest into root. diff --git a/internal/objectstoreplugin/objectstoreplugintest/objectstoreplugintest.go b/internal/objectstoreplugin/objectstoreplugintest/objectstoreplugintest.go index 8b672da707..1e4b592d2e 100644 --- a/internal/objectstoreplugin/objectstoreplugintest/objectstoreplugintest.go +++ b/internal/objectstoreplugin/objectstoreplugintest/objectstoreplugintest.go @@ -22,12 +22,12 @@ import ( "github.com/agent-substrate/substrate/internal/objectstore" "github.com/agent-substrate/substrate/internal/objectstoreplugin" - objectstoresnapshotv1 "github.com/agent-substrate/substrate/pkg/proto/objectstoresnapshotpb/v1" + objectstorev1 "github.com/agent-substrate/substrate/pkg/proto/objectstorepb/v1" "google.golang.org/grpc" ) // NodeClient returns a client that calls p directly. -func NodeClient(p *objectstoreplugin.NodePlugin) objectstoresnapshotv1.NodeProviderClient { +func NodeClient(p *objectstoreplugin.NodePlugin) objectstorev1.NodeProviderClient { return nodeClient{p} } @@ -35,16 +35,16 @@ type nodeClient struct { p *objectstoreplugin.NodePlugin } -func (c nodeClient) FetchSnapshot(ctx context.Context, in *objectstoresnapshotv1.FetchSnapshotRequest, _ ...grpc.CallOption) (*objectstoresnapshotv1.FetchSnapshotResponse, error) { +func (c nodeClient) FetchSnapshot(ctx context.Context, in *objectstorev1.FetchSnapshotRequest, _ ...grpc.CallOption) (*objectstorev1.FetchSnapshotResponse, error) { return c.p.FetchSnapshot(ctx, in) } -func (c nodeClient) UploadSnapshot(ctx context.Context, in *objectstoresnapshotv1.UploadSnapshotRequest, _ ...grpc.CallOption) (*objectstoresnapshotv1.UploadSnapshotResponse, error) { +func (c nodeClient) UploadSnapshot(ctx context.Context, in *objectstorev1.UploadSnapshotRequest, _ ...grpc.CallOption) (*objectstorev1.UploadSnapshotResponse, error) { return c.p.UploadSnapshot(ctx, in) } // ControlClient returns a client that calls a ControlPlugin on store directly. -func ControlClient(store objectstore.Store) objectstoresnapshotv1.ControlProviderClient { +func ControlClient(store objectstore.Store) objectstorev1.ControlProviderClient { return controlClient{objectstoreplugin.NewControlPlugin(store)} } @@ -52,10 +52,10 @@ type controlClient struct { p *objectstoreplugin.ControlPlugin } -func (c controlClient) CleanupSnapshot(ctx context.Context, in *objectstoresnapshotv1.CleanupSnapshotRequest, _ ...grpc.CallOption) (*objectstoresnapshotv1.CleanupSnapshotResponse, error) { +func (c controlClient) CleanupSnapshot(ctx context.Context, in *objectstorev1.CleanupSnapshotRequest, _ ...grpc.CallOption) (*objectstorev1.CleanupSnapshotResponse, error) { return c.p.CleanupSnapshot(ctx, in) } -func (c controlClient) CopySnapshot(ctx context.Context, in *objectstoresnapshotv1.CopySnapshotRequest, _ ...grpc.CallOption) (*objectstoresnapshotv1.CopySnapshotResponse, error) { +func (c controlClient) CopySnapshot(ctx context.Context, in *objectstorev1.CopySnapshotRequest, _ ...grpc.CallOption) (*objectstorev1.CopySnapshotResponse, error) { return c.p.CopySnapshot(ctx, in) } diff --git a/internal/objectstoreplugin/plugin_test.go b/internal/objectstoreplugin/plugin_test.go index 9857f8ba80..eafac6f033 100644 --- a/internal/objectstoreplugin/plugin_test.go +++ b/internal/objectstoreplugin/plugin_test.go @@ -28,7 +28,7 @@ import ( "time" "github.com/agent-substrate/substrate/pkg/objectstorage" - objectstoresnapshotv1 "github.com/agent-substrate/substrate/pkg/proto/objectstoresnapshotpb/v1" + objectstorev1 "github.com/agent-substrate/substrate/pkg/proto/objectstorepb/v1" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/health" @@ -128,8 +128,8 @@ func serve(t *testing.T, backend *memObjects, root string) *grpc.ClientConn { t.Fatal(err) } srv := grpc.NewServer() - objectstoresnapshotv1.RegisterNodeProviderServer(srv, node) - objectstoresnapshotv1.RegisterControlProviderServer(srv, NewControlPlugin(backend)) + objectstorev1.RegisterNodeProviderServer(srv, node) + objectstorev1.RegisterControlProviderServer(srv, NewControlPlugin(backend)) healthpb.RegisterHealthServer(srv, health.NewServer()) go srv.Serve(lis) t.Cleanup(srv.Stop) @@ -168,7 +168,7 @@ func TestUploadFetchRoundTrip(t *testing.T) { ctx := context.Background() backend := newMemObjects() root := t.TempDir() - client := objectstoresnapshotv1.NewNodeProviderClient(serve(t, backend, root)) + client := objectstorev1.NewNodeProviderClient(serve(t, backend, root)) src := filepath.Join(root, "src") dst := filepath.Join(root, "dst") @@ -188,12 +188,12 @@ func TestUploadFetchRoundTrip(t *testing.T) { } } - if _, err := client.UploadSnapshot(ctx, &objectstoresnapshotv1.UploadSnapshotRequest{ + if _, err := client.UploadSnapshot(ctx, &objectstorev1.UploadSnapshotRequest{ SnapshotUri: testURI, LocalPath: src, Files: []string{"memory.img", "state.bin"}, }); err != nil { t.Fatalf("UploadSnapshot(data) = %v", err) } - if _, err := client.UploadSnapshot(ctx, &objectstoresnapshotv1.UploadSnapshotRequest{ + if _, err := client.UploadSnapshot(ctx, &objectstorev1.UploadSnapshotRequest{ SnapshotUri: testURI, LocalPath: src, Files: []string{manifestFile}, }); err != nil { t.Fatalf("UploadSnapshot(manifest) = %v", err) @@ -212,7 +212,7 @@ func TestUploadFetchRoundTrip(t *testing.T) { t.Errorf("stored manifest = %q, want it uncompressed", got) } - if _, err := client.FetchSnapshot(ctx, &objectstoresnapshotv1.FetchSnapshotRequest{ + if _, err := client.FetchSnapshot(ctx, &objectstorev1.FetchSnapshotRequest{ SnapshotUri: testURI, WritePath: dst, Files: []string{manifestFile, "memory.img", "state.bin"}, }); err != nil { t.Fatalf("FetchSnapshot = %v", err) @@ -230,9 +230,9 @@ func TestUploadFetchRoundTrip(t *testing.T) { func TestFetchMissingIsNotFound(t *testing.T) { root := t.TempDir() - client := objectstoresnapshotv1.NewNodeProviderClient(serve(t, newMemObjects(), root)) + client := objectstorev1.NewNodeProviderClient(serve(t, newMemObjects(), root)) for _, file := range []string{manifestFile, "memory.img"} { - _, err := client.FetchSnapshot(context.Background(), &objectstoresnapshotv1.FetchSnapshotRequest{ + _, err := client.FetchSnapshot(context.Background(), &objectstorev1.FetchSnapshotRequest{ SnapshotUri: testURI, WritePath: root, Files: []string{file}, }) if status.Code(err) != codes.NotFound { @@ -243,7 +243,7 @@ func TestFetchMissingIsNotFound(t *testing.T) { func TestNodeRequestValidation(t *testing.T) { root := t.TempDir() - client := objectstoresnapshotv1.NewNodeProviderClient(serve(t, newMemObjects(), root)) + client := objectstorev1.NewNodeProviderClient(serve(t, newMemObjects(), root)) for _, tc := range []struct { name string uri string @@ -262,10 +262,10 @@ func TestNodeRequestValidation(t *testing.T) { {"duplicate file", testURI, root, []string{"a", "b", "a"}}, } { t.Run(tc.name, func(t *testing.T) { - _, fetchErr := client.FetchSnapshot(context.Background(), &objectstoresnapshotv1.FetchSnapshotRequest{ + _, fetchErr := client.FetchSnapshot(context.Background(), &objectstorev1.FetchSnapshotRequest{ SnapshotUri: tc.uri, WritePath: tc.dir, Files: tc.files, }) - _, uploadErr := client.UploadSnapshot(context.Background(), &objectstoresnapshotv1.UploadSnapshotRequest{ + _, uploadErr := client.UploadSnapshot(context.Background(), &objectstorev1.UploadSnapshotRequest{ SnapshotUri: tc.uri, LocalPath: tc.dir, Files: tc.files, }) for _, err := range []error{fetchErr, uploadErr} { @@ -282,7 +282,7 @@ func TestUploadRejectsSymlinkOutsideDir(t *testing.T) { t.Run(file, func(t *testing.T) { parent := t.TempDir() backend := newMemObjects() - client := objectstoresnapshotv1.NewNodeProviderClient(serve(t, backend, parent)) + client := objectstorev1.NewNodeProviderClient(serve(t, backend, parent)) checkpointDir := filepath.Join(parent, "checkpoint-state") if err := os.Mkdir(checkpointDir, 0o700); err != nil { t.Fatal(err) @@ -295,7 +295,7 @@ func TestUploadRejectsSymlinkOutsideDir(t *testing.T) { t.Fatal(err) } - _, err := client.UploadSnapshot(context.Background(), &objectstoresnapshotv1.UploadSnapshotRequest{ + _, err := client.UploadSnapshot(context.Background(), &objectstorev1.UploadSnapshotRequest{ SnapshotUri: testURI, LocalPath: checkpointDir, Files: []string{file}, }) if err == nil { @@ -314,7 +314,7 @@ func TestFetchRejectsSymlinkOutsideDir(t *testing.T) { ctx := context.Background() parent := t.TempDir() backend := newMemObjects() - client := objectstoresnapshotv1.NewNodeProviderClient(serve(t, backend, parent)) + client := objectstorev1.NewNodeProviderClient(serve(t, backend, parent)) src := filepath.Join(parent, "src") restoreDir := filepath.Join(parent, "restore-state") for _, d := range []string{src, restoreDir} { @@ -325,7 +325,7 @@ func TestFetchRejectsSymlinkOutsideDir(t *testing.T) { if err := os.WriteFile(filepath.Join(src, file), []byte("replacement"), 0o600); err != nil { t.Fatal(err) } - if _, err := client.UploadSnapshot(ctx, &objectstoresnapshotv1.UploadSnapshotRequest{ + if _, err := client.UploadSnapshot(ctx, &objectstorev1.UploadSnapshotRequest{ SnapshotUri: testURI, LocalPath: src, Files: []string{file}, }); err != nil { t.Fatal(err) @@ -338,7 +338,7 @@ func TestFetchRejectsSymlinkOutsideDir(t *testing.T) { t.Fatal(err) } - _, err := client.FetchSnapshot(ctx, &objectstoresnapshotv1.FetchSnapshotRequest{ + _, err := client.FetchSnapshot(ctx, &objectstorev1.FetchSnapshotRequest{ SnapshotUri: testURI, WritePath: restoreDir, Files: []string{file}, }) if err == nil { @@ -371,9 +371,9 @@ func TestRejectsDirSymlinkOutsideRoot(t *testing.T) { t.Fatal(err) } backend := newMemObjects() - client := objectstoresnapshotv1.NewNodeProviderClient(serve(t, backend, root)) + client := objectstorev1.NewNodeProviderClient(serve(t, backend, root)) - if _, err := client.UploadSnapshot(ctx, &objectstoresnapshotv1.UploadSnapshotRequest{ + if _, err := client.UploadSnapshot(ctx, &objectstorev1.UploadSnapshotRequest{ SnapshotUri: testURI, LocalPath: link, Files: []string{"checkpoint.img"}, }); err == nil { t.Error("UploadSnapshot read through a directory symlink outside --root") @@ -383,7 +383,7 @@ func TestRejectsDirSymlinkOutsideRoot(t *testing.T) { } backend.m[testBucket+"/"+testPrefix+"/"+manifestFile] = []byte("m") - if _, err := client.FetchSnapshot(ctx, &objectstoresnapshotv1.FetchSnapshotRequest{ + if _, err := client.FetchSnapshot(ctx, &objectstorev1.FetchSnapshotRequest{ SnapshotUri: testURI, WritePath: link, Files: []string{manifestFile}, }); err == nil { t.Error("FetchSnapshot succeeded through a directory symlink outside --root") @@ -396,7 +396,7 @@ func TestRejectsDirSymlinkOutsideRoot(t *testing.T) { func TestCleanupSnapshot(t *testing.T) { ctx := context.Background() backend := newMemObjects() - client := objectstoresnapshotv1.NewControlProviderClient(serve(t, backend, t.TempDir())) + client := objectstorev1.NewControlProviderClient(serve(t, backend, t.TempDir())) // uid10 shares a string prefix with uid1, so it survives only if cleanup // stops at a path-segment boundary. keep := testBucket + "/root/atespaces/team-a/actors/uid10/snapshots/snap1/manifest.json" @@ -406,19 +406,19 @@ func TestCleanupSnapshot(t *testing.T) { // An owner prefix collects every snapshot below it, and nothing beside it. owner := "gs://" + testBucket + "/root/atespaces/team-a/actors/uid1" - if _, err := client.CleanupSnapshot(ctx, &objectstoresnapshotv1.CleanupSnapshotRequest{SnapshotUri: owner}); err != nil { + if _, err := client.CleanupSnapshot(ctx, &objectstorev1.CleanupSnapshotRequest{SnapshotUri: owner}); err != nil { t.Fatalf("CleanupSnapshot = %v", err) } if got := backend.keys(); !equal(got, []string{keep}) { t.Errorf("objects left = %v, want only %s", got, keep) } // Cleaning up again is a no-op. - if _, err := client.CleanupSnapshot(ctx, &objectstoresnapshotv1.CleanupSnapshotRequest{SnapshotUri: owner}); err != nil { + if _, err := client.CleanupSnapshot(ctx, &objectstorev1.CleanupSnapshotRequest{SnapshotUri: owner}); err != nil { t.Errorf("repeated CleanupSnapshot = %v", err) } // A bare bucket is never a valid prefix. for _, uri := range []string{"gs://" + testBucket, "gs://" + testBucket + "/", "not a uri?x=1"} { - _, err := client.CleanupSnapshot(ctx, &objectstoresnapshotv1.CleanupSnapshotRequest{SnapshotUri: uri}) + _, err := client.CleanupSnapshot(ctx, &objectstorev1.CleanupSnapshotRequest{SnapshotUri: uri}) if status.Code(err) != codes.InvalidArgument { t.Errorf("CleanupSnapshot(%q) = %v, want InvalidArgument", uri, err) } @@ -428,11 +428,11 @@ func TestCleanupSnapshot(t *testing.T) { func TestCopySnapshot(t *testing.T) { ctx := context.Background() backend := newMemObjects() - client := objectstoresnapshotv1.NewControlProviderClient(serve(t, backend, t.TempDir())) + client := objectstorev1.NewControlProviderClient(serve(t, backend, t.TempDir())) backend.m[testBucket+"/"+testPrefix+"/manifest.json"] = []byte("m") tag := "gs://" + testBucket + "/root/atespaces/team-a/tags/tag1" - if _, err := client.CopySnapshot(ctx, &objectstoresnapshotv1.CopySnapshotRequest{SrcUri: testURI, DstUri: tag}); err != nil { + if _, err := client.CopySnapshot(ctx, &objectstorev1.CopySnapshotRequest{SrcUri: testURI, DstUri: tag}); err != nil { t.Fatalf("CopySnapshot = %v", err) } if got := backend.m[testBucket+"/root/atespaces/team-a/tags/tag1/manifest.json"]; string(got) != "m" { @@ -440,7 +440,7 @@ func TestCopySnapshot(t *testing.T) { } empty := "gs://" + testBucket + "/root/atespaces/team-a/actors/uid9/snapshots/none" - _, err := client.CopySnapshot(ctx, &objectstoresnapshotv1.CopySnapshotRequest{SrcUri: empty, DstUri: tag}) + _, err := client.CopySnapshot(ctx, &objectstorev1.CopySnapshotRequest{SrcUri: empty, DstUri: tag}) if status.Code(err) != codes.FailedPrecondition { t.Errorf("CopySnapshot(empty source) = %v, want FailedPrecondition", err) } diff --git a/internal/objectstoreplugin/socket_test.go b/internal/objectstoreplugin/socket_test.go index b849b82713..18a8f6f659 100644 --- a/internal/objectstoreplugin/socket_test.go +++ b/internal/objectstoreplugin/socket_test.go @@ -22,7 +22,7 @@ import ( "testing" "time" - objectstoresnapshotv1 "github.com/agent-substrate/substrate/pkg/proto/objectstoresnapshotpb/v1" + objectstorev1 "github.com/agent-substrate/substrate/pkg/proto/objectstorepb/v1" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/connectivity" @@ -34,7 +34,7 @@ import ( // slowControl is a ControlProvider whose CleanupSnapshot takes delay, and // closes started when a call arrives. type slowControl struct { - objectstoresnapshotv1.UnimplementedControlProviderServer + objectstorev1.UnimplementedControlProviderServer delay time.Duration once sync.Once @@ -45,11 +45,11 @@ func newSlowControl(delay time.Duration) *slowControl { return &slowControl{delay: delay, started: make(chan struct{})} } -func (c *slowControl) CleanupSnapshot(ctx context.Context, _ *objectstoresnapshotv1.CleanupSnapshotRequest) (*objectstoresnapshotv1.CleanupSnapshotResponse, error) { +func (c *slowControl) CleanupSnapshot(ctx context.Context, _ *objectstorev1.CleanupSnapshotRequest) (*objectstorev1.CleanupSnapshotResponse, error) { c.once.Do(func() { close(c.started) }) select { case <-time.After(c.delay): - return &objectstoresnapshotv1.CleanupSnapshotResponse{}, nil + return &objectstorev1.CleanupSnapshotResponse{}, nil case <-ctx.Done(): return nil, status.FromContextError(ctx.Err()).Err() } @@ -60,13 +60,13 @@ func (c *slowControl) CleanupSnapshot(ctx context.Context, _ *objectstoresnapsho type sidecar struct { t *testing.T path string - control objectstoresnapshotv1.ControlProviderServer + control objectstorev1.ControlProviderServer mu sync.Mutex srv *grpc.Server } -func newSidecar(t *testing.T, control objectstoresnapshotv1.ControlProviderServer) *sidecar { +func newSidecar(t *testing.T, control objectstorev1.ControlProviderServer) *sidecar { t.Helper() // Unix socket paths are length-limited; t.TempDir can exceed that on macOS. dir, err := os.MkdirTemp("", "snapplug") @@ -87,7 +87,7 @@ func (s *sidecar) start() { return } srv := grpc.NewServer() - objectstoresnapshotv1.RegisterControlProviderServer(srv, s.control) + objectstorev1.RegisterControlProviderServer(srv, s.control) healthpb.RegisterHealthServer(srv, health.NewServer()) go srv.Serve(lis) s.mu.Lock() @@ -109,7 +109,7 @@ func (s *sidecar) stop() { // dialReady connects to s with the given wait and passes the startup // readiness check. -func (s *sidecar) dialReady(wait time.Duration) (*grpc.ClientConn, objectstoresnapshotv1.ControlProviderClient) { +func (s *sidecar) dialReady(wait time.Duration) (*grpc.ClientConn, objectstorev1.ControlProviderClient) { s.t.Helper() conn, err := Dial(s.path, wait) if err != nil { @@ -121,7 +121,7 @@ func (s *sidecar) dialReady(wait time.Duration) (*grpc.ClientConn, objectstoresn if err := WaitReady(ctx, conn); err != nil { s.t.Fatalf("WaitReady = %v", err) } - return conn, objectstoresnapshotv1.NewControlProviderClient(conn) + return conn, objectstorev1.NewControlProviderClient(conn) } // lose stops s and waits until conn has seen the plugin go, so the next call @@ -138,7 +138,7 @@ func (s *sidecar) lose(conn *grpc.ClientConn) { } } -var cleanupReq = &objectstoresnapshotv1.CleanupSnapshotRequest{SnapshotUri: testURI} +var cleanupReq = &objectstorev1.CleanupSnapshotRequest{SnapshotUri: testURI} // A plugin lost after startup fails a call with Unavailable once the ready // wait runs out, not when the caller's deadline does. diff --git a/internal/protoredact/protoredact_test.go b/internal/protoredact/protoredact_test.go index ee482aba6b..cc1b946d4a 100644 --- a/internal/protoredact/protoredact_test.go +++ b/internal/protoredact/protoredact_test.go @@ -30,7 +30,7 @@ import ( "github.com/agent-substrate/substrate/internal/protoredact" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/agent-substrate/substrate/pkg/proto/credproviderpb" - objectstoresnapshotv1 "github.com/agent-substrate/substrate/pkg/proto/objectstoresnapshotpb/v1" + objectstorev1 "github.com/agent-substrate/substrate/pkg/proto/objectstorepb/v1" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/reflect/protodesc" "google.golang.org/protobuf/reflect/protoreflect" @@ -377,7 +377,7 @@ var ourProtoFiles = []protoreflect.FileDescriptor{ credproviderpb.File_credprovider_proto, glutton.File_glutton_proto, grpcechopb.File_grpcecho_proto, - objectstoresnapshotv1.File_objectstoresnapshot_proto, + objectstorev1.File_objectstore_proto, } // forEachField calls fn for every field of every message in ourProtoFiles, @@ -409,13 +409,13 @@ func isDebugRedact(fd protoreflect.FieldDescriptor) bool { // change is reviewed as a deliberate decision about what the logs may show. func TestDebugRedactFieldsArePinned(t *testing.T) { want := map[string]bool{ - "ateapi.EnvVar.value": true, - "ateapi.MintActorJWTResponse.actor_jwt": true, - "atelet.EnvEntry.value": true, - "ateom.ContainerSpec.env": true, - "credprovider.FetchSecretResponse.opaque_bytes": true, - "objectstoresnapshot.v1.FetchSnapshotRequest.actor_jwt": true, - "objectstoresnapshot.v1.UploadSnapshotRequest.actor_jwt": true, + "ateapi.EnvVar.value": true, + "ateapi.MintActorJWTResponse.actor_jwt": true, + "atelet.EnvEntry.value": true, + "ateom.ContainerSpec.env": true, + "credprovider.FetchSecretResponse.opaque_bytes": true, + "objectstore.v1.FetchSnapshotRequest.actor_jwt": true, + "objectstore.v1.UploadSnapshotRequest.actor_jwt": true, } got := map[string]bool{} forEachField(func(fd protoreflect.FieldDescriptor) { diff --git a/pkg/proto/objectstoresnapshotpb/v1/gen.go b/pkg/proto/objectstorepb/v1/gen.go similarity index 94% rename from pkg/proto/objectstoresnapshotpb/v1/gen.go rename to pkg/proto/objectstorepb/v1/gen.go index 8baf4835c3..25a454562d 100644 --- a/pkg/proto/objectstoresnapshotpb/v1/gen.go +++ b/pkg/proto/objectstorepb/v1/gen.go @@ -12,6 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -package objectstoresnapshotv1 +package objectstorev1 -//go:generate bash -c "../../../../hack/protoc.sh --plugin=protoc-gen-go=$(bash ../../../../hack/run-tool.sh --print-bin-path protoc-gen-go) --plugin=protoc-gen-go-grpc=$(bash ../../../../hack/run-tool.sh --print-bin-path protoc-gen-go-grpc) --go_out=paths=source_relative:. --go-grpc_out=paths=source_relative:. objectstoresnapshot.proto" +//go:generate bash -c "../../../../hack/protoc.sh --plugin=protoc-gen-go=$(bash ../../../../hack/run-tool.sh --print-bin-path protoc-gen-go) --plugin=protoc-gen-go-grpc=$(bash ../../../../hack/run-tool.sh --print-bin-path protoc-gen-go-grpc) --go_out=paths=source_relative:. --go-grpc_out=paths=source_relative:. objectstore.proto" diff --git a/pkg/proto/objectstoresnapshotpb/v1/objectstoresnapshot.pb.go b/pkg/proto/objectstorepb/v1/objectstore.pb.go similarity index 71% rename from pkg/proto/objectstoresnapshotpb/v1/objectstoresnapshot.pb.go rename to pkg/proto/objectstorepb/v1/objectstore.pb.go index eff7f79c82..bc1a9d680a 100644 --- a/pkg/proto/objectstoresnapshotpb/v1/objectstoresnapshot.pb.go +++ b/pkg/proto/objectstorepb/v1/objectstore.pb.go @@ -16,9 +16,9 @@ // versions: // protoc-gen-go v1.36.11-devel // protoc v4.25.3 -// source: objectstoresnapshot.proto +// source: objectstore.proto -package objectstoresnapshotv1 +package objectstorev1 import ( protoreflect "google.golang.org/protobuf/reflect/protoreflect" @@ -55,7 +55,7 @@ type FetchSnapshotRequest struct { func (x *FetchSnapshotRequest) Reset() { *x = FetchSnapshotRequest{} - mi := &file_objectstoresnapshot_proto_msgTypes[0] + mi := &file_objectstore_proto_msgTypes[0] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -67,7 +67,7 @@ func (x *FetchSnapshotRequest) String() string { func (*FetchSnapshotRequest) ProtoMessage() {} func (x *FetchSnapshotRequest) ProtoReflect() protoreflect.Message { - mi := &file_objectstoresnapshot_proto_msgTypes[0] + mi := &file_objectstore_proto_msgTypes[0] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -80,7 +80,7 @@ func (x *FetchSnapshotRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use FetchSnapshotRequest.ProtoReflect.Descriptor instead. func (*FetchSnapshotRequest) Descriptor() ([]byte, []int) { - return file_objectstoresnapshot_proto_rawDescGZIP(), []int{0} + return file_objectstore_proto_rawDescGZIP(), []int{0} } func (x *FetchSnapshotRequest) GetSnapshotUri() string { @@ -119,7 +119,7 @@ type FetchSnapshotResponse struct { func (x *FetchSnapshotResponse) Reset() { *x = FetchSnapshotResponse{} - mi := &file_objectstoresnapshot_proto_msgTypes[1] + mi := &file_objectstore_proto_msgTypes[1] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -131,7 +131,7 @@ func (x *FetchSnapshotResponse) String() string { func (*FetchSnapshotResponse) ProtoMessage() {} func (x *FetchSnapshotResponse) ProtoReflect() protoreflect.Message { - mi := &file_objectstoresnapshot_proto_msgTypes[1] + mi := &file_objectstore_proto_msgTypes[1] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -144,7 +144,7 @@ func (x *FetchSnapshotResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use FetchSnapshotResponse.ProtoReflect.Descriptor instead. func (*FetchSnapshotResponse) Descriptor() ([]byte, []int) { - return file_objectstoresnapshot_proto_rawDescGZIP(), []int{1} + return file_objectstore_proto_rawDescGZIP(), []int{1} } // UploadSnapshotRequest asks the plugin to upload snapshot files. @@ -166,7 +166,7 @@ type UploadSnapshotRequest struct { func (x *UploadSnapshotRequest) Reset() { *x = UploadSnapshotRequest{} - mi := &file_objectstoresnapshot_proto_msgTypes[2] + mi := &file_objectstore_proto_msgTypes[2] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -178,7 +178,7 @@ func (x *UploadSnapshotRequest) String() string { func (*UploadSnapshotRequest) ProtoMessage() {} func (x *UploadSnapshotRequest) ProtoReflect() protoreflect.Message { - mi := &file_objectstoresnapshot_proto_msgTypes[2] + mi := &file_objectstore_proto_msgTypes[2] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -191,7 +191,7 @@ func (x *UploadSnapshotRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use UploadSnapshotRequest.ProtoReflect.Descriptor instead. func (*UploadSnapshotRequest) Descriptor() ([]byte, []int) { - return file_objectstoresnapshot_proto_rawDescGZIP(), []int{2} + return file_objectstore_proto_rawDescGZIP(), []int{2} } func (x *UploadSnapshotRequest) GetSnapshotUri() string { @@ -230,7 +230,7 @@ type UploadSnapshotResponse struct { func (x *UploadSnapshotResponse) Reset() { *x = UploadSnapshotResponse{} - mi := &file_objectstoresnapshot_proto_msgTypes[3] + mi := &file_objectstore_proto_msgTypes[3] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -242,7 +242,7 @@ func (x *UploadSnapshotResponse) String() string { func (*UploadSnapshotResponse) ProtoMessage() {} func (x *UploadSnapshotResponse) ProtoReflect() protoreflect.Message { - mi := &file_objectstoresnapshot_proto_msgTypes[3] + mi := &file_objectstore_proto_msgTypes[3] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -255,7 +255,7 @@ func (x *UploadSnapshotResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use UploadSnapshotResponse.ProtoReflect.Descriptor instead. func (*UploadSnapshotResponse) Descriptor() ([]byte, []int) { - return file_objectstoresnapshot_proto_rawDescGZIP(), []int{3} + return file_objectstore_proto_rawDescGZIP(), []int{3} } // CleanupSnapshotRequest names the snapshots to delete. @@ -271,7 +271,7 @@ type CleanupSnapshotRequest struct { func (x *CleanupSnapshotRequest) Reset() { *x = CleanupSnapshotRequest{} - mi := &file_objectstoresnapshot_proto_msgTypes[4] + mi := &file_objectstore_proto_msgTypes[4] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -283,7 +283,7 @@ func (x *CleanupSnapshotRequest) String() string { func (*CleanupSnapshotRequest) ProtoMessage() {} func (x *CleanupSnapshotRequest) ProtoReflect() protoreflect.Message { - mi := &file_objectstoresnapshot_proto_msgTypes[4] + mi := &file_objectstore_proto_msgTypes[4] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -296,7 +296,7 @@ func (x *CleanupSnapshotRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use CleanupSnapshotRequest.ProtoReflect.Descriptor instead. func (*CleanupSnapshotRequest) Descriptor() ([]byte, []int) { - return file_objectstoresnapshot_proto_rawDescGZIP(), []int{4} + return file_objectstore_proto_rawDescGZIP(), []int{4} } func (x *CleanupSnapshotRequest) GetSnapshotUri() string { @@ -314,7 +314,7 @@ type CleanupSnapshotResponse struct { func (x *CleanupSnapshotResponse) Reset() { *x = CleanupSnapshotResponse{} - mi := &file_objectstoresnapshot_proto_msgTypes[5] + mi := &file_objectstore_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -326,7 +326,7 @@ func (x *CleanupSnapshotResponse) String() string { func (*CleanupSnapshotResponse) ProtoMessage() {} func (x *CleanupSnapshotResponse) ProtoReflect() protoreflect.Message { - mi := &file_objectstoresnapshot_proto_msgTypes[5] + mi := &file_objectstore_proto_msgTypes[5] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -339,7 +339,7 @@ func (x *CleanupSnapshotResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use CleanupSnapshotResponse.ProtoReflect.Descriptor instead. func (*CleanupSnapshotResponse) Descriptor() ([]byte, []int) { - return file_objectstoresnapshot_proto_rawDescGZIP(), []int{5} + return file_objectstore_proto_rawDescGZIP(), []int{5} } // CopySnapshotRequest names the source and destination snapshots. @@ -355,7 +355,7 @@ type CopySnapshotRequest struct { func (x *CopySnapshotRequest) Reset() { *x = CopySnapshotRequest{} - mi := &file_objectstoresnapshot_proto_msgTypes[6] + mi := &file_objectstore_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -367,7 +367,7 @@ func (x *CopySnapshotRequest) String() string { func (*CopySnapshotRequest) ProtoMessage() {} func (x *CopySnapshotRequest) ProtoReflect() protoreflect.Message { - mi := &file_objectstoresnapshot_proto_msgTypes[6] + mi := &file_objectstore_proto_msgTypes[6] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -380,7 +380,7 @@ func (x *CopySnapshotRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use CopySnapshotRequest.ProtoReflect.Descriptor instead. func (*CopySnapshotRequest) Descriptor() ([]byte, []int) { - return file_objectstoresnapshot_proto_rawDescGZIP(), []int{6} + return file_objectstore_proto_rawDescGZIP(), []int{6} } func (x *CopySnapshotRequest) GetSrcUri() string { @@ -405,7 +405,7 @@ type CopySnapshotResponse struct { func (x *CopySnapshotResponse) Reset() { *x = CopySnapshotResponse{} - mi := &file_objectstoresnapshot_proto_msgTypes[7] + mi := &file_objectstore_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -417,7 +417,7 @@ func (x *CopySnapshotResponse) String() string { func (*CopySnapshotResponse) ProtoMessage() {} func (x *CopySnapshotResponse) ProtoReflect() protoreflect.Message { - mi := &file_objectstoresnapshot_proto_msgTypes[7] + mi := &file_objectstore_proto_msgTypes[7] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -430,14 +430,14 @@ func (x *CopySnapshotResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use CopySnapshotResponse.ProtoReflect.Descriptor instead. func (*CopySnapshotResponse) Descriptor() ([]byte, []int) { - return file_objectstoresnapshot_proto_rawDescGZIP(), []int{7} + return file_objectstore_proto_rawDescGZIP(), []int{7} } -var File_objectstoresnapshot_proto protoreflect.FileDescriptor +var File_objectstore_proto protoreflect.FileDescriptor -const file_objectstoresnapshot_proto_rawDesc = "" + +const file_objectstore_proto_rawDesc = "" + "\n" + - "\x19objectstoresnapshot.proto\x12\x16objectstoresnapshot.v1\"\x90\x01\n" + + "\x11objectstore.proto\x12\x0eobjectstore.v1\"\x90\x01\n" + "\x14FetchSnapshotRequest\x12!\n" + "\fsnapshot_uri\x18\x01 \x01(\tR\vsnapshotUri\x12 \n" + "\tactor_jwt\x18\x02 \x01(\tB\x03\x80\x01\x01R\bactorJwt\x12\x1d\n" + @@ -458,46 +458,46 @@ const file_objectstoresnapshot_proto_rawDesc = "" + "\x13CopySnapshotRequest\x12\x17\n" + "\asrc_uri\x18\x01 \x01(\tR\x06srcUri\x12\x17\n" + "\adst_uri\x18\x02 \x01(\tR\x06dstUri\"\x16\n" + - "\x14CopySnapshotResponse2\xf1\x01\n" + - "\fNodeProvider\x12n\n" + - "\rFetchSnapshot\x12,.objectstoresnapshot.v1.FetchSnapshotRequest\x1a-.objectstoresnapshot.v1.FetchSnapshotResponse\"\x00\x12q\n" + - "\x0eUploadSnapshot\x12-.objectstoresnapshot.v1.UploadSnapshotRequest\x1a..objectstoresnapshot.v1.UploadSnapshotResponse\"\x002\xf4\x01\n" + - "\x0fControlProvider\x12t\n" + - "\x0fCleanupSnapshot\x12..objectstoresnapshot.v1.CleanupSnapshotRequest\x1a/.objectstoresnapshot.v1.CleanupSnapshotResponse\"\x00\x12k\n" + - "\fCopySnapshot\x12+.objectstoresnapshot.v1.CopySnapshotRequest\x1a,.objectstoresnapshot.v1.CopySnapshotResponse\"\x00B_Z]github.com/agent-substrate/substrate/pkg/proto/objectstoresnapshotpb/v1;objectstoresnapshotv1b\x06proto3" + "\x14CopySnapshotResponse2\xd1\x01\n" + + "\fNodeProvider\x12^\n" + + "\rFetchSnapshot\x12$.objectstore.v1.FetchSnapshotRequest\x1a%.objectstore.v1.FetchSnapshotResponse\"\x00\x12a\n" + + "\x0eUploadSnapshot\x12%.objectstore.v1.UploadSnapshotRequest\x1a&.objectstore.v1.UploadSnapshotResponse\"\x002\xd4\x01\n" + + "\x0fControlProvider\x12d\n" + + "\x0fCleanupSnapshot\x12&.objectstore.v1.CleanupSnapshotRequest\x1a'.objectstore.v1.CleanupSnapshotResponse\"\x00\x12[\n" + + "\fCopySnapshot\x12#.objectstore.v1.CopySnapshotRequest\x1a$.objectstore.v1.CopySnapshotResponse\"\x00BOZMgithub.com/agent-substrate/substrate/pkg/proto/objectstorepb/v1;objectstorev1b\x06proto3" var ( - file_objectstoresnapshot_proto_rawDescOnce sync.Once - file_objectstoresnapshot_proto_rawDescData []byte + file_objectstore_proto_rawDescOnce sync.Once + file_objectstore_proto_rawDescData []byte ) -func file_objectstoresnapshot_proto_rawDescGZIP() []byte { - file_objectstoresnapshot_proto_rawDescOnce.Do(func() { - file_objectstoresnapshot_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_objectstoresnapshot_proto_rawDesc), len(file_objectstoresnapshot_proto_rawDesc))) +func file_objectstore_proto_rawDescGZIP() []byte { + file_objectstore_proto_rawDescOnce.Do(func() { + file_objectstore_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_objectstore_proto_rawDesc), len(file_objectstore_proto_rawDesc))) }) - return file_objectstoresnapshot_proto_rawDescData -} - -var file_objectstoresnapshot_proto_msgTypes = make([]protoimpl.MessageInfo, 8) -var file_objectstoresnapshot_proto_goTypes = []any{ - (*FetchSnapshotRequest)(nil), // 0: objectstoresnapshot.v1.FetchSnapshotRequest - (*FetchSnapshotResponse)(nil), // 1: objectstoresnapshot.v1.FetchSnapshotResponse - (*UploadSnapshotRequest)(nil), // 2: objectstoresnapshot.v1.UploadSnapshotRequest - (*UploadSnapshotResponse)(nil), // 3: objectstoresnapshot.v1.UploadSnapshotResponse - (*CleanupSnapshotRequest)(nil), // 4: objectstoresnapshot.v1.CleanupSnapshotRequest - (*CleanupSnapshotResponse)(nil), // 5: objectstoresnapshot.v1.CleanupSnapshotResponse - (*CopySnapshotRequest)(nil), // 6: objectstoresnapshot.v1.CopySnapshotRequest - (*CopySnapshotResponse)(nil), // 7: objectstoresnapshot.v1.CopySnapshotResponse -} -var file_objectstoresnapshot_proto_depIdxs = []int32{ - 0, // 0: objectstoresnapshot.v1.NodeProvider.FetchSnapshot:input_type -> objectstoresnapshot.v1.FetchSnapshotRequest - 2, // 1: objectstoresnapshot.v1.NodeProvider.UploadSnapshot:input_type -> objectstoresnapshot.v1.UploadSnapshotRequest - 4, // 2: objectstoresnapshot.v1.ControlProvider.CleanupSnapshot:input_type -> objectstoresnapshot.v1.CleanupSnapshotRequest - 6, // 3: objectstoresnapshot.v1.ControlProvider.CopySnapshot:input_type -> objectstoresnapshot.v1.CopySnapshotRequest - 1, // 4: objectstoresnapshot.v1.NodeProvider.FetchSnapshot:output_type -> objectstoresnapshot.v1.FetchSnapshotResponse - 3, // 5: objectstoresnapshot.v1.NodeProvider.UploadSnapshot:output_type -> objectstoresnapshot.v1.UploadSnapshotResponse - 5, // 6: objectstoresnapshot.v1.ControlProvider.CleanupSnapshot:output_type -> objectstoresnapshot.v1.CleanupSnapshotResponse - 7, // 7: objectstoresnapshot.v1.ControlProvider.CopySnapshot:output_type -> objectstoresnapshot.v1.CopySnapshotResponse + return file_objectstore_proto_rawDescData +} + +var file_objectstore_proto_msgTypes = make([]protoimpl.MessageInfo, 8) +var file_objectstore_proto_goTypes = []any{ + (*FetchSnapshotRequest)(nil), // 0: objectstore.v1.FetchSnapshotRequest + (*FetchSnapshotResponse)(nil), // 1: objectstore.v1.FetchSnapshotResponse + (*UploadSnapshotRequest)(nil), // 2: objectstore.v1.UploadSnapshotRequest + (*UploadSnapshotResponse)(nil), // 3: objectstore.v1.UploadSnapshotResponse + (*CleanupSnapshotRequest)(nil), // 4: objectstore.v1.CleanupSnapshotRequest + (*CleanupSnapshotResponse)(nil), // 5: objectstore.v1.CleanupSnapshotResponse + (*CopySnapshotRequest)(nil), // 6: objectstore.v1.CopySnapshotRequest + (*CopySnapshotResponse)(nil), // 7: objectstore.v1.CopySnapshotResponse +} +var file_objectstore_proto_depIdxs = []int32{ + 0, // 0: objectstore.v1.NodeProvider.FetchSnapshot:input_type -> objectstore.v1.FetchSnapshotRequest + 2, // 1: objectstore.v1.NodeProvider.UploadSnapshot:input_type -> objectstore.v1.UploadSnapshotRequest + 4, // 2: objectstore.v1.ControlProvider.CleanupSnapshot:input_type -> objectstore.v1.CleanupSnapshotRequest + 6, // 3: objectstore.v1.ControlProvider.CopySnapshot:input_type -> objectstore.v1.CopySnapshotRequest + 1, // 4: objectstore.v1.NodeProvider.FetchSnapshot:output_type -> objectstore.v1.FetchSnapshotResponse + 3, // 5: objectstore.v1.NodeProvider.UploadSnapshot:output_type -> objectstore.v1.UploadSnapshotResponse + 5, // 6: objectstore.v1.ControlProvider.CleanupSnapshot:output_type -> objectstore.v1.CleanupSnapshotResponse + 7, // 7: objectstore.v1.ControlProvider.CopySnapshot:output_type -> objectstore.v1.CopySnapshotResponse 4, // [4:8] is the sub-list for method output_type 0, // [0:4] is the sub-list for method input_type 0, // [0:0] is the sub-list for extension type_name @@ -505,26 +505,26 @@ var file_objectstoresnapshot_proto_depIdxs = []int32{ 0, // [0:0] is the sub-list for field type_name } -func init() { file_objectstoresnapshot_proto_init() } -func file_objectstoresnapshot_proto_init() { - if File_objectstoresnapshot_proto != nil { +func init() { file_objectstore_proto_init() } +func file_objectstore_proto_init() { + if File_objectstore_proto != nil { return } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: unsafe.Slice(unsafe.StringData(file_objectstoresnapshot_proto_rawDesc), len(file_objectstoresnapshot_proto_rawDesc)), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_objectstore_proto_rawDesc), len(file_objectstore_proto_rawDesc)), NumEnums: 0, NumMessages: 8, NumExtensions: 0, NumServices: 2, }, - GoTypes: file_objectstoresnapshot_proto_goTypes, - DependencyIndexes: file_objectstoresnapshot_proto_depIdxs, - MessageInfos: file_objectstoresnapshot_proto_msgTypes, + GoTypes: file_objectstore_proto_goTypes, + DependencyIndexes: file_objectstore_proto_depIdxs, + MessageInfos: file_objectstore_proto_msgTypes, }.Build() - File_objectstoresnapshot_proto = out.File - file_objectstoresnapshot_proto_goTypes = nil - file_objectstoresnapshot_proto_depIdxs = nil + File_objectstore_proto = out.File + file_objectstore_proto_goTypes = nil + file_objectstore_proto_depIdxs = nil } diff --git a/pkg/proto/objectstoresnapshotpb/v1/objectstoresnapshot.proto b/pkg/proto/objectstorepb/v1/objectstore.proto similarity index 98% rename from pkg/proto/objectstoresnapshotpb/v1/objectstoresnapshot.proto rename to pkg/proto/objectstorepb/v1/objectstore.proto index b99ff868b9..6e20d27d6c 100644 --- a/pkg/proto/objectstoresnapshotpb/v1/objectstoresnapshot.proto +++ b/pkg/proto/objectstorepb/v1/objectstore.proto @@ -14,9 +14,9 @@ syntax = "proto3"; -package objectstoresnapshot.v1; +package objectstore.v1; -option go_package = "github.com/agent-substrate/substrate/pkg/proto/objectstoresnapshotpb/v1;objectstoresnapshotv1"; +option go_package = "github.com/agent-substrate/substrate/pkg/proto/objectstorepb/v1;objectstorev1"; // NodeProvider is the plugin API an object storage platform implements // on every node. atelet calls it to move actor snapshots between the node and diff --git a/pkg/proto/objectstoresnapshotpb/v1/objectstoresnapshot_grpc.pb.go b/pkg/proto/objectstorepb/v1/objectstore_grpc.pb.go similarity index 96% rename from pkg/proto/objectstoresnapshotpb/v1/objectstoresnapshot_grpc.pb.go rename to pkg/proto/objectstorepb/v1/objectstore_grpc.pb.go index e9066c3a3b..acda323a34 100644 --- a/pkg/proto/objectstoresnapshotpb/v1/objectstoresnapshot_grpc.pb.go +++ b/pkg/proto/objectstorepb/v1/objectstore_grpc.pb.go @@ -16,9 +16,9 @@ // versions: // - protoc-gen-go-grpc v1.6.2 // - protoc v4.25.3 -// source: objectstoresnapshot.proto +// source: objectstore.proto -package objectstoresnapshotv1 +package objectstorev1 import ( context "context" @@ -33,8 +33,8 @@ import ( const _ = grpc.SupportPackageIsVersion9 const ( - NodeProvider_FetchSnapshot_FullMethodName = "/objectstoresnapshot.v1.NodeProvider/FetchSnapshot" - NodeProvider_UploadSnapshot_FullMethodName = "/objectstoresnapshot.v1.NodeProvider/UploadSnapshot" + NodeProvider_FetchSnapshot_FullMethodName = "/objectstore.v1.NodeProvider/FetchSnapshot" + NodeProvider_UploadSnapshot_FullMethodName = "/objectstore.v1.NodeProvider/UploadSnapshot" ) // NodeProviderClient is the client API for NodeProvider service. @@ -206,7 +206,7 @@ func _NodeProvider_UploadSnapshot_Handler(srv interface{}, ctx context.Context, // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) var NodeProvider_ServiceDesc = grpc.ServiceDesc{ - ServiceName: "objectstoresnapshot.v1.NodeProvider", + ServiceName: "objectstore.v1.NodeProvider", HandlerType: (*NodeProviderServer)(nil), Methods: []grpc.MethodDesc{ { @@ -219,12 +219,12 @@ var NodeProvider_ServiceDesc = grpc.ServiceDesc{ }, }, Streams: []grpc.StreamDesc{}, - Metadata: "objectstoresnapshot.proto", + Metadata: "objectstore.proto", } const ( - ControlProvider_CleanupSnapshot_FullMethodName = "/objectstoresnapshot.v1.ControlProvider/CleanupSnapshot" - ControlProvider_CopySnapshot_FullMethodName = "/objectstoresnapshot.v1.ControlProvider/CopySnapshot" + ControlProvider_CleanupSnapshot_FullMethodName = "/objectstore.v1.ControlProvider/CleanupSnapshot" + ControlProvider_CopySnapshot_FullMethodName = "/objectstore.v1.ControlProvider/CopySnapshot" ) // ControlProviderClient is the client API for ControlProvider service. @@ -364,7 +364,7 @@ func _ControlProvider_CopySnapshot_Handler(srv interface{}, ctx context.Context, // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) var ControlProvider_ServiceDesc = grpc.ServiceDesc{ - ServiceName: "objectstoresnapshot.v1.ControlProvider", + ServiceName: "objectstore.v1.ControlProvider", HandlerType: (*ControlProviderServer)(nil), Methods: []grpc.MethodDesc{ { @@ -377,5 +377,5 @@ var ControlProvider_ServiceDesc = grpc.ServiceDesc{ }, }, Streams: []grpc.StreamDesc{}, - Metadata: "objectstoresnapshot.proto", + Metadata: "objectstore.proto", } From 153983499734ee639527ee25a5eadfa5767d14d5 Mon Sep 17 00:00:00 2001 From: Chenyi Wang Date: Fri, 9 Oct 2026 22:44:02 +0000 Subject: [PATCH 39/44] podcertcontroller: do not rate-limit PCRs assigned to other replicas (#2353) ### Summary Every replica receives every PodCertificateRequest, but only the replica chosen by rendezvous hashing should sign it. The others return `ErrNotAssigned` and were requeued with `AddRateLimited`, which treats a normal condition as a failure: - consumes the shared failure rate limiter (10 QPS bucket), and - grows the per-item exponential backoff (up to 1000s). This slows retries of genuinely assigned PCRs and delays takeover when a replica dies (the PCRs it owned may already be deep in backoff on the surviving replicas). Unassigned PCRs are now requeued with `AddAfter(rendezvous.LeaseDuration)`. Assignments only change when a replica joins or its lease expires, so re-checking more often gains nothing. `LeaseDuration` is exported from `rendezvous` for this purpose; its value is unchanged. A follow-up could make this event-driven (re-enqueue pending PCRs when the active replica set changes) instead of polling. ### Testing `go test ./cmd/podcertcontroller/...` - [x] Tests pass - [ ] Documentation updated (n/a) --- cmd/podcertcontroller/internal/rendezvous/rendezvous.go | 6 ++++-- .../internal/signercontroller/signercontroller.go | 5 ++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/cmd/podcertcontroller/internal/rendezvous/rendezvous.go b/cmd/podcertcontroller/internal/rendezvous/rendezvous.go index eb8ca35317..34bf8bad9a 100644 --- a/cmd/podcertcontroller/internal/rendezvous/rendezvous.go +++ b/cmd/podcertcontroller/internal/rendezvous/rendezvous.go @@ -44,7 +44,9 @@ import ( const labelKey = "rendezvous.ate.dev/application" var ( - leaseDuration = 15 * time.Second + // LeaseDuration is how long a replica's lease stays valid without renewal, + // and therefore the upper bound on how long a dead replica keeps its assignments. + LeaseDuration = 15 * time.Second leaseRenewalPeriod = 10 * time.Second ) @@ -170,7 +172,7 @@ func (h *Hasher) ensureLease(ctx context.Context) error { }, Spec: coordinationv1.LeaseSpec{ HolderIdentity: &h.replicaName, - LeaseDurationSeconds: ptr.To(int32(int64(leaseDuration) / 1_000_000_000)), + LeaseDurationSeconds: ptr.To(int32(int64(LeaseDuration) / 1_000_000_000)), AcquireTime: ptr.To(metav1.NewMicroTime(now)), RenewTime: ptr.To(metav1.NewMicroTime(now)), LeaseTransitions: ptr.To[int32](1), diff --git a/cmd/podcertcontroller/internal/signercontroller/signercontroller.go b/cmd/podcertcontroller/internal/signercontroller/signercontroller.go index fd844c670e..b768086247 100644 --- a/cmd/podcertcontroller/internal/signercontroller/signercontroller.go +++ b/cmd/podcertcontroller/internal/signercontroller/signercontroller.go @@ -149,7 +149,10 @@ func (c *Controller) processNextWorkItem(ctx context.Context) bool { err = c.handlePCR(ctx, pcr) if errors.Is(err, rendezvous.ErrNotAssigned) { - c.pcrQueue.AddRateLimited(key) + // Not an error: don't consume the failure rate limiter or inflate + // this item's backoff. Assignments only change when a replica joins + // or its lease expires, so re-examine after the lease duration. + c.pcrQueue.AddAfter(key, rendezvous.LeaseDuration) return true } if err != nil { From 24248ce6d7266fca0295addd7a13a9ae68457d5d Mon Sep 17 00:00:00 2001 From: Haven Xia Date: Fri, 9 Oct 2026 23:23:18 +0000 Subject: [PATCH 40/44] Add an eviction section to the API guide (#2417) We should have this section clearly describe the best practice. Part of #1686 - [x] Tests pass - [x] Appropriate changes to documentation are included in the PR --- docs/api-guide.md | 19 ++++++++++++++++++- tools/setup-gcp/README.md | 28 ---------------------------- 2 files changed, 18 insertions(+), 29 deletions(-) diff --git a/docs/api-guide.md b/docs/api-guide.md index 73d7602157..b2f6d956bc 100644 --- a/docs/api-guide.md +++ b/docs/api-guide.md @@ -468,13 +468,30 @@ When an `ActorTemplate` is created: ### Resumption Lifecycle Once a template is `Ready`, creating an actor logically (via `kubectl ate create actor`) allows it to be resumed instantly on any free worker in the referenced `WorkerPool`. Substrate bypasses the standard container boot and restores the process directly from its last saved state. +### Eviction +A worker pod can go away while it hosts running actors: when its `WorkerPool` rolls to a new worker image or pod template, as in an [upgrade](upgrade.md), or scales down, when its node is drained, or when its Spot VM is reclaimed. + +When the pod gets `SIGTERM`, its worker stops accepting actors and sends `SIGTERM` to the main process of each container of every actor it hosts. Each actor then has 30 minutes, counted from the pod's `SIGTERM`, to be suspended. An actor suspended in that window keeps its state and resumes on another worker like any suspended actor. + +Substrate does not suspend the actor for you. Whatever drives the actor, usually its harness, has to call `SuspendActor` (`kubectl ate suspend`) within the 30 minutes. Only the actor gets the `SIGTERM`, so it has to pass that on, for example on an endpoint the harness polls. + +Every actor that must keep its state through an eviction needs a `SIGTERM` handler that: + +- **does not exit.** A process that has exited cannot be suspended. +- **runs in the container's main process.** Substrate signals only to the main process. +- **gets the actor suspended within the 30 minutes.** + +An actor still running when the 30 minutes are up is killed. Once the pod is gone, the actor moves to `ACTOR_STATE_CRASHED`, and everything since its last snapshot is lost. `RevertActor` (`kubectl ate revert`) returns a crashed actor to `ACTOR_STATE_SUSPENDED` at its last external snapshot. + +The node can cut the 30 minutes short. GKE node upgrades and cluster autoscaler scale-downs wait up to an hour for a pod, so they leave the full 30 minutes. A reclaimed GKE [Spot VM](https://docs.cloud.google.com/kubernetes-engine/docs/concepts/spot-vms) gives pods 15 seconds by default and at most 2 minutes, so an actor there has to be suspended in that time or it crashes. + --- ## 5. Best Practices * **Startup Logic:** Place expensive initialization (loading large models, establishing baseline connections) in your application's entry point. These will be captured in the Golden Snapshot and won't need to be repeated on every resumption. * **Placement:** Ensure your `ActorTemplate`'s `sandboxClass` matches your `WorkerPool`'s `sandboxClasses[].name`, and use the template's `workerSelector` to target specific pools — pool selection is by label match, not by namespace or RBAC. * **Version Management:** When updating code, create a new `ActorTemplate` (e.g. `v2`). Substrate treats each template as an immutable state root. -* **Eviction:** When its worker pod is evicted, an actor gets `SIGTERM` and 30 minutes to be suspended. After that it is killed and moves to `ACTOR_STATE_CRASHED`, and everything since its last snapshot is lost. So an actor that runs for more than 30 minutes without a suspend can lose data. A `CRASHED` actor can be recovered back to `ACTOR_STATE_SUSPENDED` at its last external snapshot using `RevertActor` (`kubectl ate revert`). +* **Eviction:** Give every actor a `SIGTERM` handler that does not exit, and have the actor suspended within 30 minutes of it. See [Eviction](#eviction). --- diff --git a/tools/setup-gcp/README.md b/tools/setup-gcp/README.md index 9d86f5a1e9..c675dc3526 100644 --- a/tools/setup-gcp/README.md +++ b/tools/setup-gcp/README.md @@ -96,34 +96,6 @@ Filestore CSI driver disabled). > podcertificate ClusterTrustBundles to be ready" and `kubectl get > clustertrustbundles` reports the resource type is not served. -> [!WARNING] -> **Turn node auto-upgrade off on any node pool that runs workers, and do not -> use spot or preemptible nodes for them.** When a worker pod is deleted, -> `SIGTERM` is forwarded into the actor's containers and the control plane keeps -> accepting a suspend for 30 minutes. An actor suspended inside that -> window keeps its state. One still awake when the window closes is moved to -> `ACTOR_STATE_CRASHED` with its worker assignment cleared, and `CRASHED` is -> terminal: `resume` and `suspend` are both refused, there is no recover verb, -> and the snapshot the actor still holds cannot be used to start it. The only -> way out is to delete the actor and create a new one, which loses its state. -> -> Auto-upgrade is the trigger to plan for, because GKE enables it by default and -> it fires on Google's maintenance schedule rather than yours. `create cluster` -> does not disable it, so do it yourself on every pool that runs workers: -> -> ```bash -> gcloud container node-pools update "${NODE_POOL}" \ -> --cluster "${CLUSTER_NAME}" --location "${CLUSTER_LOCATION}" \ -> --no-enable-autoupgrade -> ``` -> -> This is a management setting, so it takes effect without recreating nodes and -> is safe to apply to a serving cluster. Node auto-repair, preemption and OOM -> kills reach the same path and cannot be configured away, so treat the setting -> as removing the scheduled risk rather than all of it. Change versions through -> the [rolling upgrade runbook](../../docs/upgrade.md), which has you suspend -> every actor on a node at your own pace before the node moves. - ```bash go run ./tools/setup-gcp create cluster [flags] ``` From 2eda16a73cfc2e7b180a6f66800554a0877c535b Mon Sep 17 00:00:00 2001 From: shrutiyam-glitch Date: Fri, 9 Oct 2026 23:25:14 +0000 Subject: [PATCH 41/44] Worker Self-Reported Sandbox & Hardware Compatibility (#2381) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #2348 Restoring a sandboxed actor from a checkpointed snapshot requires strict compatibility between the worker that created the snapshot and the worker restoring it — across the sandbox class (`gvisor`, `microvm`), the host hardware (`architecture`, `cpu_features`), and the sandbox runtime assets (`gvisor_asset_hash`, microVM kernel/rootfs digests). - **Unified compatibility identity (`SandboxRuntime` & `VersionedSandboxCompat`)**: Combines `sandbox_class`, an informational version `name`, and a `compat_version` (`schema_version` + `repeated AttributeEntry attributes`) into a single self-contained message. When an actor is checkpointed, the control plane can copy the worker's `default_runtime` directly onto the snapshot, and during warm restore the scheduler can match the snapshot's `SandboxRuntime` against candidate workers without inspecting runtime-specific fields. - **Zero-downtime runtime rollouts (`default_runtime` vs. `restorable_runtimes`)**: Workers report both a `default_runtime` (used for cold starts and new checkpoints) and `restorable_runtimes` (other enabled runtime versions on the worker that can still warm-restore existing snapshots while a new default version rolls out). - **Schema-versioned key-value attributes (`schema_version` + `AttributeEntry`)**: `compat_version.schema_version` (`"v1"`) versions the compatibility attribute schema itself, while `attributes` uses a list-map of `AttributeEntry` (`+k8s:listType=map`, `+k8s:listMapKey=key`) so declarative validation enforces required keys, values, and length bounds on every entry. #### Scope of This PR vs. Follow-ups - **In this PR**: Establishes the `SandboxRuntime` / `VersionedSandboxCompat` proto schema and declarative validation, updates `internal/hardware` (`ProbeHost` and `Matches`), and wires end-to-end registration of `default_runtime` (with `sandbox_class` and host architecture) and `restorable_runtimes` through `ateom`, `atelet`, `ate-api-server`, and `fake-workersync`. - **Follow-ups**: - Enrich `default_runtime` (`name`, `gvisor_asset_hash`) and `restorable_runtimes` from `SandboxConfig` once `SandboxConfig` moves to `WorkerPool`. - Probe CPU feature attributes (`cpu_features`) in `hardware.ProbeHost()` (#2345 - WIP) - Stamp `worker.status.default_runtime` onto snapshots at pause/checkpoint time and filter candidate workers with `hardware.Matches` during warm-restore scheduling. - [ ] Tests pass - [ ] Appropriate changes to documentation are included in the PR --- .../internal/apivalidation/worker_test.go | 202 ++++- .../apivalidation/zz_generated.validation.go | 390 +++++++-- cmd/ateapi/internal/controlapi/worker.go | 9 +- cmd/ateapi/internal/workerservice/register.go | 71 +- .../internal/workerservice/register_test.go | 240 +++++- .../internal/workerservice/service_test.go | 28 +- cmd/atelet/ateomsupport.go | 49 +- cmd/atelet/ateomsupport_test.go | 64 +- .../internal/apivalidation/validation_test.go | 232 +++++- .../apivalidation/zz_generated.validation.go | 274 ++++++- cmd/ateom-gvisor/main.go | 2 + cmd/ateom-microvm/main.go | 2 + .../isolate/fake-workersync/controller.go | 14 +- .../fake-workersync/controller_test.go | 18 +- internal/ateom/register.go | 41 +- internal/ateom/register_test.go | 32 +- internal/hardware/hardware.go | 81 +- internal/hardware/hardware_test.go | 100 ++- internal/proto/ateletpb/atelet.pb.go | 739 +++++++++++------- internal/proto/ateletpb/atelet.proto | 88 ++- internal/proto/ateletpb/atelet_grpc.pb.go | 4 +- pkg/proto/ateapipb/ateapi.pb.go | 684 ++++++++++------ pkg/proto/ateapipb/ateapi.proto | 113 ++- pkg/proto/ateapipb/ateapi_grpc.pb.go | 28 +- 24 files changed, 2645 insertions(+), 860 deletions(-) diff --git a/cmd/ateapi/internal/apivalidation/worker_test.go b/cmd/ateapi/internal/apivalidation/worker_test.go index b867c7dda5..70667446d1 100644 --- a/cmd/ateapi/internal/apivalidation/worker_test.go +++ b/cmd/ateapi/internal/apivalidation/worker_test.go @@ -324,6 +324,7 @@ func TestValidateWorkerUpdate_RequireStatus(t *testing.T) { } func TestValidateRegisterWorkerRequest(t *testing.T) { + validRuntime := validSandboxRuntime valid := func(mutate ...func(*ateapipb.RegisterWorkerRequest)) *ateapipb.RegisterWorkerRequest { r := &ateapipb.RegisterWorkerRequest{ Worker: workerRef(apiWorkerName), @@ -333,13 +334,15 @@ func TestValidateRegisterWorkerRequest(t *testing.T) { Limits: []*ateapipb.Limits{{Name: "cpu", Quantity: "2"}, {Name: "memory", Quantity: "4Gi"}}, }, }, - Hardware: &ateapipb.HardwareIdentity{Attributes: map[string]string{"architecture": "amd64"}}, + DefaultRuntime: validRuntime(), } for _, m := range mutate { m(r) } return r } + defaultRuntimePath := field.NewPath("default_runtime") + restorablePath := field.NewPath("restorable_runtimes") tests := []struct { name string req *ateapipb.RegisterWorkerRequest @@ -347,6 +350,19 @@ func TestValidateRegisterWorkerRequest(t *testing.T) { }{{ name: "valid", req: valid(), + }, { + name: "valid with restorable runtimes", + req: valid(func(r *ateapipb.RegisterWorkerRequest) { + r.RestorableRuntimes = []*ateapipb.SandboxRuntime{{ + SandboxClass: "gvisor", + Version: &ateapipb.VersionedSandboxCompat{ + SchemaVersion: "v1", + Attributes: []*ateapipb.AttributeEntry{ + {Key: "architecture", Value: "amd64"}, + }, + }, + }} + }), }, { name: "valid empty capacity", req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.Capacity = &ateapipb.WorkerResources{} }), @@ -381,28 +397,96 @@ func TestValidateRegisterWorkerRequest(t *testing.T) { }), want: field.ErrorList{field.Invalid(field.NewPath("capacity", "resources", "limits").Index(0).Child("quantity"), nil, "")}, }, { - name: "missing hardware", - req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.Hardware = nil }), - want: field.ErrorList{field.Required(field.NewPath("hardware"), "")}, + name: "missing default_runtime", + req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.DefaultRuntime = nil }), + want: field.ErrorList{field.Required(defaultRuntimePath, "")}, }, { - name: "valid with empty hardware attributes", - req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.Hardware = &ateapipb.HardwareIdentity{} }), + name: "missing default_runtime.sandbox_class", + req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.DefaultRuntime.SandboxClass = "" }), + want: field.ErrorList{field.Required(defaultRuntimePath.Child("sandbox_class"), "")}, }, { - name: "hardware attribute key too long", + name: "default_runtime.sandbox_class too long", + req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.DefaultRuntime.SandboxClass = strings.Repeat("c", 64) }), + want: field.ErrorList{field.TooLong(defaultRuntimePath.Child("sandbox_class"), "", 63).WithOrigin("maxLength")}, + }, { + name: "missing default_runtime.version", + req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.DefaultRuntime.Version = nil }), + want: field.ErrorList{field.Required(defaultRuntimePath.Child("version"), "")}, + }, { + name: "missing default_runtime.version.schema_version", + req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.DefaultRuntime.Version.SchemaVersion = "" }), + want: field.ErrorList{field.Required(defaultRuntimePath.Child("version", "schema_version"), "")}, + }, { + name: "default_runtime.version.schema_version too long", req: valid(func(r *ateapipb.RegisterWorkerRequest) { - r.Hardware = &ateapipb.HardwareIdentity{ - Attributes: map[string]string{strings.Repeat("k", 129): "v"}, - } + r.DefaultRuntime.Version.SchemaVersion = strings.Repeat("v", 65) }), - want: field.ErrorList{field.TooLong(field.NewPath("hardware", "attributes"), "", 128).WithOrigin("maxLength")}, + want: field.ErrorList{field.TooLong(defaultRuntimePath.Child("version", "schema_version"), "", 64).WithOrigin("maxLength")}, }, { - name: "hardware attribute value too long", + name: "missing default_runtime.version.attributes", + req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.DefaultRuntime.Version.Attributes = nil }), + want: field.ErrorList{field.Required(defaultRuntimePath.Child("version", "attributes"), "")}, + }, { + name: "duplicate attribute key", req: valid(func(r *ateapipb.RegisterWorkerRequest) { - r.Hardware = &ateapipb.HardwareIdentity{ - Attributes: map[string]string{"k": strings.Repeat("v", 257)}, + r.DefaultRuntime.Version.Attributes = []*ateapipb.AttributeEntry{ + {Key: "architecture", Value: "amd64"}, + {Key: "architecture", Value: "arm64"}, } }), - want: field.ErrorList{field.TooLong(field.NewPath("hardware", "attributes").Key("k"), "", 256).WithOrigin("maxLength")}, + want: field.ErrorList{field.Duplicate(defaultRuntimePath.Child("version", "attributes").Index(1), nil)}, + }, { + name: "missing attribute key", + req: valid(func(r *ateapipb.RegisterWorkerRequest) { + r.DefaultRuntime.Version.Attributes = []*ateapipb.AttributeEntry{{Value: "amd64"}} + }), + want: field.ErrorList{field.Required(defaultRuntimePath.Child("version", "attributes").Index(0).Child("key"), "")}, + }, { + name: "attribute key too long", + req: valid(func(r *ateapipb.RegisterWorkerRequest) { + r.DefaultRuntime.Version.Attributes = []*ateapipb.AttributeEntry{{Key: strings.Repeat("k", 129), Value: "v"}} + }), + want: field.ErrorList{field.TooLong(defaultRuntimePath.Child("version", "attributes").Index(0).Child("key"), "", 128).WithOrigin("maxLength")}, + }, { + name: "missing attribute value", + req: valid(func(r *ateapipb.RegisterWorkerRequest) { + r.DefaultRuntime.Version.Attributes = []*ateapipb.AttributeEntry{{Key: "k"}} + }), + want: field.ErrorList{field.Required(defaultRuntimePath.Child("version", "attributes").Index(0).Child("value"), "")}, + }, { + name: "attribute value too long", + req: valid(func(r *ateapipb.RegisterWorkerRequest) { + r.DefaultRuntime.Version.Attributes = []*ateapipb.AttributeEntry{{Key: "k", Value: strings.Repeat("v", 257)}} + }), + want: field.ErrorList{field.TooLong(defaultRuntimePath.Child("version", "attributes").Index(0).Child("value"), "", 256).WithOrigin("maxLength")}, + }, { + name: "attributes at the bound", + req: valid(func(r *ateapipb.RegisterWorkerRequest) { + r.DefaultRuntime.Version.Attributes = attributeEntries(32) + }), + }, { + name: "too many attributes", + req: valid(func(r *ateapipb.RegisterWorkerRequest) { + r.DefaultRuntime.Version.Attributes = attributeEntries(33) + }), + want: field.ErrorList{field.TooMany(defaultRuntimePath.Child("version", "attributes"), 33, 32).WithOrigin("maxItems")}, + }, { + name: "nil restorable_runtimes entry", + req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.RestorableRuntimes = []*ateapipb.SandboxRuntime{nil} }), + want: field.ErrorList{field.Required(restorablePath.Index(0), "")}, + }, { + name: "invalid restorable_runtimes entry", + req: valid(func(r *ateapipb.RegisterWorkerRequest) { + r.RestorableRuntimes = []*ateapipb.SandboxRuntime{{SandboxClass: "gvisor"}} + }), + want: field.ErrorList{field.Required(restorablePath.Index(0).Child("version"), "")}, + }, { + name: "restorable_runtimes at the bound", + req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.RestorableRuntimes = sandboxRuntimes(32) }), + }, { + name: "too many restorable_runtimes", + req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.RestorableRuntimes = sandboxRuntimes(33) }), + want: field.ErrorList{field.TooMany(restorablePath, 33, 32).WithOrigin("maxItems")}, }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -411,6 +495,64 @@ func TestValidateRegisterWorkerRequest(t *testing.T) { } } +// TestValidateWorkerUpdate_SandboxRuntimes pins the status-side checks on the +// runtimes RegisterWorker records. They reach the store as status, which no +// request validator sees, so ValidateWorkerUpdate is the only check on them. +func TestValidateWorkerUpdate_SandboxRuntimes(t *testing.T) { + statusPath := field.NewPath("worker", "status") + oldVal := validWorker(apiWorkerName, withWorkerStatus()) + tests := []struct { + name string + mutate func(s *ateapipb.WorkerStatus) + want field.ErrorList + }{{ + name: "valid runtimes", + mutate: func(s *ateapipb.WorkerStatus) { + s.DefaultRuntime = validSandboxRuntime() + s.RestorableRuntimes = sandboxRuntimes(32) + }, + }, { + name: "missing default_runtime.sandbox_class", + mutate: func(s *ateapipb.WorkerStatus) { + s.DefaultRuntime = validSandboxRuntime() + s.DefaultRuntime.SandboxClass = "" + }, + want: field.ErrorList{field.Required(statusPath.Child("default_runtime", "sandbox_class"), "")}, + }, { + name: "missing default_runtime.version", + mutate: func(s *ateapipb.WorkerStatus) { + s.DefaultRuntime = &ateapipb.SandboxRuntime{SandboxClass: "gvisor"} + }, + want: field.ErrorList{field.Required(statusPath.Child("default_runtime", "version"), "")}, + }, { + name: "too many default_runtime attributes", + mutate: func(s *ateapipb.WorkerStatus) { + s.DefaultRuntime = validSandboxRuntime() + s.DefaultRuntime.Version.Attributes = attributeEntries(33) + }, + want: field.ErrorList{field.TooMany(statusPath.Child("default_runtime", "version", "attributes"), 33, 32).WithOrigin("maxItems")}, + }, { + name: "invalid restorable_runtimes entry", + mutate: func(s *ateapipb.WorkerStatus) { + s.RestorableRuntimes = []*ateapipb.SandboxRuntime{{SandboxClass: "gvisor"}} + }, + want: field.ErrorList{field.Required(statusPath.Child("restorable_runtimes").Index(0).Child("version"), "")}, + }, { + name: "too many restorable_runtimes", + mutate: func(s *ateapipb.WorkerStatus) { + s.RestorableRuntimes = sandboxRuntimes(33) + }, + want: field.ErrorList{field.TooMany(statusPath.Child("restorable_runtimes"), 33, 32).WithOrigin("maxItems")}, + }} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + newVal := proto.Clone(oldVal).(*ateapipb.Worker) + tt.mutate(newVal.Status) + assertValidateErr(t, ValidateWorkerUpdate(context.Background(), field.NewPath("worker"), newVal, oldVal, true), tt.want) + }) + } +} + func TestValidateRequestActorSuspendRequest(t *testing.T) { valid := func(mutate ...func(*ateapipb.RequestActorSuspendRequest)) *ateapipb.RequestActorSuspendRequest { r := &ateapipb.RequestActorSuspendRequest{ @@ -523,3 +665,33 @@ func withWorkerStatus(mods ...func(*ateapipb.WorkerStatus)) func(*ateapipb.Worke func workerRef(name string) *ateapipb.ObjectRef { return &ateapipb.ObjectRef{Name: name} } + +// validSandboxRuntime returns a SandboxRuntime in the shape RegisterWorker +// accepts: a class and a v1 identity with one attribute. +func validSandboxRuntime() *ateapipb.SandboxRuntime { + return &ateapipb.SandboxRuntime{ + SandboxClass: "gvisor", + Version: &ateapipb.VersionedSandboxCompat{ + SchemaVersion: "v1", + Attributes: []*ateapipb.AttributeEntry{{Key: "architecture", Value: "amd64"}}, + }, + } +} + +// attributeEntries returns n valid attributes with distinct keys. +func attributeEntries(n int) []*ateapipb.AttributeEntry { + attrs := make([]*ateapipb.AttributeEntry, n) + for i := range attrs { + attrs[i] = &ateapipb.AttributeEntry{Key: fmt.Sprintf("k%d", i), Value: "v"} + } + return attrs +} + +// sandboxRuntimes returns n valid runtimes. +func sandboxRuntimes(n int) []*ateapipb.SandboxRuntime { + rts := make([]*ateapipb.SandboxRuntime, n) + for i := range rts { + rts[i] = validSandboxRuntime() + } + return rts +} diff --git a/cmd/ateapi/internal/apivalidation/zz_generated.validation.go b/cmd/ateapi/internal/apivalidation/zz_generated.validation.go index e083f0d18b..f754fa9d6c 100644 --- a/cmd/ateapi/internal/apivalidation/zz_generated.validation.go +++ b/cmd/ateapi/internal/apivalidation/zz_generated.validation.go @@ -1307,6 +1307,79 @@ func Validate_Atespace( return errs } +// Validate_AttributeEntry validates an instance of AttributeEntry according +// to declarative validation rules in the API schema. +func Validate_AttributeEntry( + ctx context.Context, op operation.Operation, fldPath *field.Path, + obj, oldObj *ateapipb.AttributeEntry) (errs field.ErrorList) { + + { // field ateapipb.AttributeEntry.Key + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.MaxLength(ctx, op, fldPath, obj, oldObj, 128); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.AttributeEntry) *string { + return &oldObj.Key + }) + errs = append(errs, fn(fldPath.Child("key"), &obj.Key, oldVal, oldObj != nil)...) + } + + { // field ateapipb.AttributeEntry.Value + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.MaxLength(ctx, op, fldPath, obj, oldObj, 256); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.AttributeEntry) *string { + return &oldObj.Value + }) + errs = append(errs, fn(fldPath.Child("value"), &obj.Value, oldVal, oldObj != nil)...) + } + + return errs +} + // Validate_Binding validates an instance of Binding according // to declarative validation rules in the API schema. func Validate_Binding( @@ -4642,59 +4715,6 @@ func Validate_HTTPSRule( return errs } -// Validate_HardwareIdentity validates an instance of HardwareIdentity according -// to declarative validation rules in the API schema. -func Validate_HardwareIdentity( - ctx context.Context, op operation.Operation, fldPath *field.Path, - obj, oldObj *ateapipb.HardwareIdentity) (errs field.ErrorList) { - - { // field ateapipb.HardwareIdentity.Attributes - fn := func( - fldPath *field.Path, - obj, oldObj map[string]string, - oldValueCorrelated bool) (errs field.ErrorList) { - // don't revalidate unchanged data - if oldValueCorrelated && op.Type == operation.Update { - if ateDeepEqual(obj, oldObj) { - return nil - } - } - // call field-attached validations - earlyReturn := false - if e := validate.MaxProperties(ctx, op, fldPath, obj, oldObj, 32).MarkShortCircuit(); len(e) != 0 { - errs = append(errs, e...) - earlyReturn = true - } - if e := validate.OptionalMap(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { - earlyReturn = true - } - if earlyReturn { - return // do not proceed - } - if e := validate.EachMapKey(ctx, op, fldPath, obj, oldObj, - func(ctx context.Context, op operation.Operation, fldPath *field.Path, obj, oldObj *string) field.ErrorList { - return validate.MaxLength(ctx, op, fldPath, obj, oldObj, 128) - }); len(e) != 0 { - errs = append(errs, e...) - } - if e := validate.EachMapVal(ctx, op, fldPath, obj, oldObj, validate.DirectEqual, - func(ctx context.Context, op operation.Operation, fldPath *field.Path, obj, oldObj *string) field.ErrorList { - return validate.MaxLength(ctx, op, fldPath, obj, oldObj, 256) - }); len(e) != 0 { - errs = append(errs, e...) - } - return - } - oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.HardwareIdentity) map[string]string { - return oldObj.Attributes - }) - errs = append(errs, fn(fldPath.Child("attributes"), obj.Attributes, oldVal, oldObj != nil)...) - } - - return errs -} - // Validate_HttpRuleEffects validates an instance of HttpRuleEffects according // to declarative validation rules in the API schema. func Validate_HttpRuleEffects( @@ -6187,10 +6207,10 @@ func Validate_RegisterWorkerRequest( errs = append(errs, fn(fldPath.Child("capacity"), obj.Capacity, oldVal, oldObj != nil)...) } - { // field ateapipb.RegisterWorkerRequest.Hardware + { // field ateapipb.RegisterWorkerRequest.DefaultRuntime fn := func( fldPath *field.Path, - obj, oldObj *ateapipb.HardwareIdentity, + obj, oldObj *ateapipb.SandboxRuntime, oldValueCorrelated bool) (errs field.ErrorList) { // don't revalidate unchanged data if oldValueCorrelated && op.Type == operation.Update { @@ -6208,14 +6228,54 @@ func Validate_RegisterWorkerRequest( return // do not proceed } // call the type's validation function - errs = append(errs, Validate_HardwareIdentity(ctx, op, fldPath, obj, oldObj)...) + errs = append(errs, Validate_SandboxRuntime(ctx, op, fldPath, obj, oldObj)...) + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.RegisterWorkerRequest) *ateapipb.SandboxRuntime { + return oldObj.DefaultRuntime + }) + errs = append(errs, fn(fldPath.Child("default_runtime"), obj.DefaultRuntime, oldVal, oldObj != nil)...) + } + + { // field ateapipb.RegisterWorkerRequest.RestorableRuntimes + fn := func( + fldPath *field.Path, + obj, oldObj []*ateapipb.SandboxRuntime, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.PtrSliceNoNils[ateapipb.SandboxRuntime](ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.MaxItems(ctx, op, fldPath, obj, oldObj, 32).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.OptionalSlice(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // iterate the list and call the type's validation function + if e := validate.EachPtrSliceVal(ctx, op, fldPath, obj, oldObj, nil, nil, Validate_SandboxRuntime); len(e) != 0 { + errs = append(errs, e...) + } return } oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.RegisterWorkerRequest) *ateapipb.HardwareIdentity { - return oldObj.Hardware + func(oldObj *ateapipb.RegisterWorkerRequest) []*ateapipb.SandboxRuntime { + return oldObj.RestorableRuntimes }) - errs = append(errs, fn(fldPath.Child("hardware"), obj.Hardware, oldVal, oldObj != nil)...) + errs = append(errs, fn(fldPath.Child("restorable_runtimes"), obj.RestorableRuntimes, oldVal, oldObj != nil)...) } return errs @@ -6822,6 +6882,78 @@ func Validate_SandboxConfig( return errs } +// Validate_SandboxRuntime validates an instance of SandboxRuntime according +// to declarative validation rules in the API schema. +func Validate_SandboxRuntime( + ctx context.Context, op operation.Operation, fldPath *field.Path, + obj, oldObj *ateapipb.SandboxRuntime) (errs field.ErrorList) { + + { // field ateapipb.SandboxRuntime.SandboxClass + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.MaxLength(ctx, op, fldPath, obj, oldObj, 63); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.SandboxRuntime) *string { + return &oldObj.SandboxClass + }) + errs = append(errs, fn(fldPath.Child("sandbox_class"), &obj.SandboxClass, oldVal, oldObj != nil)...) + } + + { // field ateapipb.SandboxRuntime.Version + fn := func( + fldPath *field.Path, + obj, oldObj *ateapipb.VersionedSandboxCompat, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // call the type's validation function + errs = append(errs, Validate_VersionedSandboxCompat(ctx, op, fldPath, obj, oldObj)...) + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.SandboxRuntime) *ateapipb.VersionedSandboxCompat { + return oldObj.Version + }) + errs = append(errs, fn(fldPath.Child("version"), obj.Version, oldVal, oldObj != nil)...) + } + + return errs +} + // Validate_SecurityContext validates an instance of SecurityContext according // to declarative validation rules in the API schema. func Validate_SecurityContext( @@ -8359,6 +8491,94 @@ func Validate_UpdateWorkerRequest( return errs } +// Validate_VersionedSandboxCompat validates an instance of VersionedSandboxCompat according +// to declarative validation rules in the API schema. +func Validate_VersionedSandboxCompat( + ctx context.Context, op operation.Operation, fldPath *field.Path, + obj, oldObj *ateapipb.VersionedSandboxCompat) (errs field.ErrorList) { + + { // field ateapipb.VersionedSandboxCompat.SchemaVersion + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.MaxLength(ctx, op, fldPath, obj, oldObj, 64); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.VersionedSandboxCompat) *string { + return &oldObj.SchemaVersion + }) + errs = append(errs, fn(fldPath.Child("schema_version"), &obj.SchemaVersion, oldVal, oldObj != nil)...) + } + + { // field ateapipb.VersionedSandboxCompat.Attributes + fn := func( + fldPath *field.Path, + obj, oldObj []*ateapipb.AttributeEntry, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.PtrSliceNoNils[ateapipb.AttributeEntry](ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.MaxItems(ctx, op, fldPath, obj, oldObj, 32).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.RequiredSlice(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // lists with map semantics require unique keys + if e := validate.PtrSliceUnique(ctx, op, fldPath, obj, oldObj, + func(a *ateapipb.AttributeEntry, b *ateapipb.AttributeEntry) bool { return a.Key == b.Key }); len(e) != 0 { + errs = append(errs, e...) + } + // iterate the list and call the type's validation function + if e := validate.EachPtrSliceVal(ctx, op, fldPath, obj, oldObj, + func(a *ateapipb.AttributeEntry, b *ateapipb.AttributeEntry) bool { return a.Key == b.Key }, ateDeepEqual, Validate_AttributeEntry); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.VersionedSandboxCompat) []*ateapipb.AttributeEntry { + return oldObj.Attributes + }) + errs = append(errs, fn(fldPath.Child("attributes"), obj.Attributes, oldVal, oldObj != nil)...) + } + + return errs +} + var unionMembershipFor_github_com_agent_substrate_substrate_pkg_proto_ateapipb_Volume_ = validate.NewUnionMembership(validate.NewUnionMember("durable_dir"), validate.NewUnionMember("external_volume_template"), validate.NewUnionMember("system_info"), validate.NewUnionMember("image")) // Validate_Volume validates an instance of Volume according @@ -9541,10 +9761,10 @@ func Validate_WorkerStatus( errs = append(errs, fn(fldPath.Child("observed_epoch"), &obj.ObservedEpoch, oldVal, oldObj != nil)...) } - { // field ateapipb.WorkerStatus.Hardware + { // field ateapipb.WorkerStatus.DefaultRuntime fn := func( fldPath *field.Path, - obj, oldObj *ateapipb.HardwareIdentity, + obj, oldObj *ateapipb.SandboxRuntime, oldValueCorrelated bool) (errs field.ErrorList) { // don't revalidate unchanged data if oldValueCorrelated && op.Type == operation.Update { @@ -9561,14 +9781,54 @@ func Validate_WorkerStatus( return // do not proceed } // call the type's validation function - errs = append(errs, Validate_HardwareIdentity(ctx, op, fldPath, obj, oldObj)...) + errs = append(errs, Validate_SandboxRuntime(ctx, op, fldPath, obj, oldObj)...) + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.WorkerStatus) *ateapipb.SandboxRuntime { + return oldObj.DefaultRuntime + }) + errs = append(errs, fn(fldPath.Child("default_runtime"), obj.DefaultRuntime, oldVal, oldObj != nil)...) + } + + { // field ateapipb.WorkerStatus.RestorableRuntimes + fn := func( + fldPath *field.Path, + obj, oldObj []*ateapipb.SandboxRuntime, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.PtrSliceNoNils[ateapipb.SandboxRuntime](ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.MaxItems(ctx, op, fldPath, obj, oldObj, 32).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.OptionalSlice(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // iterate the list and call the type's validation function + if e := validate.EachPtrSliceVal(ctx, op, fldPath, obj, oldObj, nil, nil, Validate_SandboxRuntime); len(e) != 0 { + errs = append(errs, e...) + } return } oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.WorkerStatus) *ateapipb.HardwareIdentity { - return oldObj.Hardware + func(oldObj *ateapipb.WorkerStatus) []*ateapipb.SandboxRuntime { + return oldObj.RestorableRuntimes }) - errs = append(errs, fn(fldPath.Child("hardware"), obj.Hardware, oldVal, oldObj != nil)...) + errs = append(errs, fn(fldPath.Child("restorable_runtimes"), obj.RestorableRuntimes, oldVal, oldObj != nil)...) } return errs diff --git a/cmd/ateapi/internal/controlapi/worker.go b/cmd/ateapi/internal/controlapi/worker.go index f66d6f5634..b67253ee3a 100644 --- a/cmd/ateapi/internal/controlapi/worker.go +++ b/cmd/ateapi/internal/controlapi/worker.go @@ -132,10 +132,11 @@ func (s *ServiceImpl) CreateWorker(ctx context.Context, inWorker *ateapipb.Worke ObservedEpoch: inWorker.GetEpoch(), } - // Capacity and hardware are left unset: a Worker holds nothing until its - // own ateom says what it has, through WorkerService.RegisterWorker. Nothing is placed - // on it in the meantime, which is the point -- the alternative is guessing - // on the Worker's behalf and placing against the guess. + // Capacity and runtimes are left unset: a Worker holds nothing and runs no + // sandbox until its own ateom says what it has, through + // WorkerService.RegisterWorker. Nothing is placed on it in the meantime, + // which is the point -- the alternative is guessing on the Worker's behalf + // and placing against the guess. // Verify that the result is properly valid before storing it. if errs := apivalidation.ValidateWorkerUpdate(ctx, field.NewPath("worker"), outWorker, inWorker, true); len(errs) > 0 { diff --git a/cmd/ateapi/internal/workerservice/register.go b/cmd/ateapi/internal/workerservice/register.go index ad6b6aee1a..2ae394e889 100644 --- a/cmd/ateapi/internal/workerservice/register.go +++ b/cmd/ateapi/internal/workerservice/register.go @@ -15,10 +15,12 @@ package workerservice import ( + "cmp" "context" "errors" "fmt" "log/slog" + "slices" "github.com/agent-substrate/substrate/cmd/ateapi/internal/apivalidation" "github.com/agent-substrate/substrate/cmd/ateapi/internal/ateletauth" @@ -29,9 +31,9 @@ import ( "google.golang.org/protobuf/proto" ) -// RegisterWorker records a Worker's reported capacity and the hardware identity of -// its node in one write. As with MintCert, the caller must be an atelet -// running on the Worker's node. +// RegisterWorker records a Worker's reported capacity and sandbox runtimes in +// one write. As with MintCert, the caller must be an atelet running on the +// Worker's node. func (s *Server) RegisterWorker(ctx context.Context, req *ateapipb.RegisterWorkerRequest) (*ateapipb.RegisterWorkerResponse, error) { // TODO(identity): This check should be handled by OpenFGA. caller, err := ateletauth.Authenticate(ctx, s.ateletSPIFFEID) @@ -42,8 +44,13 @@ func (s *Server) RegisterWorker(ctx context.Context, req *ateapipb.RegisterWorke return nil, resources.ToAPIError(errs) } reported := req.GetCapacity() - reportedHardware := req.GetHardware() + defaultRuntime := req.GetDefaultRuntime() + restorable := req.GetRestorableRuntimes() name := req.GetWorker().GetName() + sortAttributes(defaultRuntime) + for _, rt := range restorable { + sortAttributes(rt) + } // Use authoritative state to authorize the write. worker, err := s.store.GetWorker(ctx, name) @@ -62,16 +69,24 @@ func (s *Server) RegisterWorker(ctx context.Context, req *ateapipb.RegisterWorke slog.String("caller_pod", caller.PodName)) return nil, apierror.NotFound("Worker %s not found", name) } + if err := validateRuntimeClasses(worker.GetSandboxClass(), req); err != nil { + return nil, err + } - if proto.Equal(worker.GetStatus().GetCapacity(), reported) && proto.Equal(worker.GetStatus().GetHardware(), reportedHardware) { + status := worker.GetStatus() + if proto.Equal(status.GetCapacity(), reported) && + proto.Equal(status.GetDefaultRuntime(), defaultRuntime) && + sameRuntimes(status.GetRestorableRuntimes(), restorable) { return &ateapipb.RegisterWorkerResponse{Worker: worker}, nil } updated, err := s.store.UpdateWorker(ctx, name, store.PreconditionFrom(worker), func(toUpdate *ateapipb.Worker) error { // Replaces rather than merges: a Worker reports everything it has, so a - // dimension this report leaves out is one it no longer supplies. + // dimension or runtime this report leaves out is one it no longer + // supplies. toUpdate.Status.Capacity = reported - toUpdate.Status.Hardware = reportedHardware + toUpdate.Status.DefaultRuntime = defaultRuntime + toUpdate.Status.RestorableRuntimes = restorable return nil }) switch { @@ -83,10 +98,48 @@ func (s *Server) RegisterWorker(ctx context.Context, req *ateapipb.RegisterWorke default: return nil, fmt.Errorf("while recording capacity for worker %s: %w", name, err) } - slog.InfoContext(ctx, "Worker registered its capacity and hardware", + slog.InfoContext(ctx, "Worker registered its capacity and sandbox runtimes", slog.String("worker", name), slog.String("was", worker.GetStatus().GetCapacity().String()), slog.String("now", updated.GetStatus().GetCapacity().String()), - slog.String("hardware", updated.GetStatus().GetHardware().String())) + slog.String("default_runtime", updated.GetStatus().GetDefaultRuntime().String()), + slog.Int("restorable_runtimes", len(updated.GetStatus().GetRestorableRuntimes()))) return &ateapipb.RegisterWorkerResponse{Worker: updated}, nil } + +// validateRuntimeClasses rejects a runtime of a class other than the Worker's +// own. The class is fixed when the pool creates the Worker and names the +// sandbox its ateom runs, so a runtime of another class contradicts the +// recorded Worker state. A Worker recorded without a class has nothing for a +// report to contradict. +func validateRuntimeClasses(class string, req *ateapipb.RegisterWorkerRequest) error { + if class == "" { + return nil + } + if got := req.GetDefaultRuntime().GetSandboxClass(); got != class { + return apierror.FailedPrecondition("default_runtime.sandbox_class %q does not match Worker's sandbox_class %q", got, class) + } + for i, rt := range req.GetRestorableRuntimes() { + if got := rt.GetSandboxClass(); got != class { + return apierror.FailedPrecondition("restorable_runtimes[%d].sandbox_class %q does not match Worker's sandbox_class %q", i, got, class) + } + } + return nil +} + +// sameRuntimes reports whether two runtime lists are identical entry for +// entry, in order. A report is compared as sent, so a Worker that reorders its +// runtimes between reports is recorded again. +func sameRuntimes(a, b []*ateapipb.SandboxRuntime) bool { + return slices.EqualFunc(a, b, func(x, y *ateapipb.SandboxRuntime) bool { return proto.Equal(x, y) }) +} + +// sortAttributes orders a runtime's compatibility attributes by key, in place. +// Attribute order carries no meaning, but proto equality is order-sensitive, so +// a report is canonicalized before it is compared with what is stored: the same +// attributes in another order are the same runtime and write nothing. +func sortAttributes(rt *ateapipb.SandboxRuntime) { + slices.SortFunc(rt.GetVersion().GetAttributes(), func(a, b *ateapipb.AttributeEntry) int { + return cmp.Compare(a.GetKey(), b.GetKey()) + }) +} diff --git a/cmd/ateapi/internal/workerservice/register_test.go b/cmd/ateapi/internal/workerservice/register_test.go index 2761cbce4f..52e0cc15be 100644 --- a/cmd/ateapi/internal/workerservice/register_test.go +++ b/cmd/ateapi/internal/workerservice/register_test.go @@ -16,9 +16,11 @@ package workerservice import ( "context" + "fmt" "testing" "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" "google.golang.org/protobuf/testing/protocmp" "github.com/agent-substrate/substrate/cmd/ateapi/internal/ateletauth/ateletauthtest" @@ -29,13 +31,28 @@ import ( "google.golang.org/grpc/codes" ) -var testHardware = &ateapipb.HardwareIdentity{Attributes: map[string]string{"architecture": "amd64"}} +// testRuntime is a SandboxRuntime of the given class whose compatibility +// identity is a single architecture attribute, which is enough to tell two +// runtimes apart. +func testRuntime(class, arch string) *ateapipb.SandboxRuntime { + return &ateapipb.SandboxRuntime{ + SandboxClass: class, + Version: &ateapipb.VersionedSandboxCompat{ + SchemaVersion: "v1", + Attributes: []*ateapipb.AttributeEntry{{Key: "architecture", Value: arch}}, + }, + } +} + +// testDefaultRuntime is what seedReportedWorker records as already reported, +// of the class it gives the Worker. +var testDefaultRuntime = testRuntime("gvisor", "amd64") func setRequest(actors int32) *ateapipb.RegisterWorkerRequest { return &ateapipb.RegisterWorkerRequest{ - Worker: &ateapipb.ObjectRef{Name: testWorkerName}, - Capacity: &ateapipb.WorkerResources{Actors: actors}, - Hardware: testHardware, + Worker: &ateapipb.ObjectRef{Name: testWorkerName}, + Capacity: &ateapipb.WorkerResources{Actors: actors}, + DefaultRuntime: testDefaultRuntime, } } @@ -62,35 +79,138 @@ func TestRegisterWorker(t *testing.T) { } } -func TestRegisterWorker_RecordsHardware(t *testing.T) { +// A report's runtimes are checked against the Worker's class and recorded only +// when they differ from what is stored. Capacity is held at what was seeded, +// so a write here is one the runtimes alone caused. +func TestRegisterWorker_Runtimes(t *testing.T) { st, cleanup := storetest.SetupTestStore(t) defer cleanup() s := New(st, &fakeSuspender{}, testAteletSPIFFEID, nil) - seedReportedWorker(t, st, testNode, &ateapipb.WorkerResources{Actors: 1}) authed := ateletauthtest.ContextWith(ateletauthtest.CertOn(t, testNode)) - wantHW := &ateapipb.HardwareIdentity{Attributes: map[string]string{"architecture": "arm64"}} - req := &ateapipb.RegisterWorkerRequest{ - Worker: &ateapipb.ObjectRef{Name: testWorkerName}, - Capacity: &ateapipb.WorkerResources{Actors: 4094}, - Hardware: wantHW, - } - got, err := s.RegisterWorker(authed, req) - if err != nil { - t.Fatalf("RegisterWorker() failed: %v", err) - } - if diff := cmp.Diff(wantHW, got.GetWorker().GetStatus().GetHardware(), protocmp.Transform()); diff != "" { - t.Errorf("hardware mismatch (-want +got):\n%s", diff) + gvisorAMD64, gvisorARM64 := testRuntime("gvisor", "amd64"), testRuntime("gvisor", "arm64") + gvisorV8, gvisorV9 := testRuntime("gvisor", "arm64-v8"), testRuntime("gvisor", "arm64-v9") + microvm := testRuntime("microvm", "amd64") + // Every Worker here has already reported gvisorAMD64 with gvisorV8 and + // gvisorV9 restorable, in that order, at a capacity of one actor. + seeded := func() *ateapipb.WorkerStatus { + return &ateapipb.WorkerStatus{ + State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, + Capacity: &ateapipb.WorkerResources{Actors: 1}, + DefaultRuntime: gvisorAMD64, + RestorableRuntimes: []*ateapipb.SandboxRuntime{gvisorV8, gvisorV9}, + } } - // Repeating the identical capacity and hardware must not bump version. - v1 := got.GetWorker().GetMetadata().GetVersion() - again, err := s.RegisterWorker(authed, req) - if err != nil { - t.Fatalf("RegisterWorker() repeat failed: %v", err) - } - if gotV := again.GetWorker().GetMetadata().GetVersion(); gotV != v1 { - t.Errorf("version = %d after identical capacity+hardware report, want %d unchanged", gotV, v1) + tests := []struct { + name string + // workerClass is the class the pool created the Worker with; "" is + // a Worker recorded without one. + workerClass string + def *ateapipb.SandboxRuntime + restorable []*ateapipb.SandboxRuntime + wantCode codes.Code + // wantWrite is whether the report changes the record. + wantWrite bool + }{{ + name: "identical report writes nothing", + workerClass: "gvisor", + def: gvisorAMD64, + restorable: []*ateapipb.SandboxRuntime{gvisorV8, gvisorV9}, + }, { + name: "changed default runtime writes", + workerClass: "gvisor", + def: gvisorARM64, + restorable: []*ateapipb.SandboxRuntime{gvisorV8, gvisorV9}, + wantWrite: true, + }, { + name: "changed restorable runtime writes", + workerClass: "gvisor", + def: gvisorAMD64, + restorable: []*ateapipb.SandboxRuntime{gvisorV8, gvisorARM64}, + wantWrite: true, + }, { + // A report is compared as sent: the list is atomic, so a new order + // is a new report. + name: "reordered restorable runtimes write", + workerClass: "gvisor", + def: gvisorAMD64, + restorable: []*ateapipb.SandboxRuntime{gvisorV9, gvisorV8}, + wantWrite: true, + }, { + // Runtimes are replaced like capacity: a report that lists no + // restorable runtimes is a Worker that can restore from none but its + // default, as when a version is disabled, and the record must not + // keep advertising them. + name: "dropped restorable runtimes write", + workerClass: "gvisor", + def: gvisorAMD64, + wantWrite: true, + }, { + // The class is fixed by the pool that created the Worker. A runtime + // of another class is one it cannot run, so the report is refused + // rather than recorded for the scheduler to act on. + name: "default runtime of another class is refused", + workerClass: "gvisor", + def: microvm, + restorable: []*ateapipb.SandboxRuntime{gvisorV8, gvisorV9}, + wantCode: codes.FailedPrecondition, + }, { + name: "restorable runtime of another class is refused", + workerClass: "gvisor", + def: gvisorAMD64, + restorable: []*ateapipb.SandboxRuntime{gvisorV8, microvm}, + wantCode: codes.FailedPrecondition, + }, { + // A Worker recorded without a class has nothing for a report to + // contradict. + name: "unclassed Worker takes a default runtime of any class", + def: microvm, + restorable: []*ateapipb.SandboxRuntime{gvisorV8, gvisorV9}, + wantWrite: true, + }, { + name: "unclassed Worker takes restorable runtimes of mixed classes", + def: gvisorAMD64, + restorable: []*ateapipb.SandboxRuntime{gvisorV8, microvm}, + wantWrite: true, + }} + for i, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + name := fmt.Sprintf("1a7e4c83-6d20-4f95-b3c8-%012d", i) + before := seedWorker(t, st, name, testNode, tt.workerClass, seeded()) + + got, err := s.RegisterWorker(authed, &ateapipb.RegisterWorkerRequest{ + Worker: &ateapipb.ObjectRef{Name: name}, + Capacity: &ateapipb.WorkerResources{Actors: 1}, + DefaultRuntime: tt.def, + RestorableRuntimes: tt.restorable, + }) + if code := apierror.Code(err); code != tt.wantCode { + t.Fatalf("code = %v (err %v), want %v", code, err, tt.wantCode) + } + after, err := st.GetWorker(context.Background(), name) + if err != nil { + t.Fatalf("GetWorker: %v", err) + } + if wrote := after.GetMetadata().GetVersion() != before.GetMetadata().GetVersion(); wrote != tt.wantWrite { + t.Fatalf("version %d -> %d, want a write: %v", before.GetMetadata().GetVersion(), after.GetMetadata().GetVersion(), tt.wantWrite) + } + if !tt.wantWrite { + if diff := cmp.Diff(before.GetStatus(), after.GetStatus(), protocmp.Transform()); diff != "" { + t.Errorf("status changed without a write (-want +got):\n%s", diff) + } + return + } + if diff := cmp.Diff(tt.def, after.GetStatus().GetDefaultRuntime(), protocmp.Transform()); diff != "" { + t.Errorf("default_runtime mismatch (-want +got):\n%s", diff) + } + if diff := cmp.Diff(tt.restorable, after.GetStatus().GetRestorableRuntimes(), protocmp.Transform(), cmpopts.EquateEmpty()); diff != "" { + t.Errorf("restorable_runtimes mismatch (-want +got):\n%s", diff) + } + if diff := cmp.Diff(after, got.GetWorker(), protocmp.Transform()); diff != "" { + t.Errorf("response is not the stored Worker (-want +got):\n%s", diff) + } + }) } } @@ -141,6 +261,52 @@ func TestRegisterWorker_UnchangedDoesNotWrite(t *testing.T) { } } +// Attribute order is the producer's business, not the record's: the same +// attributes in another order are the same runtime, so they are stored sorted +// by key and a reordered repeat writes nothing. +func TestRegisterWorker_SortsAttributes(t *testing.T) { + st, cleanup := storetest.SetupTestStore(t) + defer cleanup() + s := New(st, &fakeSuspender{}, testAteletSPIFFEID, nil) + seedReportedWorker(t, st, testNode, &ateapipb.WorkerResources{Actors: 1}) + authed := ateletauthtest.ContextWith(ateletauthtest.CertOn(t, testNode)) + + runtimeWith := func(keys ...string) *ateapipb.SandboxRuntime { + rt := testRuntime("gvisor", "amd64") + rt.Version.Attributes = nil + for _, k := range keys { + rt.Version.Attributes = append(rt.Version.Attributes, &ateapipb.AttributeEntry{Key: k, Value: "v"}) + } + return rt + } + register := func(def, restorable *ateapipb.SandboxRuntime) *ateapipb.Worker { + t.Helper() + got, err := s.RegisterWorker(authed, &ateapipb.RegisterWorkerRequest{ + Worker: &ateapipb.ObjectRef{Name: testWorkerName}, + Capacity: &ateapipb.WorkerResources{Actors: 1}, + DefaultRuntime: def, + RestorableRuntimes: []*ateapipb.SandboxRuntime{restorable}, + }) + if err != nil { + t.Fatalf("RegisterWorker() failed: %v", err) + } + return got.GetWorker() + } + + first := register(runtimeWith("cpu_features", "architecture"), runtimeWith("gvisor_asset_hash", "architecture")) + if diff := cmp.Diff(runtimeWith("architecture", "cpu_features"), first.GetStatus().GetDefaultRuntime(), protocmp.Transform()); diff != "" { + t.Errorf("default_runtime not stored sorted by key (-want +got):\n%s", diff) + } + if diff := cmp.Diff([]*ateapipb.SandboxRuntime{runtimeWith("architecture", "gvisor_asset_hash")}, first.GetStatus().GetRestorableRuntimes(), protocmp.Transform()); diff != "" { + t.Errorf("restorable_runtimes not stored sorted by key (-want +got):\n%s", diff) + } + + again := register(runtimeWith("architecture", "cpu_features"), runtimeWith("architecture", "gvisor_asset_hash")) + if got, want := again.GetMetadata().GetVersion(), first.GetMetadata().GetVersion(); got != want { + t.Errorf("version = %d after a reordered repeat, want %d unchanged", got, want) + } +} + func TestRegisterWorker_Errors(t *testing.T) { st, cleanup := storetest.SetupTestStore(t) defer cleanup() @@ -156,21 +322,21 @@ func TestRegisterWorker_Errors(t *testing.T) { }{ {"unauthenticated", ateletauthtest.ContextWith(nil), setRequest(2), codes.Unauthenticated}, {"no worker ref", authed, &ateapipb.RegisterWorkerRequest{ - Capacity: &ateapipb.WorkerResources{Actors: 2}, - Hardware: testHardware, + Capacity: &ateapipb.WorkerResources{Actors: 2}, + DefaultRuntime: testDefaultRuntime, }, codes.InvalidArgument}, {"no capacity", authed, &ateapipb.RegisterWorkerRequest{ - Worker: &ateapipb.ObjectRef{Name: testWorkerName}, - Hardware: testHardware, + Worker: &ateapipb.ObjectRef{Name: testWorkerName}, + DefaultRuntime: testDefaultRuntime, }, codes.InvalidArgument}, - {"no hardware", authed, &ateapipb.RegisterWorkerRequest{ + {"no default runtime", authed, &ateapipb.RegisterWorkerRequest{ Worker: &ateapipb.ObjectRef{Name: testWorkerName}, Capacity: &ateapipb.WorkerResources{Actors: 2}, }, codes.InvalidArgument}, {"absent worker", authed, &ateapipb.RegisterWorkerRequest{ - Worker: &ateapipb.ObjectRef{Name: "3b9f1e77-2c4d-4a80-91be-6d5c8f0a7e21"}, - Capacity: &ateapipb.WorkerResources{Actors: 2}, - Hardware: testHardware, + Worker: &ateapipb.ObjectRef{Name: "3b9f1e77-2c4d-4a80-91be-6d5c8f0a7e21"}, + Capacity: &ateapipb.WorkerResources{Actors: 2}, + DefaultRuntime: testDefaultRuntime, }, codes.NotFound}, } for _, tc := range tests { @@ -206,9 +372,9 @@ func TestRegisterWorker_RejectsNonsense(t *testing.T) { } { t.Run(tc.name, func(t *testing.T) { _, err := s.RegisterWorker(authed, &ateapipb.RegisterWorkerRequest{ - Worker: &ateapipb.ObjectRef{Name: testWorkerName}, - Capacity: tc.capacity, - Hardware: testHardware, + Worker: &ateapipb.ObjectRef{Name: testWorkerName}, + Capacity: tc.capacity, + DefaultRuntime: testDefaultRuntime, }) if got := apierror.Code(err); got != codes.InvalidArgument { t.Fatalf("code = %v (err %v), want %v", got, err, codes.InvalidArgument) diff --git a/cmd/ateapi/internal/workerservice/service_test.go b/cmd/ateapi/internal/workerservice/service_test.go index 1ad20d5e2c..8e26beebdc 100644 --- a/cmd/ateapi/internal/workerservice/service_test.go +++ b/cmd/ateapi/internal/workerservice/service_test.go @@ -49,25 +49,35 @@ func (f *fakeSuspender) SuspendActor(_ context.Context, req *ateapipb.SuspendAct } // seedReportedWorker registers a Worker on nodeName that has already reported -// capacity and hardware: identity from the pod, the way the syncer writes it, -// plus the result of an earlier registration. A Worker that has never -// registered carries none, so what a fresh registration replaces is what this -// seeds. +// capacity and its default runtime: identity from the pod, the way the syncer +// writes it, plus the result of an earlier registration. A Worker that has +// never registered carries none, so what a fresh registration replaces is what +// this seeds. func seedReportedWorker(t *testing.T, st store.Interface, nodeName string, capacity *ateapipb.WorkerResources) *ateapipb.Worker { + t.Helper() + return seedWorker(t, st, testWorkerName, nodeName, testDefaultRuntime.GetSandboxClass(), &ateapipb.WorkerStatus{ + State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, Capacity: capacity, DefaultRuntime: testDefaultRuntime, + }) +} + +// seedWorker registers a Worker named name, of sandbox class class, on +// nodeName, with status already recorded. name doubles as the pod UID, so it +// must be UUID-shaped. class "" is a Worker recorded without one. +func seedWorker(t *testing.T, st store.Interface, name, nodeName, class string, status *ateapipb.WorkerStatus) *ateapipb.Worker { t.Helper() created, err := st.CreateWorker(context.Background(), &ateapipb.Worker{ - Metadata: &ateapipb.ResourceMetadata{Name: testWorkerName}, + Metadata: &ateapipb.ResourceMetadata{Name: name}, WorkerNamespace: "ate-system", WorkerPool: "pool-1", WorkerPod: "worker-pod-1", - WorkerPodUid: testWorkerName, + WorkerPodUid: name, NodeName: nodeName, Ips: []string{"10.1.2.3"}, - SandboxClass: "gvisor", - Status: &ateapipb.WorkerStatus{State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, Capacity: capacity, Hardware: testHardware}, + SandboxClass: class, + Status: status, }) if err != nil { - t.Fatalf("seeding worker: %v", err) + t.Fatalf("seeding worker %s: %v", name, err) } return created } diff --git a/cmd/atelet/ateomsupport.go b/cmd/atelet/ateomsupport.go index 4481257b91..c6e778a91a 100644 --- a/cmd/atelet/ateomsupport.go +++ b/cmd/atelet/ateomsupport.go @@ -102,16 +102,16 @@ func verifyClientOnSameNode(node *substratex509.PodIdentity) func(tls.Connection } // RegisterWorker registers the calling worker with the control plane: what the -// worker says its capacity and hardware identity are, in one -// WorkerService.RegisterWorker call so capacity and hardware land atomically. +// worker says its capacity and sandbox runtimes are, in one +// WorkerService.RegisterWorker call so capacity and runtimes land atomically. // // It returns the control plane's error unwrapped so the caller retries: a // worker reports once, so an accepted call is the only thing that puts -// capacity and hardware on the Worker, and a Worker record the syncer has not +// capacity and runtimes on the Worker, and a Worker record the syncer has not // created yet is the ordinary reason for a first attempt to fail. func (s *ateomSupportServer) RegisterWorker(ctx context.Context, req *ateletpb.RegisterWorkerRequest) (*ateletpb.RegisterWorkerResponse, error) { // Identity comes only from the mTLS certificate, never from the request: - // a worker can report its own capacity and hardware and no one else's. + // a worker can report its own capacity and runtimes and no one else's. workerIdentity, err := authenticatedWorkerIdentity(ctx) if err != nil { return nil, err @@ -124,14 +124,18 @@ func (s *ateomSupportServer) RegisterWorker(ctx context.Context, req *ateletpb.R } if _, err := s.workers.RegisterWorker(ctx, &ateapipb.RegisterWorkerRequest{ // Workers are global-scoped and named by their pod UID. - Worker: &ateapipb.ObjectRef{Name: workerIdentity.PodUID}, - Capacity: toWorkerResources(req.GetCapacity()), - Hardware: toHardwareIdentity(req.GetHardware()), + Worker: &ateapipb.ObjectRef{Name: workerIdentity.PodUID}, + Capacity: toWorkerResources(req.GetCapacity()), + DefaultRuntime: toAteAPISandboxRuntime(req.GetDefaultRuntime()), + RestorableRuntimes: toAteAPISandboxRuntimes(req.GetRestorableRuntimes()), }); err != nil { return nil, err } - slog.InfoContext(ctx, "Registered worker capacity and hardware", - slog.String("pod_uid", workerIdentity.PodUID), slog.Any("capacity", req.GetCapacity()), slog.Any("hardware", req.GetHardware())) + slog.InfoContext(ctx, "Registered worker capacity and sandbox runtimes", + slog.String("pod_uid", workerIdentity.PodUID), + slog.Any("capacity", req.GetCapacity()), + slog.Any("default_runtime", req.GetDefaultRuntime()), + slog.Any("restorable_runtimes", req.GetRestorableRuntimes())) return &ateletpb.RegisterWorkerResponse{}, nil } @@ -151,22 +155,33 @@ func toWorkerResources(in *ateletpb.WorkerResources) *ateapipb.WorkerResources { return out } -// toHardwareIdentity converts atelet's HardwareIdentity to the control plane's, -// which it mirrors field for field. -func toHardwareIdentity(in *ateletpb.HardwareIdentity) *ateapipb.HardwareIdentity { +// toAteAPISandboxRuntime converts atelet's SandboxRuntime to the control +// plane's, which it mirrors field for field. +func toAteAPISandboxRuntime(in *ateletpb.SandboxRuntime) *ateapipb.SandboxRuntime { if in == nil { return nil } - out := &ateapipb.HardwareIdentity{} - if attrs := in.GetAttributes(); attrs != nil { - out.Attributes = make(map[string]string, len(attrs)) - for k, v := range attrs { - out.Attributes[k] = v + out := &ateapipb.SandboxRuntime{SandboxClass: in.GetSandboxClass()} + if v := in.GetVersion(); v != nil { + out.Version = &ateapipb.VersionedSandboxCompat{SchemaVersion: v.GetSchemaVersion()} + for _, a := range v.GetAttributes() { + out.Version.Attributes = append(out.Version.Attributes, &ateapipb.AttributeEntry{Key: a.GetKey(), Value: a.GetValue()}) } } return out } +func toAteAPISandboxRuntimes(in []*ateletpb.SandboxRuntime) []*ateapipb.SandboxRuntime { + if in == nil { + return nil + } + out := make([]*ateapipb.SandboxRuntime, 0, len(in)) + for _, r := range in { + out = append(out, toAteAPISandboxRuntime(r)) + } + return out +} + // RequestActorSuspend forwards a worker's request to suspend an actor it hosts // to the control plane, which owns their lifecycle. The worker observes; the // control plane decides. diff --git a/cmd/atelet/ateomsupport_test.go b/cmd/atelet/ateomsupport_test.go index 8c704209a6..d6f096be19 100644 --- a/cmd/atelet/ateomsupport_test.go +++ b/cmd/atelet/ateomsupport_test.go @@ -114,13 +114,41 @@ func (s *fakeWorkerService) MintAteomActorCertificate(_ context.Context, in *ate } func TestRegisterWorker(t *testing.T) { - reqHW := &ateletpb.HardwareIdentity{Attributes: map[string]string{"architecture": "amd64"}} - wantHW := &ateapipb.HardwareIdentity{Attributes: map[string]string{"architecture": "amd64"}} - forwarded := func(capacity *ateapipb.WorkerResources) []*ateapipb.RegisterWorkerRequest { + reqRuntime := func(hash string) *ateletpb.SandboxRuntime { + return &ateletpb.SandboxRuntime{ + SandboxClass: "gvisor", + Version: &ateletpb.VersionedSandboxCompat{ + SchemaVersion: "v1", + Attributes: []*ateletpb.AttributeEntry{ + {Key: "architecture", Value: "amd64"}, + {Key: "gvisor_asset_hash", Value: hash}, + }, + }, + } + } + wantRuntime := func(hash string) *ateapipb.SandboxRuntime { + return &ateapipb.SandboxRuntime{ + SandboxClass: "gvisor", + Version: &ateapipb.VersionedSandboxCompat{ + SchemaVersion: "v1", + Attributes: []*ateapipb.AttributeEntry{ + {Key: "architecture", Value: "amd64"}, + {Key: "gvisor_asset_hash", Value: hash}, + }, + }, + } + } + reqDefault, wantDefault := reqRuntime("9988"), wantRuntime("9988") + forwarded := func(capacity *ateapipb.WorkerResources, restorable ...*ateapipb.SandboxRuntime) []*ateapipb.RegisterWorkerRequest { // The Worker is named after the worker pod UID, taken from the - // certificate rather than the request; capacity and hardware come from - // what the worker reported. - return []*ateapipb.RegisterWorkerRequest{{Worker: &ateapipb.ObjectRef{Name: "pod-a"}, Capacity: capacity, Hardware: wantHW}} + // certificate rather than the request; capacity and runtimes come + // from what the worker reported. + return []*ateapipb.RegisterWorkerRequest{{ + Worker: &ateapipb.ObjectRef{Name: "pod-a"}, + Capacity: capacity, + DefaultRuntime: wantDefault, + RestorableRuntimes: restorable, + }} } tests := []struct { @@ -138,16 +166,24 @@ func TestRegisterWorker(t *testing.T) { Capacity: &ateletpb.WorkerResources{Actors: 4, Resources: &ateletpb.Resources{ Limits: []*ateletpb.Limits{{Name: "cpu", Quantity: "2"}, {Name: "memory", Quantity: "4Gi"}}, }}, - Hardware: reqHW, + DefaultRuntime: reqDefault, }, wantForwarded: forwarded(&ateapipb.WorkerResources{Actors: 4, Resources: resources.CPUMemory(2000, 4294967296)}), + }, { + name: "forwards restorable runtimes", + req: &ateletpb.RegisterWorkerRequest{ + Capacity: &ateletpb.WorkerResources{Actors: 1}, + DefaultRuntime: reqDefault, + RestorableRuntimes: []*ateletpb.SandboxRuntime{reqRuntime("d547")}, + }, + wantForwarded: forwarded(&ateapipb.WorkerResources{Actors: 1}, wantRuntime("d547")), }, { name: "omits undetermined compute", - req: &ateletpb.RegisterWorkerRequest{Capacity: &ateletpb.WorkerResources{Actors: 1}, Hardware: reqHW}, + req: &ateletpb.RegisterWorkerRequest{Capacity: &ateletpb.WorkerResources{Actors: 1}, DefaultRuntime: reqDefault}, wantForwarded: forwarded(&ateapipb.WorkerResources{Actors: 1}), }, { name: "forwards empty resources", - req: &ateletpb.RegisterWorkerRequest{Capacity: &ateletpb.WorkerResources{Resources: &ateletpb.Resources{}}, Hardware: reqHW}, + req: &ateletpb.RegisterWorkerRequest{Capacity: &ateletpb.WorkerResources{Resources: &ateletpb.Resources{}}, DefaultRuntime: reqDefault}, wantForwarded: forwarded(&ateapipb.WorkerResources{Resources: &ateapipb.Resources{}}), }, { name: "rejects invalid capacity without forwarding", @@ -155,22 +191,22 @@ func TestRegisterWorker(t *testing.T) { Capacity: &ateletpb.WorkerResources{Resources: &ateletpb.Resources{ Limits: []*ateletpb.Limits{{Name: "gpu", Quantity: "1"}}, }}, - Hardware: reqHW, + DefaultRuntime: reqDefault, }, wantCode: codes.InvalidArgument, }, { name: "rejects missing capacity without forwarding", - req: &ateletpb.RegisterWorkerRequest{Hardware: reqHW}, + req: &ateletpb.RegisterWorkerRequest{DefaultRuntime: reqDefault}, wantCode: codes.InvalidArgument, }, { - name: "rejects missing hardware without forwarding", + name: "rejects missing default runtime without forwarding", req: &ateletpb.RegisterWorkerRequest{Capacity: &ateletpb.WorkerResources{Actors: 1}}, wantCode: codes.InvalidArgument, }, { // A worker may report only what its certificate proves it is. name: "requires a certificate", unauthenticated: true, - req: &ateletpb.RegisterWorkerRequest{Capacity: &ateletpb.WorkerResources{Actors: 1}, Hardware: reqHW}, + req: &ateletpb.RegisterWorkerRequest{Capacity: &ateletpb.WorkerResources{Actors: 1}, DefaultRuntime: reqDefault}, wantCode: codes.Unauthenticated, }, { // The Worker record may not exist yet. The error must reach the @@ -178,7 +214,7 @@ func TestRegisterWorker(t *testing.T) { // leaves the Worker with no capacity forever. name: "surfaces the control plane's rejection", serviceErr: errors.New("no such worker"), - req: &ateletpb.RegisterWorkerRequest{Capacity: &ateletpb.WorkerResources{Actors: 1}, Hardware: reqHW}, + req: &ateletpb.RegisterWorkerRequest{Capacity: &ateletpb.WorkerResources{Actors: 1}, DefaultRuntime: reqDefault}, wantCode: codes.Unknown, }} for _, tt := range tests { diff --git a/cmd/atelet/internal/apivalidation/validation_test.go b/cmd/atelet/internal/apivalidation/validation_test.go index 7e38efb0bd..938a5b3e28 100644 --- a/cmd/atelet/internal/apivalidation/validation_test.go +++ b/cmd/atelet/internal/apivalidation/validation_test.go @@ -16,6 +16,7 @@ package apivalidation import ( "context" + "fmt" "strings" "testing" @@ -28,6 +29,28 @@ func assertValidateErr(t *testing.T, got field.ErrorList, want field.ErrorList) field.ErrorMatcher{}.ByType().ByField().ByOrigin().Test(t, want, got) } +// runtimeWithAttributes returns a valid gVisor runtime carrying n attributes +// with distinct keys. +func runtimeWithAttributes(n int) *ateletpb.SandboxRuntime { + attrs := make([]*ateletpb.AttributeEntry, n) + for i := range attrs { + attrs[i] = &ateletpb.AttributeEntry{Key: fmt.Sprintf("k%d", i), Value: "v"} + } + return &ateletpb.SandboxRuntime{ + SandboxClass: "gvisor", + Version: &ateletpb.VersionedSandboxCompat{SchemaVersion: "v1", Attributes: attrs}, + } +} + +// sandboxRuntimes returns n runtimes built by mk. +func sandboxRuntimes(n int, mk func() *ateletpb.SandboxRuntime) []*ateletpb.SandboxRuntime { + rts := make([]*ateletpb.SandboxRuntime, n) + for i := range rts { + rts[i] = mk() + } + return rts +} + func TestValidateRequestActorSuspendRequest(t *testing.T) { valid := func(mutate ...func(*ateletpb.RequestActorSuspendRequest)) *ateletpb.RequestActorSuspendRequest { r := &ateletpb.RequestActorSuspendRequest{ @@ -149,14 +172,26 @@ func TestValidateMintActorCertificateRequest(t *testing.T) { } func TestValidateRegisterWorkerRequest(t *testing.T) { - testHW := &ateletpb.HardwareIdentity{Attributes: map[string]string{"architecture": "amd64"}} + testRuntime := func() *ateletpb.SandboxRuntime { + return &ateletpb.SandboxRuntime{ + SandboxClass: "gvisor", + Version: &ateletpb.VersionedSandboxCompat{ + SchemaVersion: "v1", + Attributes: []*ateletpb.AttributeEntry{ + {Key: "architecture", Value: "amd64"}, + }, + }, + } + } withLimits := func(limits ...*ateletpb.Limits) *ateletpb.RegisterWorkerRequest { return &ateletpb.RegisterWorkerRequest{ - Capacity: &ateletpb.WorkerResources{Resources: &ateletpb.Resources{Limits: limits}}, - Hardware: testHW, + Capacity: &ateletpb.WorkerResources{Resources: &ateletpb.Resources{Limits: limits}}, + DefaultRuntime: testRuntime(), } } limitsPath := field.NewPath("capacity", "resources", "limits") + defaultRuntimePath := field.NewPath("default_runtime") + restorablePath := field.NewPath("restorable_runtimes") tests := []struct { name string @@ -164,43 +199,206 @@ func TestValidateRegisterWorkerRequest(t *testing.T) { want field.ErrorList }{{ name: "valid", - obj: &ateletpb.RegisterWorkerRequest{Capacity: &ateletpb.WorkerResources{}, Hardware: testHW}, + obj: &ateletpb.RegisterWorkerRequest{Capacity: &ateletpb.WorkerResources{}, DefaultRuntime: testRuntime()}, + }, { + name: "valid with restorable runtimes", + obj: &ateletpb.RegisterWorkerRequest{ + Capacity: &ateletpb.WorkerResources{}, + DefaultRuntime: testRuntime(), + RestorableRuntimes: []*ateletpb.SandboxRuntime{testRuntime()}, + }, }, { name: "missing capacity", - obj: &ateletpb.RegisterWorkerRequest{Hardware: testHW}, + obj: &ateletpb.RegisterWorkerRequest{DefaultRuntime: testRuntime()}, want: field.ErrorList{field.Required(field.NewPath("capacity"), "")}, }, { - name: "missing hardware", + name: "missing default_runtime", obj: &ateletpb.RegisterWorkerRequest{Capacity: &ateletpb.WorkerResources{}}, - want: field.ErrorList{field.Required(field.NewPath("hardware"), "")}, + want: field.ErrorList{field.Required(defaultRuntimePath, "")}, + }, { + name: "missing default_runtime.sandbox_class", + obj: &ateletpb.RegisterWorkerRequest{ + Capacity: &ateletpb.WorkerResources{}, + DefaultRuntime: &ateletpb.SandboxRuntime{ + Version: testRuntime().Version, + }, + }, + want: field.ErrorList{field.Required(defaultRuntimePath.Child("sandbox_class"), "")}, + }, { + name: "default_runtime.sandbox_class too long", + obj: &ateletpb.RegisterWorkerRequest{ + Capacity: &ateletpb.WorkerResources{}, + DefaultRuntime: &ateletpb.SandboxRuntime{ + SandboxClass: strings.Repeat("c", 64), + Version: testRuntime().Version, + }, + }, + want: field.ErrorList{field.TooLong(defaultRuntimePath.Child("sandbox_class"), "", 63).WithOrigin("maxLength")}, + }, { + name: "missing default_runtime.version", + obj: &ateletpb.RegisterWorkerRequest{ + Capacity: &ateletpb.WorkerResources{}, + DefaultRuntime: &ateletpb.SandboxRuntime{SandboxClass: "gvisor"}, + }, + want: field.ErrorList{field.Required(defaultRuntimePath.Child("version"), "")}, + }, { + name: "missing default_runtime.version.schema_version", + obj: &ateletpb.RegisterWorkerRequest{ + Capacity: &ateletpb.WorkerResources{}, + DefaultRuntime: &ateletpb.SandboxRuntime{ + SandboxClass: "gvisor", + Version: &ateletpb.VersionedSandboxCompat{ + Attributes: []*ateletpb.AttributeEntry{{Key: "architecture", Value: "amd64"}}, + }, + }, + }, + want: field.ErrorList{field.Required(defaultRuntimePath.Child("version", "schema_version"), "")}, + }, { + name: "default_runtime.version.schema_version too long", + obj: &ateletpb.RegisterWorkerRequest{ + Capacity: &ateletpb.WorkerResources{}, + DefaultRuntime: &ateletpb.SandboxRuntime{ + SandboxClass: "gvisor", + Version: &ateletpb.VersionedSandboxCompat{ + SchemaVersion: strings.Repeat("v", 65), + Attributes: []*ateletpb.AttributeEntry{{Key: "architecture", Value: "amd64"}}, + }, + }, + }, + want: field.ErrorList{field.TooLong(defaultRuntimePath.Child("version", "schema_version"), "", 64).WithOrigin("maxLength")}, + }, { + name: "missing default_runtime.version.attributes", + obj: &ateletpb.RegisterWorkerRequest{ + Capacity: &ateletpb.WorkerResources{}, + DefaultRuntime: &ateletpb.SandboxRuntime{ + SandboxClass: "gvisor", + Version: &ateletpb.VersionedSandboxCompat{SchemaVersion: "v1"}, + }, + }, + want: field.ErrorList{field.Required(defaultRuntimePath.Child("version", "attributes"), "")}, + }, { + name: "duplicate attribute key", + obj: &ateletpb.RegisterWorkerRequest{ + Capacity: &ateletpb.WorkerResources{}, + DefaultRuntime: &ateletpb.SandboxRuntime{ + SandboxClass: "gvisor", + Version: &ateletpb.VersionedSandboxCompat{ + SchemaVersion: "v1", + Attributes: []*ateletpb.AttributeEntry{ + {Key: "architecture", Value: "amd64"}, + {Key: "architecture", Value: "arm64"}, + }, + }, + }, + }, + want: field.ErrorList{field.Duplicate(defaultRuntimePath.Child("version", "attributes").Index(1), nil)}, + }, { + name: "missing attribute key", + obj: &ateletpb.RegisterWorkerRequest{ + Capacity: &ateletpb.WorkerResources{}, + DefaultRuntime: &ateletpb.SandboxRuntime{ + SandboxClass: "gvisor", + Version: &ateletpb.VersionedSandboxCompat{ + SchemaVersion: "v1", + Attributes: []*ateletpb.AttributeEntry{{Value: "v"}}, + }, + }, + }, + want: field.ErrorList{field.Required(defaultRuntimePath.Child("version", "attributes").Index(0).Child("key"), "")}, }, { - name: "valid empty hardware attributes", - obj: &ateletpb.RegisterWorkerRequest{Capacity: &ateletpb.WorkerResources{}, Hardware: &ateletpb.HardwareIdentity{}}, + name: "attribute key too long", + obj: &ateletpb.RegisterWorkerRequest{ + Capacity: &ateletpb.WorkerResources{}, + DefaultRuntime: &ateletpb.SandboxRuntime{ + SandboxClass: "gvisor", + Version: &ateletpb.VersionedSandboxCompat{ + SchemaVersion: "v1", + Attributes: []*ateletpb.AttributeEntry{{Key: strings.Repeat("k", 129), Value: "v"}}, + }, + }, + }, + want: field.ErrorList{field.TooLong(defaultRuntimePath.Child("version", "attributes").Index(0).Child("key"), "", 128).WithOrigin("maxLength")}, }, { - name: "hardware attribute key too long", + name: "missing attribute value", obj: &ateletpb.RegisterWorkerRequest{ Capacity: &ateletpb.WorkerResources{}, - Hardware: &ateletpb.HardwareIdentity{Attributes: map[string]string{strings.Repeat("k", 129): "v"}}, + DefaultRuntime: &ateletpb.SandboxRuntime{ + SandboxClass: "gvisor", + Version: &ateletpb.VersionedSandboxCompat{ + SchemaVersion: "v1", + Attributes: []*ateletpb.AttributeEntry{{Key: "k"}}, + }, + }, }, - want: field.ErrorList{field.TooLong(field.NewPath("hardware", "attributes"), "", 128).WithOrigin("maxLength")}, + want: field.ErrorList{field.Required(defaultRuntimePath.Child("version", "attributes").Index(0).Child("value"), "")}, }, { - name: "hardware attribute value too long", + name: "attribute value too long", obj: &ateletpb.RegisterWorkerRequest{ Capacity: &ateletpb.WorkerResources{}, - Hardware: &ateletpb.HardwareIdentity{Attributes: map[string]string{"k": strings.Repeat("v", 257)}}, + DefaultRuntime: &ateletpb.SandboxRuntime{ + SandboxClass: "gvisor", + Version: &ateletpb.VersionedSandboxCompat{ + SchemaVersion: "v1", + Attributes: []*ateletpb.AttributeEntry{{Key: "k", Value: strings.Repeat("v", 257)}}, + }, + }, + }, + want: field.ErrorList{field.TooLong(defaultRuntimePath.Child("version", "attributes").Index(0).Child("value"), "", 256).WithOrigin("maxLength")}, + }, { + name: "attributes at the bound", + obj: &ateletpb.RegisterWorkerRequest{ + Capacity: &ateletpb.WorkerResources{}, + DefaultRuntime: runtimeWithAttributes(32), + }, + }, { + name: "too many attributes", + obj: &ateletpb.RegisterWorkerRequest{ + Capacity: &ateletpb.WorkerResources{}, + DefaultRuntime: runtimeWithAttributes(33), + }, + want: field.ErrorList{field.TooMany(defaultRuntimePath.Child("version", "attributes"), 33, 32).WithOrigin("maxItems")}, + }, { + name: "nil restorable_runtimes entry", + obj: &ateletpb.RegisterWorkerRequest{ + Capacity: &ateletpb.WorkerResources{}, + DefaultRuntime: testRuntime(), + RestorableRuntimes: []*ateletpb.SandboxRuntime{nil}, + }, + want: field.ErrorList{field.Required(restorablePath.Index(0), "")}, + }, { + name: "invalid restorable_runtimes entry", + obj: &ateletpb.RegisterWorkerRequest{ + Capacity: &ateletpb.WorkerResources{}, + DefaultRuntime: testRuntime(), + RestorableRuntimes: []*ateletpb.SandboxRuntime{{SandboxClass: "gvisor"}}, + }, + want: field.ErrorList{field.Required(restorablePath.Index(0).Child("version"), "")}, + }, { + name: "restorable_runtimes at the bound", + obj: &ateletpb.RegisterWorkerRequest{ + Capacity: &ateletpb.WorkerResources{}, + DefaultRuntime: testRuntime(), + RestorableRuntimes: sandboxRuntimes(32, testRuntime), + }, + }, { + name: "too many restorable_runtimes", + obj: &ateletpb.RegisterWorkerRequest{ + Capacity: &ateletpb.WorkerResources{}, + DefaultRuntime: testRuntime(), + RestorableRuntimes: sandboxRuntimes(33, testRuntime), }, - want: field.ErrorList{field.TooLong(field.NewPath("hardware", "attributes").Key("k"), "", 256).WithOrigin("maxLength")}, + want: field.ErrorList{field.TooMany(restorablePath, 33, 32).WithOrigin("maxItems")}, }, { name: "full capacity", obj: &ateletpb.RegisterWorkerRequest{ Capacity: &ateletpb.WorkerResources{Actors: 4, Resources: &ateletpb.Resources{ Limits: []*ateletpb.Limits{{Name: "cpu", Quantity: "4"}, {Name: "memory", Quantity: "8Gi"}}, }}, - Hardware: testHW, + DefaultRuntime: testRuntime(), }, }, { name: "negative actors", - obj: &ateletpb.RegisterWorkerRequest{Capacity: &ateletpb.WorkerResources{Actors: -1}, Hardware: testHW}, + obj: &ateletpb.RegisterWorkerRequest{Capacity: &ateletpb.WorkerResources{Actors: -1}, DefaultRuntime: testRuntime()}, want: field.ErrorList{field.Invalid(field.NewPath("capacity", "actors"), nil, "").WithOrigin("minimum")}, }, { name: "unsupported resource name", diff --git a/cmd/atelet/internal/apivalidation/zz_generated.validation.go b/cmd/atelet/internal/apivalidation/zz_generated.validation.go index 6f59c7bb20..c77dd28826 100644 --- a/cmd/atelet/internal/apivalidation/zz_generated.validation.go +++ b/cmd/atelet/internal/apivalidation/zz_generated.validation.go @@ -29,54 +29,74 @@ import ( field "k8s.io/apimachinery/pkg/util/validation/field" ) -// Validate_HardwareIdentity validates an instance of HardwareIdentity according +// Validate_AttributeEntry validates an instance of AttributeEntry according // to declarative validation rules in the API schema. -func Validate_HardwareIdentity( +func Validate_AttributeEntry( ctx context.Context, op operation.Operation, fldPath *field.Path, - obj, oldObj *ateletpb.HardwareIdentity) (errs field.ErrorList) { + obj, oldObj *ateletpb.AttributeEntry) (errs field.ErrorList) { - { // field ateletpb.HardwareIdentity.Attributes + { // field ateletpb.AttributeEntry.Key fn := func( fldPath *field.Path, - obj, oldObj map[string]string, + obj, oldObj *string, oldValueCorrelated bool) (errs field.ErrorList) { // don't revalidate unchanged data if oldValueCorrelated && op.Type == operation.Update { - if ateDeepEqual(obj, oldObj) { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { return nil } } // call field-attached validations earlyReturn := false - if e := validate.MaxProperties(ctx, op, fldPath, obj, oldObj, 32).MarkShortCircuit(); len(e) != 0 { + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { errs = append(errs, e...) earlyReturn = true } - if e := validate.OptionalMap(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { - earlyReturn = true - } if earlyReturn { return // do not proceed } - if e := validate.EachMapKey(ctx, op, fldPath, obj, oldObj, - func(ctx context.Context, op operation.Operation, fldPath *field.Path, obj, oldObj *string) field.ErrorList { - return validate.MaxLength(ctx, op, fldPath, obj, oldObj, 128) - }); len(e) != 0 { + if e := validate.MaxLength(ctx, op, fldPath, obj, oldObj, 128); len(e) != 0 { errs = append(errs, e...) } - if e := validate.EachMapVal(ctx, op, fldPath, obj, oldObj, validate.DirectEqual, - func(ctx context.Context, op operation.Operation, fldPath *field.Path, obj, oldObj *string) field.ErrorList { - return validate.MaxLength(ctx, op, fldPath, obj, oldObj, 256) - }); len(e) != 0 { + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateletpb.AttributeEntry) *string { + return &oldObj.Key + }) + errs = append(errs, fn(fldPath.Child("key"), &obj.Key, oldVal, oldObj != nil)...) + } + + { // field ateletpb.AttributeEntry.Value + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.MaxLength(ctx, op, fldPath, obj, oldObj, 256); len(e) != 0 { errs = append(errs, e...) } return } oldVal := safe.Field(oldObj, - func(oldObj *ateletpb.HardwareIdentity) map[string]string { - return oldObj.Attributes + func(oldObj *ateletpb.AttributeEntry) *string { + return &oldObj.Value }) - errs = append(errs, fn(fldPath.Child("attributes"), obj.Attributes, oldVal, oldObj != nil)...) + errs = append(errs, fn(fldPath.Child("value"), &obj.Value, oldVal, oldObj != nil)...) } return errs @@ -334,10 +354,10 @@ func Validate_RegisterWorkerRequest( errs = append(errs, fn(fldPath.Child("capacity"), obj.Capacity, oldVal, oldObj != nil)...) } - { // field ateletpb.RegisterWorkerRequest.Hardware + { // field ateletpb.RegisterWorkerRequest.DefaultRuntime fn := func( fldPath *field.Path, - obj, oldObj *ateletpb.HardwareIdentity, + obj, oldObj *ateletpb.SandboxRuntime, oldValueCorrelated bool) (errs field.ErrorList) { // don't revalidate unchanged data if oldValueCorrelated && op.Type == operation.Update { @@ -355,14 +375,54 @@ func Validate_RegisterWorkerRequest( return // do not proceed } // call the type's validation function - errs = append(errs, Validate_HardwareIdentity(ctx, op, fldPath, obj, oldObj)...) + errs = append(errs, Validate_SandboxRuntime(ctx, op, fldPath, obj, oldObj)...) + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateletpb.RegisterWorkerRequest) *ateletpb.SandboxRuntime { + return oldObj.DefaultRuntime + }) + errs = append(errs, fn(fldPath.Child("default_runtime"), obj.DefaultRuntime, oldVal, oldObj != nil)...) + } + + { // field ateletpb.RegisterWorkerRequest.RestorableRuntimes + fn := func( + fldPath *field.Path, + obj, oldObj []*ateletpb.SandboxRuntime, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.PtrSliceNoNils[ateletpb.SandboxRuntime](ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.MaxItems(ctx, op, fldPath, obj, oldObj, 32).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.OptionalSlice(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // iterate the list and call the type's validation function + if e := validate.EachPtrSliceVal(ctx, op, fldPath, obj, oldObj, nil, nil, Validate_SandboxRuntime); len(e) != 0 { + errs = append(errs, e...) + } return } oldVal := safe.Field(oldObj, - func(oldObj *ateletpb.RegisterWorkerRequest) *ateletpb.HardwareIdentity { - return oldObj.Hardware + func(oldObj *ateletpb.RegisterWorkerRequest) []*ateletpb.SandboxRuntime { + return oldObj.RestorableRuntimes }) - errs = append(errs, fn(fldPath.Child("hardware"), obj.Hardware, oldVal, oldObj != nil)...) + errs = append(errs, fn(fldPath.Child("restorable_runtimes"), obj.RestorableRuntimes, oldVal, oldObj != nil)...) } return errs @@ -528,6 +588,166 @@ func Validate_Resources( return errs } +// Validate_SandboxRuntime validates an instance of SandboxRuntime according +// to declarative validation rules in the API schema. +func Validate_SandboxRuntime( + ctx context.Context, op operation.Operation, fldPath *field.Path, + obj, oldObj *ateletpb.SandboxRuntime) (errs field.ErrorList) { + + { // field ateletpb.SandboxRuntime.SandboxClass + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.MaxLength(ctx, op, fldPath, obj, oldObj, 63); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateletpb.SandboxRuntime) *string { + return &oldObj.SandboxClass + }) + errs = append(errs, fn(fldPath.Child("sandbox_class"), &obj.SandboxClass, oldVal, oldObj != nil)...) + } + + { // field ateletpb.SandboxRuntime.Version + fn := func( + fldPath *field.Path, + obj, oldObj *ateletpb.VersionedSandboxCompat, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // call the type's validation function + errs = append(errs, Validate_VersionedSandboxCompat(ctx, op, fldPath, obj, oldObj)...) + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateletpb.SandboxRuntime) *ateletpb.VersionedSandboxCompat { + return oldObj.Version + }) + errs = append(errs, fn(fldPath.Child("version"), obj.Version, oldVal, oldObj != nil)...) + } + + return errs +} + +// Validate_VersionedSandboxCompat validates an instance of VersionedSandboxCompat according +// to declarative validation rules in the API schema. +func Validate_VersionedSandboxCompat( + ctx context.Context, op operation.Operation, fldPath *field.Path, + obj, oldObj *ateletpb.VersionedSandboxCompat) (errs field.ErrorList) { + + { // field ateletpb.VersionedSandboxCompat.SchemaVersion + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.MaxLength(ctx, op, fldPath, obj, oldObj, 64); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateletpb.VersionedSandboxCompat) *string { + return &oldObj.SchemaVersion + }) + errs = append(errs, fn(fldPath.Child("schema_version"), &obj.SchemaVersion, oldVal, oldObj != nil)...) + } + + { // field ateletpb.VersionedSandboxCompat.Attributes + fn := func( + fldPath *field.Path, + obj, oldObj []*ateletpb.AttributeEntry, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.PtrSliceNoNils[ateletpb.AttributeEntry](ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.MaxItems(ctx, op, fldPath, obj, oldObj, 32).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.RequiredSlice(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // lists with map semantics require unique keys + if e := validate.PtrSliceUnique(ctx, op, fldPath, obj, oldObj, + func(a *ateletpb.AttributeEntry, b *ateletpb.AttributeEntry) bool { return a.Key == b.Key }); len(e) != 0 { + errs = append(errs, e...) + } + // iterate the list and call the type's validation function + if e := validate.EachPtrSliceVal(ctx, op, fldPath, obj, oldObj, + func(a *ateletpb.AttributeEntry, b *ateletpb.AttributeEntry) bool { return a.Key == b.Key }, ateDeepEqual, Validate_AttributeEntry); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateletpb.VersionedSandboxCompat) []*ateletpb.AttributeEntry { + return oldObj.Attributes + }) + errs = append(errs, fn(fldPath.Child("attributes"), obj.Attributes, oldVal, oldObj != nil)...) + } + + return errs +} + // Validate_WorkerResources validates an instance of WorkerResources according // to declarative validation rules in the API schema. func Validate_WorkerResources( diff --git a/cmd/ateom-gvisor/main.go b/cmd/ateom-gvisor/main.go index a54f4a6539..3e21122300 100644 --- a/cmd/ateom-gvisor/main.go +++ b/cmd/ateom-gvisor/main.go @@ -54,6 +54,7 @@ import ( "github.com/agent-substrate/substrate/internal/sizing" "github.com/agent-substrate/substrate/internal/version" "github.com/agent-substrate/substrate/internal/wakeupprobe" + atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" "github.com/spf13/pflag" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "google.golang.org/grpc" @@ -266,6 +267,7 @@ func do(ctx context.Context) error { TrustBundlePath: tunnelConfig.TrustBundle, AteletSPIFFEID: tunnelConfig.BrokerIdentity, Actors: *maxActors, + SandboxClass: string(atev1alpha1.SandboxClassGvisor), }) if err != nil && ctx.Err() == nil { serverboot.Fatal(ctx, "Failed to report worker capacity", err) diff --git a/cmd/ateom-microvm/main.go b/cmd/ateom-microvm/main.go index 501d907001..443754be0c 100644 --- a/cmd/ateom-microvm/main.go +++ b/cmd/ateom-microvm/main.go @@ -52,6 +52,7 @@ import ( "github.com/agent-substrate/substrate/internal/resources" "github.com/agent-substrate/substrate/internal/serverboot" "github.com/agent-substrate/substrate/internal/version" + atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" "github.com/spf13/pflag" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "golang.org/x/sys/unix" @@ -282,6 +283,7 @@ func do(ctx context.Context) error { TrustBundlePath: tunnelConfig.TrustBundle, AteletSPIFFEID: tunnelConfig.BrokerIdentity, Actors: *maxActors, + SandboxClass: string(atev1alpha1.SandboxClassMicroVM), }) if err != nil && ctx.Err() == nil { serverboot.Fatal(ctx, "Failed to report worker capacity", err) diff --git a/cmd/benchmarking/isolate/fake-workersync/controller.go b/cmd/benchmarking/isolate/fake-workersync/controller.go index 1b41ce5aab..8920072373 100644 --- a/cmd/benchmarking/isolate/fake-workersync/controller.go +++ b/cmd/benchmarking/isolate/fake-workersync/controller.go @@ -378,7 +378,10 @@ func (c *controller) ensure(ctx context.Context, pod string, d desiredWorker, w if err := c.relay.Report(ctx, addr, &ateapipb.RegisterWorkerRequest{ Worker: &ateapipb.ObjectRef{Name: name}, Capacity: fixed, - Hardware: hardware.ProbeHost(), + DefaultRuntime: &ateapipb.SandboxRuntime{ + SandboxClass: string(d.pool.Spec.DefaultSandboxClass()), + Version: hostCompat(), + }, }); err != nil { return fmt.Errorf("while reporting capacity for Worker %s: %w", name, err) } @@ -388,6 +391,15 @@ func (c *controller) ensure(ctx context.Context, pod string, d desiredWorker, w return nil } +func hostCompat() *ateapipb.VersionedSandboxCompat { + in := hardware.ProbeHost() + out := &ateapipb.VersionedSandboxCompat{SchemaVersion: in.GetSchemaVersion()} + for _, a := range in.GetAttributes() { + out.Attributes = append(out.Attributes, &ateapipb.AttributeEntry{Key: a.GetKey(), Value: a.GetValue()}) + } + return out +} + // create registers the Worker with the fields the worker syncer would copy // from the pool and its pod. Epoch stays 0: a rising epoch makes // ate-api-server crash every Actor on the Worker. diff --git a/cmd/benchmarking/isolate/fake-workersync/controller_test.go b/cmd/benchmarking/isolate/fake-workersync/controller_test.go index 9a1a14661d..8d43bebaf2 100644 --- a/cmd/benchmarking/isolate/fake-workersync/controller_test.go +++ b/cmd/benchmarking/isolate/fake-workersync/controller_test.go @@ -25,7 +25,6 @@ import ( "time" "github.com/agent-substrate/substrate/cmd/benchmarking/isolate/internal/fakeworker" - "github.com/agent-substrate/substrate/internal/hardware" atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "google.golang.org/grpc" @@ -216,16 +215,16 @@ func (f *fakeControl) uid(pod string) string { type fakeRelay struct { control *fakeControl mu sync.Mutex - reported map[string]*ateapipb.WorkerResources // by Worker name - hardware map[string]*ateapipb.HardwareIdentity // by Worker name - via map[string]string // relay address, by Worker name + reported map[string]*ateapipb.WorkerResources // by Worker name + runtimes map[string]*ateapipb.SandboxRuntime // default runtime, by Worker name + via map[string]string // relay address, by Worker name reject map[string]codes.Code calls int live map[string]string // as last pruned to } func newFakeRelay() *fakeRelay { - return &fakeRelay{reported: map[string]*ateapipb.WorkerResources{}, hardware: map[string]*ateapipb.HardwareIdentity{}, via: map[string]string{}, reject: map[string]codes.Code{}} + return &fakeRelay{reported: map[string]*ateapipb.WorkerResources{}, runtimes: map[string]*ateapipb.SandboxRuntime{}, via: map[string]string{}, reject: map[string]codes.Code{}} } func (f *fakeRelay) Report(_ context.Context, addr string, req *ateapipb.RegisterWorkerRequest) error { @@ -237,12 +236,12 @@ func (f *fakeRelay) Report(_ context.Context, addr string, req *ateapipb.Registe return status.Error(code, "rejected") } f.reported[name] = req.GetCapacity() - f.hardware[name] = req.GetHardware() + f.runtimes[name] = req.GetDefaultRuntime() f.via[name] = addr if f.control != nil { f.control.mu.Lock() if w, ok := f.control.workers[name]; ok { - w.Status.Capacity, w.Status.Hardware = req.GetCapacity(), req.GetHardware() + w.Status.Capacity, w.Status.DefaultRuntime = req.GetCapacity(), req.GetDefaultRuntime() } f.control.mu.Unlock() } @@ -408,8 +407,9 @@ func TestReconcileHonorsReplicasAndLimits(t *testing.T) { if got, want := rel.reported[ctl.uid(name)], wantCapacity(1000, "1500m", "4Gi"); !proto.Equal(got, want) { t.Errorf("reported %v, want %v from the pool's limits", got, want) } - if got, want := rel.hardware[ctl.uid(name)], hardware.ProbeHost(); !proto.Equal(got, want) { - t.Errorf("hardware %v, want %v, which ate-api-server requires", got, want) + wantRuntime := &ateapipb.SandboxRuntime{SandboxClass: "gvisor", Version: hostCompat()} + if got := rel.runtimes[ctl.uid(name)]; !proto.Equal(got, wantRuntime) { + t.Errorf("default runtime %v, want %v: the pool's class on this host, which ate-api-server requires", got, wantRuntime) } if got, want := cl.statuses["bench"], (atev1alpha1.WorkerPoolStatus{Replicas: 3, ReadyReplicas: 3, Selector: "ate.dev/worker-pool=bench"}); got != want { t.Errorf("status = %+v, want %+v", got, want) diff --git a/internal/ateom/register.go b/internal/ateom/register.go index 05bce5f288..4457fcba19 100644 --- a/internal/ateom/register.go +++ b/internal/ateom/register.go @@ -13,12 +13,14 @@ // limitations under the License. // Package ateom registers an ateom worker with the control plane through the -// node-local atelet, reporting its compute capacity and hardware identity. +// node-local atelet, reporting its compute capacity and the sandbox runtime +// its actors start under. package ateom import ( "context" "crypto/tls" + "errors" "fmt" "log/slog" "os" @@ -73,11 +75,16 @@ func fromDir(dir string, actors int) *ateletpb.WorkerResources { } } -// probeHardware returns the hardware identity that actors hosted by this ateom -// observe. -func probeHardware() *ateletpb.HardwareIdentity { - return &ateletpb.HardwareIdentity{ - Attributes: hardware.ProbeHost().GetAttributes(), +// defaultRuntime is the runtime actors hosted by this ateom start under: its +// sandbox class on this host's hardware. +// +// TODO: Report the pool's SandboxConfig so atelet can complete the identity +// with the digests of the default version's assets and list its other +// enabled versions as restorable_runtimes. +func defaultRuntime(sandboxClass string) *ateletpb.SandboxRuntime { + return &ateletpb.SandboxRuntime{ + SandboxClass: sandboxClass, + Version: hardware.ProbeHost(), } } @@ -112,7 +119,8 @@ func readLimit(path string) int64 { return value } -// ReportConfig is what an ateom needs to reach the atelet on its node. +// ReportConfig configures how an ateom reaches the node-local atelet and what +// capacity and sandbox runtime class it registers with. type ReportConfig struct { SocketPath string CredentialBundlePath string @@ -123,23 +131,30 @@ type ReportConfig struct { AteletSPIFFEID string // Actors is how many actors this ateom will host at once. Actors int + // SandboxClass is the class this ateom implements ("gvisor", "microvm"). + SandboxClass string } -// Report tells the node-local atelet what this ateom can supply and its -// hardware identity, retrying until it is accepted or ctx ends. +// Report tells the node-local atelet what this ateom can supply and the +// runtime its actors start under, retrying until it is accepted or ctx ends. // // Retrying is what makes a single report durable: atelet only accepts once the // control plane has recorded it, and the Worker record may not exist yet when // an ateom first comes up. Nothing else reports this, so giving up would leave // the Worker holding no capacity and hosting nothing. func Report(ctx context.Context, cfg ReportConfig) error { + if cfg.SandboxClass == "" { + // The control plane refuses a runtime without a class, and retrying + // would not change that. + return errors.New("capacity report: SandboxClass is required") + } tlsConfig, err := ateletdial.TLSConfig(cfg.CredentialBundlePath, cfg.TrustBundlePath, cfg.AteletSPIFFEID) if err != nil { return fmt.Errorf("capacity report: %w", err) } req := &ateletpb.RegisterWorkerRequest{ - Capacity: FromFiles(cfg.Actors), - Hardware: probeHardware(), + Capacity: FromFiles(cfg.Actors), + DefaultRuntime: defaultRuntime(cfg.SandboxClass), } err = retryReport(ctx, func() error { return reportOnce(ctx, cfg.SocketPath, tlsConfig, req) @@ -147,9 +162,9 @@ func Report(ctx context.Context, cfg ReportConfig) error { if err != nil { return err } - slog.InfoContext(ctx, "Registered worker capacity and hardware", + slog.InfoContext(ctx, "Registered worker capacity and sandbox runtime", slog.Any("capacity", req.GetCapacity()), - slog.Any("hardware", req.GetHardware())) + slog.Any("default_runtime", req.GetDefaultRuntime())) return nil } diff --git a/internal/ateom/register_test.go b/internal/ateom/register_test.go index ce39bd9b26..48b32bcaa0 100644 --- a/internal/ateom/register_test.go +++ b/internal/ateom/register_test.go @@ -72,13 +72,19 @@ func TestFromFilesMissing(t *testing.T) { } } -func TestProbeHardware(t *testing.T) { - got := probeHardware() - want := &ateletpb.HardwareIdentity{ - Attributes: map[string]string{hardware.AttrArchitecture: runtime.GOARCH}, +func TestDefaultRuntime(t *testing.T) { + got := defaultRuntime("gvisor") + // The class is the ateom's own; the compat identity is the host's, under + // the probe's schema version. + want := &ateletpb.SandboxRuntime{ + SandboxClass: "gvisor", + Version: &ateletpb.VersionedSandboxCompat{ + SchemaVersion: hardware.SchemaVersionV1, + Attributes: []*ateletpb.AttributeEntry{{Key: hardware.AttrArchitecture, Value: runtime.GOARCH}}, + }, } if diff := cmp.Diff(want, got, protocmp.Transform()); diff != "" { - t.Errorf("probeHardware() mismatch (-want +got):\n%s", diff) + t.Errorf("defaultRuntime() mismatch (-want +got):\n%s", diff) } } @@ -127,6 +133,7 @@ func TestReportFailsFastOnBadCredentials(t *testing.T) { SocketPath: filepath.Join(t.TempDir(), "atelet.sock"), CredentialBundlePath: filepath.Join(t.TempDir(), "does-not-exist.pem"), TrustBundlePath: filepath.Join(t.TempDir(), "also-missing.pem"), + SandboxClass: "gvisor", }) if err == nil { t.Fatal("Report() with unreadable credentials succeeded, want an error the caller can exit on") @@ -135,3 +142,18 @@ func TestReportFailsFastOnBadCredentials(t *testing.T) { t.Errorf("Report() retried a permanent failure until the deadline: %v", err) } } + +// A runtime without a class is refused by the control plane on every attempt, +// so an ateom that names none must exit rather than retry. +func TestReportFailsFastWithoutSandboxClass(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + err := Report(ctx, ReportConfig{SocketPath: filepath.Join(t.TempDir(), "atelet.sock")}) + if err == nil { + t.Fatal("Report() without a SandboxClass succeeded, want an error the caller can exit on") + } + if errors.Is(err, context.DeadlineExceeded) { + t.Errorf("Report() retried a permanent failure until the deadline: %v", err) + } +} diff --git a/internal/hardware/hardware.go b/internal/hardware/hardware.go index 56f1e6c2e3..4fab18123c 100644 --- a/internal/hardware/hardware.go +++ b/internal/hardware/hardware.go @@ -12,45 +12,86 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package hardware probes host hardware identity and evaluates snapshot -// hardware compatibility between workers and snapshots. +// Package hardware probes host hardware compatibility attributes and evaluates +// sandbox runtime compatibility between workers and snapshots. package hardware import ( "runtime" + "github.com/agent-substrate/substrate/internal/proto/ateletpb" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" ) const ( + // SchemaVersionV1 is the initial compatibility schema version. + SchemaVersionV1 = "v1" + // AttrArchitecture is the CPU architecture ("amd64", "arm64", etc.). AttrArchitecture = "architecture" - // TODO: Add AttrCPUVendor ("cpu_vendor"), AttrCPUModel ("cpu_model"), and - // other hardware attributes as snapshot compatibility expands. + // TODO: Add AttrCPUFeatures ("cpu_features"), AttrGVisorAssetHash + // ("gvisor_asset_hash"), and other compatibility attributes as snapshot + // compatibility expands. ) -// ProbeHost inspects the current host and returns its HardwareIdentity. +// ProbeHost inspects the current host and returns its v1 VersionedSandboxCompat +// in atelet wire format. // -// TODO: Probe and populate cpu_vendor, cpu_model, and other host hardware -// attributes (e.g. via CPUID on amd64 and MIDR_EL1 on arm64). -func ProbeHost() *ateapipb.HardwareIdentity { - return &ateapipb.HardwareIdentity{ - Attributes: map[string]string{ - AttrArchitecture: runtime.GOARCH, +// TODO: Probe and populate cpu_features and other host hardware attributes +// (e.g. via CPUID on amd64 and MIDR_EL1 on arm64). +func ProbeHost() *ateletpb.VersionedSandboxCompat { + return &ateletpb.VersionedSandboxCompat{ + SchemaVersion: SchemaVersionV1, + Attributes: []*ateletpb.AttributeEntry{ + { + Key: AttrArchitecture, + Value: runtime.GOARCH, + }, }, } } -// Matches reports whether worker satisfies the hardware identity recorded on -// snap. A nil or empty snapshot HardwareIdentity imposes no constraint. -// -// TODO: Distinguish memory-restore matching (full CPU vendor/model) from -// cold-boot fallback matching (architecture only) once additional hardware -// attributes are populated. -func Matches(worker, snap *ateapipb.HardwareIdentity) bool { - for k, v := range snap.GetAttributes() { - if worker.GetAttributes()[k] != v { +// Matches reports whether a worker's SandboxRuntime can restore a snapshot +// stamped with snap. A nil snapshot SandboxRuntime imposes no constraint. +// Otherwise the sandbox classes must be equal and MatchesCompat must hold. +func Matches(worker, snap *ateapipb.SandboxRuntime) bool { + if snap == nil { + return true + } + if worker == nil || worker.GetSandboxClass() != snap.GetSandboxClass() { + return false + } + return MatchesCompat(worker.GetVersion(), snap.GetVersion()) +} + +// MatchesCompat reports whether worker can restore a snapshot stamped with +// snap: same non-empty schema version, and every attribute on snap present on +// worker with an equal value. Keys only the worker has are ignored; a new +// schema version is how older snapshots are excluded. +func MatchesCompat(worker, snap *ateapipb.VersionedSandboxCompat) bool { + if worker == nil || snap == nil { + return false + } + if worker.GetSchemaVersion() == "" || worker.GetSchemaVersion() != snap.GetSchemaVersion() { + return false + } + wAttrs := worker.GetAttributes() + sAttrs := snap.GetAttributes() + // API validation bounds attributes to at most 32 entries with unique keys, + // so the nested scan is small and avoids map allocation on the hot path. + for _, sa := range sAttrs { + found := false + for _, wa := range wAttrs { + if wa.GetKey() == sa.GetKey() { + if wa.GetValue() != sa.GetValue() { + return false + } + found = true + break + } + } + if !found { return false } } diff --git a/internal/hardware/hardware_test.go b/internal/hardware/hardware_test.go index 3fb46757cb..5f0da993b3 100644 --- a/internal/hardware/hardware_test.go +++ b/internal/hardware/hardware_test.go @@ -22,27 +22,68 @@ import ( ) func TestProbeHost(t *testing.T) { - hw := ProbeHost() - if got := hw.GetAttributes()[AttrArchitecture]; got != runtime.GOARCH { - t.Errorf("ProbeHost() architecture = %q, want %q", got, runtime.GOARCH) + compat := ProbeHost() + if got := compat.GetSchemaVersion(); got != SchemaVersionV1 { + t.Errorf("ProbeHost() schema_version = %q, want %q", got, SchemaVersionV1) + } + attrs := compat.GetAttributes() + if len(attrs) != 1 || attrs[0].GetKey() != AttrArchitecture || attrs[0].GetValue() != runtime.GOARCH { + t.Errorf("ProbeHost() attributes = %v, want [{%s: %s}]", attrs, AttrArchitecture, runtime.GOARCH) } } func TestMatches(t *testing.T) { - amd64Worker := &ateapipb.HardwareIdentity{Attributes: map[string]string{AttrArchitecture: "amd64"}} - arm64Worker := &ateapipb.HardwareIdentity{Attributes: map[string]string{AttrArchitecture: "arm64"}} + runtimeWith := func(class, ver string, attrs ...*ateapipb.AttributeEntry) *ateapipb.SandboxRuntime { + return &ateapipb.SandboxRuntime{ + SandboxClass: class, + Version: &ateapipb.VersionedSandboxCompat{ + SchemaVersion: ver, + Attributes: attrs, + }, + } + } + amd64GVisor := runtimeWith("gvisor", "v1", + &ateapipb.AttributeEntry{Key: AttrArchitecture, Value: "amd64"}, + &ateapipb.AttributeEntry{Key: "cpu_features", Value: "GenuineIntel"}, + ) + amd64GVisorReordered := runtimeWith("gvisor", "v1", + &ateapipb.AttributeEntry{Key: "cpu_features", Value: "GenuineIntel"}, + &ateapipb.AttributeEntry{Key: AttrArchitecture, Value: "amd64"}, + ) + arm64GVisor := runtimeWith("gvisor", "v1", + &ateapipb.AttributeEntry{Key: AttrArchitecture, Value: "arm64"}, + &ateapipb.AttributeEntry{Key: "cpu_features", Value: "GenuineIntel"}, + ) + amd64MicroVM := runtimeWith("microvm", "v1", + &ateapipb.AttributeEntry{Key: AttrArchitecture, Value: "amd64"}, + &ateapipb.AttributeEntry{Key: "cpu_features", Value: "GenuineIntel"}, + ) + amd64GVisorV2 := runtimeWith("gvisor", "v2", + &ateapipb.AttributeEntry{Key: AttrArchitecture, Value: "amd64"}, + &ateapipb.AttributeEntry{Key: "cpu_features", Value: "GenuineIntel"}, + ) + amd64GVisorExtraAttr := runtimeWith("gvisor", "v1", + &ateapipb.AttributeEntry{Key: AttrArchitecture, Value: "amd64"}, + &ateapipb.AttributeEntry{Key: "cpu_features", Value: "GenuineIntel"}, + &ateapipb.AttributeEntry{Key: "gvisor_asset_hash", Value: "abc"}, + ) tests := []struct { name string - worker *ateapipb.HardwareIdentity - snap *ateapipb.HardwareIdentity + worker *ateapipb.SandboxRuntime + snap *ateapipb.SandboxRuntime want bool }{ - {"nil snapshot imposes no constraint", amd64Worker, nil, true}, - {"empty snapshot imposes no constraint", amd64Worker, &ateapipb.HardwareIdentity{}, true}, - {"same architecture matches", amd64Worker, amd64Worker, true}, - {"different architecture fails", arm64Worker, amd64Worker, false}, - {"nil worker fails when snapshot is stamped", nil, amd64Worker, false}, + {"nil snapshot imposes no constraint", amd64GVisor, nil, true}, + {"exact match", amd64GVisor, amd64GVisor, true}, + {"attribute order ignored", amd64GVisor, amd64GVisorReordered, true}, + {"different attribute value fails", arm64GVisor, amd64GVisor, false}, + {"different sandbox class fails", amd64MicroVM, amd64GVisor, false}, + {"different schema version fails", amd64GVisorV2, amd64GVisor, false}, + {"attribute the snapshot predates is not checked", amd64GVisorExtraAttr, amd64GVisor, true}, + {"attribute missing on worker fails", amd64GVisor, amd64GVisorExtraAttr, false}, + {"nil worker fails when snapshot is stamped", nil, amd64GVisor, false}, + {"nil version fails", &ateapipb.SandboxRuntime{SandboxClass: "gvisor"}, amd64GVisor, false}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { @@ -52,3 +93,38 @@ func TestMatches(t *testing.T) { }) } } + +func TestMatchesCompat(t *testing.T) { + compat := func(ver string, attrs ...*ateapipb.AttributeEntry) *ateapipb.VersionedSandboxCompat { + return &ateapipb.VersionedSandboxCompat{SchemaVersion: ver, Attributes: attrs} + } + amd64 := &ateapipb.AttributeEntry{Key: AttrArchitecture, Value: "amd64"} + arm64 := &ateapipb.AttributeEntry{Key: AttrArchitecture, Value: "arm64"} + cpu := &ateapipb.AttributeEntry{Key: "cpu_features", Value: "GenuineIntel"} + + tests := []struct { + name string + worker, snap *ateapipb.VersionedSandboxCompat + want bool + }{ + {"equal", compat("v1", amd64), compat("v1", amd64), true}, + {"key only the worker has is ignored", compat("v1", amd64, cpu), compat("v1", amd64), true}, + {"key only the snapshot has fails", compat("v1", amd64), compat("v1", amd64, cpu), false}, + {"value differs", compat("v1", arm64), compat("v1", amd64), false}, + {"schema version differs", compat("v2", amd64), compat("v1", amd64), false}, + {"both schema versions empty", compat("", amd64), compat("", amd64), false}, + {"no attributes on either", compat("v1"), compat("v1"), true}, + // Unlike Matches, a nil snapshot identity here is a stamped snapshot + // missing its identity, not the absence of a constraint. + {"nil snapshot identity", compat("v1", amd64), nil, false}, + {"nil worker identity", nil, compat("v1", amd64), false}, + {"both nil", nil, nil, false}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := MatchesCompat(tc.worker, tc.snap); got != tc.want { + t.Errorf("MatchesCompat() = %v, want %v", got, tc.want) + } + }) + } +} diff --git a/internal/proto/ateletpb/atelet.pb.go b/internal/proto/ateletpb/atelet.pb.go index 5d7f8cb916..e11f9cc298 100644 --- a/internal/proto/ateletpb/atelet.pb.go +++ b/internal/proto/ateletpb/atelet.pb.go @@ -209,13 +209,21 @@ type RegisterWorkerRequest struct { // // +k8s:required Capacity *WorkerResources `protobuf:"bytes,1,opt,name=capacity,proto3" json:"capacity,omitempty"` - // Hardware identity of the worker. atelet converts it to the control - // plane's HardwareIdentity when forwarding. + // Runtime that new and cold-booted actors start under. + // Snapshots matching default_runtime can also warm-boot on this worker. + // atelet converts it to the control plane's SandboxRuntime when forwarding. // // +k8s:required - Hardware *HardwareIdentity `protobuf:"bytes,2,opt,name=hardware,proto3" json:"hardware,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + DefaultRuntime *SandboxRuntime `protobuf:"bytes,2,opt,name=default_runtime,json=defaultRuntime,proto3" json:"default_runtime,omitempty"` + // Additional runtimes (excluding default_runtime) that this worker can + // restore snapshots from right now. + // + // +k8s:optional + // +k8s:maxItems=32 + // +k8s:listType=atomic + RestorableRuntimes []*SandboxRuntime `protobuf:"bytes,3,rep,name=restorable_runtimes,json=restorableRuntimes,proto3" json:"restorable_runtimes,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *RegisterWorkerRequest) Reset() { @@ -255,43 +263,55 @@ func (x *RegisterWorkerRequest) GetCapacity() *WorkerResources { return nil } -func (x *RegisterWorkerRequest) GetHardware() *HardwareIdentity { +func (x *RegisterWorkerRequest) GetDefaultRuntime() *SandboxRuntime { + if x != nil { + return x.DefaultRuntime + } + return nil +} + +func (x *RegisterWorkerRequest) GetRestorableRuntimes() []*SandboxRuntime { if x != nil { - return x.Hardware + return x.RestorableRuntimes } return nil } -// HardwareIdentity identifies the hardware properties of a worker. It mirrors -// the control plane's HardwareIdentity and is validated by the same rules. -type HardwareIdentity struct { +// SandboxRuntime describes a sandbox runtime and its compatibility identity as +// reported by a worker. It mirrors the control plane's SandboxRuntime and is +// validated by the same rules. +type SandboxRuntime struct { state protoimpl.MessageState `protogen:"open.v1"` - // Key-value pairs describing hardware properties (currently "architecture"; - // TODO: add "cpu_vendor", "cpu_model", etc.). + // e.g. "gvisor", "microvm" // - // +k8s:optional - // +k8s:maxProperties=32 - // +k8s:eachKey=+k8s:maxLength=128 - // +k8s:eachVal=+k8s:maxLength=256 - Attributes map[string]string `protobuf:"bytes,1,rep,name=attributes,proto3" json:"attributes,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + // +k8s:required + // +k8s:maxLength=63 + SandboxClass string `protobuf:"bytes,1,opt,name=sandbox_class,json=sandboxClass,proto3" json:"sandbox_class,omitempty"` + // Primary compatibility version stamped onto an actor's snapshot when a + // checkpoint is taken on this runtime. Snapshots matching `version` + // can warm-boot on this runtime. + // TODO: Add support for additional compatibility versions. + // + // +k8s:required + Version *VersionedSandboxCompat `protobuf:"bytes,2,opt,name=version,proto3" json:"version,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } -func (x *HardwareIdentity) Reset() { - *x = HardwareIdentity{} +func (x *SandboxRuntime) Reset() { + *x = SandboxRuntime{} mi := &file_atelet_proto_msgTypes[1] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } -func (x *HardwareIdentity) String() string { +func (x *SandboxRuntime) String() string { return protoimpl.X.MessageStringOf(x) } -func (*HardwareIdentity) ProtoMessage() {} +func (*SandboxRuntime) ProtoMessage() {} -func (x *HardwareIdentity) ProtoReflect() protoreflect.Message { +func (x *SandboxRuntime) ProtoReflect() protoreflect.Message { mi := &file_atelet_proto_msgTypes[1] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -303,18 +323,159 @@ func (x *HardwareIdentity) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use HardwareIdentity.ProtoReflect.Descriptor instead. -func (*HardwareIdentity) Descriptor() ([]byte, []int) { +// Deprecated: Use SandboxRuntime.ProtoReflect.Descriptor instead. +func (*SandboxRuntime) Descriptor() ([]byte, []int) { return file_atelet_proto_rawDescGZIP(), []int{1} } -func (x *HardwareIdentity) GetAttributes() map[string]string { +func (x *SandboxRuntime) GetSandboxClass() string { + if x != nil { + return x.SandboxClass + } + return "" +} + +func (x *SandboxRuntime) GetVersion() *VersionedSandboxCompat { + if x != nil { + return x.Version + } + return nil +} + +// VersionedSandboxCompat holds the compatibility schema version and the +// key-value compatibility attributes for a SandboxRuntime. +type VersionedSandboxCompat struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Compatibility schema version (e.g. "v1") defining which runtime and + // hardware attributes are recorded and how they are computed. + // + // +k8s:required + // +k8s:maxLength=64 + SchemaVersion string `protobuf:"bytes,1,opt,name=schema_version,json=schemaVersion,proto3" json:"schema_version,omitempty"` + // Key-value compatibility attributes defined by this schema version, covering + // both runtime assets (e.g. "gvisor_asset_hash") and effective guest hardware + // features after masking (e.g. "architecture", "cpu_features"). + // Matched against a snapshot under the same schema_version: every attribute + // on the snapshot must be present on the worker with an equal value; keys + // only the worker has are ignored. Bump schema_version when a key's meaning + // changes or older snapshots must no longer match. + // Order as sent does not matter: ate-api-server sorts attributes by key + // before recording them. + // + // +k8s:required + // +k8s:maxItems=32 + // +k8s:listType=map + // +k8s:listMapKey=key + Attributes []*AttributeEntry `protobuf:"bytes,2,rep,name=attributes,proto3" json:"attributes,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *VersionedSandboxCompat) Reset() { + *x = VersionedSandboxCompat{} + mi := &file_atelet_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *VersionedSandboxCompat) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*VersionedSandboxCompat) ProtoMessage() {} + +func (x *VersionedSandboxCompat) ProtoReflect() protoreflect.Message { + mi := &file_atelet_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use VersionedSandboxCompat.ProtoReflect.Descriptor instead. +func (*VersionedSandboxCompat) Descriptor() ([]byte, []int) { + return file_atelet_proto_rawDescGZIP(), []int{2} +} + +func (x *VersionedSandboxCompat) GetSchemaVersion() string { + if x != nil { + return x.SchemaVersion + } + return "" +} + +func (x *VersionedSandboxCompat) GetAttributes() []*AttributeEntry { if x != nil { return x.Attributes } return nil } +// AttributeEntry is a single key-value compatibility attribute. +type AttributeEntry struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Attribute name (e.g. "architecture", "cpu_features", "gvisor_asset_hash"). + // + // +k8s:required + // +k8s:maxLength=128 + Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` + // Attribute value corresponding to key. + // + // +k8s:required + // +k8s:maxLength=256 + Value string `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *AttributeEntry) Reset() { + *x = AttributeEntry{} + mi := &file_atelet_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *AttributeEntry) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AttributeEntry) ProtoMessage() {} + +func (x *AttributeEntry) ProtoReflect() protoreflect.Message { + mi := &file_atelet_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AttributeEntry.ProtoReflect.Descriptor instead. +func (*AttributeEntry) Descriptor() ([]byte, []int) { + return file_atelet_proto_rawDescGZIP(), []int{3} +} + +func (x *AttributeEntry) GetKey() string { + if x != nil { + return x.Key + } + return "" +} + +func (x *AttributeEntry) GetValue() string { + if x != nil { + return x.Value + } + return "" +} + // WorkerResources is what a worker can supply. It mirrors the control plane's // WorkerResources and is validated by the same rules. type WorkerResources struct { @@ -336,7 +497,7 @@ type WorkerResources struct { func (x *WorkerResources) Reset() { *x = WorkerResources{} - mi := &file_atelet_proto_msgTypes[2] + mi := &file_atelet_proto_msgTypes[4] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -348,7 +509,7 @@ func (x *WorkerResources) String() string { func (*WorkerResources) ProtoMessage() {} func (x *WorkerResources) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[2] + mi := &file_atelet_proto_msgTypes[4] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -361,7 +522,7 @@ func (x *WorkerResources) ProtoReflect() protoreflect.Message { // Deprecated: Use WorkerResources.ProtoReflect.Descriptor instead. func (*WorkerResources) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{2} + return file_atelet_proto_rawDescGZIP(), []int{4} } func (x *WorkerResources) GetResources() *Resources { @@ -395,7 +556,7 @@ type Resources struct { func (x *Resources) Reset() { *x = Resources{} - mi := &file_atelet_proto_msgTypes[3] + mi := &file_atelet_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -407,7 +568,7 @@ func (x *Resources) String() string { func (*Resources) ProtoMessage() {} func (x *Resources) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[3] + mi := &file_atelet_proto_msgTypes[5] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -420,7 +581,7 @@ func (x *Resources) ProtoReflect() protoreflect.Message { // Deprecated: Use Resources.ProtoReflect.Descriptor instead. func (*Resources) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{3} + return file_atelet_proto_rawDescGZIP(), []int{5} } func (x *Resources) GetLimits() []*Limits { @@ -448,7 +609,7 @@ type Limits struct { func (x *Limits) Reset() { *x = Limits{} - mi := &file_atelet_proto_msgTypes[4] + mi := &file_atelet_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -460,7 +621,7 @@ func (x *Limits) String() string { func (*Limits) ProtoMessage() {} func (x *Limits) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[4] + mi := &file_atelet_proto_msgTypes[6] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -473,7 +634,7 @@ func (x *Limits) ProtoReflect() protoreflect.Message { // Deprecated: Use Limits.ProtoReflect.Descriptor instead. func (*Limits) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{4} + return file_atelet_proto_rawDescGZIP(), []int{6} } func (x *Limits) GetName() string { @@ -498,7 +659,7 @@ type RegisterWorkerResponse struct { func (x *RegisterWorkerResponse) Reset() { *x = RegisterWorkerResponse{} - mi := &file_atelet_proto_msgTypes[5] + mi := &file_atelet_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -510,7 +671,7 @@ func (x *RegisterWorkerResponse) String() string { func (*RegisterWorkerResponse) ProtoMessage() {} func (x *RegisterWorkerResponse) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[5] + mi := &file_atelet_proto_msgTypes[7] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -523,7 +684,7 @@ func (x *RegisterWorkerResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RegisterWorkerResponse.ProtoReflect.Descriptor instead. func (*RegisterWorkerResponse) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{5} + return file_atelet_proto_rawDescGZIP(), []int{7} } type RequestActorSuspendRequest struct { @@ -550,7 +711,7 @@ type RequestActorSuspendRequest struct { func (x *RequestActorSuspendRequest) Reset() { *x = RequestActorSuspendRequest{} - mi := &file_atelet_proto_msgTypes[6] + mi := &file_atelet_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -562,7 +723,7 @@ func (x *RequestActorSuspendRequest) String() string { func (*RequestActorSuspendRequest) ProtoMessage() {} func (x *RequestActorSuspendRequest) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[6] + mi := &file_atelet_proto_msgTypes[8] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -575,7 +736,7 @@ func (x *RequestActorSuspendRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use RequestActorSuspendRequest.ProtoReflect.Descriptor instead. func (*RequestActorSuspendRequest) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{6} + return file_atelet_proto_rawDescGZIP(), []int{8} } func (x *RequestActorSuspendRequest) GetActorAtespace() string { @@ -607,7 +768,7 @@ type RequestActorSuspendResponse struct { func (x *RequestActorSuspendResponse) Reset() { *x = RequestActorSuspendResponse{} - mi := &file_atelet_proto_msgTypes[7] + mi := &file_atelet_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -619,7 +780,7 @@ func (x *RequestActorSuspendResponse) String() string { func (*RequestActorSuspendResponse) ProtoMessage() {} func (x *RequestActorSuspendResponse) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[7] + mi := &file_atelet_proto_msgTypes[9] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -632,7 +793,7 @@ func (x *RequestActorSuspendResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RequestActorSuspendResponse.ProtoReflect.Descriptor instead. func (*RequestActorSuspendResponse) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{7} + return file_atelet_proto_rawDescGZIP(), []int{9} } type MintActorCertificateRequest struct { @@ -663,7 +824,7 @@ type MintActorCertificateRequest struct { func (x *MintActorCertificateRequest) Reset() { *x = MintActorCertificateRequest{} - mi := &file_atelet_proto_msgTypes[8] + mi := &file_atelet_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -675,7 +836,7 @@ func (x *MintActorCertificateRequest) String() string { func (*MintActorCertificateRequest) ProtoMessage() {} func (x *MintActorCertificateRequest) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[8] + mi := &file_atelet_proto_msgTypes[10] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -688,7 +849,7 @@ func (x *MintActorCertificateRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use MintActorCertificateRequest.ProtoReflect.Descriptor instead. func (*MintActorCertificateRequest) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{8} + return file_atelet_proto_rawDescGZIP(), []int{10} } func (x *MintActorCertificateRequest) GetActorAtespace() string { @@ -729,7 +890,7 @@ type MintActorCertificateResponse struct { func (x *MintActorCertificateResponse) Reset() { *x = MintActorCertificateResponse{} - mi := &file_atelet_proto_msgTypes[9] + mi := &file_atelet_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -741,7 +902,7 @@ func (x *MintActorCertificateResponse) String() string { func (*MintActorCertificateResponse) ProtoMessage() {} func (x *MintActorCertificateResponse) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[9] + mi := &file_atelet_proto_msgTypes[11] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -754,7 +915,7 @@ func (x *MintActorCertificateResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use MintActorCertificateResponse.ProtoReflect.Descriptor instead. func (*MintActorCertificateResponse) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{9} + return file_atelet_proto_rawDescGZIP(), []int{11} } func (x *MintActorCertificateResponse) GetActorCertificates() [][]byte { @@ -780,7 +941,7 @@ type TerminateRequest struct { func (x *TerminateRequest) Reset() { *x = TerminateRequest{} - mi := &file_atelet_proto_msgTypes[10] + mi := &file_atelet_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -792,7 +953,7 @@ func (x *TerminateRequest) String() string { func (*TerminateRequest) ProtoMessage() {} func (x *TerminateRequest) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[10] + mi := &file_atelet_proto_msgTypes[12] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -805,7 +966,7 @@ func (x *TerminateRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use TerminateRequest.ProtoReflect.Descriptor instead. func (*TerminateRequest) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{10} + return file_atelet_proto_rawDescGZIP(), []int{12} } func (x *TerminateRequest) GetWorkerPodUid() string { @@ -865,7 +1026,7 @@ type TerminateResponse struct { func (x *TerminateResponse) Reset() { *x = TerminateResponse{} - mi := &file_atelet_proto_msgTypes[11] + mi := &file_atelet_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -877,7 +1038,7 @@ func (x *TerminateResponse) String() string { func (*TerminateResponse) ProtoMessage() {} func (x *TerminateResponse) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[11] + mi := &file_atelet_proto_msgTypes[13] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -890,7 +1051,7 @@ func (x *TerminateResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use TerminateResponse.ProtoReflect.Descriptor instead. func (*TerminateResponse) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{11} + return file_atelet_proto_rawDescGZIP(), []int{13} } type RunRequest struct { @@ -919,7 +1080,7 @@ type RunRequest struct { func (x *RunRequest) Reset() { *x = RunRequest{} - mi := &file_atelet_proto_msgTypes[12] + mi := &file_atelet_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -931,7 +1092,7 @@ func (x *RunRequest) String() string { func (*RunRequest) ProtoMessage() {} func (x *RunRequest) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[12] + mi := &file_atelet_proto_msgTypes[14] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -944,7 +1105,7 @@ func (x *RunRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use RunRequest.ProtoReflect.Descriptor instead. func (*RunRequest) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{12} + return file_atelet_proto_rawDescGZIP(), []int{14} } func (x *RunRequest) GetWorkerPodUid() string { @@ -1036,7 +1197,7 @@ type EgressGateway struct { func (x *EgressGateway) Reset() { *x = EgressGateway{} - mi := &file_atelet_proto_msgTypes[13] + mi := &file_atelet_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1048,7 +1209,7 @@ func (x *EgressGateway) String() string { func (*EgressGateway) ProtoMessage() {} func (x *EgressGateway) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[13] + mi := &file_atelet_proto_msgTypes[15] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1061,7 +1222,7 @@ func (x *EgressGateway) ProtoReflect() protoreflect.Message { // Deprecated: Use EgressGateway.ProtoReflect.Descriptor instead. func (*EgressGateway) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{13} + return file_atelet_proto_rawDescGZIP(), []int{15} } func (x *EgressGateway) GetAddress() string { @@ -1085,7 +1246,7 @@ type AssetFile struct { func (x *AssetFile) Reset() { *x = AssetFile{} - mi := &file_atelet_proto_msgTypes[14] + mi := &file_atelet_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1097,7 +1258,7 @@ func (x *AssetFile) String() string { func (*AssetFile) ProtoMessage() {} func (x *AssetFile) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[14] + mi := &file_atelet_proto_msgTypes[16] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1110,7 +1271,7 @@ func (x *AssetFile) ProtoReflect() protoreflect.Message { // Deprecated: Use AssetFile.ProtoReflect.Descriptor instead. func (*AssetFile) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{14} + return file_atelet_proto_rawDescGZIP(), []int{16} } func (x *AssetFile) GetUrl() string { @@ -1138,7 +1299,7 @@ type ArchAssets struct { func (x *ArchAssets) Reset() { *x = ArchAssets{} - mi := &file_atelet_proto_msgTypes[15] + mi := &file_atelet_proto_msgTypes[17] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1150,7 +1311,7 @@ func (x *ArchAssets) String() string { func (*ArchAssets) ProtoMessage() {} func (x *ArchAssets) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[15] + mi := &file_atelet_proto_msgTypes[17] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1163,7 +1324,7 @@ func (x *ArchAssets) ProtoReflect() protoreflect.Message { // Deprecated: Use ArchAssets.ProtoReflect.Descriptor instead. func (*ArchAssets) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{15} + return file_atelet_proto_rawDescGZIP(), []int{17} } func (x *ArchAssets) GetFiles() map[string]*AssetFile { @@ -1194,7 +1355,7 @@ type SandboxAssets struct { func (x *SandboxAssets) Reset() { *x = SandboxAssets{} - mi := &file_atelet_proto_msgTypes[16] + mi := &file_atelet_proto_msgTypes[18] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1206,7 +1367,7 @@ func (x *SandboxAssets) String() string { func (*SandboxAssets) ProtoMessage() {} func (x *SandboxAssets) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[16] + mi := &file_atelet_proto_msgTypes[18] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1219,7 +1380,7 @@ func (x *SandboxAssets) ProtoReflect() protoreflect.Message { // Deprecated: Use SandboxAssets.ProtoReflect.Descriptor instead. func (*SandboxAssets) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{16} + return file_atelet_proto_rawDescGZIP(), []int{18} } func (x *SandboxAssets) GetSandboxClass() string { @@ -1254,7 +1415,7 @@ type WorkloadSpec struct { func (x *WorkloadSpec) Reset() { *x = WorkloadSpec{} - mi := &file_atelet_proto_msgTypes[17] + mi := &file_atelet_proto_msgTypes[19] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1266,7 +1427,7 @@ func (x *WorkloadSpec) String() string { func (*WorkloadSpec) ProtoMessage() {} func (x *WorkloadSpec) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[17] + mi := &file_atelet_proto_msgTypes[19] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1279,7 +1440,7 @@ func (x *WorkloadSpec) ProtoReflect() protoreflect.Message { // Deprecated: Use WorkloadSpec.ProtoReflect.Descriptor instead. func (*WorkloadSpec) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{17} + return file_atelet_proto_rawDescGZIP(), []int{19} } func (x *WorkloadSpec) GetContainers() []*Container { @@ -1304,7 +1465,7 @@ type DurableDirVolume struct { func (x *DurableDirVolume) Reset() { *x = DurableDirVolume{} - mi := &file_atelet_proto_msgTypes[18] + mi := &file_atelet_proto_msgTypes[20] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1316,7 +1477,7 @@ func (x *DurableDirVolume) String() string { func (*DurableDirVolume) ProtoMessage() {} func (x *DurableDirVolume) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[18] + mi := &file_atelet_proto_msgTypes[20] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1329,7 +1490,7 @@ func (x *DurableDirVolume) ProtoReflect() protoreflect.Message { // Deprecated: Use DurableDirVolume.ProtoReflect.Descriptor instead. func (*DurableDirVolume) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{18} + return file_atelet_proto_rawDescGZIP(), []int{20} } type ExternalVolumeSource struct { @@ -1346,7 +1507,7 @@ type ExternalVolumeSource struct { func (x *ExternalVolumeSource) Reset() { *x = ExternalVolumeSource{} - mi := &file_atelet_proto_msgTypes[19] + mi := &file_atelet_proto_msgTypes[21] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1358,7 +1519,7 @@ func (x *ExternalVolumeSource) String() string { func (*ExternalVolumeSource) ProtoMessage() {} func (x *ExternalVolumeSource) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[19] + mi := &file_atelet_proto_msgTypes[21] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1371,7 +1532,7 @@ func (x *ExternalVolumeSource) ProtoReflect() protoreflect.Message { // Deprecated: Use ExternalVolumeSource.ProtoReflect.Descriptor instead. func (*ExternalVolumeSource) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{19} + return file_atelet_proto_rawDescGZIP(), []int{21} } func (x *ExternalVolumeSource) GetStorageVolumeId() string { @@ -1411,7 +1572,7 @@ type ImageVolumeSource struct { func (x *ImageVolumeSource) Reset() { *x = ImageVolumeSource{} - mi := &file_atelet_proto_msgTypes[20] + mi := &file_atelet_proto_msgTypes[22] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1423,7 +1584,7 @@ func (x *ImageVolumeSource) String() string { func (*ImageVolumeSource) ProtoMessage() {} func (x *ImageVolumeSource) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[20] + mi := &file_atelet_proto_msgTypes[22] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1436,7 +1597,7 @@ func (x *ImageVolumeSource) ProtoReflect() protoreflect.Message { // Deprecated: Use ImageVolumeSource.ProtoReflect.Descriptor instead. func (*ImageVolumeSource) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{20} + return file_atelet_proto_rawDescGZIP(), []int{22} } func (x *ImageVolumeSource) GetReference() string { @@ -1458,7 +1619,7 @@ type ActorMetadataItem struct { func (x *ActorMetadataItem) Reset() { *x = ActorMetadataItem{} - mi := &file_atelet_proto_msgTypes[21] + mi := &file_atelet_proto_msgTypes[23] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1470,7 +1631,7 @@ func (x *ActorMetadataItem) String() string { func (*ActorMetadataItem) ProtoMessage() {} func (x *ActorMetadataItem) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[21] + mi := &file_atelet_proto_msgTypes[23] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1483,7 +1644,7 @@ func (x *ActorMetadataItem) ProtoReflect() protoreflect.Message { // Deprecated: Use ActorMetadataItem.ProtoReflect.Descriptor instead. func (*ActorMetadataItem) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{21} + return file_atelet_proto_rawDescGZIP(), []int{23} } func (x *ActorMetadataItem) GetField() ActorMetadataField { @@ -1511,7 +1672,7 @@ type ActorMetadataDataSource struct { func (x *ActorMetadataDataSource) Reset() { *x = ActorMetadataDataSource{} - mi := &file_atelet_proto_msgTypes[22] + mi := &file_atelet_proto_msgTypes[24] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1523,7 +1684,7 @@ func (x *ActorMetadataDataSource) String() string { func (*ActorMetadataDataSource) ProtoMessage() {} func (x *ActorMetadataDataSource) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[22] + mi := &file_atelet_proto_msgTypes[24] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1536,7 +1697,7 @@ func (x *ActorMetadataDataSource) ProtoReflect() protoreflect.Message { // Deprecated: Use ActorMetadataDataSource.ProtoReflect.Descriptor instead. func (*ActorMetadataDataSource) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{22} + return file_atelet_proto_rawDescGZIP(), []int{24} } func (x *ActorMetadataDataSource) GetItems() []*ActorMetadataItem { @@ -1557,7 +1718,7 @@ type TrustBundleDataSource struct { func (x *TrustBundleDataSource) Reset() { *x = TrustBundleDataSource{} - mi := &file_atelet_proto_msgTypes[23] + mi := &file_atelet_proto_msgTypes[25] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1569,7 +1730,7 @@ func (x *TrustBundleDataSource) String() string { func (*TrustBundleDataSource) ProtoMessage() {} func (x *TrustBundleDataSource) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[23] + mi := &file_atelet_proto_msgTypes[25] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1582,7 +1743,7 @@ func (x *TrustBundleDataSource) ProtoReflect() protoreflect.Message { // Deprecated: Use TrustBundleDataSource.ProtoReflect.Descriptor instead. func (*TrustBundleDataSource) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{23} + return file_atelet_proto_rawDescGZIP(), []int{25} } func (x *TrustBundleDataSource) GetPath() string { @@ -1612,7 +1773,7 @@ type SystemInfoDataSource struct { func (x *SystemInfoDataSource) Reset() { *x = SystemInfoDataSource{} - mi := &file_atelet_proto_msgTypes[24] + mi := &file_atelet_proto_msgTypes[26] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1624,7 +1785,7 @@ func (x *SystemInfoDataSource) String() string { func (*SystemInfoDataSource) ProtoMessage() {} func (x *SystemInfoDataSource) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[24] + mi := &file_atelet_proto_msgTypes[26] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1637,7 +1798,7 @@ func (x *SystemInfoDataSource) ProtoReflect() protoreflect.Message { // Deprecated: Use SystemInfoDataSource.ProtoReflect.Descriptor instead. func (*SystemInfoDataSource) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{24} + return file_atelet_proto_rawDescGZIP(), []int{26} } func (x *SystemInfoDataSource) GetDataSource() isSystemInfoDataSource_DataSource { @@ -1693,7 +1854,7 @@ type SystemInfoVolume struct { func (x *SystemInfoVolume) Reset() { *x = SystemInfoVolume{} - mi := &file_atelet_proto_msgTypes[25] + mi := &file_atelet_proto_msgTypes[27] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1705,7 +1866,7 @@ func (x *SystemInfoVolume) String() string { func (*SystemInfoVolume) ProtoMessage() {} func (x *SystemInfoVolume) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[25] + mi := &file_atelet_proto_msgTypes[27] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1718,7 +1879,7 @@ func (x *SystemInfoVolume) ProtoReflect() protoreflect.Message { // Deprecated: Use SystemInfoVolume.ProtoReflect.Descriptor instead. func (*SystemInfoVolume) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{25} + return file_atelet_proto_rawDescGZIP(), []int{27} } func (x *SystemInfoVolume) GetDataSources() []*SystemInfoDataSource { @@ -1744,7 +1905,7 @@ type Volume struct { func (x *Volume) Reset() { *x = Volume{} - mi := &file_atelet_proto_msgTypes[26] + mi := &file_atelet_proto_msgTypes[28] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1756,7 +1917,7 @@ func (x *Volume) String() string { func (*Volume) ProtoMessage() {} func (x *Volume) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[26] + mi := &file_atelet_proto_msgTypes[28] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1769,7 +1930,7 @@ func (x *Volume) ProtoReflect() protoreflect.Message { // Deprecated: Use Volume.ProtoReflect.Descriptor instead. func (*Volume) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{26} + return file_atelet_proto_rawDescGZIP(), []int{28} } func (x *Volume) GetName() string { @@ -1860,7 +2021,7 @@ type VolumeMount struct { func (x *VolumeMount) Reset() { *x = VolumeMount{} - mi := &file_atelet_proto_msgTypes[27] + mi := &file_atelet_proto_msgTypes[29] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1872,7 +2033,7 @@ func (x *VolumeMount) String() string { func (*VolumeMount) ProtoMessage() {} func (x *VolumeMount) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[27] + mi := &file_atelet_proto_msgTypes[29] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1885,7 +2046,7 @@ func (x *VolumeMount) ProtoReflect() protoreflect.Message { // Deprecated: Use VolumeMount.ProtoReflect.Descriptor instead. func (*VolumeMount) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{27} + return file_atelet_proto_rawDescGZIP(), []int{29} } func (x *VolumeMount) GetName() string { @@ -1921,7 +2082,7 @@ type Container struct { func (x *Container) Reset() { *x = Container{} - mi := &file_atelet_proto_msgTypes[28] + mi := &file_atelet_proto_msgTypes[30] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1933,7 +2094,7 @@ func (x *Container) String() string { func (*Container) ProtoMessage() {} func (x *Container) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[28] + mi := &file_atelet_proto_msgTypes[30] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1946,7 +2107,7 @@ func (x *Container) ProtoReflect() protoreflect.Message { // Deprecated: Use Container.ProtoReflect.Descriptor instead. func (*Container) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{28} + return file_atelet_proto_rawDescGZIP(), []int{30} } func (x *Container) GetName() string { @@ -2022,7 +2183,7 @@ type SecurityContext struct { func (x *SecurityContext) Reset() { *x = SecurityContext{} - mi := &file_atelet_proto_msgTypes[29] + mi := &file_atelet_proto_msgTypes[31] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2034,7 +2195,7 @@ func (x *SecurityContext) String() string { func (*SecurityContext) ProtoMessage() {} func (x *SecurityContext) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[29] + mi := &file_atelet_proto_msgTypes[31] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2047,7 +2208,7 @@ func (x *SecurityContext) ProtoReflect() protoreflect.Message { // Deprecated: Use SecurityContext.ProtoReflect.Descriptor instead. func (*SecurityContext) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{29} + return file_atelet_proto_rawDescGZIP(), []int{31} } func (x *SecurityContext) GetCapabilities() *Capabilities { @@ -2069,7 +2230,7 @@ type Capabilities struct { func (x *Capabilities) Reset() { *x = Capabilities{} - mi := &file_atelet_proto_msgTypes[30] + mi := &file_atelet_proto_msgTypes[32] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2081,7 +2242,7 @@ func (x *Capabilities) String() string { func (*Capabilities) ProtoMessage() {} func (x *Capabilities) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[30] + mi := &file_atelet_proto_msgTypes[32] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2094,7 +2255,7 @@ func (x *Capabilities) ProtoReflect() protoreflect.Message { // Deprecated: Use Capabilities.ProtoReflect.Descriptor instead. func (*Capabilities) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{30} + return file_atelet_proto_rawDescGZIP(), []int{32} } func (x *Capabilities) GetAdd() []string { @@ -2125,7 +2286,7 @@ type ResourceLimits struct { func (x *ResourceLimits) Reset() { *x = ResourceLimits{} - mi := &file_atelet_proto_msgTypes[31] + mi := &file_atelet_proto_msgTypes[33] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2137,7 +2298,7 @@ func (x *ResourceLimits) String() string { func (*ResourceLimits) ProtoMessage() {} func (x *ResourceLimits) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[31] + mi := &file_atelet_proto_msgTypes[33] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2150,7 +2311,7 @@ func (x *ResourceLimits) ProtoReflect() protoreflect.Message { // Deprecated: Use ResourceLimits.ProtoReflect.Descriptor instead. func (*ResourceLimits) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{31} + return file_atelet_proto_rawDescGZIP(), []int{33} } func (x *ResourceLimits) GetMemoryBytes() int64 { @@ -2177,7 +2338,7 @@ type EnvEntry struct { func (x *EnvEntry) Reset() { *x = EnvEntry{} - mi := &file_atelet_proto_msgTypes[32] + mi := &file_atelet_proto_msgTypes[34] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2189,7 +2350,7 @@ func (x *EnvEntry) String() string { func (*EnvEntry) ProtoMessage() {} func (x *EnvEntry) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[32] + mi := &file_atelet_proto_msgTypes[34] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2202,7 +2363,7 @@ func (x *EnvEntry) ProtoReflect() protoreflect.Message { // Deprecated: Use EnvEntry.ProtoReflect.Descriptor instead. func (*EnvEntry) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{32} + return file_atelet_proto_rawDescGZIP(), []int{34} } func (x *EnvEntry) GetName() string { @@ -2232,7 +2393,7 @@ type WakeupProbe struct { func (x *WakeupProbe) Reset() { *x = WakeupProbe{} - mi := &file_atelet_proto_msgTypes[33] + mi := &file_atelet_proto_msgTypes[35] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2244,7 +2405,7 @@ func (x *WakeupProbe) String() string { func (*WakeupProbe) ProtoMessage() {} func (x *WakeupProbe) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[33] + mi := &file_atelet_proto_msgTypes[35] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2257,7 +2418,7 @@ func (x *WakeupProbe) ProtoReflect() protoreflect.Message { // Deprecated: Use WakeupProbe.ProtoReflect.Descriptor instead. func (*WakeupProbe) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{33} + return file_atelet_proto_rawDescGZIP(), []int{35} } func (x *WakeupProbe) GetHttpGet() *HTTPGetAction { @@ -2287,7 +2448,7 @@ type HTTPGetAction struct { func (x *HTTPGetAction) Reset() { *x = HTTPGetAction{} - mi := &file_atelet_proto_msgTypes[34] + mi := &file_atelet_proto_msgTypes[36] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2299,7 +2460,7 @@ func (x *HTTPGetAction) String() string { func (*HTTPGetAction) ProtoMessage() {} func (x *HTTPGetAction) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[34] + mi := &file_atelet_proto_msgTypes[36] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2312,7 +2473,7 @@ func (x *HTTPGetAction) ProtoReflect() protoreflect.Message { // Deprecated: Use HTTPGetAction.ProtoReflect.Descriptor instead. func (*HTTPGetAction) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{34} + return file_atelet_proto_rawDescGZIP(), []int{36} } func (x *HTTPGetAction) GetPath() string { @@ -2337,7 +2498,7 @@ type RunResponse struct { func (x *RunResponse) Reset() { *x = RunResponse{} - mi := &file_atelet_proto_msgTypes[35] + mi := &file_atelet_proto_msgTypes[37] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2349,7 +2510,7 @@ func (x *RunResponse) String() string { func (*RunResponse) ProtoMessage() {} func (x *RunResponse) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[35] + mi := &file_atelet_proto_msgTypes[37] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2362,7 +2523,7 @@ func (x *RunResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RunResponse.ProtoReflect.Descriptor instead. func (*RunResponse) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{35} + return file_atelet_proto_rawDescGZIP(), []int{37} } type LocalCheckpointConfiguration struct { @@ -2378,7 +2539,7 @@ type LocalCheckpointConfiguration struct { func (x *LocalCheckpointConfiguration) Reset() { *x = LocalCheckpointConfiguration{} - mi := &file_atelet_proto_msgTypes[36] + mi := &file_atelet_proto_msgTypes[38] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2390,7 +2551,7 @@ func (x *LocalCheckpointConfiguration) String() string { func (*LocalCheckpointConfiguration) ProtoMessage() {} func (x *LocalCheckpointConfiguration) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[36] + mi := &file_atelet_proto_msgTypes[38] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2403,7 +2564,7 @@ func (x *LocalCheckpointConfiguration) ProtoReflect() protoreflect.Message { // Deprecated: Use LocalCheckpointConfiguration.ProtoReflect.Descriptor instead. func (*LocalCheckpointConfiguration) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{36} + return file_atelet_proto_rawDescGZIP(), []int{38} } func (x *LocalCheckpointConfiguration) GetSnapshotName() string { @@ -2424,7 +2585,7 @@ type ExternalCheckpointConfiguration struct { func (x *ExternalCheckpointConfiguration) Reset() { *x = ExternalCheckpointConfiguration{} - mi := &file_atelet_proto_msgTypes[37] + mi := &file_atelet_proto_msgTypes[39] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2436,7 +2597,7 @@ func (x *ExternalCheckpointConfiguration) String() string { func (*ExternalCheckpointConfiguration) ProtoMessage() {} func (x *ExternalCheckpointConfiguration) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[37] + mi := &file_atelet_proto_msgTypes[39] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2449,7 +2610,7 @@ func (x *ExternalCheckpointConfiguration) ProtoReflect() protoreflect.Message { // Deprecated: Use ExternalCheckpointConfiguration.ProtoReflect.Descriptor instead. func (*ExternalCheckpointConfiguration) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{37} + return file_atelet_proto_rawDescGZIP(), []int{39} } func (x *ExternalCheckpointConfiguration) GetSnapshotUri() string { @@ -2474,7 +2635,7 @@ type ExternalRestoreConfiguration struct { func (x *ExternalRestoreConfiguration) Reset() { *x = ExternalRestoreConfiguration{} - mi := &file_atelet_proto_msgTypes[38] + mi := &file_atelet_proto_msgTypes[40] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2486,7 +2647,7 @@ func (x *ExternalRestoreConfiguration) String() string { func (*ExternalRestoreConfiguration) ProtoMessage() {} func (x *ExternalRestoreConfiguration) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[38] + mi := &file_atelet_proto_msgTypes[40] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2499,7 +2660,7 @@ func (x *ExternalRestoreConfiguration) ProtoReflect() protoreflect.Message { // Deprecated: Use ExternalRestoreConfiguration.ProtoReflect.Descriptor instead. func (*ExternalRestoreConfiguration) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{38} + return file_atelet_proto_rawDescGZIP(), []int{40} } func (x *ExternalRestoreConfiguration) GetSnapshotUri() string { @@ -2537,7 +2698,7 @@ type CheckpointRequest struct { func (x *CheckpointRequest) Reset() { *x = CheckpointRequest{} - mi := &file_atelet_proto_msgTypes[39] + mi := &file_atelet_proto_msgTypes[41] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2549,7 +2710,7 @@ func (x *CheckpointRequest) String() string { func (*CheckpointRequest) ProtoMessage() {} func (x *CheckpointRequest) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[39] + mi := &file_atelet_proto_msgTypes[41] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2562,7 +2723,7 @@ func (x *CheckpointRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use CheckpointRequest.ProtoReflect.Descriptor instead. func (*CheckpointRequest) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{39} + return file_atelet_proto_rawDescGZIP(), []int{41} } func (x *CheckpointRequest) GetWorkerPodUid() string { @@ -2677,7 +2838,7 @@ type CheckpointResponse struct { func (x *CheckpointResponse) Reset() { *x = CheckpointResponse{} - mi := &file_atelet_proto_msgTypes[40] + mi := &file_atelet_proto_msgTypes[42] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2689,7 +2850,7 @@ func (x *CheckpointResponse) String() string { func (*CheckpointResponse) ProtoMessage() {} func (x *CheckpointResponse) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[40] + mi := &file_atelet_proto_msgTypes[42] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2702,7 +2863,7 @@ func (x *CheckpointResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use CheckpointResponse.ProtoReflect.Descriptor instead. func (*CheckpointResponse) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{40} + return file_atelet_proto_rawDescGZIP(), []int{42} } type UploadPausedCheckpointRequest struct { @@ -2730,7 +2891,7 @@ type UploadPausedCheckpointRequest struct { func (x *UploadPausedCheckpointRequest) Reset() { *x = UploadPausedCheckpointRequest{} - mi := &file_atelet_proto_msgTypes[41] + mi := &file_atelet_proto_msgTypes[43] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2742,7 +2903,7 @@ func (x *UploadPausedCheckpointRequest) String() string { func (*UploadPausedCheckpointRequest) ProtoMessage() {} func (x *UploadPausedCheckpointRequest) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[41] + mi := &file_atelet_proto_msgTypes[43] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2755,7 +2916,7 @@ func (x *UploadPausedCheckpointRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use UploadPausedCheckpointRequest.ProtoReflect.Descriptor instead. func (*UploadPausedCheckpointRequest) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{41} + return file_atelet_proto_rawDescGZIP(), []int{43} } func (x *UploadPausedCheckpointRequest) GetAtespace() string { @@ -2822,7 +2983,7 @@ type UploadPausedCheckpointResponse struct { func (x *UploadPausedCheckpointResponse) Reset() { *x = UploadPausedCheckpointResponse{} - mi := &file_atelet_proto_msgTypes[42] + mi := &file_atelet_proto_msgTypes[44] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2834,7 +2995,7 @@ func (x *UploadPausedCheckpointResponse) String() string { func (*UploadPausedCheckpointResponse) ProtoMessage() {} func (x *UploadPausedCheckpointResponse) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[42] + mi := &file_atelet_proto_msgTypes[44] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2847,7 +3008,7 @@ func (x *UploadPausedCheckpointResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use UploadPausedCheckpointResponse.ProtoReflect.Descriptor instead. func (*UploadPausedCheckpointResponse) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{42} + return file_atelet_proto_rawDescGZIP(), []int{44} } type RestoreRequest struct { @@ -2886,7 +3047,7 @@ type RestoreRequest struct { func (x *RestoreRequest) Reset() { *x = RestoreRequest{} - mi := &file_atelet_proto_msgTypes[43] + mi := &file_atelet_proto_msgTypes[45] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2898,7 +3059,7 @@ func (x *RestoreRequest) String() string { func (*RestoreRequest) ProtoMessage() {} func (x *RestoreRequest) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[43] + mi := &file_atelet_proto_msgTypes[45] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2911,7 +3072,7 @@ func (x *RestoreRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use RestoreRequest.ProtoReflect.Descriptor instead. func (*RestoreRequest) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{43} + return file_atelet_proto_rawDescGZIP(), []int{45} } func (x *RestoreRequest) GetWorkerPodUid() string { @@ -3054,7 +3215,7 @@ type RestoreResponse struct { func (x *RestoreResponse) Reset() { *x = RestoreResponse{} - mi := &file_atelet_proto_msgTypes[44] + mi := &file_atelet_proto_msgTypes[46] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3066,7 +3227,7 @@ func (x *RestoreResponse) String() string { func (*RestoreResponse) ProtoMessage() {} func (x *RestoreResponse) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[44] + mi := &file_atelet_proto_msgTypes[46] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3079,24 +3240,29 @@ func (x *RestoreResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RestoreResponse.ProtoReflect.Descriptor instead. func (*RestoreResponse) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{44} + return file_atelet_proto_rawDescGZIP(), []int{46} } var File_atelet_proto protoreflect.FileDescriptor const file_atelet_proto_rawDesc = "" + "\n" + - "\fatelet.proto\x12\x06atelet\"\x82\x01\n" + + "\fatelet.proto\x12\x06atelet\"\xd6\x01\n" + "\x15RegisterWorkerRequest\x123\n" + - "\bcapacity\x18\x01 \x01(\v2\x17.atelet.WorkerResourcesR\bcapacity\x124\n" + - "\bhardware\x18\x02 \x01(\v2\x18.atelet.HardwareIdentityR\bhardware\"\x9b\x01\n" + - "\x10HardwareIdentity\x12H\n" + + "\bcapacity\x18\x01 \x01(\v2\x17.atelet.WorkerResourcesR\bcapacity\x12?\n" + + "\x0fdefault_runtime\x18\x02 \x01(\v2\x16.atelet.SandboxRuntimeR\x0edefaultRuntime\x12G\n" + + "\x13restorable_runtimes\x18\x03 \x03(\v2\x16.atelet.SandboxRuntimeR\x12restorableRuntimes\"o\n" + + "\x0eSandboxRuntime\x12#\n" + + "\rsandbox_class\x18\x01 \x01(\tR\fsandboxClass\x128\n" + + "\aversion\x18\x02 \x01(\v2\x1e.atelet.VersionedSandboxCompatR\aversion\"w\n" + + "\x16VersionedSandboxCompat\x12%\n" + + "\x0eschema_version\x18\x01 \x01(\tR\rschemaVersion\x126\n" + "\n" + - "attributes\x18\x01 \x03(\v2(.atelet.HardwareIdentity.AttributesEntryR\n" + - "attributes\x1a=\n" + - "\x0fAttributesEntry\x12\x10\n" + + "attributes\x18\x02 \x03(\v2\x16.atelet.AttributeEntryR\n" + + "attributes\"8\n" + + "\x0eAttributeEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + - "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"Z\n" + + "\x05value\x18\x02 \x01(\tR\x05value\"Z\n" + "\x0fWorkerResources\x12/\n" + "\tresources\x18\x01 \x01(\v2\x11.atelet.ResourcesR\tresources\x12\x16\n" + "\x06actors\x18\x02 \x01(\x05R\x06actors\"3\n" + @@ -3335,130 +3501,133 @@ func file_atelet_proto_rawDescGZIP() []byte { } var file_atelet_proto_enumTypes = make([]protoimpl.EnumInfo, 3) -var file_atelet_proto_msgTypes = make([]protoimpl.MessageInfo, 50) +var file_atelet_proto_msgTypes = make([]protoimpl.MessageInfo, 51) var file_atelet_proto_goTypes = []any{ (ActorMetadataField)(0), // 0: atelet.ActorMetadataField (CheckpointType)(0), // 1: atelet.CheckpointType (SnapshotFidelity)(0), // 2: atelet.SnapshotFidelity (*RegisterWorkerRequest)(nil), // 3: atelet.RegisterWorkerRequest - (*HardwareIdentity)(nil), // 4: atelet.HardwareIdentity - (*WorkerResources)(nil), // 5: atelet.WorkerResources - (*Resources)(nil), // 6: atelet.Resources - (*Limits)(nil), // 7: atelet.Limits - (*RegisterWorkerResponse)(nil), // 8: atelet.RegisterWorkerResponse - (*RequestActorSuspendRequest)(nil), // 9: atelet.RequestActorSuspendRequest - (*RequestActorSuspendResponse)(nil), // 10: atelet.RequestActorSuspendResponse - (*MintActorCertificateRequest)(nil), // 11: atelet.MintActorCertificateRequest - (*MintActorCertificateResponse)(nil), // 12: atelet.MintActorCertificateResponse - (*TerminateRequest)(nil), // 13: atelet.TerminateRequest - (*TerminateResponse)(nil), // 14: atelet.TerminateResponse - (*RunRequest)(nil), // 15: atelet.RunRequest - (*EgressGateway)(nil), // 16: atelet.EgressGateway - (*AssetFile)(nil), // 17: atelet.AssetFile - (*ArchAssets)(nil), // 18: atelet.ArchAssets - (*SandboxAssets)(nil), // 19: atelet.SandboxAssets - (*WorkloadSpec)(nil), // 20: atelet.WorkloadSpec - (*DurableDirVolume)(nil), // 21: atelet.DurableDirVolume - (*ExternalVolumeSource)(nil), // 22: atelet.ExternalVolumeSource - (*ImageVolumeSource)(nil), // 23: atelet.ImageVolumeSource - (*ActorMetadataItem)(nil), // 24: atelet.ActorMetadataItem - (*ActorMetadataDataSource)(nil), // 25: atelet.ActorMetadataDataSource - (*TrustBundleDataSource)(nil), // 26: atelet.TrustBundleDataSource - (*SystemInfoDataSource)(nil), // 27: atelet.SystemInfoDataSource - (*SystemInfoVolume)(nil), // 28: atelet.SystemInfoVolume - (*Volume)(nil), // 29: atelet.Volume - (*VolumeMount)(nil), // 30: atelet.VolumeMount - (*Container)(nil), // 31: atelet.Container - (*SecurityContext)(nil), // 32: atelet.SecurityContext - (*Capabilities)(nil), // 33: atelet.Capabilities - (*ResourceLimits)(nil), // 34: atelet.ResourceLimits - (*EnvEntry)(nil), // 35: atelet.EnvEntry - (*WakeupProbe)(nil), // 36: atelet.WakeupProbe - (*HTTPGetAction)(nil), // 37: atelet.HTTPGetAction - (*RunResponse)(nil), // 38: atelet.RunResponse - (*LocalCheckpointConfiguration)(nil), // 39: atelet.LocalCheckpointConfiguration - (*ExternalCheckpointConfiguration)(nil), // 40: atelet.ExternalCheckpointConfiguration - (*ExternalRestoreConfiguration)(nil), // 41: atelet.ExternalRestoreConfiguration - (*CheckpointRequest)(nil), // 42: atelet.CheckpointRequest - (*CheckpointResponse)(nil), // 43: atelet.CheckpointResponse - (*UploadPausedCheckpointRequest)(nil), // 44: atelet.UploadPausedCheckpointRequest - (*UploadPausedCheckpointResponse)(nil), // 45: atelet.UploadPausedCheckpointResponse - (*RestoreRequest)(nil), // 46: atelet.RestoreRequest - (*RestoreResponse)(nil), // 47: atelet.RestoreResponse - nil, // 48: atelet.HardwareIdentity.AttributesEntry - nil, // 49: atelet.ArchAssets.FilesEntry - nil, // 50: atelet.SandboxAssets.AssetsEntry - nil, // 51: atelet.ExternalVolumeSource.VolumeContextEntry - nil, // 52: atelet.ExternalVolumeSource.PublishContextEntry + (*SandboxRuntime)(nil), // 4: atelet.SandboxRuntime + (*VersionedSandboxCompat)(nil), // 5: atelet.VersionedSandboxCompat + (*AttributeEntry)(nil), // 6: atelet.AttributeEntry + (*WorkerResources)(nil), // 7: atelet.WorkerResources + (*Resources)(nil), // 8: atelet.Resources + (*Limits)(nil), // 9: atelet.Limits + (*RegisterWorkerResponse)(nil), // 10: atelet.RegisterWorkerResponse + (*RequestActorSuspendRequest)(nil), // 11: atelet.RequestActorSuspendRequest + (*RequestActorSuspendResponse)(nil), // 12: atelet.RequestActorSuspendResponse + (*MintActorCertificateRequest)(nil), // 13: atelet.MintActorCertificateRequest + (*MintActorCertificateResponse)(nil), // 14: atelet.MintActorCertificateResponse + (*TerminateRequest)(nil), // 15: atelet.TerminateRequest + (*TerminateResponse)(nil), // 16: atelet.TerminateResponse + (*RunRequest)(nil), // 17: atelet.RunRequest + (*EgressGateway)(nil), // 18: atelet.EgressGateway + (*AssetFile)(nil), // 19: atelet.AssetFile + (*ArchAssets)(nil), // 20: atelet.ArchAssets + (*SandboxAssets)(nil), // 21: atelet.SandboxAssets + (*WorkloadSpec)(nil), // 22: atelet.WorkloadSpec + (*DurableDirVolume)(nil), // 23: atelet.DurableDirVolume + (*ExternalVolumeSource)(nil), // 24: atelet.ExternalVolumeSource + (*ImageVolumeSource)(nil), // 25: atelet.ImageVolumeSource + (*ActorMetadataItem)(nil), // 26: atelet.ActorMetadataItem + (*ActorMetadataDataSource)(nil), // 27: atelet.ActorMetadataDataSource + (*TrustBundleDataSource)(nil), // 28: atelet.TrustBundleDataSource + (*SystemInfoDataSource)(nil), // 29: atelet.SystemInfoDataSource + (*SystemInfoVolume)(nil), // 30: atelet.SystemInfoVolume + (*Volume)(nil), // 31: atelet.Volume + (*VolumeMount)(nil), // 32: atelet.VolumeMount + (*Container)(nil), // 33: atelet.Container + (*SecurityContext)(nil), // 34: atelet.SecurityContext + (*Capabilities)(nil), // 35: atelet.Capabilities + (*ResourceLimits)(nil), // 36: atelet.ResourceLimits + (*EnvEntry)(nil), // 37: atelet.EnvEntry + (*WakeupProbe)(nil), // 38: atelet.WakeupProbe + (*HTTPGetAction)(nil), // 39: atelet.HTTPGetAction + (*RunResponse)(nil), // 40: atelet.RunResponse + (*LocalCheckpointConfiguration)(nil), // 41: atelet.LocalCheckpointConfiguration + (*ExternalCheckpointConfiguration)(nil), // 42: atelet.ExternalCheckpointConfiguration + (*ExternalRestoreConfiguration)(nil), // 43: atelet.ExternalRestoreConfiguration + (*CheckpointRequest)(nil), // 44: atelet.CheckpointRequest + (*CheckpointResponse)(nil), // 45: atelet.CheckpointResponse + (*UploadPausedCheckpointRequest)(nil), // 46: atelet.UploadPausedCheckpointRequest + (*UploadPausedCheckpointResponse)(nil), // 47: atelet.UploadPausedCheckpointResponse + (*RestoreRequest)(nil), // 48: atelet.RestoreRequest + (*RestoreResponse)(nil), // 49: atelet.RestoreResponse + nil, // 50: atelet.ArchAssets.FilesEntry + nil, // 51: atelet.SandboxAssets.AssetsEntry + nil, // 52: atelet.ExternalVolumeSource.VolumeContextEntry + nil, // 53: atelet.ExternalVolumeSource.PublishContextEntry } var file_atelet_proto_depIdxs = []int32{ - 5, // 0: atelet.RegisterWorkerRequest.capacity:type_name -> atelet.WorkerResources - 4, // 1: atelet.RegisterWorkerRequest.hardware:type_name -> atelet.HardwareIdentity - 48, // 2: atelet.HardwareIdentity.attributes:type_name -> atelet.HardwareIdentity.AttributesEntry - 6, // 3: atelet.WorkerResources.resources:type_name -> atelet.Resources - 7, // 4: atelet.Resources.limits:type_name -> atelet.Limits - 20, // 5: atelet.TerminateRequest.spec:type_name -> atelet.WorkloadSpec - 20, // 6: atelet.RunRequest.spec:type_name -> atelet.WorkloadSpec - 19, // 7: atelet.RunRequest.sandbox_assets:type_name -> atelet.SandboxAssets - 16, // 8: atelet.RunRequest.egress_gateway:type_name -> atelet.EgressGateway - 49, // 9: atelet.ArchAssets.files:type_name -> atelet.ArchAssets.FilesEntry - 50, // 10: atelet.SandboxAssets.assets:type_name -> atelet.SandboxAssets.AssetsEntry - 31, // 11: atelet.WorkloadSpec.containers:type_name -> atelet.Container - 29, // 12: atelet.WorkloadSpec.volumes:type_name -> atelet.Volume - 51, // 13: atelet.ExternalVolumeSource.volume_context:type_name -> atelet.ExternalVolumeSource.VolumeContextEntry - 52, // 14: atelet.ExternalVolumeSource.publish_context:type_name -> atelet.ExternalVolumeSource.PublishContextEntry - 0, // 15: atelet.ActorMetadataItem.field:type_name -> atelet.ActorMetadataField - 24, // 16: atelet.ActorMetadataDataSource.items:type_name -> atelet.ActorMetadataItem - 25, // 17: atelet.SystemInfoDataSource.actor_metadata:type_name -> atelet.ActorMetadataDataSource - 26, // 18: atelet.SystemInfoDataSource.trust_bundle:type_name -> atelet.TrustBundleDataSource - 27, // 19: atelet.SystemInfoVolume.data_sources:type_name -> atelet.SystemInfoDataSource - 21, // 20: atelet.Volume.durable_dir:type_name -> atelet.DurableDirVolume - 22, // 21: atelet.Volume.external:type_name -> atelet.ExternalVolumeSource - 28, // 22: atelet.Volume.system_info:type_name -> atelet.SystemInfoVolume - 23, // 23: atelet.Volume.image:type_name -> atelet.ImageVolumeSource - 35, // 24: atelet.Container.env:type_name -> atelet.EnvEntry - 36, // 25: atelet.Container.wakeup_probe:type_name -> atelet.WakeupProbe - 30, // 26: atelet.Container.volume_mounts:type_name -> atelet.VolumeMount - 32, // 27: atelet.Container.security_context:type_name -> atelet.SecurityContext - 34, // 28: atelet.Container.resources:type_name -> atelet.ResourceLimits - 33, // 29: atelet.SecurityContext.capabilities:type_name -> atelet.Capabilities - 37, // 30: atelet.WakeupProbe.http_get:type_name -> atelet.HTTPGetAction - 20, // 31: atelet.CheckpointRequest.spec:type_name -> atelet.WorkloadSpec - 1, // 32: atelet.CheckpointRequest.type:type_name -> atelet.CheckpointType - 39, // 33: atelet.CheckpointRequest.local_config:type_name -> atelet.LocalCheckpointConfiguration - 40, // 34: atelet.CheckpointRequest.external_config:type_name -> atelet.ExternalCheckpointConfiguration - 2, // 35: atelet.CheckpointRequest.fidelity:type_name -> atelet.SnapshotFidelity - 2, // 36: atelet.UploadPausedCheckpointRequest.desired_fidelity:type_name -> atelet.SnapshotFidelity - 20, // 37: atelet.RestoreRequest.spec:type_name -> atelet.WorkloadSpec - 1, // 38: atelet.RestoreRequest.type:type_name -> atelet.CheckpointType - 39, // 39: atelet.RestoreRequest.local_config:type_name -> atelet.LocalCheckpointConfiguration - 41, // 40: atelet.RestoreRequest.external_config:type_name -> atelet.ExternalRestoreConfiguration - 2, // 41: atelet.RestoreRequest.fidelity:type_name -> atelet.SnapshotFidelity - 16, // 42: atelet.RestoreRequest.egress_gateway:type_name -> atelet.EgressGateway - 19, // 43: atelet.RestoreRequest.sandbox_assets:type_name -> atelet.SandboxAssets - 17, // 44: atelet.ArchAssets.FilesEntry.value:type_name -> atelet.AssetFile - 18, // 45: atelet.SandboxAssets.AssetsEntry.value:type_name -> atelet.ArchAssets - 11, // 46: atelet.AteomSupport.MintActorCertificate:input_type -> atelet.MintActorCertificateRequest - 3, // 47: atelet.AteomSupport.RegisterWorker:input_type -> atelet.RegisterWorkerRequest - 9, // 48: atelet.AteomSupport.RequestActorSuspend:input_type -> atelet.RequestActorSuspendRequest - 15, // 49: atelet.AteomHerder.Run:input_type -> atelet.RunRequest - 42, // 50: atelet.AteomHerder.Checkpoint:input_type -> atelet.CheckpointRequest - 46, // 51: atelet.AteomHerder.Restore:input_type -> atelet.RestoreRequest - 44, // 52: atelet.AteomHerder.UploadPausedCheckpoint:input_type -> atelet.UploadPausedCheckpointRequest - 13, // 53: atelet.AteomHerder.Terminate:input_type -> atelet.TerminateRequest - 12, // 54: atelet.AteomSupport.MintActorCertificate:output_type -> atelet.MintActorCertificateResponse - 8, // 55: atelet.AteomSupport.RegisterWorker:output_type -> atelet.RegisterWorkerResponse - 10, // 56: atelet.AteomSupport.RequestActorSuspend:output_type -> atelet.RequestActorSuspendResponse - 38, // 57: atelet.AteomHerder.Run:output_type -> atelet.RunResponse - 43, // 58: atelet.AteomHerder.Checkpoint:output_type -> atelet.CheckpointResponse - 47, // 59: atelet.AteomHerder.Restore:output_type -> atelet.RestoreResponse - 45, // 60: atelet.AteomHerder.UploadPausedCheckpoint:output_type -> atelet.UploadPausedCheckpointResponse - 14, // 61: atelet.AteomHerder.Terminate:output_type -> atelet.TerminateResponse - 54, // [54:62] is the sub-list for method output_type - 46, // [46:54] is the sub-list for method input_type - 46, // [46:46] is the sub-list for extension type_name - 46, // [46:46] is the sub-list for extension extendee - 0, // [0:46] is the sub-list for field type_name + 7, // 0: atelet.RegisterWorkerRequest.capacity:type_name -> atelet.WorkerResources + 4, // 1: atelet.RegisterWorkerRequest.default_runtime:type_name -> atelet.SandboxRuntime + 4, // 2: atelet.RegisterWorkerRequest.restorable_runtimes:type_name -> atelet.SandboxRuntime + 5, // 3: atelet.SandboxRuntime.version:type_name -> atelet.VersionedSandboxCompat + 6, // 4: atelet.VersionedSandboxCompat.attributes:type_name -> atelet.AttributeEntry + 8, // 5: atelet.WorkerResources.resources:type_name -> atelet.Resources + 9, // 6: atelet.Resources.limits:type_name -> atelet.Limits + 22, // 7: atelet.TerminateRequest.spec:type_name -> atelet.WorkloadSpec + 22, // 8: atelet.RunRequest.spec:type_name -> atelet.WorkloadSpec + 21, // 9: atelet.RunRequest.sandbox_assets:type_name -> atelet.SandboxAssets + 18, // 10: atelet.RunRequest.egress_gateway:type_name -> atelet.EgressGateway + 50, // 11: atelet.ArchAssets.files:type_name -> atelet.ArchAssets.FilesEntry + 51, // 12: atelet.SandboxAssets.assets:type_name -> atelet.SandboxAssets.AssetsEntry + 33, // 13: atelet.WorkloadSpec.containers:type_name -> atelet.Container + 31, // 14: atelet.WorkloadSpec.volumes:type_name -> atelet.Volume + 52, // 15: atelet.ExternalVolumeSource.volume_context:type_name -> atelet.ExternalVolumeSource.VolumeContextEntry + 53, // 16: atelet.ExternalVolumeSource.publish_context:type_name -> atelet.ExternalVolumeSource.PublishContextEntry + 0, // 17: atelet.ActorMetadataItem.field:type_name -> atelet.ActorMetadataField + 26, // 18: atelet.ActorMetadataDataSource.items:type_name -> atelet.ActorMetadataItem + 27, // 19: atelet.SystemInfoDataSource.actor_metadata:type_name -> atelet.ActorMetadataDataSource + 28, // 20: atelet.SystemInfoDataSource.trust_bundle:type_name -> atelet.TrustBundleDataSource + 29, // 21: atelet.SystemInfoVolume.data_sources:type_name -> atelet.SystemInfoDataSource + 23, // 22: atelet.Volume.durable_dir:type_name -> atelet.DurableDirVolume + 24, // 23: atelet.Volume.external:type_name -> atelet.ExternalVolumeSource + 30, // 24: atelet.Volume.system_info:type_name -> atelet.SystemInfoVolume + 25, // 25: atelet.Volume.image:type_name -> atelet.ImageVolumeSource + 37, // 26: atelet.Container.env:type_name -> atelet.EnvEntry + 38, // 27: atelet.Container.wakeup_probe:type_name -> atelet.WakeupProbe + 32, // 28: atelet.Container.volume_mounts:type_name -> atelet.VolumeMount + 34, // 29: atelet.Container.security_context:type_name -> atelet.SecurityContext + 36, // 30: atelet.Container.resources:type_name -> atelet.ResourceLimits + 35, // 31: atelet.SecurityContext.capabilities:type_name -> atelet.Capabilities + 39, // 32: atelet.WakeupProbe.http_get:type_name -> atelet.HTTPGetAction + 22, // 33: atelet.CheckpointRequest.spec:type_name -> atelet.WorkloadSpec + 1, // 34: atelet.CheckpointRequest.type:type_name -> atelet.CheckpointType + 41, // 35: atelet.CheckpointRequest.local_config:type_name -> atelet.LocalCheckpointConfiguration + 42, // 36: atelet.CheckpointRequest.external_config:type_name -> atelet.ExternalCheckpointConfiguration + 2, // 37: atelet.CheckpointRequest.fidelity:type_name -> atelet.SnapshotFidelity + 2, // 38: atelet.UploadPausedCheckpointRequest.desired_fidelity:type_name -> atelet.SnapshotFidelity + 22, // 39: atelet.RestoreRequest.spec:type_name -> atelet.WorkloadSpec + 1, // 40: atelet.RestoreRequest.type:type_name -> atelet.CheckpointType + 41, // 41: atelet.RestoreRequest.local_config:type_name -> atelet.LocalCheckpointConfiguration + 43, // 42: atelet.RestoreRequest.external_config:type_name -> atelet.ExternalRestoreConfiguration + 2, // 43: atelet.RestoreRequest.fidelity:type_name -> atelet.SnapshotFidelity + 18, // 44: atelet.RestoreRequest.egress_gateway:type_name -> atelet.EgressGateway + 21, // 45: atelet.RestoreRequest.sandbox_assets:type_name -> atelet.SandboxAssets + 19, // 46: atelet.ArchAssets.FilesEntry.value:type_name -> atelet.AssetFile + 20, // 47: atelet.SandboxAssets.AssetsEntry.value:type_name -> atelet.ArchAssets + 13, // 48: atelet.AteomSupport.MintActorCertificate:input_type -> atelet.MintActorCertificateRequest + 3, // 49: atelet.AteomSupport.RegisterWorker:input_type -> atelet.RegisterWorkerRequest + 11, // 50: atelet.AteomSupport.RequestActorSuspend:input_type -> atelet.RequestActorSuspendRequest + 17, // 51: atelet.AteomHerder.Run:input_type -> atelet.RunRequest + 44, // 52: atelet.AteomHerder.Checkpoint:input_type -> atelet.CheckpointRequest + 48, // 53: atelet.AteomHerder.Restore:input_type -> atelet.RestoreRequest + 46, // 54: atelet.AteomHerder.UploadPausedCheckpoint:input_type -> atelet.UploadPausedCheckpointRequest + 15, // 55: atelet.AteomHerder.Terminate:input_type -> atelet.TerminateRequest + 14, // 56: atelet.AteomSupport.MintActorCertificate:output_type -> atelet.MintActorCertificateResponse + 10, // 57: atelet.AteomSupport.RegisterWorker:output_type -> atelet.RegisterWorkerResponse + 12, // 58: atelet.AteomSupport.RequestActorSuspend:output_type -> atelet.RequestActorSuspendResponse + 40, // 59: atelet.AteomHerder.Run:output_type -> atelet.RunResponse + 45, // 60: atelet.AteomHerder.Checkpoint:output_type -> atelet.CheckpointResponse + 49, // 61: atelet.AteomHerder.Restore:output_type -> atelet.RestoreResponse + 47, // 62: atelet.AteomHerder.UploadPausedCheckpoint:output_type -> atelet.UploadPausedCheckpointResponse + 16, // 63: atelet.AteomHerder.Terminate:output_type -> atelet.TerminateResponse + 56, // [56:64] is the sub-list for method output_type + 48, // [48:56] is the sub-list for method input_type + 48, // [48:48] is the sub-list for extension type_name + 48, // [48:48] is the sub-list for extension extendee + 0, // [0:48] is the sub-list for field type_name } func init() { file_atelet_proto_init() } @@ -3466,22 +3635,22 @@ func file_atelet_proto_init() { if File_atelet_proto != nil { return } - file_atelet_proto_msgTypes[12].OneofWrappers = []any{} - file_atelet_proto_msgTypes[24].OneofWrappers = []any{ + file_atelet_proto_msgTypes[14].OneofWrappers = []any{} + file_atelet_proto_msgTypes[26].OneofWrappers = []any{ (*SystemInfoDataSource_ActorMetadata)(nil), (*SystemInfoDataSource_TrustBundle)(nil), } - file_atelet_proto_msgTypes[26].OneofWrappers = []any{ + file_atelet_proto_msgTypes[28].OneofWrappers = []any{ (*Volume_DurableDir)(nil), (*Volume_External)(nil), (*Volume_SystemInfo)(nil), (*Volume_Image)(nil), } - file_atelet_proto_msgTypes[39].OneofWrappers = []any{ + file_atelet_proto_msgTypes[41].OneofWrappers = []any{ (*CheckpointRequest_LocalConfig)(nil), (*CheckpointRequest_ExternalConfig)(nil), } - file_atelet_proto_msgTypes[43].OneofWrappers = []any{ + file_atelet_proto_msgTypes[45].OneofWrappers = []any{ (*RestoreRequest_LocalConfig)(nil), (*RestoreRequest_ExternalConfig)(nil), } @@ -3491,7 +3660,7 @@ func file_atelet_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_atelet_proto_rawDesc), len(file_atelet_proto_rawDesc)), NumEnums: 3, - NumMessages: 50, + NumMessages: 51, NumExtensions: 0, NumServices: 2, }, diff --git a/internal/proto/ateletpb/atelet.proto b/internal/proto/ateletpb/atelet.proto index 4715da5d57..034e7e03dc 100644 --- a/internal/proto/ateletpb/atelet.proto +++ b/internal/proto/ateletpb/atelet.proto @@ -30,7 +30,7 @@ service AteomSupport { // the actor filesystem, when/if we support those). rpc MintActorCertificate(MintActorCertificateRequest) returns (MintActorCertificateResponse) {} - // Register capacity and hardware identity for this worker with atelet. + // Register capacity and supported sandbox runtimes for this worker with atelet. rpc RegisterWorker(RegisterWorkerRequest) returns (RegisterWorkerResponse) {} // RequestActorSuspend asks that the actor this worker hosts be suspended, @@ -52,24 +52,84 @@ message RegisterWorkerRequest { // +k8s:required WorkerResources capacity = 1; - // Hardware identity of the worker. atelet converts it to the control - // plane's HardwareIdentity when forwarding. + // Runtime that new and cold-booted actors start under. + // Snapshots matching default_runtime can also warm-boot on this worker. + // atelet converts it to the control plane's SandboxRuntime when forwarding. // // +k8s:required - HardwareIdentity hardware = 2; -} + SandboxRuntime default_runtime = 2; -// HardwareIdentity identifies the hardware properties of a worker. It mirrors -// the control plane's HardwareIdentity and is validated by the same rules. -message HardwareIdentity { - // Key-value pairs describing hardware properties (currently "architecture"; - // TODO: add "cpu_vendor", "cpu_model", etc.). + // Additional runtimes (excluding default_runtime) that this worker can + // restore snapshots from right now. // // +k8s:optional - // +k8s:maxProperties=32 - // +k8s:eachKey=+k8s:maxLength=128 - // +k8s:eachVal=+k8s:maxLength=256 - map attributes = 1; + // +k8s:maxItems=32 + // +k8s:listType=atomic + repeated SandboxRuntime restorable_runtimes = 3; +} + +// SandboxRuntime describes a sandbox runtime and its compatibility identity as +// reported by a worker. It mirrors the control plane's SandboxRuntime and is +// validated by the same rules. +message SandboxRuntime { + // e.g. "gvisor", "microvm" + // + // +k8s:required + // +k8s:maxLength=63 + string sandbox_class = 1; + + // Primary compatibility version stamped onto an actor's snapshot when a + // checkpoint is taken on this runtime. Snapshots matching `version` + // can warm-boot on this runtime. + // TODO: Add support for additional compatibility versions. + // + // +k8s:required + VersionedSandboxCompat version = 2; + + // TODO: Add an informational SandboxConfig version name (e.g. "gvisor-2") + // once workers report one. +} + +// VersionedSandboxCompat holds the compatibility schema version and the +// key-value compatibility attributes for a SandboxRuntime. +message VersionedSandboxCompat { + // Compatibility schema version (e.g. "v1") defining which runtime and + // hardware attributes are recorded and how they are computed. + // + // +k8s:required + // +k8s:maxLength=64 + string schema_version = 1; + + // Key-value compatibility attributes defined by this schema version, covering + // both runtime assets (e.g. "gvisor_asset_hash") and effective guest hardware + // features after masking (e.g. "architecture", "cpu_features"). + // Matched against a snapshot under the same schema_version: every attribute + // on the snapshot must be present on the worker with an equal value; keys + // only the worker has are ignored. Bump schema_version when a key's meaning + // changes or older snapshots must no longer match. + // Order as sent does not matter: ate-api-server sorts attributes by key + // before recording them. + // + // +k8s:required + // +k8s:maxItems=32 + // +k8s:listType=map + // +k8s:listMapKey=key + repeated AttributeEntry attributes = 2; +} + +// AttributeEntry is a single key-value compatibility attribute. +message AttributeEntry { + // Attribute name (e.g. "architecture", "cpu_features", "gvisor_asset_hash"). + // + // +k8s:required + // +k8s:maxLength=128 + string key = 1; + + // Attribute value corresponding to key. + // + // +k8s:required + // +k8s:maxLength=256 + string value = 2; } // WorkerResources is what a worker can supply. It mirrors the control plane's diff --git a/internal/proto/ateletpb/atelet_grpc.pb.go b/internal/proto/ateletpb/atelet_grpc.pb.go index 3eaecf4ea6..bec22fb739 100644 --- a/internal/proto/ateletpb/atelet_grpc.pb.go +++ b/internal/proto/ateletpb/atelet_grpc.pb.go @@ -53,7 +53,7 @@ type AteomSupportClient interface { // MintActorCertificate (which would be used for certificates projected into // the actor filesystem, when/if we support those). MintActorCertificate(ctx context.Context, in *MintActorCertificateRequest, opts ...grpc.CallOption) (*MintActorCertificateResponse, error) - // Register capacity and hardware identity for this worker with atelet. + // Register capacity and supported sandbox runtimes for this worker with atelet. RegisterWorker(ctx context.Context, in *RegisterWorkerRequest, opts ...grpc.CallOption) (*RegisterWorkerResponse, error) // RequestActorSuspend asks that the actor this worker hosts be suspended, // for atelet to forward to the control plane's WorkerService, which this @@ -120,7 +120,7 @@ type AteomSupportServer interface { // MintActorCertificate (which would be used for certificates projected into // the actor filesystem, when/if we support those). MintActorCertificate(context.Context, *MintActorCertificateRequest) (*MintActorCertificateResponse, error) - // Register capacity and hardware identity for this worker with atelet. + // Register capacity and supported sandbox runtimes for this worker with atelet. RegisterWorker(context.Context, *RegisterWorkerRequest) (*RegisterWorkerResponse, error) // RequestActorSuspend asks that the actor this worker hosts be suspended, // for atelet to forward to the control plane's WorkerService, which this diff --git a/pkg/proto/ateapipb/ateapi.pb.go b/pkg/proto/ateapipb/ateapi.pb.go index 7e063939be..6f4145c96b 100644 --- a/pkg/proto/ateapipb/ateapi.pb.go +++ b/pkg/proto/ateapipb/ateapi.pb.go @@ -7158,12 +7158,21 @@ type WorkerStatus struct { // +k8s:optional // +k8s:minimum=0 ObservedEpoch int64 `protobuf:"varint,4,opt,name=observed_epoch,json=observedEpoch,proto3" json:"observed_epoch,omitempty"` - // Hardware identity reported by the Worker at registration. + // Runtime that new and cold-booted actors start under, as reported by the + // Worker at registration. Snapshots matching default_runtime can also + // warm-boot on this worker. // // +k8s:optional - Hardware *HardwareIdentity `protobuf:"bytes,5,opt,name=hardware,proto3" json:"hardware,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + DefaultRuntime *SandboxRuntime `protobuf:"bytes,5,opt,name=default_runtime,json=defaultRuntime,proto3" json:"default_runtime,omitempty"` + // Additional runtimes (excluding default_runtime) that this worker can + // restore snapshots from right now. + // + // +k8s:optional + // +k8s:maxItems=32 + // +k8s:listType=atomic + RestorableRuntimes []*SandboxRuntime `protobuf:"bytes,6,rep,name=restorable_runtimes,json=restorableRuntimes,proto3" json:"restorable_runtimes,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *WorkerStatus) Reset() { @@ -7224,43 +7233,54 @@ func (x *WorkerStatus) GetObservedEpoch() int64 { return 0 } -func (x *WorkerStatus) GetHardware() *HardwareIdentity { +func (x *WorkerStatus) GetDefaultRuntime() *SandboxRuntime { + if x != nil { + return x.DefaultRuntime + } + return nil +} + +func (x *WorkerStatus) GetRestorableRuntimes() []*SandboxRuntime { if x != nil { - return x.Hardware + return x.RestorableRuntimes } return nil } -// HardwareIdentity identifies the hardware of a Worker or the Worker a -// snapshot was captured on. -type HardwareIdentity struct { +// SandboxRuntime describes a sandbox runtime and its compatibility identity as +// reported by a Worker or recorded on a snapshot. +type SandboxRuntime struct { state protoimpl.MessageState `protogen:"open.v1"` - // Key-value pairs describing hardware properties (currently - // "architecture"; TODO: add "cpu_vendor", "cpu_model", etc.). + // e.g. "gvisor", "microvm" // - // +k8s:optional - // +k8s:maxProperties=32 - // +k8s:eachKey=+k8s:maxLength=128 - // +k8s:eachVal=+k8s:maxLength=256 - Attributes map[string]string `protobuf:"bytes,1,rep,name=attributes,proto3" json:"attributes,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + // +k8s:required + // +k8s:maxLength=63 + SandboxClass string `protobuf:"bytes,1,opt,name=sandbox_class,json=sandboxClass,proto3" json:"sandbox_class,omitempty"` + // Primary compatibility version stamped onto an actor's snapshot when a + // checkpoint is taken on this runtime. Snapshots matching `version` + // can warm-boot on this runtime. + // TODO: Add support for additional compatibility versions. + // + // +k8s:required + Version *VersionedSandboxCompat `protobuf:"bytes,2,opt,name=version,proto3" json:"version,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } -func (x *HardwareIdentity) Reset() { - *x = HardwareIdentity{} +func (x *SandboxRuntime) Reset() { + *x = SandboxRuntime{} mi := &file_ateapi_proto_msgTypes[98] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } -func (x *HardwareIdentity) String() string { +func (x *SandboxRuntime) String() string { return protoimpl.X.MessageStringOf(x) } -func (*HardwareIdentity) ProtoMessage() {} +func (*SandboxRuntime) ProtoMessage() {} -func (x *HardwareIdentity) ProtoReflect() protoreflect.Message { +func (x *SandboxRuntime) ProtoReflect() protoreflect.Message { mi := &file_ateapi_proto_msgTypes[98] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -7272,18 +7292,159 @@ func (x *HardwareIdentity) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use HardwareIdentity.ProtoReflect.Descriptor instead. -func (*HardwareIdentity) Descriptor() ([]byte, []int) { +// Deprecated: Use SandboxRuntime.ProtoReflect.Descriptor instead. +func (*SandboxRuntime) Descriptor() ([]byte, []int) { return file_ateapi_proto_rawDescGZIP(), []int{98} } -func (x *HardwareIdentity) GetAttributes() map[string]string { +func (x *SandboxRuntime) GetSandboxClass() string { + if x != nil { + return x.SandboxClass + } + return "" +} + +func (x *SandboxRuntime) GetVersion() *VersionedSandboxCompat { + if x != nil { + return x.Version + } + return nil +} + +// VersionedSandboxCompat holds the compatibility schema version and the +// key-value compatibility attributes for a SandboxRuntime. +type VersionedSandboxCompat struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Compatibility schema version (e.g. "v1") defining which runtime and + // hardware attributes are recorded and how they are computed. + // + // +k8s:required + // +k8s:maxLength=64 + SchemaVersion string `protobuf:"bytes,1,opt,name=schema_version,json=schemaVersion,proto3" json:"schema_version,omitempty"` + // Key-value compatibility attributes defined by this schema version, covering + // both runtime assets (e.g. "gvisor_asset_hash") and effective guest hardware + // features after masking (e.g. "architecture", "cpu_features"). + // Matched against a snapshot under the same schema_version: every attribute + // on the snapshot must be present on the worker with an equal value; keys + // only the worker has are ignored. Bump schema_version when a key's meaning + // changes or older snapshots must no longer match. + // Order as sent does not matter: ate-api-server sorts attributes by key + // before recording them. + // + // +k8s:required + // +k8s:maxItems=32 + // +k8s:listType=map + // +k8s:listMapKey=key + Attributes []*AttributeEntry `protobuf:"bytes,2,rep,name=attributes,proto3" json:"attributes,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *VersionedSandboxCompat) Reset() { + *x = VersionedSandboxCompat{} + mi := &file_ateapi_proto_msgTypes[99] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *VersionedSandboxCompat) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*VersionedSandboxCompat) ProtoMessage() {} + +func (x *VersionedSandboxCompat) ProtoReflect() protoreflect.Message { + mi := &file_ateapi_proto_msgTypes[99] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use VersionedSandboxCompat.ProtoReflect.Descriptor instead. +func (*VersionedSandboxCompat) Descriptor() ([]byte, []int) { + return file_ateapi_proto_rawDescGZIP(), []int{99} +} + +func (x *VersionedSandboxCompat) GetSchemaVersion() string { + if x != nil { + return x.SchemaVersion + } + return "" +} + +func (x *VersionedSandboxCompat) GetAttributes() []*AttributeEntry { if x != nil { return x.Attributes } return nil } +// AttributeEntry is a single key-value compatibility attribute. +type AttributeEntry struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Attribute name (e.g. "architecture", "cpu_features", "gvisor_asset_hash"). + // + // +k8s:required + // +k8s:maxLength=128 + Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` + // Attribute value corresponding to key. + // + // +k8s:required + // +k8s:maxLength=256 + Value string `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *AttributeEntry) Reset() { + *x = AttributeEntry{} + mi := &file_ateapi_proto_msgTypes[100] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *AttributeEntry) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AttributeEntry) ProtoMessage() {} + +func (x *AttributeEntry) ProtoReflect() protoreflect.Message { + mi := &file_ateapi_proto_msgTypes[100] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AttributeEntry.ProtoReflect.Descriptor instead. +func (*AttributeEntry) Descriptor() ([]byte, []int) { + return file_ateapi_proto_rawDescGZIP(), []int{100} +} + +func (x *AttributeEntry) GetKey() string { + if x != nil { + return x.Key + } + return "" +} + +func (x *AttributeEntry) GetValue() string { + if x != nil { + return x.Value + } + return "" +} + // WorkerResources represents a set of resources. type WorkerResources struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -7306,7 +7467,7 @@ type WorkerResources struct { func (x *WorkerResources) Reset() { *x = WorkerResources{} - mi := &file_ateapi_proto_msgTypes[99] + mi := &file_ateapi_proto_msgTypes[101] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7318,7 +7479,7 @@ func (x *WorkerResources) String() string { func (*WorkerResources) ProtoMessage() {} func (x *WorkerResources) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[99] + mi := &file_ateapi_proto_msgTypes[101] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7331,7 +7492,7 @@ func (x *WorkerResources) ProtoReflect() protoreflect.Message { // Deprecated: Use WorkerResources.ProtoReflect.Descriptor instead. func (*WorkerResources) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{99} + return file_ateapi_proto_rawDescGZIP(), []int{101} } func (x *WorkerResources) GetResources() *Resources { @@ -7384,7 +7545,7 @@ type ActorAssignment struct { func (x *ActorAssignment) Reset() { *x = ActorAssignment{} - mi := &file_ateapi_proto_msgTypes[100] + mi := &file_ateapi_proto_msgTypes[102] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7396,7 +7557,7 @@ func (x *ActorAssignment) String() string { func (*ActorAssignment) ProtoMessage() {} func (x *ActorAssignment) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[100] + mi := &file_ateapi_proto_msgTypes[102] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7409,7 +7570,7 @@ func (x *ActorAssignment) ProtoReflect() protoreflect.Message { // Deprecated: Use ActorAssignment.ProtoReflect.Descriptor instead. func (*ActorAssignment) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{100} + return file_ateapi_proto_rawDescGZIP(), []int{102} } func (x *ActorAssignment) GetMetadata() *ResourceMetadata { @@ -7461,18 +7622,26 @@ type RegisterWorkerRequest struct { // // +k8s:required Capacity *WorkerResources `protobuf:"bytes,2,opt,name=capacity,proto3" json:"capacity,omitempty"` - // Hardware identity reported by the Worker. Like capacity, this replaces - // what is recorded on the Worker. + // Runtime that new and cold-booted actors start under. + // Snapshots matching default_runtime can also warm-boot on this worker. + // Like capacity, this replaces what is recorded on the Worker. // // +k8s:required - Hardware *HardwareIdentity `protobuf:"bytes,3,opt,name=hardware,proto3" json:"hardware,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + DefaultRuntime *SandboxRuntime `protobuf:"bytes,3,opt,name=default_runtime,json=defaultRuntime,proto3" json:"default_runtime,omitempty"` + // Additional runtimes (excluding default_runtime) that this worker can + // restore snapshots from right now. + // + // +k8s:optional + // +k8s:maxItems=32 + // +k8s:listType=atomic + RestorableRuntimes []*SandboxRuntime `protobuf:"bytes,4,rep,name=restorable_runtimes,json=restorableRuntimes,proto3" json:"restorable_runtimes,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *RegisterWorkerRequest) Reset() { *x = RegisterWorkerRequest{} - mi := &file_ateapi_proto_msgTypes[101] + mi := &file_ateapi_proto_msgTypes[103] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7484,7 +7653,7 @@ func (x *RegisterWorkerRequest) String() string { func (*RegisterWorkerRequest) ProtoMessage() {} func (x *RegisterWorkerRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[101] + mi := &file_ateapi_proto_msgTypes[103] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7497,7 +7666,7 @@ func (x *RegisterWorkerRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use RegisterWorkerRequest.ProtoReflect.Descriptor instead. func (*RegisterWorkerRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{101} + return file_ateapi_proto_rawDescGZIP(), []int{103} } func (x *RegisterWorkerRequest) GetWorker() *ObjectRef { @@ -7514,9 +7683,16 @@ func (x *RegisterWorkerRequest) GetCapacity() *WorkerResources { return nil } -func (x *RegisterWorkerRequest) GetHardware() *HardwareIdentity { +func (x *RegisterWorkerRequest) GetDefaultRuntime() *SandboxRuntime { if x != nil { - return x.Hardware + return x.DefaultRuntime + } + return nil +} + +func (x *RegisterWorkerRequest) GetRestorableRuntimes() []*SandboxRuntime { + if x != nil { + return x.RestorableRuntimes } return nil } @@ -7531,7 +7707,7 @@ type RegisterWorkerResponse struct { func (x *RegisterWorkerResponse) Reset() { *x = RegisterWorkerResponse{} - mi := &file_ateapi_proto_msgTypes[102] + mi := &file_ateapi_proto_msgTypes[104] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7543,7 +7719,7 @@ func (x *RegisterWorkerResponse) String() string { func (*RegisterWorkerResponse) ProtoMessage() {} func (x *RegisterWorkerResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[102] + mi := &file_ateapi_proto_msgTypes[104] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7556,7 +7732,7 @@ func (x *RegisterWorkerResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RegisterWorkerResponse.ProtoReflect.Descriptor instead. func (*RegisterWorkerResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{102} + return file_ateapi_proto_rawDescGZIP(), []int{104} } func (x *RegisterWorkerResponse) GetWorker() *Worker { @@ -7595,7 +7771,7 @@ type MintAteomActorCertificateRequest struct { func (x *MintAteomActorCertificateRequest) Reset() { *x = MintAteomActorCertificateRequest{} - mi := &file_ateapi_proto_msgTypes[103] + mi := &file_ateapi_proto_msgTypes[105] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7607,7 +7783,7 @@ func (x *MintAteomActorCertificateRequest) String() string { func (*MintAteomActorCertificateRequest) ProtoMessage() {} func (x *MintAteomActorCertificateRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[103] + mi := &file_ateapi_proto_msgTypes[105] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7620,7 +7796,7 @@ func (x *MintAteomActorCertificateRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use MintAteomActorCertificateRequest.ProtoReflect.Descriptor instead. func (*MintAteomActorCertificateRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{103} + return file_ateapi_proto_rawDescGZIP(), []int{105} } func (x *MintAteomActorCertificateRequest) GetActor() *ObjectRef { @@ -7657,7 +7833,7 @@ type MintAteomActorCertificateResponse struct { func (x *MintAteomActorCertificateResponse) Reset() { *x = MintAteomActorCertificateResponse{} - mi := &file_ateapi_proto_msgTypes[104] + mi := &file_ateapi_proto_msgTypes[106] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7669,7 +7845,7 @@ func (x *MintAteomActorCertificateResponse) String() string { func (*MintAteomActorCertificateResponse) ProtoMessage() {} func (x *MintAteomActorCertificateResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[104] + mi := &file_ateapi_proto_msgTypes[106] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7682,7 +7858,7 @@ func (x *MintAteomActorCertificateResponse) ProtoReflect() protoreflect.Message // Deprecated: Use MintAteomActorCertificateResponse.ProtoReflect.Descriptor instead. func (*MintAteomActorCertificateResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{104} + return file_ateapi_proto_rawDescGZIP(), []int{106} } func (x *MintAteomActorCertificateResponse) GetActorCertificates() [][]byte { @@ -7720,7 +7896,7 @@ type RequestActorSuspendRequest struct { func (x *RequestActorSuspendRequest) Reset() { *x = RequestActorSuspendRequest{} - mi := &file_ateapi_proto_msgTypes[105] + mi := &file_ateapi_proto_msgTypes[107] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7732,7 +7908,7 @@ func (x *RequestActorSuspendRequest) String() string { func (*RequestActorSuspendRequest) ProtoMessage() {} func (x *RequestActorSuspendRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[105] + mi := &file_ateapi_proto_msgTypes[107] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7745,7 +7921,7 @@ func (x *RequestActorSuspendRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use RequestActorSuspendRequest.ProtoReflect.Descriptor instead. func (*RequestActorSuspendRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{105} + return file_ateapi_proto_rawDescGZIP(), []int{107} } func (x *RequestActorSuspendRequest) GetWorker() *ObjectRef { @@ -7779,7 +7955,7 @@ type RequestActorSuspendResponse struct { func (x *RequestActorSuspendResponse) Reset() { *x = RequestActorSuspendResponse{} - mi := &file_ateapi_proto_msgTypes[106] + mi := &file_ateapi_proto_msgTypes[108] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7791,7 +7967,7 @@ func (x *RequestActorSuspendResponse) String() string { func (*RequestActorSuspendResponse) ProtoMessage() {} func (x *RequestActorSuspendResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[106] + mi := &file_ateapi_proto_msgTypes[108] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7804,7 +7980,7 @@ func (x *RequestActorSuspendResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RequestActorSuspendResponse.ProtoReflect.Descriptor instead. func (*RequestActorSuspendResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{106} + return file_ateapi_proto_rawDescGZIP(), []int{108} } func (x *RequestActorSuspendResponse) GetActor() *Actor { @@ -7839,7 +8015,7 @@ type AccessPolicy struct { func (x *AccessPolicy) Reset() { *x = AccessPolicy{} - mi := &file_ateapi_proto_msgTypes[107] + mi := &file_ateapi_proto_msgTypes[109] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7851,7 +8027,7 @@ func (x *AccessPolicy) String() string { func (*AccessPolicy) ProtoMessage() {} func (x *AccessPolicy) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[107] + mi := &file_ateapi_proto_msgTypes[109] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7864,7 +8040,7 @@ func (x *AccessPolicy) ProtoReflect() protoreflect.Message { // Deprecated: Use AccessPolicy.ProtoReflect.Descriptor instead. func (*AccessPolicy) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{107} + return file_ateapi_proto_rawDescGZIP(), []int{109} } func (x *AccessPolicy) GetMetadata() *ResourceMetadata { @@ -7906,7 +8082,7 @@ type Binding struct { func (x *Binding) Reset() { *x = Binding{} - mi := &file_ateapi_proto_msgTypes[108] + mi := &file_ateapi_proto_msgTypes[110] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7918,7 +8094,7 @@ func (x *Binding) String() string { func (*Binding) ProtoMessage() {} func (x *Binding) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[108] + mi := &file_ateapi_proto_msgTypes[110] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7931,7 +8107,7 @@ func (x *Binding) ProtoReflect() protoreflect.Message { // Deprecated: Use Binding.ProtoReflect.Descriptor instead. func (*Binding) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{108} + return file_ateapi_proto_rawDescGZIP(), []int{110} } func (x *Binding) GetRole() string { @@ -7957,7 +8133,7 @@ type GetGlobalAccessPolicyRequest struct { func (x *GetGlobalAccessPolicyRequest) Reset() { *x = GetGlobalAccessPolicyRequest{} - mi := &file_ateapi_proto_msgTypes[109] + mi := &file_ateapi_proto_msgTypes[111] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7969,7 +8145,7 @@ func (x *GetGlobalAccessPolicyRequest) String() string { func (*GetGlobalAccessPolicyRequest) ProtoMessage() {} func (x *GetGlobalAccessPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[109] + mi := &file_ateapi_proto_msgTypes[111] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7982,7 +8158,7 @@ func (x *GetGlobalAccessPolicyRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetGlobalAccessPolicyRequest.ProtoReflect.Descriptor instead. func (*GetGlobalAccessPolicyRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{109} + return file_ateapi_proto_rawDescGZIP(), []int{111} } // CreateGlobalAccessPolicyRequest creates the deployment-wide global access policy singleton. @@ -7999,7 +8175,7 @@ type CreateGlobalAccessPolicyRequest struct { func (x *CreateGlobalAccessPolicyRequest) Reset() { *x = CreateGlobalAccessPolicyRequest{} - mi := &file_ateapi_proto_msgTypes[110] + mi := &file_ateapi_proto_msgTypes[112] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8011,7 +8187,7 @@ func (x *CreateGlobalAccessPolicyRequest) String() string { func (*CreateGlobalAccessPolicyRequest) ProtoMessage() {} func (x *CreateGlobalAccessPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[110] + mi := &file_ateapi_proto_msgTypes[112] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8024,7 +8200,7 @@ func (x *CreateGlobalAccessPolicyRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use CreateGlobalAccessPolicyRequest.ProtoReflect.Descriptor instead. func (*CreateGlobalAccessPolicyRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{110} + return file_ateapi_proto_rawDescGZIP(), []int{112} } func (x *CreateGlobalAccessPolicyRequest) GetAccessPolicy() *AccessPolicy { @@ -8048,7 +8224,7 @@ type UpdateGlobalAccessPolicyRequest struct { func (x *UpdateGlobalAccessPolicyRequest) Reset() { *x = UpdateGlobalAccessPolicyRequest{} - mi := &file_ateapi_proto_msgTypes[111] + mi := &file_ateapi_proto_msgTypes[113] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8060,7 +8236,7 @@ func (x *UpdateGlobalAccessPolicyRequest) String() string { func (*UpdateGlobalAccessPolicyRequest) ProtoMessage() {} func (x *UpdateGlobalAccessPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[111] + mi := &file_ateapi_proto_msgTypes[113] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8073,7 +8249,7 @@ func (x *UpdateGlobalAccessPolicyRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use UpdateGlobalAccessPolicyRequest.ProtoReflect.Descriptor instead. func (*UpdateGlobalAccessPolicyRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{111} + return file_ateapi_proto_rawDescGZIP(), []int{113} } func (x *UpdateGlobalAccessPolicyRequest) GetAccessPolicy() *AccessPolicy { @@ -8097,7 +8273,7 @@ type GetAtespaceAccessPolicyRequest struct { func (x *GetAtespaceAccessPolicyRequest) Reset() { *x = GetAtespaceAccessPolicyRequest{} - mi := &file_ateapi_proto_msgTypes[112] + mi := &file_ateapi_proto_msgTypes[114] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8109,7 +8285,7 @@ func (x *GetAtespaceAccessPolicyRequest) String() string { func (*GetAtespaceAccessPolicyRequest) ProtoMessage() {} func (x *GetAtespaceAccessPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[112] + mi := &file_ateapi_proto_msgTypes[114] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8122,7 +8298,7 @@ func (x *GetAtespaceAccessPolicyRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetAtespaceAccessPolicyRequest.ProtoReflect.Descriptor instead. func (*GetAtespaceAccessPolicyRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{112} + return file_ateapi_proto_rawDescGZIP(), []int{114} } func (x *GetAtespaceAccessPolicyRequest) GetAtespace() *ObjectRef { @@ -8151,7 +8327,7 @@ type CreateAtespaceAccessPolicyRequest struct { func (x *CreateAtespaceAccessPolicyRequest) Reset() { *x = CreateAtespaceAccessPolicyRequest{} - mi := &file_ateapi_proto_msgTypes[113] + mi := &file_ateapi_proto_msgTypes[115] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8163,7 +8339,7 @@ func (x *CreateAtespaceAccessPolicyRequest) String() string { func (*CreateAtespaceAccessPolicyRequest) ProtoMessage() {} func (x *CreateAtespaceAccessPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[113] + mi := &file_ateapi_proto_msgTypes[115] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8176,7 +8352,7 @@ func (x *CreateAtespaceAccessPolicyRequest) ProtoReflect() protoreflect.Message // Deprecated: Use CreateAtespaceAccessPolicyRequest.ProtoReflect.Descriptor instead. func (*CreateAtespaceAccessPolicyRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{113} + return file_ateapi_proto_rawDescGZIP(), []int{115} } func (x *CreateAtespaceAccessPolicyRequest) GetAtespace() *ObjectRef { @@ -8212,7 +8388,7 @@ type UpdateAtespaceAccessPolicyRequest struct { func (x *UpdateAtespaceAccessPolicyRequest) Reset() { *x = UpdateAtespaceAccessPolicyRequest{} - mi := &file_ateapi_proto_msgTypes[114] + mi := &file_ateapi_proto_msgTypes[116] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8224,7 +8400,7 @@ func (x *UpdateAtespaceAccessPolicyRequest) String() string { func (*UpdateAtespaceAccessPolicyRequest) ProtoMessage() {} func (x *UpdateAtespaceAccessPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[114] + mi := &file_ateapi_proto_msgTypes[116] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8237,7 +8413,7 @@ func (x *UpdateAtespaceAccessPolicyRequest) ProtoReflect() protoreflect.Message // Deprecated: Use UpdateAtespaceAccessPolicyRequest.ProtoReflect.Descriptor instead. func (*UpdateAtespaceAccessPolicyRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{114} + return file_ateapi_proto_rawDescGZIP(), []int{116} } func (x *UpdateAtespaceAccessPolicyRequest) GetAtespace() *ObjectRef { @@ -8272,7 +8448,7 @@ type DeleteAtespaceAccessPolicyRequest struct { func (x *DeleteAtespaceAccessPolicyRequest) Reset() { *x = DeleteAtespaceAccessPolicyRequest{} - mi := &file_ateapi_proto_msgTypes[115] + mi := &file_ateapi_proto_msgTypes[117] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8284,7 +8460,7 @@ func (x *DeleteAtespaceAccessPolicyRequest) String() string { func (*DeleteAtespaceAccessPolicyRequest) ProtoMessage() {} func (x *DeleteAtespaceAccessPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[115] + mi := &file_ateapi_proto_msgTypes[117] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8297,7 +8473,7 @@ func (x *DeleteAtespaceAccessPolicyRequest) ProtoReflect() protoreflect.Message // Deprecated: Use DeleteAtespaceAccessPolicyRequest.ProtoReflect.Descriptor instead. func (*DeleteAtespaceAccessPolicyRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{115} + return file_ateapi_proto_rawDescGZIP(), []int{117} } func (x *DeleteAtespaceAccessPolicyRequest) GetAtespace() *ObjectRef { @@ -8682,20 +8858,25 @@ const file_ateapi_proto_rawDesc = "" + "\x06status\x18\v \x01(\v2\x14.ateapi.WorkerStatusR\x06status\x1a9\n" + "\vLabelsEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + - "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\x82\x02\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\xd6\x02\n" + "\fWorkerStatus\x12)\n" + "\x05state\x18\x01 \x01(\x0e2\x13.ateapi.WorkerStateR\x05state\x123\n" + "\bcapacity\x18\x02 \x01(\v2\x17.ateapi.WorkerResourcesR\bcapacity\x125\n" + "\tallocated\x18\x03 \x01(\v2\x17.ateapi.WorkerResourcesR\tallocated\x12%\n" + - "\x0eobserved_epoch\x18\x04 \x01(\x03R\robservedEpoch\x124\n" + - "\bhardware\x18\x05 \x01(\v2\x18.ateapi.HardwareIdentityR\bhardware\"\x9b\x01\n" + - "\x10HardwareIdentity\x12H\n" + + "\x0eobserved_epoch\x18\x04 \x01(\x03R\robservedEpoch\x12?\n" + + "\x0fdefault_runtime\x18\x05 \x01(\v2\x16.ateapi.SandboxRuntimeR\x0edefaultRuntime\x12G\n" + + "\x13restorable_runtimes\x18\x06 \x03(\v2\x16.ateapi.SandboxRuntimeR\x12restorableRuntimes\"o\n" + + "\x0eSandboxRuntime\x12#\n" + + "\rsandbox_class\x18\x01 \x01(\tR\fsandboxClass\x128\n" + + "\aversion\x18\x02 \x01(\v2\x1e.ateapi.VersionedSandboxCompatR\aversion\"w\n" + + "\x16VersionedSandboxCompat\x12%\n" + + "\x0eschema_version\x18\x01 \x01(\tR\rschemaVersion\x126\n" + "\n" + - "attributes\x18\x01 \x03(\v2(.ateapi.HardwareIdentity.AttributesEntryR\n" + - "attributes\x1a=\n" + - "\x0fAttributesEntry\x12\x10\n" + + "attributes\x18\x02 \x03(\v2\x16.ateapi.AttributeEntryR\n" + + "attributes\"8\n" + + "\x0eAttributeEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + - "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"Z\n" + + "\x05value\x18\x02 \x01(\tR\x05value\"Z\n" + "\x0fWorkerResources\x12/\n" + "\tresources\x18\x01 \x01(\v2\x11.ateapi.ResourcesR\tresources\x12\x16\n" + "\x06actors\x18\x02 \x01(\x05R\x06actors\"\xe1\x01\n" + @@ -8704,11 +8885,12 @@ const file_ateapi_proto_rawDesc = "" + "\x05actor\x18\x02 \x01(\v2\x11.ateapi.ObjectRefR\x05actor\x12\x1b\n" + "\tactor_uid\x18\x03 \x01(\tR\bactorUid\x12/\n" + "\tresources\x18\x05 \x01(\v2\x11.ateapi.ResourcesR\tresources\x12!\n" + - "\fworker_epoch\x18\a \x01(\x03R\vworkerEpoch\"\xad\x01\n" + + "\fworker_epoch\x18\a \x01(\x03R\vworkerEpoch\"\x81\x02\n" + "\x15RegisterWorkerRequest\x12)\n" + "\x06worker\x18\x01 \x01(\v2\x11.ateapi.ObjectRefR\x06worker\x123\n" + - "\bcapacity\x18\x02 \x01(\v2\x17.ateapi.WorkerResourcesR\bcapacity\x124\n" + - "\bhardware\x18\x03 \x01(\v2\x18.ateapi.HardwareIdentityR\bhardware\"@\n" + + "\bcapacity\x18\x02 \x01(\v2\x17.ateapi.WorkerResourcesR\bcapacity\x12?\n" + + "\x0fdefault_runtime\x18\x03 \x01(\v2\x16.ateapi.SandboxRuntimeR\x0edefaultRuntime\x12G\n" + + "\x13restorable_runtimes\x18\x04 \x03(\v2\x16.ateapi.SandboxRuntimeR\x12restorableRuntimes\"@\n" + "\x16RegisterWorkerResponse\x12&\n" + "\x06worker\x18\x01 \x01(\v2\x0e.ateapi.WorkerR\x06worker\"\xa8\x01\n" + " MintAteomActorCertificateRequest\x12'\n" + @@ -8854,7 +9036,7 @@ func file_ateapi_proto_rawDescGZIP() []byte { } var file_ateapi_proto_enumTypes = make([]protoimpl.EnumInfo, 10) -var file_ateapi_proto_msgTypes = make([]protoimpl.MessageInfo, 120) +var file_ateapi_proto_msgTypes = make([]protoimpl.MessageInfo, 121) var file_ateapi_proto_goTypes = []any{ (SnapshotOwner)(0), // 0: ateapi.SnapshotOwner (SnapshotDurability)(0), // 1: ateapi.SnapshotDurability @@ -8964,29 +9146,30 @@ var file_ateapi_proto_goTypes = []any{ (*ListActorsResponse)(nil), // 105: ateapi.ListActorsResponse (*Worker)(nil), // 106: ateapi.Worker (*WorkerStatus)(nil), // 107: ateapi.WorkerStatus - (*HardwareIdentity)(nil), // 108: ateapi.HardwareIdentity - (*WorkerResources)(nil), // 109: ateapi.WorkerResources - (*ActorAssignment)(nil), // 110: ateapi.ActorAssignment - (*RegisterWorkerRequest)(nil), // 111: ateapi.RegisterWorkerRequest - (*RegisterWorkerResponse)(nil), // 112: ateapi.RegisterWorkerResponse - (*MintAteomActorCertificateRequest)(nil), // 113: ateapi.MintAteomActorCertificateRequest - (*MintAteomActorCertificateResponse)(nil), // 114: ateapi.MintAteomActorCertificateResponse - (*RequestActorSuspendRequest)(nil), // 115: ateapi.RequestActorSuspendRequest - (*RequestActorSuspendResponse)(nil), // 116: ateapi.RequestActorSuspendResponse - (*AccessPolicy)(nil), // 117: ateapi.AccessPolicy - (*Binding)(nil), // 118: ateapi.Binding - (*GetGlobalAccessPolicyRequest)(nil), // 119: ateapi.GetGlobalAccessPolicyRequest - (*CreateGlobalAccessPolicyRequest)(nil), // 120: ateapi.CreateGlobalAccessPolicyRequest - (*UpdateGlobalAccessPolicyRequest)(nil), // 121: ateapi.UpdateGlobalAccessPolicyRequest - (*GetAtespaceAccessPolicyRequest)(nil), // 122: ateapi.GetAtespaceAccessPolicyRequest - (*CreateAtespaceAccessPolicyRequest)(nil), // 123: ateapi.CreateAtespaceAccessPolicyRequest - (*UpdateAtespaceAccessPolicyRequest)(nil), // 124: ateapi.UpdateAtespaceAccessPolicyRequest - (*DeleteAtespaceAccessPolicyRequest)(nil), // 125: ateapi.DeleteAtespaceAccessPolicyRequest - nil, // 126: ateapi.Selector.MatchLabelsEntry - nil, // 127: ateapi.ExternalVolume.VolumeContextEntry - nil, // 128: ateapi.Worker.LabelsEntry - nil, // 129: ateapi.HardwareIdentity.AttributesEntry - (*timestamppb.Timestamp)(nil), // 130: google.protobuf.Timestamp + (*SandboxRuntime)(nil), // 108: ateapi.SandboxRuntime + (*VersionedSandboxCompat)(nil), // 109: ateapi.VersionedSandboxCompat + (*AttributeEntry)(nil), // 110: ateapi.AttributeEntry + (*WorkerResources)(nil), // 111: ateapi.WorkerResources + (*ActorAssignment)(nil), // 112: ateapi.ActorAssignment + (*RegisterWorkerRequest)(nil), // 113: ateapi.RegisterWorkerRequest + (*RegisterWorkerResponse)(nil), // 114: ateapi.RegisterWorkerResponse + (*MintAteomActorCertificateRequest)(nil), // 115: ateapi.MintAteomActorCertificateRequest + (*MintAteomActorCertificateResponse)(nil), // 116: ateapi.MintAteomActorCertificateResponse + (*RequestActorSuspendRequest)(nil), // 117: ateapi.RequestActorSuspendRequest + (*RequestActorSuspendResponse)(nil), // 118: ateapi.RequestActorSuspendResponse + (*AccessPolicy)(nil), // 119: ateapi.AccessPolicy + (*Binding)(nil), // 120: ateapi.Binding + (*GetGlobalAccessPolicyRequest)(nil), // 121: ateapi.GetGlobalAccessPolicyRequest + (*CreateGlobalAccessPolicyRequest)(nil), // 122: ateapi.CreateGlobalAccessPolicyRequest + (*UpdateGlobalAccessPolicyRequest)(nil), // 123: ateapi.UpdateGlobalAccessPolicyRequest + (*GetAtespaceAccessPolicyRequest)(nil), // 124: ateapi.GetAtespaceAccessPolicyRequest + (*CreateAtespaceAccessPolicyRequest)(nil), // 125: ateapi.CreateAtespaceAccessPolicyRequest + (*UpdateAtespaceAccessPolicyRequest)(nil), // 126: ateapi.UpdateAtespaceAccessPolicyRequest + (*DeleteAtespaceAccessPolicyRequest)(nil), // 127: ateapi.DeleteAtespaceAccessPolicyRequest + nil, // 128: ateapi.Selector.MatchLabelsEntry + nil, // 129: ateapi.ExternalVolume.VolumeContextEntry + nil, // 130: ateapi.Worker.LabelsEntry + (*timestamppb.Timestamp)(nil), // 131: google.protobuf.Timestamp } var file_ateapi_proto_depIdxs = []int32{ 1, // 0: ateapi.SnapshotStorage.durability:type_name -> ateapi.SnapshotDurability @@ -8996,11 +9179,11 @@ var file_ateapi_proto_depIdxs = []int32{ 10, // 4: ateapi.SnapshotStorage.object:type_name -> ateapi.ObjectSnapshot 0, // 5: ateapi.Snapshot.owner:type_name -> ateapi.SnapshotOwner 12, // 6: ateapi.Snapshot.storage:type_name -> ateapi.SnapshotStorage - 126, // 7: ateapi.Selector.match_labels:type_name -> ateapi.Selector.MatchLabelsEntry - 130, // 8: ateapi.ResourceMetadata.create_time:type_name -> google.protobuf.Timestamp - 130, // 9: ateapi.ResourceMetadata.update_time:type_name -> google.protobuf.Timestamp + 128, // 7: ateapi.Selector.match_labels:type_name -> ateapi.Selector.MatchLabelsEntry + 131, // 8: ateapi.ResourceMetadata.create_time:type_name -> google.protobuf.Timestamp + 131, // 9: ateapi.ResourceMetadata.update_time:type_name -> google.protobuf.Timestamp 9, // 10: ateapi.ExternalVolume.status:type_name -> ateapi.ExternalVolume.Status - 127, // 11: ateapi.ExternalVolume.volume_context:type_name -> ateapi.ExternalVolume.VolumeContextEntry + 129, // 11: ateapi.ExternalVolume.volume_context:type_name -> ateapi.ExternalVolume.VolumeContextEntry 15, // 12: ateapi.Actor.metadata:type_name -> ateapi.ResourceMetadata 34, // 13: ateapi.Actor.actor_template:type_name -> ateapi.ObjectRef 14, // 14: ateapi.Actor.worker_selector:type_name -> ateapi.Selector @@ -9024,7 +9207,7 @@ var file_ateapi_proto_depIdxs = []int32{ 16, // 32: ateapi.ActorStatus.external_volumes:type_name -> ateapi.ExternalVolume 29, // 33: ateapi.ActorStatus.crash:type_name -> ateapi.ActorCrash 13, // 34: ateapi.ActorStatus.snapshots:type_name -> ateapi.Snapshot - 130, // 35: ateapi.ActorCrash.crash_time:type_name -> google.protobuf.Timestamp + 131, // 35: ateapi.ActorCrash.crash_time:type_name -> google.protobuf.Timestamp 34, // 36: ateapi.WorkerAssignment.worker:type_name -> ateapi.ObjectRef 13, // 37: ateapi.TagStatus.snapshot:type_name -> ateapi.Snapshot 15, // 38: ateapi.Tag.metadata:type_name -> ateapi.ResourceMetadata @@ -9042,7 +9225,7 @@ var file_ateapi_proto_depIdxs = []int32{ 39, // 50: ateapi.ActorTemplate.status:type_name -> ateapi.ActorTemplateStatus 37, // 51: ateapi.Resources.limits:type_name -> ateapi.Limits 34, // 52: ateapi.GoldenSnapshotStatus.golden_tag:type_name -> ateapi.ObjectRef - 130, // 53: ateapi.GoldenSnapshotStatus.take_golden_snapshot_at:type_name -> google.protobuf.Timestamp + 131, // 53: ateapi.GoldenSnapshotStatus.take_golden_snapshot_at:type_name -> google.protobuf.Timestamp 38, // 54: ateapi.ActorTemplateStatus.golden_snapshot_status:type_name -> ateapi.GoldenSnapshotStatus 6, // 55: ateapi.SandboxConfig.sandbox_class:type_name -> ateapi.SandboxClass 3, // 56: ateapi.SnapshotConfig.preferred_fidelity:type_name -> ateapi.SnapshotFidelity @@ -9094,7 +9277,7 @@ var file_ateapi_proto_depIdxs = []int32{ 94, // 102: ateapi.DeleteActorEgressPolicyRequest.options:type_name -> ateapi.DeleteOptions 34, // 103: ateapi.GetTagRequest.tag:type_name -> ateapi.ObjectRef 34, // 104: ateapi.MintActorJWTRequest.actor:type_name -> ateapi.ObjectRef - 130, // 105: ateapi.MintActorJWTResponse.expires_at:type_name -> google.protobuf.Timestamp + 131, // 105: ateapi.MintActorJWTResponse.expires_at:type_name -> google.protobuf.Timestamp 34, // 106: ateapi.MintActorCertificateRequest.actor:type_name -> ateapi.ObjectRef 32, // 107: ateapi.ListTagsResponse.tags:type_name -> ateapi.Tag 32, // 108: ateapi.CreateTagRequest.tag:type_name -> ateapi.Tag @@ -9102,7 +9285,7 @@ var file_ateapi_proto_depIdxs = []int32{ 34, // 110: ateapi.DeleteTagRequest.tag:type_name -> ateapi.ObjectRef 94, // 111: ateapi.DeleteTagRequest.options:type_name -> ateapi.DeleteOptions 34, // 112: ateapi.ListWorkerActorAssignmentsRequest.worker:type_name -> ateapi.ObjectRef - 110, // 113: ateapi.ListWorkerActorAssignmentsResponse.actor_assignments:type_name -> ateapi.ActorAssignment + 112, // 113: ateapi.ListWorkerActorAssignmentsResponse.actor_assignments:type_name -> ateapi.ActorAssignment 106, // 114: ateapi.ListWorkersResponse.workers:type_name -> ateapi.Worker 34, // 115: ateapi.GetWorkerRequest.worker:type_name -> ateapi.ObjectRef 106, // 116: ateapi.CreateWorkerRequest.worker:type_name -> ateapi.Worker @@ -9112,131 +9295,134 @@ var file_ateapi_proto_depIdxs = []int32{ 34, // 120: ateapi.DrainWorkerRequest.worker:type_name -> ateapi.ObjectRef 17, // 121: ateapi.ListActorsResponse.actors:type_name -> ateapi.Actor 15, // 122: ateapi.Worker.metadata:type_name -> ateapi.ResourceMetadata - 128, // 123: ateapi.Worker.labels:type_name -> ateapi.Worker.LabelsEntry + 130, // 123: ateapi.Worker.labels:type_name -> ateapi.Worker.LabelsEntry 107, // 124: ateapi.Worker.status:type_name -> ateapi.WorkerStatus 8, // 125: ateapi.WorkerStatus.state:type_name -> ateapi.WorkerState - 109, // 126: ateapi.WorkerStatus.capacity:type_name -> ateapi.WorkerResources - 109, // 127: ateapi.WorkerStatus.allocated:type_name -> ateapi.WorkerResources - 108, // 128: ateapi.WorkerStatus.hardware:type_name -> ateapi.HardwareIdentity - 129, // 129: ateapi.HardwareIdentity.attributes:type_name -> ateapi.HardwareIdentity.AttributesEntry - 36, // 130: ateapi.WorkerResources.resources:type_name -> ateapi.Resources - 15, // 131: ateapi.ActorAssignment.metadata:type_name -> ateapi.ResourceMetadata - 34, // 132: ateapi.ActorAssignment.actor:type_name -> ateapi.ObjectRef - 36, // 133: ateapi.ActorAssignment.resources:type_name -> ateapi.Resources - 34, // 134: ateapi.RegisterWorkerRequest.worker:type_name -> ateapi.ObjectRef - 109, // 135: ateapi.RegisterWorkerRequest.capacity:type_name -> ateapi.WorkerResources - 108, // 136: ateapi.RegisterWorkerRequest.hardware:type_name -> ateapi.HardwareIdentity - 106, // 137: ateapi.RegisterWorkerResponse.worker:type_name -> ateapi.Worker - 34, // 138: ateapi.MintAteomActorCertificateRequest.actor:type_name -> ateapi.ObjectRef - 34, // 139: ateapi.RequestActorSuspendRequest.worker:type_name -> ateapi.ObjectRef - 34, // 140: ateapi.RequestActorSuspendRequest.actor:type_name -> ateapi.ObjectRef - 17, // 141: ateapi.RequestActorSuspendResponse.actor:type_name -> ateapi.Actor - 15, // 142: ateapi.AccessPolicy.metadata:type_name -> ateapi.ResourceMetadata - 118, // 143: ateapi.AccessPolicy.bindings:type_name -> ateapi.Binding - 117, // 144: ateapi.CreateGlobalAccessPolicyRequest.access_policy:type_name -> ateapi.AccessPolicy - 117, // 145: ateapi.UpdateGlobalAccessPolicyRequest.access_policy:type_name -> ateapi.AccessPolicy - 34, // 146: ateapi.GetAtespaceAccessPolicyRequest.atespace:type_name -> ateapi.ObjectRef - 34, // 147: ateapi.CreateAtespaceAccessPolicyRequest.atespace:type_name -> ateapi.ObjectRef - 117, // 148: ateapi.CreateAtespaceAccessPolicyRequest.access_policy:type_name -> ateapi.AccessPolicy - 34, // 149: ateapi.UpdateAtespaceAccessPolicyRequest.atespace:type_name -> ateapi.ObjectRef - 117, // 150: ateapi.UpdateAtespaceAccessPolicyRequest.access_policy:type_name -> ateapi.AccessPolicy - 34, // 151: ateapi.DeleteAtespaceAccessPolicyRequest.atespace:type_name -> ateapi.ObjectRef - 94, // 152: ateapi.DeleteAtespaceAccessPolicyRequest.options:type_name -> ateapi.DeleteOptions - 68, // 153: ateapi.Control.GetActor:input_type -> ateapi.GetActorRequest - 69, // 154: ateapi.Control.CreateActor:input_type -> ateapi.CreateActorRequest - 70, // 155: ateapi.Control.UpdateActor:input_type -> ateapi.UpdateActorRequest - 71, // 156: ateapi.Control.SuspendActor:input_type -> ateapi.SuspendActorRequest - 73, // 157: ateapi.Control.PauseActor:input_type -> ateapi.PauseActorRequest - 75, // 158: ateapi.Control.ResumeActor:input_type -> ateapi.ResumeActorRequest - 77, // 159: ateapi.Control.RevertActor:input_type -> ateapi.RevertActorRequest - 79, // 160: ateapi.Control.DeleteActor:input_type -> ateapi.DeleteActorRequest - 80, // 161: ateapi.Control.GetActorEgressPolicy:input_type -> ateapi.GetActorEgressPolicyRequest - 81, // 162: ateapi.Control.CreateActorEgressPolicy:input_type -> ateapi.CreateActorEgressPolicyRequest - 82, // 163: ateapi.Control.UpdateActorEgressPolicy:input_type -> ateapi.UpdateActorEgressPolicyRequest - 83, // 164: ateapi.Control.DeleteActorEgressPolicy:input_type -> ateapi.DeleteActorEgressPolicyRequest - 85, // 165: ateapi.Control.MintActorJWT:input_type -> ateapi.MintActorJWTRequest - 87, // 166: ateapi.Control.MintActorCertificate:input_type -> ateapi.MintActorCertificateRequest - 91, // 167: ateapi.Control.CreateTag:input_type -> ateapi.CreateTagRequest - 84, // 168: ateapi.Control.GetTag:input_type -> ateapi.GetTagRequest - 89, // 169: ateapi.Control.ListTags:input_type -> ateapi.ListTagsRequest - 92, // 170: ateapi.Control.UpdateTag:input_type -> ateapi.UpdateTagRequest - 93, // 171: ateapi.Control.DeleteTag:input_type -> ateapi.DeleteTagRequest - 97, // 172: ateapi.Control.ListWorkers:input_type -> ateapi.ListWorkersRequest - 99, // 173: ateapi.Control.GetWorker:input_type -> ateapi.GetWorkerRequest - 100, // 174: ateapi.Control.CreateWorker:input_type -> ateapi.CreateWorkerRequest - 101, // 175: ateapi.Control.UpdateWorker:input_type -> ateapi.UpdateWorkerRequest - 102, // 176: ateapi.Control.DeleteWorker:input_type -> ateapi.DeleteWorkerRequest - 103, // 177: ateapi.Control.DrainWorker:input_type -> ateapi.DrainWorkerRequest - 95, // 178: ateapi.Control.ListWorkerActorAssignments:input_type -> ateapi.ListWorkerActorAssignmentsRequest - 104, // 179: ateapi.Control.ListActors:input_type -> ateapi.ListActorsRequest - 58, // 180: ateapi.Control.CreateAtespace:input_type -> ateapi.CreateAtespaceRequest - 59, // 181: ateapi.Control.GetAtespace:input_type -> ateapi.GetAtespaceRequest - 60, // 182: ateapi.Control.ListAtespaces:input_type -> ateapi.ListAtespacesRequest - 62, // 183: ateapi.Control.DeleteAtespace:input_type -> ateapi.DeleteAtespaceRequest - 63, // 184: ateapi.Control.CreateActorTemplate:input_type -> ateapi.CreateActorTemplateRequest - 64, // 185: ateapi.Control.GetActorTemplate:input_type -> ateapi.GetActorTemplateRequest - 65, // 186: ateapi.Control.ListActorTemplates:input_type -> ateapi.ListActorTemplatesRequest - 67, // 187: ateapi.Control.DeleteActorTemplate:input_type -> ateapi.DeleteActorTemplateRequest - 119, // 188: ateapi.Control.GetGlobalAccessPolicy:input_type -> ateapi.GetGlobalAccessPolicyRequest - 120, // 189: ateapi.Control.CreateGlobalAccessPolicy:input_type -> ateapi.CreateGlobalAccessPolicyRequest - 121, // 190: ateapi.Control.UpdateGlobalAccessPolicy:input_type -> ateapi.UpdateGlobalAccessPolicyRequest - 122, // 191: ateapi.Control.GetAtespaceAccessPolicy:input_type -> ateapi.GetAtespaceAccessPolicyRequest - 123, // 192: ateapi.Control.CreateAtespaceAccessPolicy:input_type -> ateapi.CreateAtespaceAccessPolicyRequest - 124, // 193: ateapi.Control.UpdateAtespaceAccessPolicy:input_type -> ateapi.UpdateAtespaceAccessPolicyRequest - 125, // 194: ateapi.Control.DeleteAtespaceAccessPolicy:input_type -> ateapi.DeleteAtespaceAccessPolicyRequest - 111, // 195: ateapi.WorkerService.RegisterWorker:input_type -> ateapi.RegisterWorkerRequest - 113, // 196: ateapi.WorkerService.MintAteomActorCertificate:input_type -> ateapi.MintAteomActorCertificateRequest - 115, // 197: ateapi.WorkerService.RequestActorSuspend:input_type -> ateapi.RequestActorSuspendRequest - 17, // 198: ateapi.Control.GetActor:output_type -> ateapi.Actor - 17, // 199: ateapi.Control.CreateActor:output_type -> ateapi.Actor - 17, // 200: ateapi.Control.UpdateActor:output_type -> ateapi.Actor - 72, // 201: ateapi.Control.SuspendActor:output_type -> ateapi.SuspendActorResponse - 74, // 202: ateapi.Control.PauseActor:output_type -> ateapi.PauseActorResponse - 76, // 203: ateapi.Control.ResumeActor:output_type -> ateapi.ResumeActorResponse - 78, // 204: ateapi.Control.RevertActor:output_type -> ateapi.RevertActorResponse - 17, // 205: ateapi.Control.DeleteActor:output_type -> ateapi.Actor - 18, // 206: ateapi.Control.GetActorEgressPolicy:output_type -> ateapi.EgressPolicy - 18, // 207: ateapi.Control.CreateActorEgressPolicy:output_type -> ateapi.EgressPolicy - 18, // 208: ateapi.Control.UpdateActorEgressPolicy:output_type -> ateapi.EgressPolicy - 18, // 209: ateapi.Control.DeleteActorEgressPolicy:output_type -> ateapi.EgressPolicy - 86, // 210: ateapi.Control.MintActorJWT:output_type -> ateapi.MintActorJWTResponse - 88, // 211: ateapi.Control.MintActorCertificate:output_type -> ateapi.MintActorCertificateResponse - 32, // 212: ateapi.Control.CreateTag:output_type -> ateapi.Tag - 32, // 213: ateapi.Control.GetTag:output_type -> ateapi.Tag - 90, // 214: ateapi.Control.ListTags:output_type -> ateapi.ListTagsResponse - 32, // 215: ateapi.Control.UpdateTag:output_type -> ateapi.Tag - 32, // 216: ateapi.Control.DeleteTag:output_type -> ateapi.Tag - 98, // 217: ateapi.Control.ListWorkers:output_type -> ateapi.ListWorkersResponse - 106, // 218: ateapi.Control.GetWorker:output_type -> ateapi.Worker - 106, // 219: ateapi.Control.CreateWorker:output_type -> ateapi.Worker - 106, // 220: ateapi.Control.UpdateWorker:output_type -> ateapi.Worker - 106, // 221: ateapi.Control.DeleteWorker:output_type -> ateapi.Worker - 106, // 222: ateapi.Control.DrainWorker:output_type -> ateapi.Worker - 96, // 223: ateapi.Control.ListWorkerActorAssignments:output_type -> ateapi.ListWorkerActorAssignmentsResponse - 105, // 224: ateapi.Control.ListActors:output_type -> ateapi.ListActorsResponse - 33, // 225: ateapi.Control.CreateAtespace:output_type -> ateapi.Atespace - 33, // 226: ateapi.Control.GetAtespace:output_type -> ateapi.Atespace - 61, // 227: ateapi.Control.ListAtespaces:output_type -> ateapi.ListAtespacesResponse - 33, // 228: ateapi.Control.DeleteAtespace:output_type -> ateapi.Atespace - 35, // 229: ateapi.Control.CreateActorTemplate:output_type -> ateapi.ActorTemplate - 35, // 230: ateapi.Control.GetActorTemplate:output_type -> ateapi.ActorTemplate - 66, // 231: ateapi.Control.ListActorTemplates:output_type -> ateapi.ListActorTemplatesResponse - 35, // 232: ateapi.Control.DeleteActorTemplate:output_type -> ateapi.ActorTemplate - 117, // 233: ateapi.Control.GetGlobalAccessPolicy:output_type -> ateapi.AccessPolicy - 117, // 234: ateapi.Control.CreateGlobalAccessPolicy:output_type -> ateapi.AccessPolicy - 117, // 235: ateapi.Control.UpdateGlobalAccessPolicy:output_type -> ateapi.AccessPolicy - 117, // 236: ateapi.Control.GetAtespaceAccessPolicy:output_type -> ateapi.AccessPolicy - 117, // 237: ateapi.Control.CreateAtespaceAccessPolicy:output_type -> ateapi.AccessPolicy - 117, // 238: ateapi.Control.UpdateAtespaceAccessPolicy:output_type -> ateapi.AccessPolicy - 117, // 239: ateapi.Control.DeleteAtespaceAccessPolicy:output_type -> ateapi.AccessPolicy - 112, // 240: ateapi.WorkerService.RegisterWorker:output_type -> ateapi.RegisterWorkerResponse - 114, // 241: ateapi.WorkerService.MintAteomActorCertificate:output_type -> ateapi.MintAteomActorCertificateResponse - 116, // 242: ateapi.WorkerService.RequestActorSuspend:output_type -> ateapi.RequestActorSuspendResponse - 198, // [198:243] is the sub-list for method output_type - 153, // [153:198] is the sub-list for method input_type - 153, // [153:153] is the sub-list for extension type_name - 153, // [153:153] is the sub-list for extension extendee - 0, // [0:153] is the sub-list for field type_name + 111, // 126: ateapi.WorkerStatus.capacity:type_name -> ateapi.WorkerResources + 111, // 127: ateapi.WorkerStatus.allocated:type_name -> ateapi.WorkerResources + 108, // 128: ateapi.WorkerStatus.default_runtime:type_name -> ateapi.SandboxRuntime + 108, // 129: ateapi.WorkerStatus.restorable_runtimes:type_name -> ateapi.SandboxRuntime + 109, // 130: ateapi.SandboxRuntime.version:type_name -> ateapi.VersionedSandboxCompat + 110, // 131: ateapi.VersionedSandboxCompat.attributes:type_name -> ateapi.AttributeEntry + 36, // 132: ateapi.WorkerResources.resources:type_name -> ateapi.Resources + 15, // 133: ateapi.ActorAssignment.metadata:type_name -> ateapi.ResourceMetadata + 34, // 134: ateapi.ActorAssignment.actor:type_name -> ateapi.ObjectRef + 36, // 135: ateapi.ActorAssignment.resources:type_name -> ateapi.Resources + 34, // 136: ateapi.RegisterWorkerRequest.worker:type_name -> ateapi.ObjectRef + 111, // 137: ateapi.RegisterWorkerRequest.capacity:type_name -> ateapi.WorkerResources + 108, // 138: ateapi.RegisterWorkerRequest.default_runtime:type_name -> ateapi.SandboxRuntime + 108, // 139: ateapi.RegisterWorkerRequest.restorable_runtimes:type_name -> ateapi.SandboxRuntime + 106, // 140: ateapi.RegisterWorkerResponse.worker:type_name -> ateapi.Worker + 34, // 141: ateapi.MintAteomActorCertificateRequest.actor:type_name -> ateapi.ObjectRef + 34, // 142: ateapi.RequestActorSuspendRequest.worker:type_name -> ateapi.ObjectRef + 34, // 143: ateapi.RequestActorSuspendRequest.actor:type_name -> ateapi.ObjectRef + 17, // 144: ateapi.RequestActorSuspendResponse.actor:type_name -> ateapi.Actor + 15, // 145: ateapi.AccessPolicy.metadata:type_name -> ateapi.ResourceMetadata + 120, // 146: ateapi.AccessPolicy.bindings:type_name -> ateapi.Binding + 119, // 147: ateapi.CreateGlobalAccessPolicyRequest.access_policy:type_name -> ateapi.AccessPolicy + 119, // 148: ateapi.UpdateGlobalAccessPolicyRequest.access_policy:type_name -> ateapi.AccessPolicy + 34, // 149: ateapi.GetAtespaceAccessPolicyRequest.atespace:type_name -> ateapi.ObjectRef + 34, // 150: ateapi.CreateAtespaceAccessPolicyRequest.atespace:type_name -> ateapi.ObjectRef + 119, // 151: ateapi.CreateAtespaceAccessPolicyRequest.access_policy:type_name -> ateapi.AccessPolicy + 34, // 152: ateapi.UpdateAtespaceAccessPolicyRequest.atespace:type_name -> ateapi.ObjectRef + 119, // 153: ateapi.UpdateAtespaceAccessPolicyRequest.access_policy:type_name -> ateapi.AccessPolicy + 34, // 154: ateapi.DeleteAtespaceAccessPolicyRequest.atespace:type_name -> ateapi.ObjectRef + 94, // 155: ateapi.DeleteAtespaceAccessPolicyRequest.options:type_name -> ateapi.DeleteOptions + 68, // 156: ateapi.Control.GetActor:input_type -> ateapi.GetActorRequest + 69, // 157: ateapi.Control.CreateActor:input_type -> ateapi.CreateActorRequest + 70, // 158: ateapi.Control.UpdateActor:input_type -> ateapi.UpdateActorRequest + 71, // 159: ateapi.Control.SuspendActor:input_type -> ateapi.SuspendActorRequest + 73, // 160: ateapi.Control.PauseActor:input_type -> ateapi.PauseActorRequest + 75, // 161: ateapi.Control.ResumeActor:input_type -> ateapi.ResumeActorRequest + 77, // 162: ateapi.Control.RevertActor:input_type -> ateapi.RevertActorRequest + 79, // 163: ateapi.Control.DeleteActor:input_type -> ateapi.DeleteActorRequest + 80, // 164: ateapi.Control.GetActorEgressPolicy:input_type -> ateapi.GetActorEgressPolicyRequest + 81, // 165: ateapi.Control.CreateActorEgressPolicy:input_type -> ateapi.CreateActorEgressPolicyRequest + 82, // 166: ateapi.Control.UpdateActorEgressPolicy:input_type -> ateapi.UpdateActorEgressPolicyRequest + 83, // 167: ateapi.Control.DeleteActorEgressPolicy:input_type -> ateapi.DeleteActorEgressPolicyRequest + 85, // 168: ateapi.Control.MintActorJWT:input_type -> ateapi.MintActorJWTRequest + 87, // 169: ateapi.Control.MintActorCertificate:input_type -> ateapi.MintActorCertificateRequest + 91, // 170: ateapi.Control.CreateTag:input_type -> ateapi.CreateTagRequest + 84, // 171: ateapi.Control.GetTag:input_type -> ateapi.GetTagRequest + 89, // 172: ateapi.Control.ListTags:input_type -> ateapi.ListTagsRequest + 92, // 173: ateapi.Control.UpdateTag:input_type -> ateapi.UpdateTagRequest + 93, // 174: ateapi.Control.DeleteTag:input_type -> ateapi.DeleteTagRequest + 97, // 175: ateapi.Control.ListWorkers:input_type -> ateapi.ListWorkersRequest + 99, // 176: ateapi.Control.GetWorker:input_type -> ateapi.GetWorkerRequest + 100, // 177: ateapi.Control.CreateWorker:input_type -> ateapi.CreateWorkerRequest + 101, // 178: ateapi.Control.UpdateWorker:input_type -> ateapi.UpdateWorkerRequest + 102, // 179: ateapi.Control.DeleteWorker:input_type -> ateapi.DeleteWorkerRequest + 103, // 180: ateapi.Control.DrainWorker:input_type -> ateapi.DrainWorkerRequest + 95, // 181: ateapi.Control.ListWorkerActorAssignments:input_type -> ateapi.ListWorkerActorAssignmentsRequest + 104, // 182: ateapi.Control.ListActors:input_type -> ateapi.ListActorsRequest + 58, // 183: ateapi.Control.CreateAtespace:input_type -> ateapi.CreateAtespaceRequest + 59, // 184: ateapi.Control.GetAtespace:input_type -> ateapi.GetAtespaceRequest + 60, // 185: ateapi.Control.ListAtespaces:input_type -> ateapi.ListAtespacesRequest + 62, // 186: ateapi.Control.DeleteAtespace:input_type -> ateapi.DeleteAtespaceRequest + 63, // 187: ateapi.Control.CreateActorTemplate:input_type -> ateapi.CreateActorTemplateRequest + 64, // 188: ateapi.Control.GetActorTemplate:input_type -> ateapi.GetActorTemplateRequest + 65, // 189: ateapi.Control.ListActorTemplates:input_type -> ateapi.ListActorTemplatesRequest + 67, // 190: ateapi.Control.DeleteActorTemplate:input_type -> ateapi.DeleteActorTemplateRequest + 121, // 191: ateapi.Control.GetGlobalAccessPolicy:input_type -> ateapi.GetGlobalAccessPolicyRequest + 122, // 192: ateapi.Control.CreateGlobalAccessPolicy:input_type -> ateapi.CreateGlobalAccessPolicyRequest + 123, // 193: ateapi.Control.UpdateGlobalAccessPolicy:input_type -> ateapi.UpdateGlobalAccessPolicyRequest + 124, // 194: ateapi.Control.GetAtespaceAccessPolicy:input_type -> ateapi.GetAtespaceAccessPolicyRequest + 125, // 195: ateapi.Control.CreateAtespaceAccessPolicy:input_type -> ateapi.CreateAtespaceAccessPolicyRequest + 126, // 196: ateapi.Control.UpdateAtespaceAccessPolicy:input_type -> ateapi.UpdateAtespaceAccessPolicyRequest + 127, // 197: ateapi.Control.DeleteAtespaceAccessPolicy:input_type -> ateapi.DeleteAtespaceAccessPolicyRequest + 113, // 198: ateapi.WorkerService.RegisterWorker:input_type -> ateapi.RegisterWorkerRequest + 115, // 199: ateapi.WorkerService.MintAteomActorCertificate:input_type -> ateapi.MintAteomActorCertificateRequest + 117, // 200: ateapi.WorkerService.RequestActorSuspend:input_type -> ateapi.RequestActorSuspendRequest + 17, // 201: ateapi.Control.GetActor:output_type -> ateapi.Actor + 17, // 202: ateapi.Control.CreateActor:output_type -> ateapi.Actor + 17, // 203: ateapi.Control.UpdateActor:output_type -> ateapi.Actor + 72, // 204: ateapi.Control.SuspendActor:output_type -> ateapi.SuspendActorResponse + 74, // 205: ateapi.Control.PauseActor:output_type -> ateapi.PauseActorResponse + 76, // 206: ateapi.Control.ResumeActor:output_type -> ateapi.ResumeActorResponse + 78, // 207: ateapi.Control.RevertActor:output_type -> ateapi.RevertActorResponse + 17, // 208: ateapi.Control.DeleteActor:output_type -> ateapi.Actor + 18, // 209: ateapi.Control.GetActorEgressPolicy:output_type -> ateapi.EgressPolicy + 18, // 210: ateapi.Control.CreateActorEgressPolicy:output_type -> ateapi.EgressPolicy + 18, // 211: ateapi.Control.UpdateActorEgressPolicy:output_type -> ateapi.EgressPolicy + 18, // 212: ateapi.Control.DeleteActorEgressPolicy:output_type -> ateapi.EgressPolicy + 86, // 213: ateapi.Control.MintActorJWT:output_type -> ateapi.MintActorJWTResponse + 88, // 214: ateapi.Control.MintActorCertificate:output_type -> ateapi.MintActorCertificateResponse + 32, // 215: ateapi.Control.CreateTag:output_type -> ateapi.Tag + 32, // 216: ateapi.Control.GetTag:output_type -> ateapi.Tag + 90, // 217: ateapi.Control.ListTags:output_type -> ateapi.ListTagsResponse + 32, // 218: ateapi.Control.UpdateTag:output_type -> ateapi.Tag + 32, // 219: ateapi.Control.DeleteTag:output_type -> ateapi.Tag + 98, // 220: ateapi.Control.ListWorkers:output_type -> ateapi.ListWorkersResponse + 106, // 221: ateapi.Control.GetWorker:output_type -> ateapi.Worker + 106, // 222: ateapi.Control.CreateWorker:output_type -> ateapi.Worker + 106, // 223: ateapi.Control.UpdateWorker:output_type -> ateapi.Worker + 106, // 224: ateapi.Control.DeleteWorker:output_type -> ateapi.Worker + 106, // 225: ateapi.Control.DrainWorker:output_type -> ateapi.Worker + 96, // 226: ateapi.Control.ListWorkerActorAssignments:output_type -> ateapi.ListWorkerActorAssignmentsResponse + 105, // 227: ateapi.Control.ListActors:output_type -> ateapi.ListActorsResponse + 33, // 228: ateapi.Control.CreateAtespace:output_type -> ateapi.Atespace + 33, // 229: ateapi.Control.GetAtespace:output_type -> ateapi.Atespace + 61, // 230: ateapi.Control.ListAtespaces:output_type -> ateapi.ListAtespacesResponse + 33, // 231: ateapi.Control.DeleteAtespace:output_type -> ateapi.Atespace + 35, // 232: ateapi.Control.CreateActorTemplate:output_type -> ateapi.ActorTemplate + 35, // 233: ateapi.Control.GetActorTemplate:output_type -> ateapi.ActorTemplate + 66, // 234: ateapi.Control.ListActorTemplates:output_type -> ateapi.ListActorTemplatesResponse + 35, // 235: ateapi.Control.DeleteActorTemplate:output_type -> ateapi.ActorTemplate + 119, // 236: ateapi.Control.GetGlobalAccessPolicy:output_type -> ateapi.AccessPolicy + 119, // 237: ateapi.Control.CreateGlobalAccessPolicy:output_type -> ateapi.AccessPolicy + 119, // 238: ateapi.Control.UpdateGlobalAccessPolicy:output_type -> ateapi.AccessPolicy + 119, // 239: ateapi.Control.GetAtespaceAccessPolicy:output_type -> ateapi.AccessPolicy + 119, // 240: ateapi.Control.CreateAtespaceAccessPolicy:output_type -> ateapi.AccessPolicy + 119, // 241: ateapi.Control.UpdateAtespaceAccessPolicy:output_type -> ateapi.AccessPolicy + 119, // 242: ateapi.Control.DeleteAtespaceAccessPolicy:output_type -> ateapi.AccessPolicy + 114, // 243: ateapi.WorkerService.RegisterWorker:output_type -> ateapi.RegisterWorkerResponse + 116, // 244: ateapi.WorkerService.MintAteomActorCertificate:output_type -> ateapi.MintAteomActorCertificateResponse + 118, // 245: ateapi.WorkerService.RequestActorSuspend:output_type -> ateapi.RequestActorSuspendResponse + 201, // [201:246] is the sub-list for method output_type + 156, // [156:201] is the sub-list for method input_type + 156, // [156:156] is the sub-list for extension type_name + 156, // [156:156] is the sub-list for extension extendee + 0, // [0:156] is the sub-list for field type_name } func init() { file_ateapi_proto_init() } @@ -9250,7 +9436,7 @@ func file_ateapi_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_ateapi_proto_rawDesc), len(file_ateapi_proto_rawDesc)), NumEnums: 10, - NumMessages: 120, + NumMessages: 121, NumExtensions: 0, NumServices: 2, }, diff --git a/pkg/proto/ateapipb/ateapi.proto b/pkg/proto/ateapipb/ateapi.proto index 4541749380..138cb6967c 100644 --- a/pkg/proto/ateapipb/ateapi.proto +++ b/pkg/proto/ateapipb/ateapi.proto @@ -2223,23 +2223,83 @@ message WorkerStatus { // +k8s:minimum=0 int64 observed_epoch = 4; - // Hardware identity reported by the Worker at registration. + // Runtime that new and cold-booted actors start under, as reported by the + // Worker at registration. Snapshots matching default_runtime can also + // warm-boot on this worker. // // +k8s:optional - HardwareIdentity hardware = 5; -} + SandboxRuntime default_runtime = 5; -// HardwareIdentity identifies the hardware of a Worker or the Worker a -// snapshot was captured on. -message HardwareIdentity { - // Key-value pairs describing hardware properties (currently - // "architecture"; TODO: add "cpu_vendor", "cpu_model", etc.). + // Additional runtimes (excluding default_runtime) that this worker can + // restore snapshots from right now. // // +k8s:optional - // +k8s:maxProperties=32 - // +k8s:eachKey=+k8s:maxLength=128 - // +k8s:eachVal=+k8s:maxLength=256 - map attributes = 1; + // +k8s:maxItems=32 + // +k8s:listType=atomic + repeated SandboxRuntime restorable_runtimes = 6; +} + +// SandboxRuntime describes a sandbox runtime and its compatibility identity as +// reported by a Worker or recorded on a snapshot. +message SandboxRuntime { + // e.g. "gvisor", "microvm" + // + // +k8s:required + // +k8s:maxLength=63 + string sandbox_class = 1; + + // Primary compatibility version stamped onto an actor's snapshot when a + // checkpoint is taken on this runtime. Snapshots matching `version` + // can warm-boot on this runtime. + // TODO: Add support for additional compatibility versions. + // + // +k8s:required + VersionedSandboxCompat version = 2; + + // TODO: Add an informational SandboxConfig version name (e.g. "gvisor-2") + // once workers report one. +} + +// VersionedSandboxCompat holds the compatibility schema version and the +// key-value compatibility attributes for a SandboxRuntime. +message VersionedSandboxCompat { + // Compatibility schema version (e.g. "v1") defining which runtime and + // hardware attributes are recorded and how they are computed. + // + // +k8s:required + // +k8s:maxLength=64 + string schema_version = 1; + + // Key-value compatibility attributes defined by this schema version, covering + // both runtime assets (e.g. "gvisor_asset_hash") and effective guest hardware + // features after masking (e.g. "architecture", "cpu_features"). + // Matched against a snapshot under the same schema_version: every attribute + // on the snapshot must be present on the worker with an equal value; keys + // only the worker has are ignored. Bump schema_version when a key's meaning + // changes or older snapshots must no longer match. + // Order as sent does not matter: ate-api-server sorts attributes by key + // before recording them. + // + // +k8s:required + // +k8s:maxItems=32 + // +k8s:listType=map + // +k8s:listMapKey=key + repeated AttributeEntry attributes = 2; +} + +// AttributeEntry is a single key-value compatibility attribute. +message AttributeEntry { + // Attribute name (e.g. "architecture", "cpu_features", "gvisor_asset_hash"). + // + // +k8s:required + // +k8s:maxLength=128 + string key = 1; + + // Attribute value corresponding to key. + // + // +k8s:required + // +k8s:maxLength=256 + string value = 2; } // WorkerResources represents a set of resources. @@ -2300,15 +2360,15 @@ message ActorAssignment { // authorization: Control is the client-facing API, while these RPCs are served // only to an atelet, and only for the Workers on its own node. service WorkerService { - // RegisterWorker records what a Worker can hold and its hardware identity in - // one write, so a Worker is never schedulable without hardware to match - // snapshots against. Capacity and hardware are the Worker's to report rather - // than the control plane's to infer: they are what the ateom can actually - // supply and expose, only its node can observe them, and a fleet may run - // mixed ateom versions. + // RegisterWorker records what a Worker can hold and its supported sandbox + // runtimes in one write, so a Worker is never schedulable without runtime + // compatibility information to match snapshots against. Capacity and + // runtimes are the Worker's to report rather than the control plane's to + // infer: they are what the worker can actually supply and expose, only its + // node can observe them, and a fleet may run mixed versions. // // atelet calls this with its own client certificate, as it does for - // MintCert. Idempotent: re-sending the same capacity and hardware is not a + // MintCert. Idempotent: re-sending the same capacity and runtimes is not a // write. rpc RegisterWorker(RegisterWorkerRequest) returns (RegisterWorkerResponse); @@ -2355,11 +2415,20 @@ message RegisterWorkerRequest { // +k8s:required WorkerResources capacity = 2; - // Hardware identity reported by the Worker. Like capacity, this replaces - // what is recorded on the Worker. + // Runtime that new and cold-booted actors start under. + // Snapshots matching default_runtime can also warm-boot on this worker. + // Like capacity, this replaces what is recorded on the Worker. // // +k8s:required - HardwareIdentity hardware = 3; + SandboxRuntime default_runtime = 3; + + // Additional runtimes (excluding default_runtime) that this worker can + // restore snapshots from right now. + // + // +k8s:optional + // +k8s:maxItems=32 + // +k8s:listType=atomic + repeated SandboxRuntime restorable_runtimes = 4; } message RegisterWorkerResponse { diff --git a/pkg/proto/ateapipb/ateapi_grpc.pb.go b/pkg/proto/ateapipb/ateapi_grpc.pb.go index 549fdb2550..682ebb0e46 100644 --- a/pkg/proto/ateapipb/ateapi_grpc.pb.go +++ b/pkg/proto/ateapipb/ateapi_grpc.pb.go @@ -1839,15 +1839,15 @@ const ( // authorization: Control is the client-facing API, while these RPCs are served // only to an atelet, and only for the Workers on its own node. type WorkerServiceClient interface { - // RegisterWorker records what a Worker can hold and its hardware identity in - // one write, so a Worker is never schedulable without hardware to match - // snapshots against. Capacity and hardware are the Worker's to report rather - // than the control plane's to infer: they are what the ateom can actually - // supply and expose, only its node can observe them, and a fleet may run - // mixed ateom versions. + // RegisterWorker records what a Worker can hold and its supported sandbox + // runtimes in one write, so a Worker is never schedulable without runtime + // compatibility information to match snapshots against. Capacity and + // runtimes are the Worker's to report rather than the control plane's to + // infer: they are what the worker can actually supply and expose, only its + // node can observe them, and a fleet may run mixed versions. // // atelet calls this with its own client certificate, as it does for - // MintCert. Idempotent: re-sending the same capacity and hardware is not a + // MintCert. Idempotent: re-sending the same capacity and runtimes is not a // write. RegisterWorker(ctx context.Context, in *RegisterWorkerRequest, opts ...grpc.CallOption) (*RegisterWorkerResponse, error) // Create a Substrate-issued SPIFFE certificate that asserts an ateom acting @@ -1924,15 +1924,15 @@ func (c *workerServiceClient) RequestActorSuspend(ctx context.Context, in *Reque // authorization: Control is the client-facing API, while these RPCs are served // only to an atelet, and only for the Workers on its own node. type WorkerServiceServer interface { - // RegisterWorker records what a Worker can hold and its hardware identity in - // one write, so a Worker is never schedulable without hardware to match - // snapshots against. Capacity and hardware are the Worker's to report rather - // than the control plane's to infer: they are what the ateom can actually - // supply and expose, only its node can observe them, and a fleet may run - // mixed ateom versions. + // RegisterWorker records what a Worker can hold and its supported sandbox + // runtimes in one write, so a Worker is never schedulable without runtime + // compatibility information to match snapshots against. Capacity and + // runtimes are the Worker's to report rather than the control plane's to + // infer: they are what the worker can actually supply and expose, only its + // node can observe them, and a fleet may run mixed versions. // // atelet calls this with its own client certificate, as it does for - // MintCert. Idempotent: re-sending the same capacity and hardware is not a + // MintCert. Idempotent: re-sending the same capacity and runtimes is not a // write. RegisterWorker(context.Context, *RegisterWorkerRequest) (*RegisterWorkerResponse, error) // Create a Substrate-issued SPIFFE certificate that asserts an ateom acting From 59072f7b68d3bb0a241459aa567275e016d19a4d Mon Sep 17 00:00:00 2001 From: Taahir Ahmed Date: Fri, 9 Oct 2026 23:31:31 +0000 Subject: [PATCH 42/44] Update COLLABORATING.md to reflect our stronger processes (#2437) We now have technical controls that obsolete some of the content in COLLABORATING.md: * We use branch protection rules to prevent creation of arbitrary branches (even by maintainers and owners). * We now require GitHub reviews and use the GitHub merge queue functionality, so we no longer need to caution not to merge your own PRs. --- COLLABORATING.md | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/COLLABORATING.md b/COLLABORATING.md index b59db85d28..634dd929ab 100644 --- a/COLLABORATING.md +++ b/COLLABORATING.md @@ -35,17 +35,6 @@ When writing, please strive for clarity and conciseness. Not everyone in the community will have the same proficiency in English, so please be mindful of that when writing. -## Always use Pull Requests - -Do not push directly to the main branch. Always use pull requests to propose -changes to the codebase. This allows for code review and discussion before -changes are merged, which helps maintain the quality of the codebase and -fosters collaboration. - -### Merging PRs - -PRs should be merged by the reviewer of a PR, not the author. - ### Using AI For the time being, the Substrate project uses the same AI contribution policy @@ -76,8 +65,16 @@ project](https://www.kubernetes.dev/docs/guide/pull-requests/#ai-guidance): ## Code reviews We want to move quickly, but we need to balance that with quality and -maintainability. Code reviews are an important part of that process. All PRs -should be reviewed BEFORE they are merged. +maintainability. Code reviews are an important part of that process. + +Agent Substrate requires at least one approving GitHub review from a maintainer +in order for the PR to be eligible for the merge queue. It is allowable to use +the "merge when ready" functionality to mark the PR as eligible for the merge +queue as soon as all preconditions are met. + +Note that most changes to a PR will require a new approving review. Use the +"re-request review" button to automatically place the PR back into your +reviewer's GitHub inbox. ### Code reviews across companies From fd7a9cdef65e5a6793b835fb1ca0b686e4397273 Mon Sep 17 00:00:00 2001 From: Greg Castle Date: Sat, 10 Oct 2026 00:51:19 +0000 Subject: [PATCH 43/44] atenet-router no longer exposes its plaintext xDS server on the pod IP (#2443) Binds the atenet-router's xDS server (ADS/CDS/LDS/RDS/SDS on `--port-xds`) to `127.0.0.1` instead of all interfaces. It is plaintext and unauthenticated, so any pod that could reach the router pod IP could stream the dataplane's listeners, routes, clusters and secret paths. Envoy in the same pod already dials `127.0.0.1:18000`, so nothing changes for it. Also drops the `xds` containerPort from the Deployment. Fixes #2276 - [x] Tests: `xds_listen_test.go` checks the bound address is loopback (including a real listen), and that the shipped manifest's Envoy bootstrap dials loopback on `--port-xds` with no container declaring that port. Both tests fail against the old code. - [ ] Docs: n/a --- cmd/atenet/internal/router/dataplane.go | 17 ++- cmd/atenet/internal/router/xds_listen_test.go | 110 ++++++++++++++++++ manifests/ate-install/atenet-router.yaml | 4 +- 3 files changed, 126 insertions(+), 5 deletions(-) create mode 100644 cmd/atenet/internal/router/xds_listen_test.go diff --git a/cmd/atenet/internal/router/dataplane.go b/cmd/atenet/internal/router/dataplane.go index 1169b93df9..e56f6417a8 100644 --- a/cmd/atenet/internal/router/dataplane.go +++ b/cmd/atenet/internal/router/dataplane.go @@ -19,6 +19,7 @@ import ( "fmt" "log/slog" "net" + "strconv" "time" "golang.org/x/sync/errgroup" @@ -87,10 +88,11 @@ func (s *RouterServer) startEnvoyDataplane(ctx context.Context, g *errgroup.Grou // Envoy receives all routing configuration from the local xDS server. g.Go(func() error { - slog.InfoContext(ctx, "Starting Envoy xDS Server", slog.Int("port", s.cfg.XdsPort)) - lis, err := net.Listen("tcp", fmt.Sprintf(":%d", s.cfg.XdsPort)) + addr := xdsListenAddress(s.cfg.XdsPort) + slog.InfoContext(ctx, "Starting Envoy xDS Server", slog.String("address", addr)) + lis, err := net.Listen("tcp", addr) if err != nil { - return fmt.Errorf("failed to listen on port %d: %w", s.cfg.XdsPort, err) + return fmt.Errorf("failed to listen on %s: %w", addr, err) } defer lis.Close() @@ -98,3 +100,12 @@ func (s *RouterServer) startEnvoyDataplane(ctx context.Context, g *errgroup.Grou }) return nil } + +// xdsListenAddress is where the router serves xDS to the Envoy in its own pod. +// The xDS server is plaintext and unauthenticated, and its ADS and SDS streams +// describe every listener, route, cluster and secret path in the dataplane, so +// it binds loopback only: Envoy dials it at 127.0.0.1 from the same network +// namespace, and nothing else in the cluster has any business reaching it. +func xdsListenAddress(port int) string { + return net.JoinHostPort("127.0.0.1", strconv.Itoa(port)) +} diff --git a/cmd/atenet/internal/router/xds_listen_test.go b/cmd/atenet/internal/router/xds_listen_test.go new file mode 100644 index 0000000000..0248374e59 --- /dev/null +++ b/cmd/atenet/internal/router/xds_listen_test.go @@ -0,0 +1,110 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package router + +import ( + "net" + "strconv" + "strings" + "testing" +) + +// TestXdsListenAddressIsLoopback guards #2276: the xDS server is plaintext +// and unauthenticated, and ADS/SDS hand out the whole dataplane topology and +// secret paths, so it must never bind a pod-IP-reachable address. +func TestXdsListenAddressIsLoopback(t *testing.T) { + for _, port := range []int{0, 18000} { + addr := xdsListenAddress(port) + host, p, err := net.SplitHostPort(addr) + if err != nil { + t.Fatalf("xdsListenAddress(%d) = %q, not host:port: %v", port, addr, err) + } + if ip := net.ParseIP(host); ip == nil || !ip.IsLoopback() { + t.Errorf("xdsListenAddress(%d) binds host %q; it must be a loopback IP", port, host) + } + if p != strconv.Itoa(port) { + t.Errorf("xdsListenAddress(%d) uses port %q", port, p) + } + } + + // The kernel must agree: listen on an ephemeral port and check what was + // actually bound, so a wildcard sneaking back in fails here. + lis, err := net.Listen("tcp", xdsListenAddress(0)) + if err != nil { + t.Fatalf("listening on %s: %v", xdsListenAddress(0), err) + } + defer lis.Close() + if ip := lis.Addr().(*net.TCPAddr).IP; !ip.IsLoopback() { + t.Errorf("xDS listener bound %v; it must be loopback", lis.Addr()) + } +} + +// TestXdsManifestIsLoopbackOnly checks that the shipped atenet-router manifest +// still works with a loopback-only xDS server, and does not advertise it: the +// Envoy bootstrap must dial a loopback IP on the --port-xds port, and no +// container in the pod may declare that port. +func TestXdsManifestIsLoopbackOnly(t *testing.T) { + pod := findDeployment(t, routerManifestPath, "atenet-router").Spec.Template.Spec + + xdsPort := 0 + for _, c := range pod.Containers { + if c.Name != "atenet-router" { + continue + } + for _, a := range c.Args { + if v, ok := strings.CutPrefix(a, "--port-xds="); ok { + n, err := strconv.Atoi(v) + if err != nil { + t.Fatalf("--port-xds=%q is not a number: %v", v, err) + } + xdsPort = n + } + } + } + if xdsPort == 0 { + t.Fatal("the atenet-router container sets no --port-xds; the manifest changed shape and this test is checking nothing") + } + + b := parseEnvoyAdminBootstrap(t, envoyConfigFrom(t, routerManifestPath, "atenet-router-envoy-config")) + found := false + for _, cl := range b.StaticResources.Clusters { + if cl.Name != "xds_cluster" { + continue + } + for _, ep := range cl.LoadAssignment.Endpoints { + for _, lb := range ep.LbEndpoints { + sa := lb.Endpoint.Address.SocketAddress + found = true + if ip := net.ParseIP(sa.Address); ip == nil || !ip.IsLoopback() { + t.Errorf("Envoy xds_cluster dials %q; the xDS server only listens on loopback", sa.Address) + } + if int(sa.PortValue) != xdsPort { + t.Errorf("Envoy xds_cluster dials port %d, but the router serves xDS on --port-xds=%d", sa.PortValue, xdsPort) + } + } + } + } + if !found { + t.Fatal("the Envoy bootstrap has no xds_cluster endpoint; the manifest changed shape and this test is checking nothing") + } + + for _, c := range append(pod.InitContainers, pod.Containers...) { + for _, p := range c.Ports { + if int(p.ContainerPort) == xdsPort { + t.Errorf("container %s declares port %d (%q), the loopback-only xDS port", c.Name, p.ContainerPort, p.Name) + } + } + } +} diff --git a/manifests/ate-install/atenet-router.yaml b/manifests/ate-install/atenet-router.yaml index fb1a551610..800ebf2ef7 100644 --- a/manifests/ate-install/atenet-router.yaml +++ b/manifests/ate-install/atenet-router.yaml @@ -265,9 +265,9 @@ spec: envFrom: - configMapRef: name: ate-otel-config + # The xDS server (--port-xds) binds 127.0.0.1 and serves only the Envoy + # in this pod, so it is deliberately not declared as a container port. ports: - - name: xds - containerPort: 18000 - name: extproc containerPort: 50051 - name: status From f20063021e1779fddc0340ffd22b1ce836877580 Mon Sep 17 00:00:00 2001 From: Adhita Selvaraj Date: Sat, 10 Oct 2026 05:00:22 +0000 Subject: [PATCH 44/44] Add E2E HTTPS egress tests and reject WebSocket upgrades (#1634) Addresses #1017 by adding E2E tests for HTTPS and WebSocket egress. Added scope after the egress policy was finalized: [review comment](https://github.com/agent-substrate/substrate/pull/1634#issuecomment-5769046214). Updates the gateways to reject HTTP/1.1 WebSocket upgrades with 403. PRs merge order/stack: #1945 #1944 and then this PR --- .github/workflows/pr-workflow.yaml | 2 + cmd/ate-setup/internal/steps/overlay_test.go | 17 +- cmd/atenet/internal/router/egress/request.go | 21 + .../internal/router/egress/request_test.go | 39 ++ .../router/egresspolicymanifest_test.go | 61 +++ .../router/extproc/egress_websocket_test.go | 98 ++++ demos/egress/README.md | 13 +- demos/egress/main.go | 208 ++++++++- demos/egress/main_test.go | 274 ++++++++++- docs/dev/mitm-egress-test-runbook.md | 80 ++++ hack/setup-e2e-egress-tls-kind.sh | 84 ++++ .../e2e/fixtures/testserver/common_test.go | 96 ++++ internal/e2e/fixtures/testserver/http.go | 39 +- internal/e2e/fixtures/testserver/http_test.go | 88 ++++ internal/e2e/fixtures/testserver/tls.go | 55 +++ internal/e2e/fixtures/testserver/tls_test.go | 35 ++ internal/e2e/fixtures/testserver/websocket.go | 87 ++-- .../e2e/fixtures/testserver/websocket_test.go | 109 +++++ internal/e2e/serverpod.go | 110 ++++- internal/e2e/serverpod_test.go | 39 ++ .../e2e/suites/networking/accesslog_test.go | 143 ++++++ .../suites/networking/microvmclock_test.go | 140 ++++++ .../e2e/suites/networking/networking_test.go | 428 +++++++++++++++++- .../suites/networking/websocketegress_test.go | 95 ++++ internal/testcert/servertls.go | 81 ++++ internal/testcert/servertls_test.go | 86 ++++ internal/volume/csi/tls_test.go | 6 + manifests/ate-install/atenet-egress.yaml | 62 +-- .../agentgateway-egress/kustomization.yaml | 47 +- .../components/agentgateway/configmap.yaml | 30 +- .../agentgateway/kustomization.yaml | 4 + 31 files changed, 2566 insertions(+), 111 deletions(-) create mode 100644 cmd/atenet/internal/router/extproc/egress_websocket_test.go create mode 100644 docs/dev/mitm-egress-test-runbook.md create mode 100755 hack/setup-e2e-egress-tls-kind.sh create mode 100644 internal/e2e/fixtures/testserver/common_test.go create mode 100644 internal/e2e/fixtures/testserver/http_test.go create mode 100644 internal/e2e/fixtures/testserver/tls.go create mode 100644 internal/e2e/fixtures/testserver/tls_test.go create mode 100644 internal/e2e/fixtures/testserver/websocket_test.go create mode 100644 internal/e2e/suites/networking/accesslog_test.go create mode 100644 internal/e2e/suites/networking/microvmclock_test.go create mode 100644 internal/e2e/suites/networking/websocketegress_test.go create mode 100644 internal/testcert/servertls.go create mode 100644 internal/testcert/servertls_test.go diff --git a/.github/workflows/pr-workflow.yaml b/.github/workflows/pr-workflow.yaml index da4889330b..be08a8dd70 100644 --- a/.github/workflows/pr-workflow.yaml +++ b/.github/workflows/pr-workflow.yaml @@ -252,6 +252,8 @@ jobs: run: | hack/install-ate-kind.sh --deploy-demo-egress hack/install-ate-kind.sh --deploy-demo-egress-microvm + - name: Trust local TLS origins in egress gateway + run: hack/setup-e2e-egress-tls-kind.sh # scale the idle sandbox class's demo pools to zero so each lane has the node - name: Focus the node on gVisor run: hack/e2e-focus-sandbox-class.sh gvisor diff --git a/cmd/ate-setup/internal/steps/overlay_test.go b/cmd/ate-setup/internal/steps/overlay_test.go index c047a029c3..3cfd8610e7 100644 --- a/cmd/ate-setup/internal/steps/overlay_test.go +++ b/cmd/ate-setup/internal/steps/overlay_test.go @@ -651,9 +651,20 @@ func TestRenderAgentgatewayCredentialProvider(t *testing.T) { t.Fatal(err) } for _, protocol := range []string{"HTTP", "HTTPS"} { - policy, err := gateway.Pipe(kyaml.Lookup("binds", "[mode=internal]", "listeners", "[protocol="+protocol+"]", "routes", "0", "policies", "substrateEgress")) - if err != nil || policy == nil { - t.Fatalf("%s egress policy missing: %v", protocol, err) + routes, err := gateway.Pipe(kyaml.Lookup("binds", "[mode=internal]", "listeners", "[protocol="+protocol+"]", "routes")) + if err != nil || routes == nil { + t.Fatalf("%s routes missing: %v", protocol, err) + } + var policy *kyaml.RNode + for _, route := range routes.YNode().Content { + candidate, err := kyaml.NewRNode(route).Pipe(kyaml.Lookup("policies", "substrateEgress")) + if err == nil && candidate != nil { + policy = candidate + break + } + } + if policy == nil { + t.Fatalf("%s egress policy missing", protocol) } var got struct { Providers []map[string]any `yaml:"credentialProviders"` diff --git a/cmd/atenet/internal/router/egress/request.go b/cmd/atenet/internal/router/egress/request.go index 45e30d22aa..2c34fe2bb1 100644 --- a/cmd/atenet/internal/router/egress/request.go +++ b/cmd/atenet/internal/router/egress/request.go @@ -19,6 +19,7 @@ import ( "errors" "fmt" "log/slog" + "strings" extprocv3 "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" envoy_type "github.com/envoyproxy/go-control-plane/envoy/type/v3" @@ -28,6 +29,8 @@ import ( "github.com/agent-substrate/substrate/internal/resources" ) +const websocketDeniedBody = "WebSocket egress is not supported" + // handleRequest authorizes one request the gateway can read: cleartext HTTP, // or HTTPS the gateway terminated. It runs per request, because the // Host can change between requests on one connection. @@ -38,6 +41,11 @@ import ( // request is dialed by the name it was decided on, on the port the actor // dialed. func (h *Handler) handleRequest(ctx context.Context, md *extproc.RequestMetadata, leg string) (extproc.Result, error) { + if isWebSocketUpgrade(md) { + slog.WarnContext(ctx, "egress denied: WebSocket upgrades are not supported", slog.String("leg", leg), slog.String("host", md.Host)) + return extproc.Result{}, extproc.NewReqError(envoy_type.StatusCode_Forbidden, websocketDeniedBody) + } + ref, err := actorFromFilterState(md) if err != nil { slog.WarnContext(ctx, "egress denied: request carries no actor identity", slog.String("leg", leg), slog.Any("err", err)) @@ -121,6 +129,19 @@ func dialHost(dest egresspolicy.Destination) string { } } +func isWebSocketUpgrade(md *extproc.RequestMetadata) bool { + return md.Method == "GET" && hasHTTPToken(md.Header("connection"), "upgrade") && hasHTTPToken(md.Header("upgrade"), "websocket") +} + +func hasHTTPToken(value, want string) bool { + for _, token := range strings.Split(value, ",") { + if strings.EqualFold(strings.TrimSpace(token), want) { + return true + } + } + return false +} + // connectionDestination is the SNI a decrypted connection presented, on the // port the actor dialed, in the form the https rules are matched against. An // SNI that is not a DNS name, or none at all, matches nothing. diff --git a/cmd/atenet/internal/router/egress/request_test.go b/cmd/atenet/internal/router/egress/request_test.go index 3037d2733a..43eed50ebd 100644 --- a/cmd/atenet/internal/router/egress/request_test.go +++ b/cmd/atenet/internal/router/egress/request_test.go @@ -328,6 +328,45 @@ func TestRequestLegPicksRulesByLeg(t *testing.T) { } } +func TestRequestLegDeniesHTTP1WebSocketUpgrade(t *testing.T) { + h := policyHandler(allowAllPolicy()) + for _, leg := range []string{extproc.EgressCleartextFilterChainName, extproc.EgressTLSMITMFilterChainName} { + t.Run(leg, func(t *testing.T) { + md := innerMetadata(leg, "GET", "api.example.com", nil) + md.Headers["connection"] = "keep-alive, UpGrAdE" + md.Headers["upgrade"] = "WebSocket" + _, err := h.HandleRequestHeaders(context.Background(), md) + wantStatus(t, err, envoy_type.StatusCode_Forbidden) + if err.Error() != websocketDeniedBody { + t.Errorf("denial body = %q, want %q", err, websocketDeniedBody) + } + }) + } +} + +func TestRequestLegAllowsOrdinaryHTTPSRequest(t *testing.T) { + h := policyHandler(allowAllPolicy()) + for _, tc := range []struct { + name string + method string + connection string + upgrade string + }{ + {name: "ordinary GET", method: "GET"}, + {name: "other upgrade", method: "GET", connection: "Upgrade", upgrade: "h2c"}, + {name: "upgrade without connection token", method: "GET", connection: "keep-alive", upgrade: "websocket"}, + {name: "non-GET", method: "POST", connection: "Upgrade", upgrade: "websocket"}, + } { + t.Run(tc.name, func(t *testing.T) { + md := innerMetadata(extproc.EgressTLSMITMFilterChainName, tc.method, "api.example.com", nil) + md.Headers["connection"] = tc.connection + md.Headers["upgrade"] = tc.upgrade + res, err := h.HandleRequestHeaders(context.Background(), md) + wantDial(t, res, err, extproc.EgressDialName) + }) + } +} + // Without the identity the outer chain shares, a request cannot be attributed // to any actor and is refused. func TestRequestLegRequiresIdentity(t *testing.T) { diff --git a/cmd/atenet/internal/router/egresspolicymanifest_test.go b/cmd/atenet/internal/router/egresspolicymanifest_test.go index 84e4c9f511..ba73a0b573 100644 --- a/cmd/atenet/internal/router/egresspolicymanifest_test.go +++ b/cmd/atenet/internal/router/egresspolicymanifest_test.go @@ -192,6 +192,67 @@ func TestEgressManifestsNameEveryExtProcChain(t *testing.T) { } } +func TestEgressManifestsDisableWebSocketUpgrades(t *testing.T) { + want := map[string][]string{ + egressManifests[0]: {extproc.EgressCleartextFilterChainName, extproc.EgressTLSMITMFilterChainName}, + } + for _, path := range egressManifests { + t.Run(path, func(t *testing.T) { + tree := bootstrapTree(t, path) + chains := map[string]node{} + for _, lc := range allChains(tree) { + name := str(lc.chain, "name") + if slices.Contains(requestLegs, name) { + chains[name] = lc.chain + } + } + for _, name := range want[path] { + chain, present := chains[name] + if !present { + t.Errorf("expected request-leg chain %q is absent", name) + continue + } + var disabled bool + for _, upgrade := range list(hcm(chain), "upgrade_configs") { + if str(upgrade, "upgrade_type") == "websocket" { + enabled, ok := upgrade["enabled"].(bool) + disabled = ok && !enabled + break + } + } + if !disabled { + t.Errorf("chain %q must explicitly disable websocket upgrades", name) + } + var routes []node + for _, virtualHost := range list(child(hcm(chain), "route_config"), "virtual_hosts") { + routes = append(routes, list(virtualHost, "routes")...) + } + if len(routes) == 0 { + t.Errorf("chain %q has no routes", name) + } + for i, route := range routes { + action := child(route, "route") + if action == nil { + t.Errorf("chain %q route %d has no route action", name, i) + continue + } + disabled := false + for _, upgrade := range list(action, "upgrade_configs") { + if str(upgrade, "upgrade_type") == "websocket" { + enabled, ok := upgrade["enabled"].(bool) + disabled = ok && !enabled + break + } + } + if !disabled { + t.Errorf("chain %q route %d must explicitly disable websocket upgrades", name, i) + } + } + } + }) + } +} + // extProcOf returns the ext_proc filter config of a chain's HCM and its index // among the http_filters, or nil and -1. func extProcOf(chain node) (node, int, []node) { diff --git a/cmd/atenet/internal/router/extproc/egress_websocket_test.go b/cmd/atenet/internal/router/extproc/egress_websocket_test.go new file mode 100644 index 0000000000..f82116830b --- /dev/null +++ b/cmd/atenet/internal/router/extproc/egress_websocket_test.go @@ -0,0 +1,98 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extproc_test + +import ( + "context" + "io" + "testing" + + corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + extprocv3 "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" + "google.golang.org/protobuf/types/known/structpb" + + "github.com/agent-substrate/substrate/cmd/atenet/internal/router/egress" + "github.com/agent-substrate/substrate/cmd/atenet/internal/router/extproc" +) + +const websocketDenialBody = "WebSocket egress is not supported" + +type websocketProcessStream struct { + extprocv3.ExternalProcessor_ProcessServer + ctx context.Context + requests []*extprocv3.ProcessingRequest + responses []*extprocv3.ProcessingResponse +} + +func (s *websocketProcessStream) Context() context.Context { return s.ctx } + +func (s *websocketProcessStream) Recv() (*extprocv3.ProcessingRequest, error) { + if len(s.requests) == 0 { + return nil, io.EOF + } + req := s.requests[0] + s.requests = s.requests[1:] + return req, nil +} + +func (s *websocketProcessStream) Send(resp *extprocv3.ProcessingResponse) error { + s.responses = append(s.responses, resp) + return nil +} + +func TestProcessReturnsWebSocketDenialAsImmediateResponse(t *testing.T) { + for _, leg := range []string{extproc.EgressCleartextFilterChainName, extproc.EgressTLSMITMFilterChainName} { + t.Run(leg, func(t *testing.T) { + request := &extprocv3.ProcessingRequest{ + Request: &extprocv3.ProcessingRequest_RequestHeaders{ + RequestHeaders: &extprocv3.HttpHeaders{ + Headers: &corev3.HeaderMap{Headers: []*corev3.HeaderValue{ + {Key: ":method", RawValue: []byte("GET")}, + {Key: ":authority", RawValue: []byte("api.example.com")}, + {Key: "connection", RawValue: []byte("keep-alive, Upgrade")}, + {Key: "upgrade", RawValue: []byte("websocket")}, + }}, + }, + }, + Attributes: map[string]*structpb.Struct{ + "envoy.filters.http.ext_proc": {Fields: map[string]*structpb.Value{ + extproc.FilterChainNameAttribute: structpb.NewStringValue(leg), + }}, + }, + } + stream := &websocketProcessStream{ctx: context.Background(), requests: []*extprocv3.ProcessingRequest{request}} + server := extproc.NewServer(0, nil, extproc.Handlers{ + extproc.DirectionEgress: egress.New(nil, nil, 0, nil, ""), + }) + + if err := server.Process(stream); err != nil { + t.Fatalf("Process() error = %v", err) + } + if len(stream.responses) != 1 { + t.Fatalf("responses = %d, want 1", len(stream.responses)) + } + response := stream.responses[0].GetImmediateResponse() + if response == nil { + t.Fatalf("response = %v, want immediate denial", stream.responses[0]) + } + if got := response.GetStatus().GetCode(); got != 403 { + t.Errorf("status = %d, want 403", got) + } + if got := string(response.GetBody()); got != websocketDenialBody { + t.Errorf("body = %q, want %q", got, websocketDenialBody) + } + }) + } +} diff --git a/demos/egress/README.md b/demos/egress/README.md index c2e0d13a87..229f54bead 100644 --- a/demos/egress/README.md +++ b/demos/egress/README.md @@ -77,8 +77,9 @@ rejects that option with agentgateway rather than silently omitting it. ## Components -- **Egress app (`main.go`)** — the Actor: `POST /` with `{"url":"..."}` → fetches it → returns - status + body. It also serves `POST /grpc`, described below. +- **Egress app (`main.go`)** — the Actor: `POST /` fetches an HTTP(S) URL and returns the status, + body, and observed protocol. `POST /websocket` probes a `ws` or `wss` origin; `POST /grpc` makes + unary and streaming gRPC calls, described below. - **Egress gateway** — the `atenet-egress` Deployment. Envoy uses a co-located atenet `ext_proc` container started with `--mode=egress`; agentgateway uses its built-in `substrateEgress` policy and does not need that sidecar. The installer renders the matching configuration and container. @@ -132,6 +133,14 @@ then asserts: Add `--cleanup` to remove everything the script created. +### HTTPS and WebSocket E2E tests + +Both supported egress dataplanes intercept HTTPS policy traffic. The Actor +trusts the gateway through its projected CA bundle; the gateway verifies each +local test origin against the service-DNS CA. The [egress protocol test +runbook](../../docs/dev/mitm-egress-test-runbook.md) sets up this trust on Kind +and runs the HTTP, HTTPS, WebSocket, and delayed-restore tests. + ## Manual walkthrough ```bash diff --git a/demos/egress/main.go b/demos/egress/main.go index 66607e054c..dcbc72daa1 100644 --- a/demos/egress/main.go +++ b/demos/egress/main.go @@ -19,6 +19,8 @@ package main import ( "context" + "crypto/tls" + "crypto/x509" "encoding/json" "errors" "fmt" @@ -31,6 +33,7 @@ import ( "strconv" "time" + "github.com/gorilla/websocket" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials/insecure" @@ -40,23 +43,50 @@ import ( ) const ( - listenAddress = ":80" - maxRequestBody = 64 << 10 - maxResponseBody = 1 << 20 - requestTimeout = 15 * time.Second + listenAddress = ":80" + maxRequestBody = 64 << 10 + maxResponseBody = 1 << 20 + requestTimeout = 15 * time.Second + maxWebSocketMessages = 16 + maxWebSocketMessageSize = 64 << 10 + maxWebSocketHandshakeBody = 4 << 10 ) type fetchRequest struct { - URL string `json:"url"` + URL string `json:"url"` + RootCA string `json:"rootCA,omitempty"` + HTTP1 bool `json:"http1,omitempty"` } type fetchResponse struct { StatusCode int `json:"statusCode,omitempty"` Body string `json:"body,omitempty"` ServerCert string `json:"serverCert,omitempty"` + Protocol string `json:"protocol,omitempty"` + TLS bool `json:"tls"` Error string `json:"error,omitempty"` } +type websocketRequest struct { + URL string `json:"url"` + RootCA string `json:"rootCA,omitempty"` + Messages []string `json:"messages"` +} + +type websocketMessage struct { + Type int `json:"type"` + Message string `json:"message"` +} + +type websocketResponse struct { + StatusCode int `json:"statusCode,omitempty"` + Protocol string `json:"protocol,omitempty"` + HandshakeBody string `json:"handshakeBody,omitempty"` + TLS bool `json:"tls"` + Messages []websocketMessage `json:"messages"` + Error string `json:"error,omitempty"` +} + // grpcRequest asks for one unary Echo against target, and additionally for a // server-stream or a bidirectional stream when the matching count is positive. type grpcRequest struct { @@ -136,7 +166,13 @@ func newHandler(client *http.Client) http.Handler { if traceparent := r.Header.Get("traceparent"); traceparent != "" { outbound.Header.Set("traceparent", traceparent) } - response, err := client.Do(outbound) + requestClient, cleanup, err := fetchClient(client, input) + if err != nil { + writeJSON(w, http.StatusBadRequest, fetchResponse{Error: fmt.Sprintf("configuring request: %v", err)}) + return + } + defer cleanup() + response, err := requestClient.Do(outbound) if err != nil { writeJSON(w, http.StatusBadGateway, fetchResponse{Error: fmt.Sprintf("request failed: %v", err)}) return @@ -149,15 +185,173 @@ func newHandler(client *http.Client) http.Handler { return } var serverCert string + verifiedTLS := response.TLS != nil && len(response.TLS.VerifiedChains) > 0 if response.TLS != nil && len(response.TLS.PeerCertificates) > 0 { serverCert = response.TLS.PeerCertificates[0].Issuer.String() } - writeJSON(w, response.StatusCode, fetchResponse{StatusCode: response.StatusCode, Body: string(body), ServerCert: serverCert}) + writeJSON(w, response.StatusCode, fetchResponse{ + StatusCode: response.StatusCode, + Body: string(body), + ServerCert: serverCert, + Protocol: response.Proto, + TLS: verifiedTLS, + }) }) + mux.HandleFunc("/websocket", handleWebSocket) mux.HandleFunc("/grpc", handleGRPC) return mux } +// fetchClient constructs a per-request client only when the request asks for +// request-scoped TLS or protocol behavior. The default path retains the +// caller's client, including its transport and redirect policy. +func fetchClient(base *http.Client, input fetchRequest) (*http.Client, func(), error) { + if input.RootCA == "" && !input.HTTP1 { + return base, func() {}, nil + } + if base == nil { + base = http.DefaultClient + } + // Start from the standard transport so request-scoped trust cannot inherit + // an insecure TLS callback or proxy from an injected/base client. + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.Proxy = nil + transport.TLSClientConfig = &tls.Config{MinVersion: tls.VersionTLS12} + if input.RootCA != "" { + roots := x509.NewCertPool() + if !roots.AppendCertsFromPEM([]byte(input.RootCA)) { + return nil, nil, errors.New("rootCA does not contain a valid certificate") + } + transport.TLSClientConfig.RootCAs = roots + } + if input.HTTP1 { + protocols := new(http.Protocols) + protocols.SetHTTP1(true) + transport.Protocols = protocols + transport.ForceAttemptHTTP2 = false + transport.TLSNextProto = nil + } + requestClient := *base + requestClient.Transport = transport + requestClient.Timeout = requestTimeout + requestClient.CheckRedirect = func(*http.Request, []*http.Request) error { + return http.ErrUseLastResponse + } + return &requestClient, transport.CloseIdleConnections, nil +} + +func handleWebSocket(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + w.Header().Set("Allow", http.MethodPost) + writeJSON(w, http.StatusMethodNotAllowed, websocketResponse{Messages: []websocketMessage{}, Error: "method must be POST"}) + return + } + + var input websocketRequest + decoder := json.NewDecoder(http.MaxBytesReader(w, r.Body, maxRequestBody)) + if err := decoder.Decode(&input); err != nil { + writeJSON(w, http.StatusBadRequest, websocketResponse{Messages: []websocketMessage{}, Error: fmt.Sprintf("invalid JSON payload: %v", err)}) + return + } + if err := validateWebSocketRequest(input); err != nil { + writeJSON(w, http.StatusBadRequest, websocketResponse{Messages: []websocketMessage{}, Error: err.Error()}) + return + } + + ctx, cancel := context.WithTimeout(r.Context(), requestTimeout) + defer cancel() + response, err := performWebSocket(ctx, input) + if err != nil { + response.Error = err.Error() + writeJSON(w, http.StatusBadGateway, response) + return + } + writeJSON(w, http.StatusOK, response) +} + +func validateWebSocketRequest(input websocketRequest) error { + parsed, err := url.Parse(input.URL) + if err != nil { + return fmt.Errorf("invalid WebSocket URL: %w", err) + } + if parsed.Scheme != "ws" && parsed.Scheme != "wss" { + return errors.New("WebSocket URL scheme must be ws or wss") + } + if parsed.Hostname() == "" { + return errors.New("WebSocket URL must include a hostname") + } + if len(input.Messages) == 0 { + return errors.New("messages must contain at least one message") + } + if len(input.Messages) > maxWebSocketMessages { + return fmt.Errorf("messages contains %d items, maximum is %d", len(input.Messages), maxWebSocketMessages) + } + for index, message := range input.Messages { + if len(message) > maxWebSocketMessageSize { + return fmt.Errorf("message %d is %d bytes, maximum is %d", index, len(message), maxWebSocketMessageSize) + } + } + return nil +} + +func performWebSocket(ctx context.Context, input websocketRequest) (websocketResponse, error) { + dialer := websocket.Dialer{HandshakeTimeout: requestTimeout, Proxy: nil} + if input.RootCA != "" { + roots := x509.NewCertPool() + if !roots.AppendCertsFromPEM([]byte(input.RootCA)) { + return websocketResponse{Messages: []websocketMessage{}}, errors.New("configuring TLS: rootCA does not contain a valid certificate") + } + dialer.TLSClientConfig = &tls.Config{RootCAs: roots, MinVersion: tls.VersionTLS12} + } + response := websocketResponse{Messages: []websocketMessage{}} + conn, handshakeResponse, err := dialer.DialContext(ctx, input.URL, nil) + if handshakeResponse != nil { + response.StatusCode = handshakeResponse.StatusCode + response.Protocol = handshakeResponse.Proto + if handshakeResponse.Body != nil { + body, _ := io.ReadAll(io.LimitReader(handshakeResponse.Body, maxWebSocketHandshakeBody)) + _ = handshakeResponse.Body.Close() + response.HandshakeBody = string(body) + } + } + if err != nil { + return response, fmt.Errorf("WebSocket handshake failed: %w", err) + } + if tlsConn, ok := conn.UnderlyingConn().(*tls.Conn); ok { + response.TLS = tlsConn.ConnectionState().VerifiedChains != nil + } + defer conn.Close() + conn.SetReadLimit(maxWebSocketMessageSize) + + stopCancellation := make(chan struct{}) + go func() { + select { + case <-ctx.Done(): + _ = conn.Close() + case <-stopCancellation: + } + }() + defer close(stopCancellation) + + for index, message := range input.Messages { + if err := conn.SetWriteDeadline(time.Now().Add(requestTimeout)); err != nil { + return response, fmt.Errorf("setting write deadline for message %d: %w", index, err) + } + if err := conn.WriteMessage(websocket.TextMessage, []byte(message)); err != nil { + return response, fmt.Errorf("writing message %d: %w", index, err) + } + if err := conn.SetReadDeadline(time.Now().Add(requestTimeout)); err != nil { + return response, fmt.Errorf("setting read deadline for message %d: %w", index, err) + } + messageType, received, err := conn.ReadMessage() + if err != nil { + return response, fmt.Errorf("reading response to message %d: %w", index, err) + } + response.Messages = append(response.Messages, websocketMessage{Type: messageType, Message: string(received)}) + } + return response, nil +} + // handleGRPC dials the requested target and echoes back what the RPCs returned. // It exists so an e2e can assert that gRPC survives the egress path: HTTP/2 // framing end to end, and a status delivered in trailers. diff --git a/demos/egress/main_test.go b/demos/egress/main_test.go index c164d3371f..c2182c471c 100644 --- a/demos/egress/main_test.go +++ b/demos/egress/main_test.go @@ -15,7 +15,9 @@ package main import ( + "bytes" "context" + "crypto/tls" "encoding/json" "errors" "fmt" @@ -26,10 +28,12 @@ import ( "strings" "testing" + "github.com/gorilla/websocket" "google.golang.org/grpc" "google.golang.org/grpc/codes" "github.com/agent-substrate/substrate/internal/proto/grpcechopb" + "github.com/agent-substrate/substrate/internal/testcert" ) func TestFetch(t *testing.T) { @@ -82,8 +86,9 @@ func TestFetchHTTPAndHTTPS(t *testing.T) { // The demo verifies origins with the roots its client is given: in the // cluster the projected trust bundle, here the test server's own. - handler := newHandler(httpsSrv.Client()) - + httpsClient := httpsSrv.Client() + httpsClient.Timeout = requestTimeout + handler := newHandler(httpsClient) for _, tc := range []struct { name string url string @@ -157,6 +162,271 @@ func TestOutboundFailure(t *testing.T) { } } +func TestFetchWithRequestScopedTLSAndHTTP1(t *testing.T) { + server, rootCA := startIPTLSServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Proto != "HTTP/1.1" { + t.Errorf("request protocol = %s, want HTTP/1.1", r.Proto) + } + _, _ = io.WriteString(w, "trusted") + })) + + recorder := postHandler(t, newHandler(http.DefaultClient), "/", fetchRequest{ + URL: server.URL, + RootCA: rootCA, + HTTP1: true, + }) + if recorder.Code != http.StatusOK { + t.Fatalf("status = %d, want %d; body = %s", recorder.Code, http.StatusOK, recorder.Body.String()) + } + var got fetchResponse + decodeRecorder(t, recorder, &got) + if got.StatusCode != http.StatusOK || got.Body != "trusted" || got.Protocol != "HTTP/1.1" || !got.TLS { + t.Errorf("response = %+v, want successful verified HTTP/1.1 TLS response", got) + } +} + +func TestFetchTLSVerificationFailure(t *testing.T) { + server, _ := startIPTLSServer(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, "should not arrive") + })) + + recorder := postHandler(t, newHandler(http.DefaultClient), "/", fetchRequest{URL: server.URL}) + if recorder.Code != http.StatusBadGateway { + t.Fatalf("status = %d, want %d; body = %s", recorder.Code, http.StatusBadGateway, recorder.Body.String()) + } + var got fetchResponse + decodeRecorder(t, recorder, &got) + if got.Error == "" || got.Body != "" || got.StatusCode != 0 { + t.Errorf("response = %+v, want TLS failure without upstream response", got) + } +} + +func TestFetchTLSWrongRootFails(t *testing.T) { + server, _ := startIPTLSServer(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, "should not arrive") + })) + _, wrongRoot := startIPTLSServer(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, "other server") + })) + + recorder := postHandler(t, newHandler(http.DefaultClient), "/", fetchRequest{URL: server.URL, RootCA: wrongRoot}) + if recorder.Code != http.StatusBadGateway { + t.Fatalf("status = %d, want %d; body = %s", recorder.Code, http.StatusBadGateway, recorder.Body.String()) + } +} + +func TestFetchDoesNotFollowRedirects(t *testing.T) { + redirected := false + target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + redirected = true + _, _ = io.WriteString(w, "redirected") + })) + t.Cleanup(target.Close) + source := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Location", target.URL) + w.WriteHeader(http.StatusFound) + })) + t.Cleanup(source.Close) + + recorder := postHandler(t, newHandler(http.DefaultClient), "/", fetchRequest{URL: source.URL, HTTP1: true}) + if recorder.Code != http.StatusFound { + t.Fatalf("status = %d, want %d; body = %s", recorder.Code, http.StatusFound, recorder.Body.String()) + } + if redirected { + t.Fatal("redirect target received a request") + } + var got fetchResponse + decodeRecorder(t, recorder, &got) + if got.StatusCode != http.StatusFound || got.Protocol == "" { + t.Errorf("response = %+v, want observed redirect response", got) + } +} + +func TestWebSocket(t *testing.T) { + server := startWebSocketServer(t, false) + messages := []string{"one", "two", "three"} + recorder := postHandler(t, newHandler(http.DefaultClient), "/websocket", websocketRequest{URL: toWebSocketURL(server), Messages: messages}) + if recorder.Code != http.StatusOK { + t.Fatalf("status = %d, want %d; body = %s", recorder.Code, http.StatusOK, recorder.Body.String()) + } + var got websocketResponse + decodeRecorder(t, recorder, &got) + if got.StatusCode != http.StatusSwitchingProtocols || got.Protocol != "HTTP/1.1" || got.TLS || got.Error != "" { + t.Errorf("response = %+v, want successful plaintext handshake", got) + } + if len(got.Messages) != len(messages) { + t.Fatalf("messages = %+v, want %d observations", got.Messages, len(messages)) + } + for i, message := range got.Messages { + if message.Type != websocket.TextMessage || message.Message != messages[i] { + t.Errorf("messages[%d] = %+v, want text %q", i, message, messages[i]) + } + } +} + +func TestWebSocketTLSAndVerificationFailure(t *testing.T) { + server, rootCA := startIPTLSServer(t, webSocketEchoHandler(false)) + messages := []string{"tls-one", "tls-two", "tls-three"} + recorder := postHandler(t, newHandler(http.DefaultClient), "/websocket", websocketRequest{URL: toWebSocketURL(server.URL), RootCA: rootCA, Messages: messages}) + if recorder.Code != http.StatusOK { + t.Fatalf("status = %d, want %d; body = %s", recorder.Code, http.StatusOK, recorder.Body.String()) + } + var got websocketResponse + decodeRecorder(t, recorder, &got) + if got.StatusCode != http.StatusSwitchingProtocols || got.Protocol != "HTTP/1.1" || !got.TLS || len(got.Messages) != len(messages) { + t.Errorf("response = %+v, want verified TLS observations", got) + } + + recorder = postHandler(t, newHandler(http.DefaultClient), "/websocket", websocketRequest{URL: toWebSocketURL(server.URL), Messages: messages}) + if recorder.Code != http.StatusBadGateway { + t.Fatalf("untrusted status = %d, want %d; body = %s", recorder.Code, http.StatusBadGateway, recorder.Body.String()) + } +} + +func TestWebSocketRejectsNonUpgrade(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = io.WriteString(w, "not websocket") + })) + t.Cleanup(server.Close) + + recorder := postHandler(t, newHandler(http.DefaultClient), "/websocket", websocketRequest{URL: toWebSocketURL(server.URL), Messages: []string{"one", "two", "three"}}) + if recorder.Code != http.StatusBadGateway { + t.Fatalf("status = %d, want %d; body = %s", recorder.Code, http.StatusBadGateway, recorder.Body.String()) + } + var got websocketResponse + decodeRecorder(t, recorder, &got) + if got.StatusCode != http.StatusOK || got.Protocol != "HTTP/1.1" || got.TLS || got.Error == "" { + t.Errorf("response = %+v, want preserved non-upgrade response and stage error", got) + } +} + +func TestWebSocketPreservesBoundedHandshakeBody(t *testing.T) { + for _, tc := range []struct { + name string + body string + want string + wantPrefix bool + }{ + {name: "exact denial body", body: "WebSocket egress is not supported", want: "WebSocket egress is not supported"}, + {name: "oversized body", body: strings.Repeat("x", maxWebSocketHandshakeBody+1), wantPrefix: true}, + } { + t.Run(tc.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusForbidden) + _, _ = io.WriteString(w, tc.body) + })) + t.Cleanup(server.Close) + + recorder := postHandler(t, newHandler(http.DefaultClient), "/websocket", websocketRequest{URL: toWebSocketURL(server.URL), Messages: []string{"probe"}}) + if recorder.Code != http.StatusBadGateway { + t.Fatalf("status = %d, want %d; body = %s", recorder.Code, http.StatusBadGateway, recorder.Body.String()) + } + var got websocketResponse + decodeRecorder(t, recorder, &got) + if got.StatusCode != http.StatusForbidden { + t.Errorf("handshake status = %d, want %d", got.StatusCode, http.StatusForbidden) + } + if tc.wantPrefix { + if len(got.HandshakeBody) == 0 || len(got.HandshakeBody) > maxWebSocketHandshakeBody || !strings.HasPrefix(tc.body, got.HandshakeBody) { + t.Errorf("handshake body length = %d, want a prefix no longer than %d bytes", len(got.HandshakeBody), maxWebSocketHandshakeBody) + } + } else if got.HandshakeBody != tc.want { + t.Errorf("handshake body = %q, want %q", got.HandshakeBody, tc.want) + } + }) + } +} + +func TestWebSocketPreservesPartialObservationsOnBadResponse(t *testing.T) { + server := startWebSocketServer(t, true) + recorder := postHandler(t, newHandler(http.DefaultClient), "/websocket", websocketRequest{URL: toWebSocketURL(server), Messages: []string{"one", "two", "three"}}) + if recorder.Code != http.StatusBadGateway { + t.Fatalf("status = %d, want %d; body = %s", recorder.Code, http.StatusBadGateway, recorder.Body.String()) + } + var got websocketResponse + decodeRecorder(t, recorder, &got) + if len(got.Messages) != 1 || got.Messages[0].Message != "one" || got.Messages[0].Type != websocket.TextMessage || got.Error == "" { + t.Errorf("response = %+v, want first observation and read failure", got) + } +} + +func postHandler(t *testing.T, handler http.Handler, path string, body any) *httptest.ResponseRecorder { + t.Helper() + payload, err := json.Marshal(body) + if err != nil { + t.Fatal(err) + } + recorder := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodPost, path, bytes.NewReader(payload)) + handler.ServeHTTP(recorder, request) + return recorder +} + +func decodeRecorder(t *testing.T, recorder *httptest.ResponseRecorder, dst any) { + t.Helper() + if err := json.NewDecoder(recorder.Body).Decode(dst); err != nil { + t.Fatalf("decoding response (HTTP %d): %v", recorder.Code, err) + } +} + +func startWebSocketServer(t *testing.T, badSecondResponse bool) string { + t.Helper() + server := httptest.NewServer(webSocketEchoHandler(badSecondResponse)) + t.Cleanup(server.Close) + return server.URL +} + +func webSocketEchoHandler(badSecondResponse bool) http.Handler { + upgrader := websocket.Upgrader{CheckOrigin: func(*http.Request) bool { return true }} + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := upgrader.Upgrade(w, r, nil) + if err != nil { + return + } + defer conn.Close() + for index := 0; ; index++ { + messageType, message, err := conn.ReadMessage() + if err != nil { + return + } + if badSecondResponse && index == 1 { + _ = conn.Close() + return + } + if err := conn.WriteMessage(messageType, message); err != nil { + return + } + } + }) +} + +func startIPTLSServer(t *testing.T, handler http.Handler) (*httptest.Server, string) { + t.Helper() + material := testcert.NewServerTLS(t, net.ParseIP("127.0.0.1")) + cert, err := tls.X509KeyPair(material.Certificate, material.PrivateKey) + if err != nil { + t.Fatalf("loading server certificate: %v", err) + } + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listening on loopback: %v", err) + } + server := httptest.NewUnstartedServer(handler) + server.Listener = listener + server.TLS = &tls.Config{Certificates: []tls.Certificate{cert}} + server.StartTLS() + t.Cleanup(server.Close) + return server, string(material.RootCA) +} + +func toWebSocketURL(raw string) string { + if strings.HasPrefix(raw, "https://") { + return strings.Replace(raw, "https://", "wss://", 1) + } + return strings.Replace(raw, "http://", "ws://", 1) +} + type roundTripFunc func(*http.Request) (*http.Response, error) func (f roundTripFunc) RoundTrip(request *http.Request) (*http.Response, error) { diff --git a/docs/dev/mitm-egress-test-runbook.md b/docs/dev/mitm-egress-test-runbook.md new file mode 100644 index 0000000000..912e823104 --- /dev/null +++ b/docs/dev/mitm-egress-test-runbook.md @@ -0,0 +1,80 @@ +# Egress protocol tests on Kind + +The unified egress gateway intercepts TLS for HTTPS policy rules. It presents +an egress CA certificate to the Actor and verifies the origin certificate on +its upstream connection. The E2E networking tests use local origins signed by +the cluster service-DNS CA, so the gateway must trust that CA as well as its +normal public roots. + +The CI workflow installs both egress ActorTemplates, adds the service-DNS CA +to the gateway's upstream roots with `hack/setup-e2e-egress-tls-kind.sh`, and +runs the networking suite on gVisor and micro-VM. The commands below reproduce +that setup locally. + +## Prerequisites + +Follow the [development quickstart](../../README.md#quickstart-development) +for Go, Docker, `kubectl`, and Kind. The micro-VM lane also needs KVM and the +setup in [Running the microVM runtime locally](microvm-local.md). + +## Install the cluster and fixtures + +```bash +./hack/create-kind-cluster.sh +kubectl config use-context kind-kind +ATE_CREDENTIAL_PROVIDER='{"name":"k8s.io"}' ./hack/install-ate-kind.sh --deploy-ate-system +``` + +For micro-VM, install its assets before deploying the fixture: + +```bash +NO_DEV_ENV=true ATE_INSTALL_KIND=true KUBECTL_CONTEXT=kind-kind \ + ./hack/install-microvm-deps.sh --install +``` + +Deploy both egress ActorTemplates and add local-origin trust to the current +gateway: + +```bash +ATE_CREDENTIAL_PROVIDER='{"name":"k8s.io"}' ./hack/install-ate-kind.sh --deploy-demo-egress +ATE_CREDENTIAL_PROVIDER='{"name":"k8s.io"}' ./hack/install-ate-kind.sh --deploy-demo-egress-microvm +./hack/setup-e2e-egress-tls-kind.sh +``` + +The trust setup retains public roots, adds the service-DNS CA, and waits for +the egress deployment to roll out. The test origins use their Service DNS name +for SNI. The Actor verifies the gateway's minted certificate through its +projected egress trust bundle. + +## Run the networking tests + +Run the gVisor lane: + +```bash +hack/run-e2e-kind.sh ./internal/e2e/suites/networking -run '^TestActorEgress' -v -args --no-color +``` + +Run the micro-VM lane sequentially on the same cluster: + +```bash +E2E_SANDBOX_CLASS=microvm hack/run-e2e-kind.sh ./internal/e2e/suites/networking -run '^TestActorEgress|^TestMicroVMClockAfterDelayedRestore' -v -args --no-color +``` + +The HTTPS protocol test checks a verified HTTP/1.1 request to a local TLS +origin on port 8443. The WebSocket tests check that both `ws` and `wss` +upgrades are denied with HTTP 403 before the origin receives any messages. +The delayed-restore test issues a fresh origin certificate after the Actor is +suspended, then checks that the restored micro-VM can complete the TLS request. + +## Troubleshooting + +```bash +kubectl --context kind-kind get pods -n ate-system -o wide +kubectl --context kind-kind get workerpools -A +kubectl --context kind-kind get pods -A -l ate.dev/worker-pool -o wide +kubectl --context kind-kind -n ate-system logs deploy/atenet-egress -c envoy --tail=300 +``` + +For AgentGateway, use `-c agentgateway` for the last command. For a local TLS +failure, check that `setup-e2e-egress-tls-kind.sh` completed and that the test +URL uses the origin's Service DNS name. diff --git a/hack/setup-e2e-egress-tls-kind.sh b/hack/setup-e2e-egress-tls-kind.sh new file mode 100755 index 0000000000..47c36a160d --- /dev/null +++ b/hack/setup-e2e-egress-tls-kind.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit -o nounset -o pipefail + +# Test setup only: keep public origin verification and also trust local origins +# signed by the cluster's service-DNS CA. Run after installing the unified egress +# gateway and before its local-origin TLS tests. No CA signing key is mounted on +# the gateway. +context="${KUBECTL_CONTEXT:-kind-${KIND_CLUSTER_NAME:-kind}}" +kubectl_args=(--context "${context}" -n ate-system) +gateway="$(kubectl "${kubectl_args[@]}" get deployment atenet-egress -o jsonpath='{.spec.template.spec.containers[?(@.name=="envoy")].name}')" +if [[ -z "${gateway}" ]]; then + gateway="$(kubectl "${kubectl_args[@]}" get deployment atenet-egress -o jsonpath='{.spec.template.spec.containers[?(@.name=="agentgateway")].name}')" +fi +image="$(kubectl "${kubectl_args[@]}" get deployment atenet-egress -o jsonpath="{.spec.template.spec.containers[?(@.name==\"${gateway}\")].image}")" +sdsmint="$(kubectl "${kubectl_args[@]}" get deployment atenet-egress -o jsonpath='{.spec.template.spec.initContainers[?(@.name=="sdsmint")].name}')" +if [[ -z "${gateway}" || -z "${image}" ]]; then + echo "error: install the unified Envoy or AgentGateway egress before configuring E2E TLS origins" >&2 + exit 1 +fi +if [[ "${gateway}" == envoy && "${sdsmint}" != sdsmint ]]; then + echo "error: the Envoy egress is missing its sdsmint certificate service" >&2 + exit 1 +fi + +# AgentGateway is distroless, so use the pinned Envoy image as a shell-capable +# init helper. Its public roots plus the service-DNS CA are mounted only for +# this E2E test setup; production keeps the gateway's normal root behavior. +root_image="${image}" +if [[ "${gateway}" == agentgateway ]]; then + root_image="envoyproxy/envoy:v1.39-latest@sha256:57e14a549d7bd43c8d3f6d03e8cfa653e037d4b38e133acd9b54f38c524401b4" +fi + +# Use the helper image's public roots on every Pod start. The separate init +# container prevents re-runs from appending to an existing bundle. +kubectl "${kubectl_args[@]}" patch deployment atenet-egress --type=strategic --patch "$(cat < /run/e2e-egress-roots/ca-certificates.crt + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: [ALL] + volumeMounts: + - name: servicedns + mountPath: /run/servicedns.podcert.ate.dev + readOnly: true + - name: e2e-egress-upstream-roots + mountPath: /run/e2e-egress-roots + containers: + - name: ${gateway} + volumeMounts: + - name: e2e-egress-upstream-roots + mountPath: /etc/ssl/certs/ca-certificates.crt + subPath: ca-certificates.crt + readOnly: true + volumes: + - name: e2e-egress-upstream-roots + emptyDir: {} +EOF +)" +kubectl "${kubectl_args[@]}" rollout status deployment/atenet-egress --timeout=120s diff --git a/internal/e2e/fixtures/testserver/common_test.go b/internal/e2e/fixtures/testserver/common_test.go new file mode 100644 index 0000000000..d6773c08f7 --- /dev/null +++ b/internal/e2e/fixtures/testserver/common_test.go @@ -0,0 +1,96 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "errors" + "net" + "net/http" + "os" + "path/filepath" + "testing" + "time" + + "github.com/agent-substrate/substrate/internal/testcert" +) + +type originCertificate struct { + caPEM []byte + certFile string + keyFile string +} + +type originServer struct { + httpURL string + httpsURL string + wsURL string + wssURL string +} + +func writeOriginCertificate(t *testing.T) originCertificate { + t.Helper() + material := testcert.NewServerTLS(t, net.ParseIP("127.0.0.1")) + + certFile := filepath.Join(t.TempDir(), "server.crt") + keyFile := filepath.Join(t.TempDir(), "server.key") + if err := os.WriteFile(certFile, material.Certificate, 0o600); err != nil { + t.Fatalf("write server certificate: %v", err) + } + if err := os.WriteFile(keyFile, material.PrivateKey, 0o600); err != nil { + t.Fatalf("write server key: %v", err) + } + return originCertificate{caPEM: material.RootCA, certFile: certFile, keyFile: keyFile} +} + +func startOriginServer(t *testing.T, handler http.Handler, certFile, keyFile string) originServer { + t.Helper() + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen on loopback: %v", err) + } + server := &http.Server{Handler: handler} + if certFile != "" || keyFile != "" { + if err := configureTLSServer(server, certFile, keyFile); err != nil { + t.Fatalf("configure TLS: %v", err) + } + } + serveErr := make(chan error, 1) + go func() { + if certFile == "" && keyFile == "" { + serveErr <- server.Serve(listener) + return + } + serveErr <- server.ServeTLS(listener, "", "") + }() + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if err := server.Shutdown(ctx); err != nil { + t.Errorf("shutdown origin: %v", err) + } + if err := <-serveErr; err != nil && !errors.Is(err, http.ErrServerClosed) { + t.Errorf("serve origin: %v", err) + } + }) + + address := listener.Addr().String() + return originServer{ + httpURL: "http://" + address, + httpsURL: "https://" + address, + wsURL: "ws://" + address, + wssURL: "wss://" + address, + } +} diff --git a/internal/e2e/fixtures/testserver/http.go b/internal/e2e/fixtures/testserver/http.go index 0ae303baa8..3cf941d8ca 100644 --- a/internal/e2e/fixtures/testserver/http.go +++ b/internal/e2e/fixtures/testserver/http.go @@ -15,6 +15,7 @@ package main import ( + "io" "log" "net/http" "time" @@ -22,34 +23,48 @@ import ( "github.com/spf13/cobra" ) -// newHTTPCmd is a plain HTTP/1.1 origin an Actor's egress lands on. It exists so +// newHTTPHandler is the HTTP/1.1 origin an Actor's egress lands on. It exists so // a test can assert the destination port is recovered from SO_ORIGINAL_DST // rather than defaulted from the URL scheme: the actor fetches its /healthz on a // non-standard port, and the gateway's access log is expected to carry that -// port. There is nothing to serve beyond readiness, so /healthz is all it -// answers. +// port. The optional body makes the same fixture useful for assertions about a +// response inside the egress tunnel. +func newHTTPHandler(body string) http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + if body != "" { + _, _ = io.WriteString(w, body) + } + }) + return mux +} + func newHTTPCmd() *cobra.Command { - var listenAddress string + var ( + listenAddress string + certFile string + keyFile string + body string + ) cmd := &cobra.Command{ Use: "http", - Short: "Serve a plain HTTP/1.1 origin answering /healthz.", + Short: "Serve an HTTP/1.1 origin answering /healthz.", Args: cobra.NoArgs, RunE: func(_ *cobra.Command, _ []string) error { - mux := http.NewServeMux() - mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) { - w.WriteHeader(http.StatusOK) - }) - server := &http.Server{ Addr: listenAddress, - Handler: mux, + Handler: newHTTPHandler(body), ReadHeaderTimeout: 10 * time.Second, WriteTimeout: 2 * time.Minute, } log.Printf("testserver http: listening on %s", listenAddress) - return server.ListenAndServe() + return serveHTTP(server, certFile, keyFile) }, } cmd.Flags().StringVar(&listenAddress, "listen", ":8080", "Address the HTTP origin listens on.") + cmd.Flags().StringVar(&certFile, "tls-cert", "", "PEM certificate file; enables HTTPS when paired with --tls-key.") + cmd.Flags().StringVar(&keyFile, "tls-key", "", "PEM private key file; enables HTTPS when paired with --tls-cert.") + cmd.Flags().StringVar(&body, "body", "", "Response body for /healthz.") return cmd } diff --git a/internal/e2e/fixtures/testserver/http_test.go b/internal/e2e/fixtures/testserver/http_test.go new file mode 100644 index 0000000000..45ab8f3713 --- /dev/null +++ b/internal/e2e/fixtures/testserver/http_test.go @@ -0,0 +1,88 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "crypto/tls" + "crypto/x509" + "io" + "net/http" + "testing" + "time" +) + +func TestHTTPHandlerDefaultsToEmptyHealthz(t *testing.T) { + server := startOriginServer(t, newHTTPHandler(""), "", "") + + client := &http.Client{Timeout: 2 * time.Second} + response, err := client.Get(server.httpURL + "/healthz") + if err != nil { + t.Fatalf("GET /healthz: %v", err) + } + defer response.Body.Close() + + if response.StatusCode != http.StatusOK { + t.Fatalf("GET /healthz status = %d, want %d", response.StatusCode, http.StatusOK) + } + body, err := io.ReadAll(response.Body) + if err != nil { + t.Fatalf("reading /healthz: %v", err) + } + if len(body) != 0 { + t.Errorf("GET /healthz body = %q, want empty", body) + } +} + +func TestHTTPSTestResponseUsesHTTP11AndVerifiedIdentity(t *testing.T) { + cert := writeOriginCertificate(t) + server := startOriginServer(t, newHTTPHandler("fixture response"), cert.certFile, cert.keyFile) + + roots := x509.NewCertPool() + if !roots.AppendCertsFromPEM(cert.caPEM) { + t.Fatal("adding fixture CA to trust pool") + } + protocols := new(http.Protocols) + protocols.SetHTTP1(true) + protocols.SetHTTP2(true) + client := &http.Client{ + Timeout: 2 * time.Second, + Transport: &http.Transport{ + Protocols: protocols, + TLSClientConfig: &tls.Config{ + RootCAs: roots, + ServerName: "127.0.0.1", + }, + }, + } + response, err := client.Get(server.httpsURL + "/healthz") + if err != nil { + t.Fatalf("GET /healthz over verified TLS: %v", err) + } + defer response.Body.Close() + + if response.Proto != "HTTP/1.1" { + t.Errorf("HTTPS response protocol with HTTP/2 offered = %q, want HTTP/1.1", response.Proto) + } + if response.StatusCode != http.StatusOK { + t.Errorf("HTTPS response status = %d, want %d", response.StatusCode, http.StatusOK) + } + body, err := io.ReadAll(response.Body) + if err != nil { + t.Fatalf("reading HTTPS response: %v", err) + } + if string(body) != "fixture response" { + t.Errorf("HTTPS response body = %q, want %q", body, "fixture response") + } +} diff --git a/internal/e2e/fixtures/testserver/tls.go b/internal/e2e/fixtures/testserver/tls.go new file mode 100644 index 0000000000..b370552dcc --- /dev/null +++ b/internal/e2e/fixtures/testserver/tls.go @@ -0,0 +1,55 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "crypto/tls" + "fmt" + "net/http" +) + +// configureTLSServer loads the origin certificate and key and offers only +// HTTP/1.1 over TLS. Invalid credentials fail setup before the listener starts. +func configureTLSServer(server *http.Server, certFile, keyFile string) error { + if certFile == "" && keyFile == "" { + return nil + } + if certFile == "" || keyFile == "" { + return fmt.Errorf("--tls-cert and --tls-key must be provided together") + } + + certificate, err := tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + return fmt.Errorf("loading TLS certificate and key: %w", err) + } + server.TLSConfig = &tls.Config{ + Certificates: []tls.Certificate{certificate}, + MinVersion: tls.VersionTLS12, + NextProtos: []string{"http/1.1"}, + } + return nil +} + +func serveHTTP(server *http.Server, certFile, keyFile string) error { + if certFile == "" && keyFile == "" { + return server.ListenAndServe() + } + + if err := configureTLSServer(server, certFile, keyFile); err != nil { + return err + } + + return server.ListenAndServeTLS("", "") +} diff --git a/internal/e2e/fixtures/testserver/tls_test.go b/internal/e2e/fixtures/testserver/tls_test.go new file mode 100644 index 0000000000..f8073cba62 --- /dev/null +++ b/internal/e2e/fixtures/testserver/tls_test.go @@ -0,0 +1,35 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "net/http" + "testing" +) + +func TestTLSConfigurationRejectsIncompleteOrUnreadableCredentials(t *testing.T) { + for name, paths := range map[string][2]string{ + "missing key": {"cert.pem", ""}, + "missing cert": {"", "key.pem"}, + "unreadable pair": {"missing-cert.pem", "missing-key.pem"}, + } { + t.Run(name, func(t *testing.T) { + server := &http.Server{} + if err := configureTLSServer(server, paths[0], paths[1]); err == nil { + t.Fatal("configureTLSServer succeeded, want error") + } + }) + } +} diff --git a/internal/e2e/fixtures/testserver/websocket.go b/internal/e2e/fixtures/testserver/websocket.go index efd189d5a8..011043c1ef 100644 --- a/internal/e2e/fixtures/testserver/websocket.go +++ b/internal/e2e/fixtures/testserver/websocket.go @@ -26,53 +26,70 @@ var upgrader = websocket.Upgrader{ CheckOrigin: func(r *http.Request) bool { return true }, } +func newWebsocketHandler(echo bool) http.Handler { + mux := http.NewServeMux() + + mux.HandleFunc("/readyz", func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("ok")) + }) + + mux.HandleFunc("/ws", func(w http.ResponseWriter, r *http.Request) { + c, err := upgrader.Upgrade(w, r, nil) + if err != nil { + log.Print("upgrade:", err) + return + } + defer c.Close() + for { + mt, message, err := c.ReadMessage() + if err != nil { + log.Println("read:", err) + break + } + log.Printf("recv: %s", message) + + var response []byte + switch { + case echo: + response = message + case string(message) == "PING": + response = []byte("PONG") + default: + continue + } + if err := c.WriteMessage(mt, response); err != nil { + log.Println("write:", err) + break + } + } + }) + return mux +} + func newWebsocketCmd() *cobra.Command { - var listenAddress string + var ( + listenAddress string + certFile string + keyFile string + echo bool + ) cmd := &cobra.Command{ Use: "websocket", Short: "Serve a websocket server for e2e tests.", Args: cobra.NoArgs, RunE: func(_ *cobra.Command, _ []string) error { - mux := http.NewServeMux() - - mux.HandleFunc("/readyz", func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusOK) - w.Write([]byte("ok")) - }) - - mux.HandleFunc("/ws", func(w http.ResponseWriter, r *http.Request) { - c, err := upgrader.Upgrade(w, r, nil) - if err != nil { - log.Print("upgrade:", err) - return - } - defer c.Close() - for { - mt, message, err := c.ReadMessage() - if err != nil { - log.Println("read:", err) - break - } - log.Printf("recv: %s", message) - - if string(message) == "PING" { - err = c.WriteMessage(mt, []byte("PONG")) - if err != nil { - log.Println("write:", err) - break - } - } - } - }) - server := &http.Server{ Addr: listenAddress, - Handler: mux, + Handler: newWebsocketHandler(echo), } log.Printf("testserver websocket: listening on %s", listenAddress) - return server.ListenAndServe() + return serveHTTP(server, certFile, keyFile) }, } cmd.Flags().StringVar(&listenAddress, "listen", ":80", "Address the websocket server listens on.") + cmd.Flags().StringVar(&certFile, "tls-cert", "", "PEM certificate file; enables HTTPS when paired with --tls-key.") + cmd.Flags().StringVar(&keyFile, "tls-key", "", "PEM private key file; enables HTTPS when paired with --tls-cert.") + cmd.Flags().BoolVar(&echo, "echo", false, "Echo every WebSocket message with its original type and payload.") return cmd } diff --git a/internal/e2e/fixtures/testserver/websocket_test.go b/internal/e2e/fixtures/testserver/websocket_test.go new file mode 100644 index 0000000000..f5f601a947 --- /dev/null +++ b/internal/e2e/fixtures/testserver/websocket_test.go @@ -0,0 +1,109 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "crypto/tls" + "crypto/x509" + "net/http" + "testing" + "time" + + "github.com/gorilla/websocket" +) + +func TestWebsocketDefaultRespondsToPingWithPong(t *testing.T) { + server := startOriginServer(t, newWebsocketHandler(false), "", "") + dialer := websocket.Dialer{HandshakeTimeout: 2 * time.Second} + conn, response, err := dialer.Dial(server.wsURL+"/ws", nil) + if err != nil { + t.Fatalf("dial websocket: %v", err) + } + defer conn.Close() + if response.StatusCode != http.StatusSwitchingProtocols { + t.Fatalf("websocket handshake status = %d, want %d", response.StatusCode, http.StatusSwitchingProtocols) + } + if err := conn.SetReadDeadline(time.Now().Add(2 * time.Second)); err != nil { + t.Fatalf("set websocket read deadline: %v", err) + } + if err := conn.SetWriteDeadline(time.Now().Add(2 * time.Second)); err != nil { + t.Fatalf("set websocket write deadline: %v", err) + } + + if err := conn.WriteMessage(websocket.TextMessage, []byte("PING")); err != nil { + t.Fatalf("write PING: %v", err) + } + messageType, message, err := conn.ReadMessage() + if err != nil { + t.Fatalf("read PONG: %v", err) + } + if messageType != websocket.TextMessage || string(message) != "PONG" { + t.Errorf("PING response = type %d, %q; want text PONG", messageType, message) + } +} + +func TestWebsocketEchoPreservesMessageTypeAndPayload(t *testing.T) { + cert := writeOriginCertificate(t) + server := startOriginServer(t, newWebsocketHandler(true), cert.certFile, cert.keyFile) + roots := x509.NewCertPool() + if !roots.AppendCertsFromPEM(cert.caPEM) { + t.Fatal("adding fixture CA to trust pool") + } + dialer := websocket.Dialer{ + HandshakeTimeout: 2 * time.Second, + TLSClientConfig: &tls.Config{ + RootCAs: roots, + ServerName: "127.0.0.1", + }, + } + conn, response, err := dialer.Dial(server.wssURL+"/ws", nil) + if err != nil { + t.Fatalf("dial secure websocket: %v", err) + } + defer conn.Close() + if response.StatusCode != http.StatusSwitchingProtocols { + t.Fatalf("secure websocket handshake status = %d, want %d", response.StatusCode, http.StatusSwitchingProtocols) + } + if err := conn.SetReadDeadline(time.Now().Add(2 * time.Second)); err != nil { + t.Fatalf("set secure websocket read deadline: %v", err) + } + if err := conn.SetWriteDeadline(time.Now().Add(2 * time.Second)); err != nil { + t.Fatalf("set secure websocket write deadline: %v", err) + } + + messages := []struct { + messageType int + payload []byte + }{ + {websocket.TextMessage, []byte("first message")}, + {websocket.BinaryMessage, []byte{0, 1, 2, 255}}, + {websocket.TextMessage, []byte("PING")}, + } + for i, want := range messages { + if err := conn.WriteMessage(want.messageType, want.payload); err != nil { + t.Fatalf("write message %d: %v", i, err) + } + gotType, gotPayload, err := conn.ReadMessage() + if err != nil { + t.Fatalf("read echo %d: %v", i, err) + } + if gotType != want.messageType { + t.Errorf("echo %d message type = %d, want %d", i, gotType, want.messageType) + } + if string(gotPayload) != string(want.payload) { + t.Errorf("echo %d payload = %q, want %q", i, gotPayload, want.payload) + } + } +} diff --git a/internal/e2e/serverpod.go b/internal/e2e/serverpod.go index ea4d186337..b94850b67b 100644 --- a/internal/e2e/serverpod.go +++ b/internal/e2e/serverpod.go @@ -18,13 +18,17 @@ import ( "context" "fmt" "net" + "os" + "slices" "strconv" "strings" "testing" "time" corev1 "k8s.io/api/core/v1" + discoveryv1 "k8s.io/api/discovery/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/yaml" ) // serverPodTemplate is the one manifest every ServerPod is rendered from. @@ -71,6 +75,8 @@ type ServerPod struct { // HealthPath is the HTTP readiness path, defaulting to /healthz. Ignored // when GRPCProbe is set. HealthPath string + // HealthScheme selects HTTP or HTTPS for the readiness probe. Empty means HTTP. + HealthScheme corev1.URIScheme // Volumes and VolumeMounts carry whatever credentials the server needs. // Typed, rather than more YAML in the template, so the Secret names here // sit beside the code that creates them instead of drifting from it. @@ -179,5 +185,107 @@ func serverReadinessProbe(spec ServerPod, targetPort string) string { if path == "" { path = "/healthz" } - return fmt.Sprintf(" httpGet:\n path: %s\n port: %s", path, targetPort) + scheme := spec.HealthScheme + if scheme == "" { + scheme = corev1.URISchemeHTTP + } + return fmt.Sprintf(" httpGet:\n path: %s\n port: %s\n scheme: %s", path, targetPort, scheme) +} + +func serverEndpointReady(endpointSlices []discoveryv1.EndpointSlice, pod *corev1.Pod, port int32) bool { + for _, slice := range endpointSlices { + hasPort := false + for _, p := range slice.Ports { + if p.Port != nil && *p.Port == port && p.Protocol != nil && *p.Protocol == corev1.ProtocolTCP { + hasPort = true + } + } + if !hasPort { + continue + } + for _, endpoint := range slice.Endpoints { + ref := endpoint.TargetRef + if ref == nil || ref.Kind != "Pod" || ref.Namespace != pod.Namespace || ref.Name != pod.Name || ref.UID != pod.UID { + continue + } + if endpoint.Conditions.Ready == nil || !*endpoint.Conditions.Ready { + continue + } + if endpoint.Conditions.Terminating != nil && *endpoint.Conditions.Terminating { + continue + } + if slices.Contains(endpoint.Addresses, pod.Status.PodIP) { + return true + } + } + } + return false +} + +// CreateServerService reserves the origin address before its Pod starts, so +// callers can issue a certificate for that IP. Namespace must already exist. +// DeployServerPod subsequently applies the same Service without replacing it. +func CreateServerService(t *testing.T, ctx context.Context, spec ServerPod) Server { + t.Helper() + if spec.Namespace == "" { + t.Fatal("CreateServerService requires a test namespace") + } + raw, err := os.ReadFile(renderServerPod(t, spec, spec.Namespace)) + if err != nil { + t.Fatalf("reading origin manifest: %v", err) + } + for doc := range strings.SplitSeq(string(raw), "\n---\n") { + var kind metav1.TypeMeta + if err := yaml.Unmarshal([]byte(doc), &kind); err != nil { + t.Fatalf("decoding origin kind: %v", err) + } + if kind.Kind != "Service" { + continue + } + var service corev1.Service + if err := yaml.UnmarshalStrict([]byte(doc), &service); err != nil { + t.Fatalf("decoding origin Service: %v", err) + } + created, err := GetClients().K8s.CoreV1().Services(spec.Namespace).Create(ctx, &service, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("creating origin Service: %v", err) + } + if net.ParseIP(created.Spec.ClusterIP) == nil { + t.Fatalf("origin Service has invalid IP %q", created.Spec.ClusterIP) + } + return Server{Namespace: spec.Namespace, ClusterIP: created.Spec.ClusterIP, Port: spec.Port} + } + t.Fatal("origin manifest has no Service") + return Server{} +} + +// WaitForServerEndpoint waits for publication of the current origin Pod as a +// ready Service backend. It does not perform the protocol exchange under test. +func WaitForServerEndpoint(t *testing.T, ctx context.Context, spec ServerPod) { + t.Helper() + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + clients := GetClients() + pod, err := clients.K8s.CoreV1().Pods(spec.Namespace).Get(ctx, spec.Name, metav1.GetOptions{}) + if err != nil { + t.Fatalf("reading origin Pod for endpoint readiness: %v", err) + } + port := spec.TargetPort + if port == 0 { + port = spec.Port + } + for { + endpoints, err := clients.K8s.DiscoveryV1().EndpointSlices(spec.Namespace).List(ctx, metav1.ListOptions{LabelSelector: discoveryv1.LabelServiceName + "=" + spec.Name}) + if err != nil { + t.Fatalf("reading origin EndpointSlices: %v", err) + } + if serverEndpointReady(endpoints.Items, pod, int32(port)) { + return + } + select { + case <-ctx.Done(): + t.Fatalf("origin %s/%s has no ready endpoint for Pod %s on port %d: %v", spec.Namespace, spec.Name, pod.UID, port, ctx.Err()) + case <-time.After(200 * time.Millisecond): + } + } } diff --git a/internal/e2e/serverpod_test.go b/internal/e2e/serverpod_test.go index 6acc1398a6..f081f48fe3 100644 --- a/internal/e2e/serverpod_test.go +++ b/internal/e2e/serverpod_test.go @@ -22,6 +22,7 @@ import ( "testing" corev1 "k8s.io/api/core/v1" + discoveryv1 "k8s.io/api/discovery/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" "sigs.k8s.io/yaml" @@ -286,3 +287,41 @@ func TestRenderServerPod_Volumes(t *testing.T) { } } } + +func TestRenderServerPod_HTTPSProbe(t *testing.T) { + pod, _ := renderServerPodDocs(t, ServerPod{Name: "tls", Port: 8443, HealthScheme: corev1.URISchemeHTTPS}) + probe := pod.Spec.Containers[0].ReadinessProbe.HTTPGet + if probe.Scheme != corev1.URISchemeHTTPS || probe.Port.IntValue() != 8443 { + t.Fatalf("probe = %+v, want HTTPS on 8443", probe) + } +} + +func TestServerEndpointReady(t *testing.T) { + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "origin", Namespace: "test", UID: "current"}, Status: corev1.PodStatus{PodIP: "10.0.0.2"}} + base := discoveryv1.EndpointSlice{ + Ports: []discoveryv1.EndpointPort{{Port: ptr.To(int32(8443)), Protocol: ptr.To(corev1.ProtocolTCP)}}, + Endpoints: []discoveryv1.Endpoint{{Addresses: []string{"10.0.0.2"}, TargetRef: &corev1.ObjectReference{Kind: "Pod", Namespace: "test", Name: "origin", UID: "current"}, Conditions: discoveryv1.EndpointConditions{Ready: ptr.To(true)}}}, + } + cases := []struct { + name string + change func(*discoveryv1.EndpointSlice) + want bool + }{ + {"ready origin", func(*discoveryv1.EndpointSlice) {}, true}, + {"not ready", func(s *discoveryv1.EndpointSlice) { s.Endpoints[0].Conditions.Ready = ptr.To(false) }, false}, + {"unknown readiness", func(s *discoveryv1.EndpointSlice) { s.Endpoints[0].Conditions.Ready = nil }, false}, + {"previous pod", func(s *discoveryv1.EndpointSlice) { s.Endpoints[0].TargetRef.UID = "previous" }, false}, + {"wrong address", func(s *discoveryv1.EndpointSlice) { s.Endpoints[0].Addresses = []string{"10.0.0.3"} }, false}, + {"wrong port", func(s *discoveryv1.EndpointSlice) { s.Ports[0].Port = ptr.To(int32(443)) }, false}, + {"terminating", func(s *discoveryv1.EndpointSlice) { s.Endpoints[0].Conditions.Terminating = ptr.To(true) }, false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + s := base.DeepCopy() + tc.change(s) + if got := serverEndpointReady([]discoveryv1.EndpointSlice{*s}, pod, 8443); got != tc.want { + t.Fatalf("ready = %v, want %v", got, tc.want) + } + }) + } +} diff --git a/internal/e2e/suites/networking/accesslog_test.go b/internal/e2e/suites/networking/accesslog_test.go new file mode 100644 index 0000000000..f9057bc5c8 --- /dev/null +++ b/internal/e2e/suites/networking/accesslog_test.go @@ -0,0 +1,143 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package networking + +import ( + "testing" + + "github.com/agent-substrate/substrate/internal/resources" +) + +func TestAccessLogField(t *testing.T) { + line := `http.status=101 ate.actor.name=ws-1 substrate.connect.authority="10.96.0.7:80" ate.atespace=networking-e2e` + for _, test := range []struct { + key, want string + }{ + {"http.status", "101"}, + {"ate.actor.name", "ws-1"}, + {"substrate.connect.authority", "10.96.0.7:80"}, + } { + if got, ok := accessLogField(line, test.key); !ok || got != test.want { + t.Errorf("accessLogField(%q, %q) = (%q, %t), want (%q, true)", line, test.key, got, ok, test.want) + } + } + if got, ok := accessLogField(`other.http.status=503 http.status=101`, "http.status"); !ok || got != "101" { + t.Errorf("accessLogField must match a complete key, got (%q, %t)", got, ok) + } +} + +func TestProtocolGatewayAccessLogMatches(t *testing.T) { + actorRef := resources.ActorRef{Atespace: "networking-e2e", Name: "ws-1"} + for _, test := range []struct { + name, container, line string + want bool + }{ + { + name: "agentgateway websocket success", + container: "agentgateway", + line: `http.status=101 ate.actor.name=ws-1 substrate.connect.authority="10.96.0.7:80" ate.atespace=networking-e2e`, + want: true, + }, + { + name: "agentgateway opaque tunnel success", + container: "agentgateway", + line: `CONNECT tunnel terminated target=10.96.0.7:80 actor_name=ws-1 actor_uid=uid-1 atespace=networking-e2e`, + want: true, + }, + { + name: "agentgateway opaque tunnel wrong identity", + container: "agentgateway", + line: `CONNECT tunnel terminated target=10.96.0.7:80 actor_name=ws-2 actor_uid=uid-1 atespace=networking-e2e substrate.connect.authority="10.96.0.7:80"`, + want: false, + }, + { + name: "agentgateway opaque tunnel wrong target", + container: "agentgateway", + line: `CONNECT tunnel terminated target=10.96.0.8:80 actor_name=ws-1 actor_uid=uid-1 atespace=networking-e2e substrate.connect.authority="10.96.0.7:80"`, + want: false, + }, + { + name: "agentgateway opaque tunnel wrong atespace", + container: "agentgateway", + line: `CONNECT tunnel terminated target=10.96.0.7:80 actor_name=ws-1 actor_uid=uid-1 atespace=other`, + want: false, + }, + { + name: "agentgateway opaque tunnel error", + container: "agentgateway", + line: `CONNECT tunnel terminated target=10.96.0.7:80 actor_name=ws-1 actor_uid=uid-1 atespace=networking-e2e error=denied`, + want: false, + }, + { + name: "agentgateway wrong destination", + container: "agentgateway", + line: `http.status=101 ate.actor.name=ws-1 substrate.connect.authority="10.96.0.8:80" ate.atespace=networking-e2e`, + want: false, + }, + { + name: "agentgateway wrong actor", + container: "agentgateway", + line: `http.status=101 ate.actor.name=ws-2 substrate.connect.authority="10.96.0.7:80" ate.atespace=networking-e2e`, + want: false, + }, + { + name: "agentgateway wrong atespace", + container: "agentgateway", + line: `http.status=101 ate.actor.name=ws-1 substrate.connect.authority="10.96.0.7:80" ate.atespace=other`, + want: false, + }, + { + name: "agentgateway rejected request", + container: "agentgateway", + line: `http.status=403 ate.actor.name=ws-1 substrate.connect.authority="10.96.0.7:80" ate.atespace=networking-e2e`, + want: false, + }, + { + name: "envoy success", + container: "envoy", + line: `authority=10.96.0.7:80 code=200 peer_san="spiffe://substrate-actor.local/ateom-for-actor/networking-e2e/ws-1"`, + want: true, + }, + { + name: "envoy wrong actor identity", + container: "envoy", + line: `authority=10.96.0.7:80 code=200 peer_san="spiffe://substrate-actor.local/ateom-for-actor/networking-e2e/ws-2"`, + want: false, + }, + { + name: "envoy wrong atespace identity", + container: "envoy", + line: `authority=10.96.0.7:80 code=200 peer_san="spiffe://substrate-actor.local/ateom-for-actor/other/ws-1"`, + want: false, + }, + { + name: "envoy actor certificate identity is not the transport peer", + container: "envoy", + line: `authority=10.96.0.7:80 code=200 peer_san="spiffe://substrate-actor.local/actor/networking-e2e/ws-1"`, + want: false, + }, + } { + t.Run(test.name, func(t *testing.T) { + got := protocolGatewayAccessLogMatches(gatewayAccessLogLine{container: test.container, text: test.line}, actorRef, "10.96.0.7:80", true) + if got != test.want { + t.Fatalf("protocolGatewayAccessLogMatches() = %t, want %t", got, test.want) + } + }) + } + valid := gatewayAccessLogLine{container: "agentgateway", text: `CONNECT tunnel terminated target=10.96.0.7:80 actor_name=ws-1 actor_uid=uid-1 atespace=networking-e2e`} + if protocolGatewayAccessLogMatches(valid, actorRef, "10.96.0.7:80", false) { + t.Fatal("opaque CONNECT evidence must not satisfy the MITM matcher") + } +} diff --git a/internal/e2e/suites/networking/microvmclock_test.go b/internal/e2e/suites/networking/microvmclock_test.go new file mode 100644 index 0000000000..22b1b25300 --- /dev/null +++ b/internal/e2e/suites/networking/microvmclock_test.go @@ -0,0 +1,140 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package networking + +import ( + "context" + "crypto/x509" + "encoding/json" + "encoding/pem" + "net/http" + "testing" + "time" + + "github.com/agent-substrate/substrate/internal/e2e" + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// TestMicroVMClockAfterDelayedRestore probes delayed-restore TLS behavior by +// issuing the origin trust chain only after a suspended actor has been idle. +// The only wait here is the deliberate interval; ingress readiness is checked +// separately before the one measured HTTPS operation. +func TestMicroVMClockAfterDelayedRestore(t *testing.T) { + if !e2e.IsMicroVM() { + t.Skip("micro-VM clock contract") + } + + ctx := t.Context() + actorName, router, actorRef := prepareProtocolActor(t, ctx, "clock") + clients := e2e.GetClients() + t.Logf("clock evidence host_utc_before_suspend=%s actor=%s/%s", time.Now().UTC().Format(time.RFC3339Nano), actorRef.Atespace, actorName) + beforeSuspend, err := clients.SubstrateAPI.GetActor(ctx, &ateapipb.GetActorRequest{Actor: actorRef.ToObjectRef()}) + if err != nil { + t.Fatalf("GetActor before SuspendActor: %v", err) + } + if assignment := beforeSuspend.GetStatus().GetWorkerAssignment(); assignment == nil || assignment.GetWorker().GetName() == "" { + t.Fatalf("actor had no worker assignment before SuspendActor: %v", assignment) + } + suspendBoundary := time.Now().UTC() + suspended, err := clients.SubstrateAPI.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: actorRef.ToObjectRef()}) + if err != nil { + t.Fatalf("SuspendActor: %v", err) + } + if actor := suspended.GetActor(); actor.GetStatus().GetState() != ateapipb.ActorState_ACTOR_STATE_SUSPENDED { + t.Fatalf("SuspendActor returned state %s, want SUSPENDED", actor.GetStatus().GetState()) + } else if actor.GetStatus().GetWorkerAssignment() != nil { + t.Fatalf("SuspendActor returned worker assignment %v, want it cleared", actor.GetStatus().GetWorkerAssignment()) + } + + // Keep this interval long enough to separate the snapshot boundary from the + // fresh certificate issuance and resumed request. + const delay = 70 * time.Second + t.Logf("clock evidence suspended; waiting %s before issuing a fresh root and leaf", delay) + timer := time.NewTimer(delay) + select { + case <-timer.C: + case <-ctx.Done(): + timer.Stop() + t.Fatal(ctx.Err()) + } + t.Logf("clock evidence host_utc_after_delay=%s", time.Now().UTC().Format(time.RFC3339Nano)) + + target, address, rootCA := prepareProtocolOrigin(t, ctx, e2e.ServerPod{ + Name: "clock-origin", ImportPath: "github.com/agent-substrate/substrate/internal/e2e/fixtures/testserver", + Args: []string{"http", "--body=delayed restore clock"}, Port: 8443, + }, true) + logOriginCertificateValidity(t, ctx, target, rootCA, suspendBoundary) + + if _, err := e2e.ResumeActorAwaitCapacity(t, ctx, clients, &ateapipb.ResumeActorRequest{Actor: actorRef.ToObjectRef()}); err != nil { + t.Fatalf("ResumeActor: %v", err) + } + waitForRouteReady(t, "restored actor ingress readiness", func() (*http.Response, error) { + return router.Get(ctx, actorRef, "/readyz") + }) + + since := metav1.NewTime(time.Now().Add(-time.Minute)) + raw := postEgressOnce(t, ctx, router, actorRef, "/", map[string]any{ + "url": "https://" + address + "/healthz", "http1": true, + }) + var got struct { + StatusCode int `json:"statusCode"` + Body string `json:"body"` + Protocol string `json:"protocol"` + TLS bool `json:"tls"` + Error string `json:"error"` + } + if err := json.Unmarshal(raw, &got); err != nil { + t.Fatalf("decoding delayed-restore HTTPS observation: %v", err) + } + if got.Error != "" || got.StatusCode != 200 || got.Body != "delayed restore clock" || got.Protocol != "HTTP/1.1" || !got.TLS { + t.Errorf("delayed-restore HTTPS observation = %+v; want one verified TLS HTTP/1.1 8443 request", got) + } + assertProtocolGateway(t, ctx, since, actorName, target.Address()) +} + +func logOriginCertificateValidity(t *testing.T, ctx context.Context, target e2e.Server, rootCA string, suspendBoundary time.Time) { + t.Helper() + rootBlock, _ := pem.Decode([]byte(rootCA)) + if rootBlock == nil { + t.Fatalf("origin root CA is not PEM") + } + root, err := x509.ParseCertificate(rootBlock.Bytes) + if err != nil { + t.Fatalf("parsing origin root CA: %v", err) + } + secret, err := e2e.GetClients().K8s.CoreV1().Secrets(target.Namespace).Get(ctx, "origin-tls", metav1.GetOptions{}) + if err != nil { + t.Fatalf("reading origin leaf metadata: %v", err) + } + leaf, err := x509.ParseCertificate(mustPEMCertificate(t, secret.Data[corev1.TLSCertKey])) + if err != nil { + t.Fatalf("parsing origin leaf: %v", err) + } + if !leaf.NotBefore.After(suspendBoundary) { + t.Fatalf("origin leaf NotBefore %s is not after suspend boundary %s", leaf.NotBefore.UTC(), suspendBoundary) + } + t.Logf("clock evidence host_utc_after_origin=%s root_not_before=%s root_not_after=%s leaf_not_before=%s leaf_not_after=%s", time.Now().UTC().Format(time.RFC3339Nano), root.NotBefore.UTC().Format(time.RFC3339Nano), root.NotAfter.UTC().Format(time.RFC3339Nano), leaf.NotBefore.UTC().Format(time.RFC3339Nano), leaf.NotAfter.UTC().Format(time.RFC3339Nano)) +} + +func mustPEMCertificate(t *testing.T, data []byte) []byte { + t.Helper() + block, _ := pem.Decode(data) + if block == nil { + t.Fatalf("origin leaf is not PEM") + } + return block.Bytes +} diff --git a/internal/e2e/suites/networking/networking_test.go b/internal/e2e/suites/networking/networking_test.go index ab0677d1cc..80e4f0fdf8 100644 --- a/internal/e2e/suites/networking/networking_test.go +++ b/internal/e2e/suites/networking/networking_test.go @@ -19,7 +19,9 @@ import ( "encoding/json" "fmt" "io" + "net" "net/http" + "slices" "strconv" "strings" "testing" @@ -27,14 +29,19 @@ import ( "github.com/agent-substrate/substrate/internal/atenet" "github.com/agent-substrate/substrate/internal/e2e" + "github.com/agent-substrate/substrate/internal/localca" "github.com/agent-substrate/substrate/internal/resources" + "github.com/agent-substrate/substrate/internal/testcert" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" corev1 "k8s.io/api/core/v1" + discoveryv1 "k8s.io/api/discovery/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) const networkingAtespace = "networking-e2e" +// egressFixture returns the unified egress demo deployed by the E2E workflow. +func egressFixture() e2e.Fixture { return e2e.EgressFixture() } func TestActorDirectAccess(t *testing.T) { ctx := context.Background() _, actorName, actor := createAndResumeSubstrateActor(t, ctx, "direct", e2e.SubstrateCounterFixture()) @@ -364,12 +371,13 @@ func waitForAccessLog(t *testing.T, ctx context.Context, since metav1.Time, want // accessLogField returns the value of the key=value field named key in an Envoy // access log line whose fields are separated by spaces. func accessLogField(line, key string) (string, bool) { - _, rest, ok := strings.Cut(line, key+"=") - if !ok { - return "", false + for field := range strings.FieldsSeq(line) { + fieldKey, value, ok := strings.Cut(field, "=") + if ok && fieldKey == key { + return strings.Trim(value, `"`), true + } } - value, _, _ := strings.Cut(rest, " ") - return value, true + return "", false } // createAndResumeActorWithEgress creates an actor from template, gives it an @@ -485,3 +493,413 @@ func assertDirectActorAccess(t *testing.T, ctx context.Context, clients *e2e.Cli } t.Logf("direct Actor access through %s/%s:80 was blocked as expected: %v", actor.GetStatus().GetWorkerAssignment().GetWorkerNamespace(), actor.GetStatus().GetWorkerAssignment().GetWorkerPod(), err) } + +// TestActorEgressHTTPSNonStandardPort verifies HTTP/1.1 inside TLS while +// preserving the original destination port through the egress tunnel. +func TestActorEgressHTTPSNonStandardPort(t *testing.T) { + ctx := t.Context() + const expectedBody = "https nonstandard origin" + target, address, _ := prepareProtocolOrigin(t, ctx, e2e.ServerPod{ + Name: "https-origin", + ImportPath: "github.com/agent-substrate/substrate/internal/e2e/fixtures/testserver", + Args: []string{"http", "--body=" + expectedBody}, + Port: 8443, + }, true) + actorName, router, actorRef := prepareProtocolActor(t, ctx, "https-port") + since := metav1.NewTime(time.Now().Add(-time.Minute)) + raw := postEgressOnce(t, ctx, router, actorRef, "/", map[string]any{ + "url": "https://" + address + "/healthz", "http1": true, + }) + var got struct { + StatusCode int `json:"statusCode"` + Body string `json:"body"` + Protocol string `json:"protocol"` + TLS bool `json:"tls"` + Error string `json:"error"` + } + if err := json.Unmarshal(raw, &got); err != nil { + t.Fatalf("decoding HTTPS observation: %v", err) + } + if got.Error != "" || got.StatusCode != http.StatusOK || got.Body != expectedBody || got.Protocol != "HTTP/1.1" || !got.TLS { + t.Errorf("HTTPS observation = %+v; want verified TLS, HTTP/1.1, status 200 and body %q", got, expectedBody) + } + assertProtocolGateway(t, ctx, since, actorName, target.Address()) +} + +func prepareProtocolOrigin(t *testing.T, ctx context.Context, spec e2e.ServerPod, encrypted bool) (e2e.Server, string, string) { + t.Helper() + spec.Namespace = e2e.CreateNamespace(t).Name + registerOriginDiagnostics(t, spec.Namespace, spec.Name) + var reserved e2e.Server + var rootCA string + if encrypted { + reserved = e2e.CreateServerService(t, ctx, spec) + // The unified gateway verifies local origins against service-DNS roots. + secret, err := e2e.GetClients().K8s.CoreV1().Secrets("podcertificate-controller-system").Get(ctx, "service-dns-ca-pool", metav1.GetOptions{}) + if err != nil { + t.Fatalf("reading service-DNS CA pool: %v", err) + } + pool, err := localca.Unmarshal(secret.Data["pool"]) + if err != nil { + t.Fatalf("parsing service-DNS CA pool: %v", err) + } + if len(pool.CAs) == 0 { + t.Fatal("service-DNS CA pool is empty") + } + material := testcert.ServerTLSWithCA(t, pool.CAs[0], net.ParseIP(reserved.ClusterIP), spec.Name+"."+spec.Namespace+".svc.cluster.local") + rootCA = string(material.RootCA) + originSecret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "origin-tls", Namespace: spec.Namespace}, Type: corev1.SecretTypeTLS, + Data: map[string][]byte{corev1.TLSCertKey: material.Certificate, corev1.TLSPrivateKeyKey: material.PrivateKey}} + if _, err := e2e.GetClients().K8s.CoreV1().Secrets(spec.Namespace).Create(ctx, originSecret, metav1.CreateOptions{}); err != nil { + t.Fatalf("creating origin credentials: %v", err) + } + spec.Volumes = []corev1.Volume{{Name: "tls", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: originSecret.Name}}}} + spec.VolumeMounts = []corev1.VolumeMount{{Name: "tls", MountPath: "/run/origin-tls", ReadOnly: true}} + spec.Args = append(spec.Args, "--tls-cert=/run/origin-tls/tls.crt", "--tls-key=/run/origin-tls/tls.key") + spec.HealthScheme = corev1.URISchemeHTTPS + } + target := e2e.DeployServerPod(t, ctx, spec) + if encrypted && target.ClusterIP != reserved.ClusterIP { + t.Fatalf("Service IP changed from %s to %s after certificate issuance", reserved.ClusterIP, target.ClusterIP) + } + e2e.WaitForServerEndpoint(t, ctx, spec) + family := "IPv6" + if net.ParseIP(target.ClusterIP).To4() != nil { + family = "IPv4" + } + t.Logf("origin %s/%s at %s (%s), TLS=%v", target.Namespace, spec.Name, target.Address(), family, encrypted) + address := protocolOriginRequest(target, spec.Name) + return target, address, rootCA +} + +const ( + originDiagnosticLogLines = 80 + originDiagnosticLogBytes = 16 << 10 + originDiagnosticFieldBytes = 1024 + originDiagnosticItems = 20 +) + +func registerOriginDiagnostics(t *testing.T, namespace, name string) { + t.Helper() + t.Cleanup(func() { + if !t.Failed() { + return + } + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + collectOriginDiagnostics(t, ctx, namespace, name) + }) +} + +func collectOriginDiagnostics(t *testing.T, ctx context.Context, namespace, name string) { + t.Helper() + clients := e2e.GetClients().K8s + observed := time.Now().UTC().Format(time.RFC3339Nano) + t.Logf("origin diagnostics observed_at=%s namespace=%s name=%s", observed, namespace, name) + + pod, err := clients.CoreV1().Pods(namespace).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + t.Logf("origin diagnostics: get Pod %s/%s: %v", namespace, name, err) + } else { + deletion := "" + if pod.DeletionTimestamp != nil { + deletion = formatOriginTimestamp(*pod.DeletionTimestamp) + } + t.Logf("origin Pod status: uid=%s phase=%s pod_ip=%s host_ip=%s reason=%s message=%q created=%s deletion=%s containers=%s", pod.UID, pod.Status.Phase, pod.Status.PodIP, pod.Status.HostIP, pod.Status.Reason, boundedOriginDiagnosticField(pod.Status.Message), formatOriginTimestamp(pod.CreationTimestamp), deletion, formatOriginContainerStatuses(pod.Status.ContainerStatuses)) + for _, previous := range []bool{false, true} { + logs, logErr := clients.CoreV1().Pods(namespace).GetLogs(name, &corev1.PodLogOptions{ + Previous: previous, + TailLines: ptrInt64(originDiagnosticLogLines), + LimitBytes: ptrInt64(originDiagnosticLogBytes), + Timestamps: true, + }).DoRaw(ctx) + label := "current" + if previous { + label = "previous" + } + if logErr != nil { + t.Logf("origin %s logs unavailable: %v", label, logErr) + continue + } + t.Logf("origin %s logs (bounded):\n%s", label, boundedOriginDiagnosticText(string(logs), originDiagnosticLogLines, originDiagnosticLogBytes)) + } + } + + service, err := clients.CoreV1().Services(namespace).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + t.Logf("origin diagnostics: get Service %s/%s: %v", namespace, name, err) + } else { + t.Logf("origin Service: cluster_ip=%s ports=%s selector=%s", service.Spec.ClusterIP, formatOriginServicePorts(service.Spec.Ports), formatOriginSelector(service.Spec.Selector)) + } + + slices, err := clients.DiscoveryV1().EndpointSlices(namespace).List(ctx, metav1.ListOptions{LabelSelector: discoveryv1.LabelServiceName + "=" + name, Limit: 20}) + if err != nil { + t.Logf("origin diagnostics: list EndpointSlices for %s/%s: %v", namespace, name, err) + } else { + for _, slice := range slices.Items { + t.Logf("origin EndpointSlice: name=%s uid=%s created=%s ports=%s endpoints=%s", slice.Name, slice.UID, formatOriginTimestamp(slice.CreationTimestamp), formatOriginEndpointPorts(slice.Ports), formatOriginEndpoints(slice.Endpoints)) + } + } + + events, err := clients.CoreV1().Events(namespace).List(ctx, metav1.ListOptions{Limit: 50}) + if err != nil { + t.Logf("origin diagnostics: list namespace events %s: %v", namespace, err) + return + } + for _, event := range events.Items { + if event.InvolvedObject.Name != name { + continue + } + t.Logf("origin event: type=%s reason=%s message=%q count=%d last=%s", event.Type, event.Reason, boundedOriginDiagnosticField(event.Message), event.Count, event.LastTimestamp.UTC().Format(time.RFC3339Nano)) + } +} + +func ptrInt64(value int) *int64 { + result := int64(value) + return &result +} + +func boundedOriginDiagnosticText(value string, maxLines, maxBytes int) string { + lines := strings.Split(value, "\n") + if len(lines) > maxLines { + lines = lines[len(lines)-maxLines:] + } + result := strings.Join(lines, "\n") + if len(result) > maxBytes { + result = result[len(result)-maxBytes:] + } + return result +} + +func boundedOriginDiagnosticField(value string) string { + if len(value) <= originDiagnosticFieldBytes { + return value + } + return value[:originDiagnosticFieldBytes-3] + "..." +} + +func formatOriginServicePorts(ports []corev1.ServicePort) string { + values := make([]string, 0, len(ports)) + for _, port := range ports[:min(len(ports), originDiagnosticItems)] { + values = append(values, fmt.Sprintf("%s/%d->%s/%s", port.Name, port.Port, port.TargetPort.String(), port.Protocol)) + } + return boundedOriginDiagnosticField(strings.Join(values, ",")) +} + +func formatOriginSelector(selector map[string]string) string { + values := make([]string, 0, len(selector)) + for key, value := range selector { + values = append(values, boundedOriginDiagnosticField(key)+"="+boundedOriginDiagnosticField(value)) + } + slices.Sort(values) + if len(values) > originDiagnosticItems { + values = values[:originDiagnosticItems] + } + return boundedOriginDiagnosticField(strings.Join(values, ",")) +} + +func formatOriginEndpointPorts(ports []discoveryv1.EndpointPort) string { + values := make([]string, 0, len(ports)) + for _, port := range ports { + protocol := "" + if port.Protocol != nil { + protocol = string(*port.Protocol) + } + values = append(values, fmt.Sprintf("%s/%d/%s", boundedOriginDiagnosticField(valueString(port.Name)), valueInt32(port.Port), protocol)) + } + if len(values) > originDiagnosticItems { + values = values[:originDiagnosticItems] + } + return boundedOriginDiagnosticField(strings.Join(values, ",")) +} + +func formatOriginEndpoints(endpoints []discoveryv1.Endpoint) string { + values := make([]string, 0, len(endpoints)) + for _, endpoint := range endpoints { + ready, terminating := "unknown", "unknown" + if endpoint.Conditions.Ready != nil { + ready = strconv.FormatBool(*endpoint.Conditions.Ready) + } + if endpoint.Conditions.Terminating != nil { + terminating = strconv.FormatBool(*endpoint.Conditions.Terminating) + } + uid := "" + if endpoint.TargetRef != nil { + uid = string(endpoint.TargetRef.UID) + } + addresses := make([]string, 0, len(endpoint.Addresses)) + for _, address := range endpoint.Addresses { + addresses = append(addresses, boundedOriginDiagnosticField(address)) + } + if len(addresses) > originDiagnosticItems { + addresses = addresses[:originDiagnosticItems] + } + values = append(values, boundedOriginDiagnosticField(fmt.Sprintf("addresses=%s uid=%s ready=%s terminating=%s", strings.Join(addresses, ","), uid, ready, terminating))) + } + if len(values) > originDiagnosticItems { + values = values[:originDiagnosticItems] + } + return strings.Join(values, "; ") +} + +func formatOriginTimestamp(value metav1.Time) string { + if value.IsZero() { + return "" + } + return value.UTC().Format(time.RFC3339Nano) +} + +func formatOriginContainerStatuses(statuses []corev1.ContainerStatus) string { + values := make([]string, 0, len(statuses)) + for _, status := range statuses[:min(len(statuses), originDiagnosticItems)] { + values = append(values, boundedOriginDiagnosticField(fmt.Sprintf("%s ready=%t restarts=%d", status.Name, status.Ready, status.RestartCount))) + } + return strings.Join(values, "; ") +} + +func valueInt32(value *int32) int32 { + if value == nil { + return 0 + } + return *value +} + +func valueString(value *string) string { + if value == nil { + return "" + } + return *value +} + +// The unified gateway needs the origin's DNS name for SNI; the outer CONNECT +// log still records target.Address(). +func protocolOriginRequest(target e2e.Server, service string) string { + return net.JoinHostPort(service+"."+target.Namespace+".svc.cluster.local", strconv.Itoa(target.Port)) +} + +func TestProtocolOriginRequest(t *testing.T) { + for _, tc := range []struct { + name, ip string + }{ + {"IPv4", "10.0.0.7"}, + {"IPv6", "fd00::7"}, + } { + t.Run(tc.name, func(t *testing.T) { + target := e2e.Server{Namespace: "test", ClusterIP: tc.ip, Port: 8443} + if got, want := protocolOriginRequest(target, "origin"), "origin.test.svc.cluster.local:8443"; got != want { + t.Fatalf("origin request = %q, want %q", got, want) + } + }) + } +} + +func TestBoundedOriginDiagnosticText(t *testing.T) { + got := boundedOriginDiagnosticText("one\ntwo\nthree\nfour", 2, 16) + if got != "three\nfour" { + t.Fatalf("bounded diagnostic text = %q, want %q", got, "three\\nfour") + } + if got := boundedOriginDiagnosticText("123456789", 10, 4); got != "6789" { + t.Fatalf("byte-bounded diagnostic text = %q, want %q", got, "6789") + } +} + +func prepareProtocolActor(t *testing.T, ctx context.Context, prefix string) (string, *e2e.RouterClient, resources.ActorRef) { + t.Helper() + https8443 := e2e.EgressAllowHTTPS("*") + https8443.Https.Ports = &ateapipb.Ports{Numbers: []int32{8443}} + rules := append(e2e.EgressAllowAll(), https8443) + _, actorName, _ := createAndResumeActorWithEgress(t, ctx, prefix, egressFixture(), rules...) + router := mustRouterClient(t, ctx) + t.Cleanup(func() { router.Close() }) + ref := resources.ActorRef{Atespace: networkingAtespace, Name: actorName} + waitForRouteReady(t, "egress actor ingress readiness", func() (*http.Response, error) { return router.Get(ctx, ref, "/readyz") }) + return actorName, router, ref +} + +// postEgressOnce starts the measured operation only after readiness. Unlike +// postThroughEgressActor, an outbound failure is never retried. +func postEgressOnce(t *testing.T, ctx context.Context, router *e2e.RouterClient, actorRef resources.ActorRef, path string, input any) []byte { + return postEgressWithStatus(t, ctx, router, actorRef, path, input, http.StatusOK) +} + +func postEgressWithStatus(t *testing.T, ctx context.Context, router *e2e.RouterClient, actorRef resources.ActorRef, path string, input any, wantStatus int) []byte { + t.Helper() + payload, err := json.Marshal(input) + if err != nil { + t.Fatalf("encoding actor operation: %v", err) + } + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + response, err := router.PostJSON(ctx, actorRef, path, payload) + if err != nil { + t.Fatalf("actor %s operation %s transport failed: %v", actorRef.Name, path, err) + } + defer response.Body.Close() + raw, err := io.ReadAll(io.LimitReader(response.Body, (1<<20)+1)) + if err != nil { + t.Fatalf("reading actor operation: %v", err) + } + if len(raw) > 1<<20 { + t.Fatal("actor operation response exceeds 1 MiB") + } + if response.StatusCode != wantStatus { + t.Fatalf("actor %s operation %s returned HTTP %d, want %d: %s", actorRef.Name, path, response.StatusCode, wantStatus, raw) + } + return raw +} + +func assertProtocolGateway(t *testing.T, ctx context.Context, since metav1.Time, actorName, authority string) { + t.Helper() + actorRef := resources.ActorRef{Atespace: networkingAtespace, Name: actorName} + identity := resources.AteomForActorSPIFFEID(actorRef).String() + waitForAccessLog(t, ctx, since, "successful gateway traffic for "+identity+" to "+authority, func(lines []gatewayAccessLogLine) bool { + for _, entry := range lines { + if protocolGatewayAccessLogMatches(entry, actorRef, authority, false) { + t.Logf("gateway evidence: %s", entry.text) + return true + } + } + return false + }) +} + +func protocolGatewayAccessLogMatches(entry gatewayAccessLogLine, actorRef resources.ActorRef, authority string, allowOpaque bool) bool { + switch entry.container { + case "envoy": + gotAuthority, _ := accessLogField(entry.text, "authority") + code, _ := accessLogField(entry.text, "code") + peers, _ := accessLogField(entry.text, "peer_san") + if gotAuthority != authority || code != "200" { + return false + } + for peer := range strings.SplitSeq(peers, ",") { + if peer == resources.AteomForActorSPIFFEID(actorRef).String() { + return true + } + } + case "agentgateway": + gotAuthority, _ := accessLogField(entry.text, "substrate.connect.authority") + if allowOpaque && strings.Contains(entry.text, "CONNECT tunnel terminated") { + // Baseline TCP routes carry opaque CONNECT traffic and therefore do + // not emit an HTTP status. The accepted CONNECT record identifies the + // authenticated actor and destination; the operation assertion above + // supplies end-to-end success. + target, targetOK := accessLogField(entry.text, "target") + tunnelActor, actorOK := accessLogField(entry.text, "actor_name") + tunnelSpace, spaceOK := accessLogField(entry.text, "atespace") + _, errorOK := accessLogField(entry.text, "error") + return targetOK && target == authority && actorOK && tunnelActor == actorRef.Name && spaceOK && tunnelSpace == actorRef.Atespace && !errorOK + } + gotActor, _ := accessLogField(entry.text, "ate.actor.name") + gotAtespace, _ := accessLogField(entry.text, "ate.atespace") + status, _ := accessLogField(entry.text, "http.status") + if gotAuthority != authority || gotActor != actorRef.Name || gotAtespace != actorRef.Atespace { + return false + } + if status == "200" || status == "101" { + return true + } + return false + } + return false +} diff --git a/internal/e2e/suites/networking/websocketegress_test.go b/internal/e2e/suites/networking/websocketegress_test.go new file mode 100644 index 0000000000..bbef856212 --- /dev/null +++ b/internal/e2e/suites/networking/websocketegress_test.go @@ -0,0 +1,95 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package networking + +import ( + "encoding/json" + "net/http" + "testing" + + "github.com/agent-substrate/substrate/internal/e2e" +) + +// egressWebSocketResponse mirrors observations returned by the actor. The +// outer HTTP status is already checked before these protocol assertions run. +type egressWebSocketResponse struct { + StatusCode int `json:"statusCode"` + Protocol string `json:"protocol"` + HandshakeBody string `json:"handshakeBody"` + TLS bool `json:"tls"` + Messages []egressWebSocketMessage `json:"messages"` + Error string `json:"error"` +} + +type egressWebSocketMessage struct { + Type int `json:"type"` + Message string `json:"message"` +} + +func TestActorEgressWebSocket(t *testing.T) { + ctx := t.Context() + _, address, _ := prepareProtocolOrigin(t, ctx, e2e.ServerPod{ + Name: "ws-origin", ImportPath: "github.com/agent-substrate/substrate/internal/e2e/fixtures/testserver", + Args: []string{"websocket", "--echo"}, Port: 80, TargetPort: 8080, HealthPath: "/readyz", + }, false) + _, router, actorRef := prepareProtocolActor(t, ctx, "ws") + raw := postEgressWithStatus(t, ctx, router, actorRef, "/websocket", map[string]any{ + "url": "ws://" + address + "/ws", "messages": []string{"ws-message"}, + }, http.StatusBadGateway) + var got egressWebSocketResponse + if err := json.Unmarshal(raw, &got); err != nil { + t.Fatalf("decoding WebSocket observation: %v", err) + } + if got.StatusCode != http.StatusForbidden { + t.Errorf("WebSocket handshake status = %d, want %d", got.StatusCode, http.StatusForbidden) + } + if got.HandshakeBody != "WebSocket egress is not supported" { + t.Errorf("WebSocket handshake body = %q, want %q", got.HandshakeBody, "WebSocket egress is not supported") + } + if got.Error == "" { + t.Error("WebSocket handshake error is empty") + } + if len(got.Messages) != 0 { + t.Errorf("WebSocket messages = %+v, want no echoes", got.Messages) + } +} + +func TestActorEgressSecureWebSocket(t *testing.T) { + ctx := t.Context() + _, address, _ := prepareProtocolOrigin(t, ctx, e2e.ServerPod{ + Name: "wss-origin", ImportPath: "github.com/agent-substrate/substrate/internal/e2e/fixtures/testserver", + Args: []string{"websocket", "--echo"}, Port: 443, TargetPort: 8443, HealthPath: "/readyz", + }, true) + _, router, actorRef := prepareProtocolActor(t, ctx, "wss") + raw := postEgressWithStatus(t, ctx, router, actorRef, "/websocket", map[string]any{ + "url": "wss://" + address + "/ws", "messages": []string{"wss-message"}, + }, http.StatusBadGateway) + var got egressWebSocketResponse + if err := json.Unmarshal(raw, &got); err != nil { + t.Fatalf("decoding secure WebSocket observation: %v", err) + } + if got.StatusCode != http.StatusForbidden { + t.Errorf("secure WebSocket handshake status = %d, want %d", got.StatusCode, http.StatusForbidden) + } + if got.HandshakeBody != "WebSocket egress is not supported" { + t.Errorf("secure WebSocket handshake body = %q, want %q", got.HandshakeBody, "WebSocket egress is not supported") + } + if got.Error == "" { + t.Error("secure WebSocket handshake error is empty") + } + if len(got.Messages) != 0 { + t.Errorf("secure WebSocket messages = %+v, want no echoes", got.Messages) + } +} diff --git a/internal/testcert/servertls.go b/internal/testcert/servertls.go new file mode 100644 index 0000000000..e5783c0371 --- /dev/null +++ b/internal/testcert/servertls.go @@ -0,0 +1,81 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testcert + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "encoding/pem" + "math/big" + "net" + "testing" + "time" + + "github.com/agent-substrate/substrate/internal/localca" +) + +// ServerTLS holds an origin key pair and the public CA trusted by its client. +type ServerTLS struct{ RootCA, Certificate, PrivateKey []byte } + +// NewServerTLS issues an origin certificate for the allocated Service IP. +// Only the public CA is returned; its signing key stays in this process. +func NewServerTLS(t *testing.T, ip net.IP) ServerTLS { + t.Helper() + ca, err := localca.GenerateCA("e2e-origin", localca.KeyTypeECDSAP256, 24*time.Hour) + if err != nil { + t.Fatalf("generating origin CA: %v", err) + } + return ServerTLSWithCA(t, ca, ip) +} + +// ServerTLSWithCA issues a Service IP and optional DNS certificate from a CA +// already trusted by the client or the MITM gateway's upstream TLS connection. +func ServerTLSWithCA(t *testing.T, ca *localca.CA, ip net.IP, dnsNames ...string) ServerTLS { + t.Helper() + if ip == nil { + t.Fatal("origin certificate requires a Service IP") + } + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatalf("generating origin key: %v", err) + } + serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) + if err != nil { + t.Fatalf("generating origin serial: %v", err) + } + leaf := &x509.Certificate{ + SerialNumber: serial, + IPAddresses: []net.IP{ip}, + DNSNames: dnsNames, + NotBefore: time.Now().Add(-time.Minute), NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + } + der, err := x509.CreateCertificate(rand.Reader, leaf, ca.RootCertificate, key.Public(), ca.SigningKey) + if err != nil { + t.Fatalf("issuing origin certificate: %v", err) + } + privateDER, err := x509.MarshalPKCS8PrivateKey(key) + if err != nil { + t.Fatalf("encoding origin key: %v", err) + } + return ServerTLS{ + RootCA: pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: ca.RootCertificate.Raw}), + Certificate: pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), + PrivateKey: pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: privateDER}), + } +} diff --git a/internal/testcert/servertls_test.go b/internal/testcert/servertls_test.go new file mode 100644 index 0000000000..fda0c12772 --- /dev/null +++ b/internal/testcert/servertls_test.go @@ -0,0 +1,86 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testcert + +import ( + "crypto/tls" + "crypto/x509" + "net" + "testing" + "time" + + "github.com/agent-substrate/substrate/internal/localca" +) + +func TestNewServerTLS(t *testing.T) { + material := NewServerTLS(t, net.ParseIP("10.0.0.7")) + pair, err := tls.X509KeyPair(material.Certificate, material.PrivateKey) + if err != nil { + t.Fatalf("loading serving key pair: %v", err) + } + leaf, err := x509.ParseCertificate(pair.Certificate[0]) + if err != nil { + t.Fatal(err) + } + roots := x509.NewCertPool() + if !roots.AppendCertsFromPEM(material.RootCA) { + t.Fatal("no CA certificate") + } + if _, err := leaf.Verify(x509.VerifyOptions{Roots: roots, DNSName: "10.0.0.7"}); err != nil { + t.Fatalf("verifying Service IP: %v", err) + } + if _, err := leaf.Verify(x509.VerifyOptions{Roots: roots, DNSName: "10.0.0.8"}); err == nil { + t.Fatal("accepted a different Service IP") + } + if _, err := leaf.Verify(x509.VerifyOptions{Roots: x509.NewCertPool(), DNSName: "10.0.0.7"}); err == nil { + t.Fatal("accepted an untrusted certificate") + } + if leaf.IsCA { + t.Fatal("origin received a CA certificate") + } +} + +func TestServerTLSWithCA(t *testing.T) { + ca, err := localca.GenerateCA("service-dns", localca.KeyTypeECDSAP256, time.Hour) + if err != nil { + t.Fatal(err) + } + const name = "https-origin.networking-test.svc.cluster.local" + material := ServerTLSWithCA(t, ca, net.ParseIP("10.0.0.7"), name) + pair, err := tls.X509KeyPair(material.Certificate, material.PrivateKey) + if err != nil { + t.Fatal(err) + } + leaf, err := x509.ParseCertificate(pair.Certificate[0]) + if err != nil { + t.Fatal(err) + } + roots := x509.NewCertPool() + roots.AddCert(ca.RootCertificate) + for _, host := range []string{name, "10.0.0.7"} { + if _, err := leaf.Verify(x509.VerifyOptions{Roots: roots, DNSName: host}); err != nil { + t.Errorf("verifying %s against supplied CA: %v", host, err) + } + } + if _, err := leaf.Verify(x509.VerifyOptions{Roots: roots, DNSName: "another.namespace.svc.cluster.local"}); err == nil { + t.Fatal("accepted a different Service DNS name") + } + if _, err := leaf.Verify(x509.VerifyOptions{Roots: x509.NewCertPool(), DNSName: name}); err == nil { + t.Fatal("accepted an untrusted Service DNS certificate") + } + if leaf.IsCA { + t.Fatal("origin received a CA certificate") + } +} diff --git a/internal/volume/csi/tls_test.go b/internal/volume/csi/tls_test.go index ad71177441..5a56aac2c6 100644 --- a/internal/volume/csi/tls_test.go +++ b/internal/volume/csi/tls_test.go @@ -391,6 +391,12 @@ func TestCAPoolCache_HitAndFileChange(t *testing.T) { // Modify the file. writeFile(t, caPath, ca.certPEM()) + // The cache intentionally keys on mtime and size; make the file change + // observable on filesystems whose timestamp granularity is coarse. + modifiedAt := time.Now().Add(time.Second) + if err := os.Chtimes(caPath, modifiedAt, modifiedAt); err != nil { + t.Fatal(err) + } // 3rd call should detect file change and return a newly parsed pool. pool3, err := cache.getCertPool() diff --git a/manifests/ate-install/atenet-egress.yaml b/manifests/ate-install/atenet-egress.yaml index 3c988052b5..ffd6f0cff8 100644 --- a/manifests/ate-install/atenet-egress.yaml +++ b/manifests/ate-install/atenet-egress.yaml @@ -377,13 +377,12 @@ data: typed_config: "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager stat_prefix: mitm_http - # Absent this, Envoy answers an Upgrade request itself instead of - # proxying it, so every WebSocket through this leg fails at the - # handshake. Note it covers the HTTP/1.1 path only; WebSockets over - # h2 are RFC 8441 extended CONNECT and would additionally need - # http2_protocol_options.allow_connect. + # The egress request filter returns the explanatory 403. Keep + # the HCM default disabled as a defense if a request ever gets + # past that decision. upgrade_configs: - upgrade_type: websocket + enabled: false # Disable stream_idle_timeout as there is already an overall timeout in the outer tunnel in the HCM. stream_idle_timeout: 0s # This is the leg that knows where the traffic actually went. @@ -420,32 +419,15 @@ data: - name: all domains: ["*"] routes: - # Upgrade is what cannot cross an h2 upstream: forwarding one - # needs RFC 8441 extended CONNECT, which almost no origin - # advertises. So the default route below stays pinned to - # HTTP/1.1, which every origin accepts, rather than letting - # upstream ALPN choose -- ALPN takes h2 against nearly - # everything today, and every WebSocket then fails with a 502. - # - # gRPC is the one thing that default cannot serve: its status - # rides in trailers, which HTTP/1.1 cannot express. - # content-type is how it opts out. The prefix also catches - # application/grpc-web, which is deliberate -- - # egress_forward_proxy_grpc reads its protocol from upstream - # ALPN, so a grpc-web endpoint that only speaks HTTP/1.1 still - # gets HTTP/1.1. - # - # The cost of the pinned default is that non-gRPC h2 traffic - # is downgraded and its trailers dropped silently. Nothing - # sends those today. If that changes, invert this: match on - # the `upgrade` header, pin that route to HTTP/1.1, and let - # everything else negotiate. - # - # Each of the two exists once per dial the sidecar answers - # with (dev.ate.egress:dial), and there is no default: a - # request it did not answer for has no route. A by-name + # There is no default route: the initial route lookup cannot + # match before ext_proc supplies dev.ate.egress:dial. This + # lets the request filter return the unsupported-upgrade + # response before route-level upgrade rejection. A by-name # route also needs the answered name (dev.ate.egress:host); # without it the forward proxy would dial the Host's port. + # Native gRPC uses upstream ALPN to negotiate h2 and carry + # status in trailers. Other requests stay on HTTP/1.1; the + # content-type match keeps those protocol choices separate. - match: prefix: "/" headers: @@ -461,6 +443,9 @@ data: value: { present_match: true } route: cluster: egress_forward_proxy_grpc + upgrade_configs: + - upgrade_type: websocket + enabled: false timeout: 0s - match: prefix: "/" @@ -474,6 +459,9 @@ data: value: { string_match: { exact: address } } route: cluster: egress_original_dst_tls_grpc + upgrade_configs: + - upgrade_type: websocket + enabled: false timeout: 0s - match: prefix: "/" @@ -486,6 +474,9 @@ data: value: { present_match: true } route: cluster: egress_forward_proxy + upgrade_configs: + - upgrade_type: websocket + enabled: false # The route timeout runs until the response completes, which # for SSE or long-poll is never, so 0. timeout: 0s @@ -497,6 +488,9 @@ data: value: { string_match: { exact: address } } route: cluster: egress_original_dst_tls + upgrade_configs: + - upgrade_type: websocket + enabled: false timeout: 0s http_filters: # Every request on this chain is authorized against the actor's @@ -623,8 +617,12 @@ data: # answers it 426 before any filter runs. http_protocol_options: accept_http_10: true + # The egress request filter returns the explanatory 403. Keep + # the HCM default disabled as a defense if a request ever gets + # past that decision. upgrade_configs: - upgrade_type: websocket + enabled: false stream_idle_timeout: 0s access_log: - name: envoy.access_loggers.file @@ -670,6 +668,9 @@ data: value: { present_match: true } route: cluster: egress_forward_proxy_cleartext + upgrade_configs: + - upgrade_type: websocket + enabled: false timeout: 0s - match: prefix: "/" @@ -679,6 +680,9 @@ data: value: { string_match: { exact: address } } route: cluster: egress_original_dst_cleartext + upgrade_configs: + - upgrade_type: websocket + enabled: false timeout: 0s http_filters: # Every request on this chain is authorized against the actor's diff --git a/manifests/ate-install/components/agentgateway-egress/kustomization.yaml b/manifests/ate-install/components/agentgateway-egress/kustomization.yaml index 6d3c32efd9..28e0012914 100644 --- a/manifests/ate-install/components/agentgateway-egress/kustomization.yaml +++ b/manifests/ate-install/components/agentgateway-egress/kustomization.yaml @@ -59,6 +59,8 @@ patches: data: config.yaml: | frontendPolicies: + connect: + mode: tunnel accessLog: add: substrate.connect.authority: source.connectHeaders["host"] @@ -70,16 +72,20 @@ patches: key: /run/podidentity.podcert.ate.dev/credential-bundle.pem root: /run/servicedns-ca/trust-bundle.pem - binds: - - port: 8443 - tunnelProtocol: connect - listeners: - - protocol: HTTPS + # Keep the outer egress gateway on 8443, but bind it to this pod's + # address. Numeric CONNECT re-entry must select the wildcard + # internal bind instead of selecting this front door again. + gateways: + egress: + port: 8443 + bindAddress: $POD_IP + protocol: HTTPS tls: cert: /run/servicedns.podcert.ate.dev/credential-bundle.pem key: /run/servicedns.podcert.ate.dev/credential-bundle.pem root: /run/actor-id-ca-certs/ca.crt - routes: [] + + binds: - mode: internal protocol: AUTO listeners: @@ -89,10 +95,24 @@ patches: cert: /run/egress-mitm/tls.crt key: /run/egress-mitm/tls.key routes: + - name: reject-websocket-upgrade + matches: + - headers: + - name: Connection + value: + regex: '(?i)(^|,)\s*upgrade\s*(,|$)' + - name: Upgrade + value: + regex: '(?i)(^|,)\s*websocket\s*(,|$)' + policies: + directResponse: + status: 403 + body: WebSocket egress is not supported - backends: - dynamic: {} policies: - backendTLS: {} + backendTLS: + root: /etc/ssl/certs/ca-certificates.crt policies: substrateEgress: &egressPolicy #ATE_AGENTGATEWAY_CREDENTIAL_PROVIDERS @@ -104,6 +124,19 @@ patches: root: /run/servicedns-ca/trust-bundle.pem - protocol: HTTP routes: + - name: reject-websocket-upgrade + matches: + - headers: + - name: Connection + value: + regex: '(?i)(^|,)\s*upgrade\s*(,|$)' + - name: Upgrade + value: + regex: '(?i)(^|,)\s*websocket\s*(,|$)' + policies: + directResponse: + status: 403 + body: WebSocket egress is not supported - backends: - dynamic: target: source.connectHeaders["host"] diff --git a/manifests/ate-install/components/agentgateway/configmap.yaml b/manifests/ate-install/components/agentgateway/configmap.yaml index 1004c26c88..38bbe9b44f 100644 --- a/manifests/ate-install/components/agentgateway/configmap.yaml +++ b/manifests/ate-install/components/agentgateway/configmap.yaml @@ -138,6 +138,8 @@ data: config.yaml: | # yaml-language-server: $schema=https://agentgateway.dev/schema/config frontendPolicies: + connect: + mode: tunnel accessLog: add: substrate.connect.authority: source.connectHeaders["host"] @@ -149,18 +151,17 @@ data: key: /run/podidentity.podcert.ate.dev/credential-bundle.pem root: /run/servicedns-ca/trust-bundle.pem - binds: - # Authenticate the actor before accepting CONNECT. - - port: 8443 - tunnelProtocol: connect - listeners: - - protocol: HTTPS + gateways: + egress: + port: 8443 + bindAddress: $POD_IP + protocol: HTTPS tls: cert: /run/servicedns.podcert.ate.dev/credential-bundle.pem key: /run/servicedns.podcert.ate.dev/credential-bundle.pem root: /run/actor-id-ca-certs/ca.crt - # CONNECT re-enters the internal bind below. - routes: [] + + binds: # Route tunneled HTTP and policy-authorized TLS passthrough. - mode: internal @@ -174,6 +175,19 @@ data: target: source.connectHeaders["host"] - protocol: HTTP routes: + - name: reject-websocket-upgrade + matches: + - headers: + - name: Connection + value: + regex: '(?i)(^|,)\s*upgrade\s*(,|$)' + - name: Upgrade + value: + regex: '(?i)(^|,)\s*websocket\s*(,|$)' + policies: + directResponse: + status: 403 + body: WebSocket egress is not supported - backends: - dynamic: target: source.connectHeaders["host"] diff --git a/manifests/ate-install/components/agentgateway/kustomization.yaml b/manifests/ate-install/components/agentgateway/kustomization.yaml index 42fd917a0c..4a2a2ccfeb 100644 --- a/manifests/ate-install/components/agentgateway/kustomization.yaml +++ b/manifests/ate-install/components/agentgateway/kustomization.yaml @@ -125,6 +125,10 @@ patches: env: - name: RUST_LOG value: debug + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP ports: - name: https containerPort: 8443