diff --git a/.agents/skills/sync-upstream-main/SKILL.md b/.agents/skills/sync-upstream-main/SKILL.md index ff0129f3fd..b7c2078519 100644 --- a/.agents/skills/sync-upstream-main/SKILL.md +++ b/.agents/skills/sync-upstream-main/SKILL.md @@ -37,8 +37,8 @@ does not authorize merging it. ## Validate and investigate failures -Run `make verify` (which includes the race-enabled unit tests), -`make verify-helm-template`, and `make verify-crd-chart`. Use Go's default cache. +Run `go test -race ./...`, `hack/verify-all.sh`, +`hack/render-manifests.sh --check`, and `hack/verify/crd-chart.sh`. Use Go's default cache. Record failures and distinguish regressions from missing prerequisites and existing failures using evidence from the base revision when needed. diff --git a/.github/workflows/helm-e2e.yaml b/.github/workflows/helm-e2e.yaml index 7d73f49aee..af00a6e4da 100644 --- a/.github/workflows/helm-e2e.yaml +++ b/.github/workflows/helm-e2e.yaml @@ -63,7 +63,7 @@ jobs: # the chart composes {registry}/{repository}/{component}, so the local # registry serves each image where the default repository expects it -- # the same path-preserving rule a production mirror follows. - for component in ateapi atecontroller atelet podcertcontroller atenet credential-provider/kubernetes-secrets; do + for component in ateapi atecontroller atelet podcertcontroller atenet snapshot-plugin credential-provider/kubernetes-secrets; do KO_DOCKER_REPO="localhost:5001/kagent-dev/substrate/${component##*/}" \ ./hack/run-tool.sh ko build --bare --tags helm-e2e \ --platform linux/amd64 "./cmd/${component}" @@ -115,6 +115,8 @@ jobs: run: hack/install-ate-kind.sh --deploy-demo-counter - name: Deploy egress demo run: hack/install-ate-kind.sh --deploy-demo-egress + - name: Trust local E2E TLS origins + run: hack/setup-e2e-egress-tls-kind.sh - name: Run E2E tests (gVisor) # The egress credential suite replaces the shared provider policy. # Run it separately after the Helm suite has checked the chart's grants. @@ -139,8 +141,7 @@ jobs: - name: Require every lane to have run tests if: always() run: | - make build-junittool - bin/junittool verify -manifest "${ARTIFACTS}/expected-junit.txt" + go -C tools/junittool run . verify -manifest "${ARTIFACTS}/expected-junit.txt" - name: Upload test results if: always() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 diff --git a/.github/workflows/helm-verify.yaml b/.github/workflows/helm-verify.yaml index c5c40966c1..c7a5254f41 100644 --- a/.github/workflows/helm-verify.yaml +++ b/.github/workflows/helm-verify.yaml @@ -32,4 +32,4 @@ jobs: - name: Lint the chart run: helm lint charts/substrate charts/substrate-crds charts/substrate-podcert - name: Verify committed manifests match the chart output - run: make verify-helm-template + run: hack/render-manifests.sh --check diff --git a/.github/workflows/pr-workflow.yaml b/.github/workflows/pr-workflow.yaml index bb9825d5c5..6159da5b65 100644 --- a/.github/workflows/pr-workflow.yaml +++ b/.github/workflows/pr-workflow.yaml @@ -97,7 +97,7 @@ jobs: run: | sudo apt-get update -qq sudo apt-get install -y -qq --no-install-recommends clang libclang-dev - make test-dynamic-modules + hack/test-dynamic-modules.sh # Root-gated tests (overlay mounts, whiteout mknod, trusted.* xattrs, ...) # skip for the unprivileged runner user above; rerun the packages that # contain them (any test importing internal/roottest) under sudo. @@ -109,9 +109,7 @@ jobs: # file that was never written fails too. - name: Require tests to have run if: always() - run: | - make build-junittool - bin/junittool verify -manifest "${ARTIFACTS}/expected-junit.txt" + run: go -C tools/junittool run . verify -manifest "${ARTIFACTS}/expected-junit.txt" - name: Upload test results if: always() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 @@ -234,6 +232,8 @@ jobs: run: | hack/install-ate-kind.sh --deploy-demo-egress hack/install-ate-kind.sh --deploy-demo-egress-microvm + - name: Trust local TLS origins in egress gateway + run: hack/setup-e2e-egress-tls-kind.sh # scale the idle sandbox class's demo pools to zero so each lane has the node - name: Focus the node on gVisor run: hack/e2e-focus-sandbox-class.sh gvisor @@ -258,9 +258,7 @@ jobs: # run-e2e.sh refuses to run in CI without E2E_JUNIT_FILE. - name: Require every lane to have run tests if: always() - run: | - make build-junittool - bin/junittool verify -manifest "${ARTIFACTS}/expected-junit.txt" + run: go -C tools/junittool run . verify -manifest "${ARTIFACTS}/expected-junit.txt" - name: Save Go build cache # Main only, so PRs restore the latest main build rather than each # storing its own copy. diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index dc78ca8548..c4f3f29d51 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -88,7 +88,7 @@ jobs: run: | set -o errexit -o nounset -o pipefail - for component in ateapi atecontroller atelet ateom-gvisor ateom-microvm podcertcontroller atenet credential-provider/kubernetes-secrets; do + for component in ateapi atecontroller atelet ateom-gvisor ateom-microvm podcertcontroller atenet snapshot-plugin credential-provider/kubernetes-secrets; do KO_DOCKER_REPO="${IMAGE_REPOSITORY}/${component##*/}" \ ./hack/run-tool.sh ko build \ --tags "${IMAGE_TAGS}" \ diff --git a/.gitignore b/.gitignore index d680cfcc50..50656545db 100644 --- a/.gitignore +++ b/.gitignore @@ -60,8 +60,8 @@ Thumbs.db /.worktrees/ # Stray outputs from `go build` without -o. Root-module tools land at the repo -# root; tools with their own go.mod land beside their source. Use -# `make build-`, which writes to /bin/ above. +# root; tools with their own go.mod land beside their source. Build with +# `-o bin/`, which is ignored above. /validate-image-cache /setup-gcp /tools/apitool/apitool diff --git a/.golangci-kal.yaml b/.golangci-kal.yaml index 19efe7d4ac..53ab0456ff 100644 --- a/.golangci-kal.yaml +++ b/.golangci-kal.yaml @@ -96,7 +96,9 @@ linters: - path: 'pkg/api/v1alpha1/sandboxconfig_types\.go' text: '^requiredfields: .*\bAssetFile\.(SHA256|URL)\b' - path: 'pkg/api/v1alpha1/sandboxconfig_types\.go' - text: '^(nomaps|optionalfields|requiredfields): .*\bSandboxConfigSpec\.(Assets|PauseImage|SandboxClass)\b' + text: '^(optionalfields|requiredfields): .*\bSandboxConfigSpec\.SandboxClass\b' + - path: 'pkg/api/v1alpha1/sandboxconfig_types\.go' + text: '^(nomaps|optionalfields): .*\bSandboxVersionConfig\.(Assets|PauseImage)\b' - path: 'pkg/api/v1alpha1/sandboxconfig_types\.go' text: '^(nonpointerstructs|requiredfields): .*\bSandboxConfig\.Spec\b' - path: 'pkg/api/v1alpha1/csidriverconfig_types\.go' @@ -112,7 +114,7 @@ linters: - path: 'pkg/api/v1alpha1/workerpool_types\.go' text: '^optionalfields: .*\bWorkerPoolPodTemplate\.PriorityClassName\b' - path: 'pkg/api/v1alpha1/workerpool_types\.go' - text: '^(optionalfields|requiredfields): .*\bWorkerPoolSpec\.(Replicas|SandboxClass|WorkerImage)\b' + text: '^(optionalfields|requiredfields): .*\bWorkerPoolSpec\.(Replicas|WorkerImage)\b' - path: 'pkg/api/v1alpha1/workerpool_types\.go' text: '^optionalfields: .*\bWorkerPoolStatus\.(ReadyReplicas|Replicas|Selector)\b' - path: 'pkg/api/v1alpha1/workerpool_types\.go' diff --git a/AGENTS.md b/AGENTS.md index 7addcfdb4e..010fb62b70 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -40,22 +40,21 @@ See `docs/dev/code-layout.md` for the full rationale and per-directory details. ## Build and Test Commands -Agent Substrate uses a `Makefile` for its build and test tasks. +The dev entry points are the scripts under `hack/` and the `cmd/ate-setup` installer. -### Building -- **Binaries**: `make build` (builds images and `kubectl-ate`) or `make build-atectl` -- **Images**: `make build-images` (uses ko to build container images) -- **Demos**: `make build-demos` -- **Release images**: `make build-release-images KO_DOCKER_REPO=REPO VERSION=TAG` (every image a pre-built install needs, all tagged `TAG`, including the docker-built `envoy-dataplane`; `make build-envoy-dataplane` builds only that one) +### Building and Installing +- **Install from source**: `go run ./cmd/ate-setup deploy ate-system` (or the `hack/install-ate.sh` shim) builds every image with ko as part of the install; see `cmd/ate-setup/README.md` +- **Binaries**: `go build -o bin/kubectl-ate ./cmd/kubectl-ate`, `go build -o bin/ate-setup ./cmd/ate-setup` +- **Release images**: `VERSION=TAG go run ./cmd/ate-setup publish release-images --ko-docker-repo REPO` (every image a pre-built install needs, all tagged `TAG`, including the docker-built `envoy-dataplane`) ### Testing and Verification -- **Run Unit Tests**: `make test` -- **Run E2E Tests**: `make e2e` (Requires GCP cluster setup and built images) -- **Run Linters and Verifiers**: `make verify` (Includes `go vet` and checks for formatting, boilerplate headers, licenses, and go modules) +- **Run Unit Tests**: `go test -race ./...` +- **Run E2E Tests**: `hack/run-e2e.sh` (Requires a cluster with Agent Substrate installed; `hack/run-e2e-kind.sh` for kind) +- **Run Linters and Verifiers**: `hack/verify-all.sh` (formatting, golangci-lint, boilerplate headers, licenses, go modules, generated code, and the metrics registry) ## Code Style Guidelines -- **Go Formatting**: Code must be formatted with `gofmt`. Run `make fmt` to automatically format all files before submitting changes. +- **Go Formatting**: Code must be formatted with `gofmt`. Run `hack/update/gofmt.sh` to automatically format all files before submitting changes. - **Copyright Headers**: All files must contain appropriate copyright and license headers. See templates in `hack/boilerplate/`. - **Modularity**: Submit small, focused Pull Requests that touch a limited part of the codebase for easier reviews and rebasing. - **Go Modules**: Ensure `go.mod` is clean. Run `go mod tidy` if adding or removing dependencies. @@ -79,7 +78,7 @@ Do not add `reserved` statements to protos, deprecated aliases, fallbacks for ol `docs/metrics/registry/` is an [OpenTelemetry Weaver](https://github.com/open-telemetry/weaver) registry. `metrics.yaml` defines every metric instrument the ate system components emit, plus every attribute any signal uses and the permitted values of each. `events.yaml` defines the log events emitted over OTLP. Read them to find an instrument, an event, or their attributes. -If you add or rename an instrument or an event, or add an attribute, follow `docs/dev/best-practices/metrics.md`, update the registry, and run `hack/verify/metrics.sh`. `make verify` runs the same check. Weaver reads the whole directory, so a new file in it is checked straight away. +If you add or rename an instrument or an event, or add an attribute, follow `docs/dev/best-practices/metrics.md`, update the registry, and run `hack/verify/metrics.sh`. `hack/verify-all.sh` runs the same check. Weaver reads the whole directory, so a new file in it is checked straight away. Some attributes are defined in the registry but barred from metric labels: actor identity is the main one. The `note` on each says so, and `cardinality_rules` in `docs/metrics/substrate.yaml` is where the bar lives. @@ -91,7 +90,7 @@ See the [metric registry](docs/observability.md#the-metric-registry) section of 1. Write tests for all new code. We will not merge code that lacks tests. 2. Ensure changes do not break existing tests. -3. Run `make verify` locally before requesting a code review to catch common issues like missed copyright headers or formatting drift. +3. Run `go test -race ./...` and `hack/verify-all.sh` locally before requesting a code review to catch common issues like missed copyright headers or formatting drift. 4. For end-to-end tests involving the actual infrastructure, ensure you have a running cluster (setup via `hack/ate-dev-env.sh.example` and `go run ./tools/setup-gcp bootstrap`). 5. A test that skips when a precondition is missing — Docker, a cluster, an artifact directory — must **fail** on it in CI, because a skip and a pass are the same exit code. Resolve strictness through a named predicate rather than an inline `os.Getenv("CI")`; `cmd/ateapi/internal/store/dockerenv.Required()` is the reference implementation. Prove the strict branch red before merging: a guard only ever observed passing is not known to guard anything. See `docs/dev/best-practices/ci-fail-closed.md`. diff --git a/COLLABORATING.md b/COLLABORATING.md index b59db85d28..634dd929ab 100644 --- a/COLLABORATING.md +++ b/COLLABORATING.md @@ -35,17 +35,6 @@ When writing, please strive for clarity and conciseness. Not everyone in the community will have the same proficiency in English, so please be mindful of that when writing. -## Always use Pull Requests - -Do not push directly to the main branch. Always use pull requests to propose -changes to the codebase. This allows for code review and discussion before -changes are merged, which helps maintain the quality of the codebase and -fosters collaboration. - -### Merging PRs - -PRs should be merged by the reviewer of a PR, not the author. - ### Using AI For the time being, the Substrate project uses the same AI contribution policy @@ -76,8 +65,16 @@ project](https://www.kubernetes.dev/docs/guide/pull-requests/#ai-guidance): ## Code reviews We want to move quickly, but we need to balance that with quality and -maintainability. Code reviews are an important part of that process. All PRs -should be reviewed BEFORE they are merged. +maintainability. Code reviews are an important part of that process. + +Agent Substrate requires at least one approving GitHub review from a maintainer +in order for the PR to be eligible for the merge queue. It is allowable to use +the "merge when ready" functionality to mark the PR as eligible for the merge +queue as soon as all preconditions are met. + +Note that most changes to a PR will require a new approving review. Use the +"re-request review" button to automatically place the PR back into your +reviewer's GitHub inbox. ### Code reviews across companies diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7668a5d809..7cb826aa83 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -31,7 +31,9 @@ The [Quickstart (Development)](README.md#quickstart-development) in the README covers bringing up a local cluster with the default (gVisor) runtime. To run the microVM runtime locally — which needs `/dev/kvm`, or Lima nested virtualization on Apple Silicon — see -[docs/dev/microvm-local.md](docs/dev/microvm-local.md). +[docs/dev/microvm-local.md](docs/dev/microvm-local.md). To measure where a +suspend or resume spends its time on such a cluster, follow +[docs/dev/suspend-resume-phase-breakdown.md](docs/dev/suspend-resume-phase-breakdown.md). ## Contribution process diff --git a/Makefile b/Makefile deleted file mode 100644 index d1f6423ae0..0000000000 --- a/Makefile +++ /dev/null @@ -1,199 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Default project ID, can be overridden -PROJECT_ID ?= $(shell echo $${USER}-gke-dev) - -# Ko configuration -export KO_DOCKER_REPO := gcr.io/$(PROJECT_ID)/ate-images - -# Go commands -GO := go -KO := hack/run-tool.sh ko - -# Flags every ko image build gets, e.g. `make build-images KO_FLAGS=--push=false`. -# Empty by default, so ko runs on its own defaults and whatever .ko.yaml configures. -KO_FLAGS ?= - -# Image naming, kept out of KO_FLAGS so that overriding those does not drop it. -# Each image is published as /, which is -# the name `ate-setup deploy --image-repo` looks for. ko's default appends an -# md5 of the full import path instead, which nothing outside ko can predict. -# cmd/ate-setup/internal/ko passes the same flag. -KO_NAMING := --base-import-paths - -# Image tags, kept out of KO_FLAGS for the same reason. Empty by default, so ko -# tags `latest`; build-release-images sets it to $(VERSION). -KO_TAGS := - -# Binaries -BINDIR := bin/ -ATECTL := $(BINDIR)/kubectl-ate -ATESETUP := $(BINDIR)/ate-setup - -# Version stamping. Override on the make command line to pin -# (e.g. `make VERSION=v0.5.0 build`). -VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo dev) -VERSION_PKG := github.com/agent-substrate/substrate/internal/version -LDFLAGS := -X=$(VERSION_PKG).Version=$(VERSION) - -# Every image the installer can deploy, defined once. These two sets together -# have to cover images.Components in cmd/ate-setup/internal/images: a package -# missing here has no image for a build from source, and one missing there has -# none for an install from a release. envoy-dataplane is built from a -# Dockerfile, not with ko, so it is in neither; see build-envoy-dataplane. -CONTROL_PLANE_IMAGES := ./cmd/ateapi \ - ./cmd/atecontroller \ - ./cmd/atelet \ - ./cmd/atenet \ - ./cmd/credential-provider/kubernetes-secrets \ - ./cmd/podcertcontroller -WORKER_IMAGES := ./cmd/ateom-gvisor \ - ./cmd/ateom-microvm -DEMO_IMAGES := ./demos/counter \ - ./demos/egress \ - ./demos/multi-template/fspersist \ - ./demos/sandbox -ALL_IMAGES := $(CONTROL_PLANE_IMAGES) $(WORKER_IMAGES) - -# Developer builds may leave components out, e.g. the microvm image, the one -# image built from a debian base rather than distroless static: -# make build-images SKIP_IMAGES=./cmd/ateom-microvm -# Overriding IMAGES or DEMOS on the command line builds exactly that set. -SKIP_IMAGES ?= -IMAGES := $(filter-out $(SKIP_IMAGES),$(ALL_IMAGES)) -DEMOS := $(filter-out $(SKIP_IMAGES),$(DEMO_IMAGES)) - -# Images ko cannot build: envoy-dataplane is Envoy plus a Rust dynamic module. -# It is pushed as $(KO_DOCKER_REPO)/envoy-dataplane:$(VERSION), beside the ko -# images, where `ate-setup deploy --image-repo --image-tag` looks for it. A -# build from source doesn't need this; ate-setup builds the image itself. -# -# Platforms default to KO_DEFAULTPLATFORMS, else linux/amd64, as in ate-setup. -# Another architecture compiles Rust under QEMU, which the build host must have -# registered with binfmt. Extra buildx flags go in DOCKER_BUILD_FLAGS. -DOCKERFILE_PLATFORMS ?= $(or $(KO_DEFAULTPLATFORMS),linux/amd64) -DOCKER_BUILD_FLAGS ?= - -.PHONY: all -all: build - -.PHONY: build -build: build-images build-atectl build-ate-setup - -.PHONY: build-images -build-images: - $(KO) build $(KO_NAMING) $(KO_TAGS) $(KO_FLAGS) \ - --ldflags="$(LDFLAGS)" \ - $(IMAGES) - -.PHONY: build-atectl -build-atectl: - $(GO) build -ldflags "$(LDFLAGS)" -o $(ATECTL) ./cmd/kubectl-ate - -# The cluster installer. hack/install-ate.sh is a shim over it; see -# cmd/ate-setup/commands.md for the flag-by-flag mapping between the two. -.PHONY: build-ate-setup -build-ate-setup: - $(GO) build -ldflags "$(LDFLAGS)" -o $(ATESETUP) ./cmd/ate-setup - -.PHONY: build-atenet -build-atenet: - $(GO) build -ldflags "$(LDFLAGS)" -o $(BINDIR)/atenet ./cmd/atenet - -# Registers the JUnit files a CI run will write, and verifies they hold tests. -# Separate module, so it builds by path. -o keeps the binary in $(BINDIR), -# which is gitignored and removed by `clean`. -.PHONY: build-junittool -build-junittool: - $(GO) -C tools/junittool build -o $(CURDIR)/$(BINDIR)/junittool . - -.PHONY: build-demos -build-demos: - $(KO) build $(KO_NAMING) $(KO_TAGS) $(KO_FLAGS) \ - --ldflags="$(LDFLAGS)" \ - $(DEMOS) - -.PHONY: build-envoy-dataplane -build-envoy-dataplane: - docker buildx build --push $(DOCKER_BUILD_FLAGS) \ - --platform=$(DOCKERFILE_PLATFORMS) \ - -t $(KO_DOCKER_REPO)/envoy-dataplane:$(VERSION) \ - cmd/dataplane/envoy - -# Every image a pre-built install needs, all tagged $(VERSION), which is what -# `ate-setup deploy --image-repo $(KO_DOCKER_REPO) --image-tag $(VERSION)` -# installs. Stage a release with e.g. -# make build-release-images KO_DOCKER_REPO=REPO VERSION=TAG -.PHONY: build-release-images -build-release-images: KO_TAGS = --tags=$(VERSION) -build-release-images: build-images build-demos build-envoy-dataplane - -.PHONY: test -test: - $(GO) test -race ./... - -# The Envoy dynamic modules are Rust. CI runs this target. -.PHONY: test-dynamic-modules -test-dynamic-modules: - hack/test-dynamic-modules.sh - -.PHONY: e2e -e2e: build build-demos - hack/run-e2e.sh - -.PHONY: fmt verify-fmt - -# Prints the Go ldflags (used for scripts to do version stamping). -ldflags: - @for flag in $(LDFLAGS); do \ - echo $$flag; \ - done - -# Formats all Go files in the project -fmt: - @./hack/update/gofmt.sh - -# Fails if any Go files are not formatted properly -verify-fmt: - @./hack/verify/gofmt.sh - -.PHONY: lint - -# Runs golangci-lint and fails on any reported issues. -lint: - @./hack/verify/golangci-lint.sh - -.PHONY: verify -verify: test - bash hack/verify-all.sh - -.PHONY: clean -clean: - rm -rf $(BINDIR) - -# Render the application and certificate-controller charts into manifests/ate-install/. -.PHONY: helm-template -helm-template: - @./hack/render-manifests.sh - -# Verify that manifests/ate-install/ matches the chart output. Used in CI. -.PHONY: verify-helm-template -verify-helm-template: - @./hack/render-manifests.sh --check - -# Verify that the CRD chart mirrors the generated CRDs. -.PHONY: verify-crd-chart -verify-crd-chart: - @./hack/verify/crd-chart.sh diff --git a/README.md b/README.md index cbae8c646b..251188fed9 100644 --- a/README.md +++ b/README.md @@ -264,7 +264,7 @@ We provide several sample applications demonstrating Agent Substrate's capabilit * [Enabling man-in-the-middle (MITM) interception for Actor Egress policy](docs/egress-trust-bundle.md): Egress policies such as header injection depend on MITM interception of Actor traffic. This guide explains how an Actor should be configured to enable interception. * [Request Parking](docs/request-parking.md): How the router parks requests through transient worker-pool saturation. * [Rolling Upgrade Runbook](docs/upgrade.md): Upgrade a running substrate node by node without losing actor state. -* [Threat Model](docs/threat-model.md): Trust boundaries, assumptions, and known risks. +* [Security Guide](docs/security.md): Security overview, security principles, and threat modeling. * [Roadmap](docs/roadmap.md): Current limitations and what is planned next. * [Benchmarking Guide](benchmarking/README.md): Locust-based load tests, monitoring stack, and the orchestrated benchmark harness. diff --git a/benchmarking/README.md b/benchmarking/README.md index 5ebe072fb9..154aad7737 100644 --- a/benchmarking/README.md +++ b/benchmarking/README.md @@ -463,6 +463,16 @@ Neither the Kubernetes API nor Prometheus is required. If either is unreachable, or discovery was skipped, the affected fields are written as `null` and the run still succeeds. A `null` means the value was not measured. It never means zero. +## Suspend/resume phase breakdown + +`SuspendActor` and `ResumeActor` latency can be attributed to the phases +inside them (object-storage transfer, hypervisor snapshot/restore, rootfs +assembly, ...) from the structured log records atelet and ateom-microvm +emit. `analysis/collect_logs.sh` dumps the node logs of a run and +`analysis/phase_report.py` aggregates them into per-phase percentiles and +waterfalls of the slowest operations. See +[analysis/README.md](analysis/README.md). + ## Optional: Prometheus + Grafana Locust provides graphs, statistics, etc. via the UI. However, you @@ -497,3 +507,10 @@ repository root: ```bash python3 -m unittest discover -s benchmarking/locust/unit_tests ``` + +`analysis/test_phase_report.py` covers the phase-log report's parser and +aggregation: + +```bash +python3 -m unittest discover -s benchmarking/analysis +``` diff --git a/benchmarking/analysis/README.md b/benchmarking/analysis/README.md new file mode 100644 index 0000000000..957b0f9894 --- /dev/null +++ b/benchmarking/analysis/README.md @@ -0,0 +1,114 @@ +# Suspend/resume phase analysis + +This directory turns the node side's developer-facing timing logs into the +percentiles a benchmark run needs, without making the phases metric API. The +`ate.actor.{restore,checkpoint}.duration` histograms stop at +`ateom_restore` / `ateom_checkpoint` / `persist` / `download` on purpose: +phases inside those are implementation details of a runtime, and metrics are +a contract. The finer breakdown is emitted as structured log records instead, +and this is the consumer that aggregates them. + +Two joinable JSON log records feed it, each written by both layers: + +| record (`msg`) | emitter | keys | +|---|---|---| +| `Restore timing breakdown` | atelet and ateom-microvm | `ate.actor.restore.duration.` / `ateom.actor.restore.duration.` | +| `Checkpoint timing breakdown` | atelet and ateom-microvm | `ate.actor.checkpoint.duration.` / `ateom.actor.checkpoint.duration.` | + +Every record carries the full actor identity (`ate.actor.uid`, name, +atespace, template) and the snapshot scope, which the histograms are barred +from, so records join per actor and per operation. Durations are float +seconds, the histograms' unit. A phase that never ran is absent, not zero. +A failed atelet operation still writes its record, marked with `error.type` +(the gRPC code); the report excludes those from the percentiles. + +## Making a measurement + +1. Run a suspend/resume-heavy load (e.g. the `glutton` or `sweperf` user + class; see `../README.md`). +2. Dump the node logs for the run's window while the pods still exist. + `automation/orchestrator.py` deletes the workload and ate-system pods right + after each test, and does not call this script yet, so run it before that + teardown (or against a cluster you manage yourself): + + ```bash + ./collect_logs.sh --dest /tmp/run1 --since-time 2026-09-24T18:00:00Z + ``` + + The script sources `.ate-dev-env.sh` from the repository root when present + (set `NO_DEV_ENV` to skip) and honors `KUBECTL_CONTEXT`, like the `hack/` + scripts. `--since-time` (or a relative `--since 30m`) should cover the run + and nothing before it. `--namespace` (default `ate-system`) selects the + atelet pods and `--worker-namespace` (default `benchmark-workloads`) the + worker pods; the ateom records land in the worker pod's stdout. `kubectl logs` + returns only a container's current log file, so on a long or busy run + kubelet's rotation can drop earlier records — collect soon after the run. + A container that restarted mid-run is dumped twice (`.previous.log`). + + On GKE the same records are in Cloud Logging, which keeps them past + rotation and teardown; an export is accepted as input directly, instead of + the kubectl dumps: + + ```bash + gcloud logging read '(jsonPayload.msg="Checkpoint timing breakdown" OR jsonPayload.msg="Restore timing breakdown") AND resource.labels.cluster_name="" AND timestamp>="2026-09-24T18:00:00Z"' \ + --project --format json > /tmp/run1/export.json + ``` + +3. Aggregate, from the kubectl dumps: + + ```bash + python3 phase_report.py /tmp/run1/*.log --csv /tmp/run1/report + ``` + + or from the Cloud Logging export: + + ```bash + python3 phase_report.py /tmp/run1/export.json --csv /tmp/run1/report + ``` + + Use one source per run: the dumps and an export of the same pods hold the + same records, and passing both would count each twice. + +## Reading the report + +**Phase percentiles.** Per layer, operation, sandbox class, snapshot kind, +scope and phase: count, p50, p90, p95 and max. A `golden` restore downloads +the golden image and a `latest` one the actor's own, so they are separate +rows, as are gVisor and micro-VM checkpoints. The atelet rows split a checkpoint between +`sandbox_assets`, `ateom_checkpoint` and `persist`, and a restore between +`volume_mount`, `manifest_fetch`, `sandbox_assets`, `download`, `oci_unpack` +and `ateom_restore`. The ateom rows split the `ateom_*` phase further: +`prep` / `pause` / `snapshot` / `durable_dir` / `rootfs_upper` / `teardown` +for a checkpoint, `prep` / `bundles` / `upper_join` / `lowers` / `tap` / +`vmm_launch` / `vm_restore` / `resume` / `wakeup_probe` for a restore. + +Concurrency matters when reading them: the atelet restore phases overlap +(the download runs alongside the asset fetch and OCI unpack), the three +ateom checkpoint captures run concurrently on the paused guest (the paused +window costs their max), and the ateom restore phases are sequential. For +the two checkpoint layers the report derives an `unattributed` row: the total +minus what the logged phases account for, counting the concurrent captures +once. It is the time the instrumentation does not yet name. The ateom +records carry no snapshot kind of their own; a paired one takes its kind from +the atelet record, so both layers split into the same rows. + +**Waterfalls.** The slowest operations, with the ateom record of the same +actor nested under the atelet `ateom_*` phase and that phase's gap to the +ateom total (RPC and queueing between the layers). The ateom record is the +one written inside that phase's window (for a checkpoint, before `persist` +began), and each ateom record pairs with at most one operation, so an +operation whose own record is missing prints without one rather than with +another cycle's. Failed operations (`error.type` set) are left out of both +the percentiles and the waterfalls. Tail outliers that blow up in one phase +every time are systematic; different phases each time are environmental. + +`--csv` also writes `phase_percentiles.csv` for run-over-run comparison. + +The parser is deliberately tolerant: it scans any line for a JSON object and +matches on `msg`, so raw `kubectl logs` dumps (even with prefixes) work. + +## Tests + +```bash +python3 -m unittest discover -s benchmarking/analysis +``` diff --git a/benchmarking/analysis/collect_logs.sh b/benchmarking/analysis/collect_logs.sh new file mode 100755 index 0000000000..e0555ca15b --- /dev/null +++ b/benchmarking/analysis/collect_logs.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Dump the node-side logs a benchmark run needs for phase_report.py: every +# atelet pod (the atelet-side timing breakdowns) and every worker pod +# (ateom-microvm writes its records to the worker pod's stdout). Point +# --since-time (RFC 3339) or --since at the run's start so the report covers +# exactly one run. +# +# kubectl logs returns only a container's current log file: once kubelet +# rotates it under load, earlier records are gone, so collect soon after the +# run. A container that restarted mid-run is dumped twice, its previous log +# under .previous.log. On GKE, Cloud Logging keeps every record past +# rotation and teardown; phase_report.py reads `gcloud logging read +# --format json` output directly. +# +# Like the other hack scripts, this sources .ate-dev-env.sh from the repo root +# for the cluster settings unless NO_DEV_ENV is set, and respects +# KUBECTL_CONTEXT. +# +# Usage: collect_logs.sh --dest DIR [--since 30m | --since-time 2026-09-24T18:00:00Z] +# [--namespace ate-system] [--worker-namespace benchmark-workloads] + +set -uo pipefail + +ROOT="$(git rev-parse --show-toplevel 2>/dev/null || pwd)" +if [[ -r "${ROOT}/.ate-dev-env.sh" ]] && [[ -z "${NO_DEV_ENV:-}" ]]; then + # shellcheck source=/dev/null + source "${ROOT}/.ate-dev-env.sh" +fi +run_kubectl() { kubectl ${KUBECTL_CONTEXT:+--context=${KUBECTL_CONTEXT}} "$@"; } + +DEST="" +SINCE="1h" +SINCE_TIME="" +NAMESPACE="ate-system" +WORKER_NAMESPACE="benchmark-workloads" + +while [[ $# -gt 0 ]]; do + case "$1" in + --dest) DEST="$2"; shift 2 ;; + --since) SINCE="$2"; shift 2 ;; + --since-time) SINCE_TIME="$2"; shift 2 ;; + --namespace) NAMESPACE="$2"; shift 2 ;; + --worker-namespace) WORKER_NAMESPACE="$2"; shift 2 ;; + *) echo "unknown flag: $1" >&2; exit 2 ;; + esac +done + +if [[ -z "${DEST}" ]]; then + echo "usage: $0 --dest DIR [--since 30m | --since-time RFC3339] [--namespace ate-system] [--worker-namespace benchmark-workloads]" >&2 + exit 2 +fi +mkdir -p "${DEST}" + +if [[ -n "${SINCE_TIME}" ]]; then + WINDOW=(--since-time="${SINCE_TIME}") +else + WINDOW=(--since="${SINCE}") +fi + +# One pod that cannot be read (still starting, evicted) must not cost the +# others, so every kubectl call warns and moves on rather than aborting. +collect() { + local ns="$1" selector="$2" + local pods + if ! pods=$(run_kubectl get pods -n "${ns}" -l "${selector}" -o name); then + echo "warn: could not list pods in ${ns} (${selector}); skipping" >&2 + return 0 + fi + for pod in ${pods}; do + local name="${pod#pod/}" + echo "collecting ${ns}/${name} (${WINDOW[*]})" + run_kubectl logs -n "${ns}" "${name}" "${WINDOW[@]}" --timestamps=false \ + > "${DEST}/${ns}-${name}.log" \ + || { echo "warn: kubectl logs ${ns}/${name} failed; skipping" >&2; rm -f "${DEST}/${ns}-${name}.log"; } + # --previous is only valid after a restart; kubectl rejects it otherwise. + local restarts + restarts=$(run_kubectl get pod -n "${ns}" "${name}" \ + -o jsonpath='{.status.containerStatuses[0].restartCount}' 2>/dev/null || echo 0) + if [[ "${restarts:-0}" -gt 0 ]]; then + echo "collecting ${ns}/${name} previous container (${restarts} restarts)" + run_kubectl logs -n "${ns}" "${name}" --previous "${WINDOW[@]}" --timestamps=false \ + > "${DEST}/${ns}-${name}.previous.log" \ + || { echo "warn: kubectl logs --previous ${ns}/${name} failed; skipping" >&2; rm -f "${DEST}/${ns}-${name}.previous.log"; } + fi + done +} + +collect "${NAMESPACE}" "app=atelet" +# Worker pods carry the pool label whatever the pool's name is. +collect "${WORKER_NAMESPACE}" "ate.dev/worker-pool" + +echo "logs in ${DEST}; next:" +echo " python3 benchmarking/analysis/phase_report.py ${DEST}/*.log" diff --git a/benchmarking/analysis/phase_report.py b/benchmarking/analysis/phase_report.py new file mode 100755 index 0000000000..962046f793 --- /dev/null +++ b/benchmarking/analysis/phase_report.py @@ -0,0 +1,451 @@ +#!/usr/bin/env python3 +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Aggregate the suspend/resume phase-breakdown log records into a report. + +Reads the atelet and worker pod logs, either as kubectl logs dumps (JSON +lines; unrelated lines are skipped) or as a Cloud Logging export +(`gcloud logging read --format json`: a JSON array of entries with the record +under jsonPayload), and aggregates the two joinable record kinds the node +side emits, each written by both layers: + + - "Restore timing breakdown" atelet (ate.actor.restore.duration.*) + and ateom (ateom.actor.restore.duration.*) + - "Checkpoint timing breakdown" atelet (ate.actor.checkpoint.duration.*) + and ateom (ateom.actor.checkpoint.duration.*) + +The report answers "where does the SuspendActor / ResumeActor time go": +per-phase percentiles at each layer, and the slowest operations as nested +waterfalls (the ateom record joined under the atelet record of the same +actor). + +Usage: + phase_report.py run-logs/*.log [--csv DEST_DIR] [--slowest N] + +Feed it one source per run: the kubectl dumps, or the Cloud Logging export, +not both, or every record counts twice. + +The records are developer-facing logs, not metric API; this reader is the +consumer that makes them percentiles. See benchmarking/analysis/README.md. +""" + +from __future__ import annotations + +import argparse +import csv +import json +import re +import statistics +import sys +from collections import defaultdict +from dataclasses import dataclass, field +from datetime import datetime +from pathlib import Path + +# The duration-key prefixes. Source of truth: cmd/atelet/metrics.go +# (restoreDurationMetric, checkpointDurationMetric) and +# cmd/ateom-microvm/phaselog.go. +BREAKDOWN_PREFIXES = { + ("atelet", "restore"): "ate.actor.restore.duration.", + ("atelet", "checkpoint"): "ate.actor.checkpoint.duration.", + ("ateom", "restore"): "ateom.actor.restore.duration.", + ("ateom", "checkpoint"): "ateom.actor.checkpoint.duration.", +} +BREAKDOWN_MSGS = {"Restore timing breakdown", "Checkpoint timing breakdown"} + +ACTOR_UID_KEY = "ate.actor.uid" +SCOPE_KEY = "ate.snapshot.scope" +PHASE_KEY = "ate.snapshot.phase" +KIND_KEY = "ate.snapshot.kind" +SANDBOX_CLASS_KEY = "ate.sandbox.class" +TEMPLATE_KEY = "ate.template.name" +ERROR_TYPE_KEY = "error.type" + +# Only ateom-microvm writes the ateom records, and they carry no sandbox +# class of their own; the atelet record of the same operation does. +ATEOM_SANDBOX_CLASS = "microvm" + +# The phase every record reports, and the one the report derives: the part of +# the total no logged phase accounts for. +TOTAL = "total" +UNATTRIBUTED = "unattributed" + +# Sequential order of the known phases, for display. Phases absent from a +# record simply don't print; unknown phases print after, in input order. +PHASE_ORDER = [ + # atelet restore + "volume_mount", "manifest_fetch", "sandbox_assets", "download", + "oci_unpack", "ateom_restore", + # ateom restore + "prep", "bundles", "upper_join", "lowers", "tap", "vmm_launch", + "vm_restore", "resume", "wakeup_probe", + # atelet + ateom checkpoint + "pause", "snapshot", "durable_dir", "rootfs_upper", + "ateom_checkpoint", "teardown", "persist", + UNATTRIBUTED, TOTAL, +] +_PHASE_RANK = {name: i for i, name in enumerate(PHASE_ORDER)} + +# Which atelet phase wraps the ateom record. +INNER_PHASE = {"restore": "ateom_restore", "checkpoint": "ateom_checkpoint"} + +# The ateom checkpoint captures that run concurrently on the paused guest: the +# paused window costs their max, not their sum. +CONCURRENT_CAPTURES = ("snapshot", "durable_dir", "rootfs_upper") + +# Slack on the window in which the ateom record of an operation must land: +# atelet does a little work after the ateom call returns before it writes its +# own record (unmounting, resetting the actor dirs), and the two clocks are +# the same node's but not the same goroutine's. +JOIN_SLACK_S = 1.0 + + +@dataclass +class Breakdown: + """One parsed timing-breakdown record.""" + source: str # atelet | ateom + op: str # restore | checkpoint + time: str + ts: float | None # epoch seconds parsed from time, None when unparseable + actor_uid: str + template: str + sandbox_class: str + scope: str + kind: str + phases: dict[str, float] # phase name -> seconds + failed: bool + + +@dataclass +class Parsed: + breakdowns: list[Breakdown] = field(default_factory=list) + lines_seen: int = 0 + lines_matched: int = 0 + bad_values: int = 0 # duration keys whose value was not a number + + +_TIME_RE = re.compile(r"^(.*T\d\d:\d\d:\d\d)(\.\d+)?(Z|[+-]\d\d:?\d\d)?$") + + +def parse_time(s: str) -> float | None: + """RFC 3339 -> epoch seconds. slog writes nanoseconds, which fromisoformat + rejects, so the fraction is trimmed to microseconds. None when unparseable + (a record then still counts, it just cannot be joined by time).""" + m = _TIME_RE.match(s.strip()) if s else None + if not m: + return None + base, frac, tz = m.groups() + frac = (frac or "")[:7] + tz = "+00:00" if tz in (None, "Z") else tz + try: + return datetime.fromisoformat(f"{base}{frac}{tz}").timestamp() + except ValueError: + return None + + +def unattributed(source: str, op: str, phases: dict[str, float]) -> float | None: + """The total minus what the logged phases account for, where that is + well-defined: the atelet checkpoint phases are sequential; the ateom + checkpoint is prep, pause, the concurrent captures (max), then teardown. + The atelet restore phases overlap by design (download runs alongside the + asset fetch and OCI unpack) and the ateom restore phases partition their + total by construction, so neither gets a residual. The phases are + sub-intervals of the total, so a negative result is a bug in the emitter + or in this formula; it is not clamped, so that it shows.""" + total = phases.get(TOTAL) + if total is None: + return None + if (source, op) == ("atelet", "checkpoint"): + spent = sum(phases.get(p, 0.0) for p in ("sandbox_assets", "ateom_checkpoint", "persist")) + elif (source, op) == ("ateom", "checkpoint"): + spent = (phases.get("prep", 0.0) + phases.get("pause", 0.0) + + max(phases.get(p, 0.0) for p in CONCURRENT_CAPTURES) + + phases.get("teardown", 0.0)) + else: + return None + return total - spent + + +def parse_line(obj: dict, out: Parsed) -> None: + # A Cloud Logging entry nests the record under jsonPayload and moves + # slog's time onto the entry's timestamp; a kubectl dump is the record. + payload = obj.get("jsonPayload") + if isinstance(payload, dict): + entry, obj = obj, dict(payload) + obj.setdefault("time", entry.get("timestamp", "")) + msg = obj.get("msg", "") + if msg in BREAKDOWN_MSGS: + for (source, op), prefix in BREAKDOWN_PREFIXES.items(): + phases: dict[str, float] = {} + for k, v in obj.items(): + if not k.startswith(prefix): + continue + try: + phases[k[len(prefix):]] = float(v) + except (TypeError, ValueError): + # One bad value costs that phase, not the run's report. + out.bad_values += 1 + if not phases: + continue + time_s = obj.get("time", "") + residual = unattributed(source, op, phases) + if residual is not None: + phases[UNATTRIBUTED] = residual + out.breakdowns.append(Breakdown( + source=source, + op=op, + time=time_s, + ts=parse_time(time_s), + actor_uid=obj.get(ACTOR_UID_KEY, ""), + template=obj.get(TEMPLATE_KEY, ""), + sandbox_class=obj.get(SANDBOX_CLASS_KEY, "") + or (ATEOM_SANDBOX_CLASS if source == "ateom" else ""), + scope=obj.get(SCOPE_KEY, ""), + kind=obj.get(KIND_KEY, ""), + phases=phases, + failed=ERROR_TYPE_KEY in obj, + )) + out.lines_matched += 1 + return + + +def parse_files(paths: list[str]) -> Parsed: + out = Parsed() + for path in paths: + f = sys.stdin if path == "-" else open(path, encoding="utf-8", errors="replace") + with f: + text = f.read() + if text.lstrip().startswith("["): + # A Cloud Logging export: one JSON array of entries, pretty-printed + # across lines, so it cannot be read line by line. + try: + entries = json.loads(text) + except json.JSONDecodeError as e: + # Not an export after all: a dump whose lines carry a bracket + # prefix, or a truncated export. Say so, then read it line by + # line like any other dump rather than drop the whole file. + print(f"warn: {path}: starts with '[' but is not a JSON array ({e}); " + f"reading it line by line", file=sys.stderr) + else: + if isinstance(entries, list): + for obj in entries: + if isinstance(obj, dict): + out.lines_seen += 1 + parse_line(obj, out) + continue + for line in text.splitlines(): + # kubectl log dumps may prefix each line (pod name, timestamp); + # recover the JSON object from the first brace. + brace = line.find("{") + if brace < 0: + continue + out.lines_seen += 1 + try: + obj = json.loads(line[brace:]) + except json.JSONDecodeError: + continue + if isinstance(obj, dict): + parse_line(obj, out) + return out + + +def percentile(values: list[float], q: float) -> float: + if not values: + return 0.0 + if len(values) == 1: + return values[0] + return statistics.quantiles(values, n=100, method="inclusive")[int(q) - 1] + + +def phase_sort_key(name: str) -> tuple[int, str]: + return (_PHASE_RANK.get(name, len(PHASE_ORDER)), name) + + +def fmt_s(seconds: float) -> str: + return f"{seconds * 1000:8.1f}" + + +def report_phases(breakdowns: list[Breakdown], writer) -> list[dict]: + """Per (source, op, sandbox class, kind, scope, phase) percentiles. A + golden restore (which downloads the golden image) and a latest restore, + or a gVisor and a micro-VM checkpoint, are different distributions and + must not be pooled. Returns the rows for CSV.""" + groups: dict[tuple, list[float]] = defaultdict(list) + for b in breakdowns: + if b.failed: + continue + for name, seconds in b.phases.items(): + groups[(b.source, b.op, b.sandbox_class or "-", b.kind or "-", b.scope or "-", name)].append(seconds) + + rows = [] + writer("== Phase percentiles (ms) ==") + writer(f"{'layer':7} {'op':11} {'class':8} {'kind':7} {'scope':15} {'phase':16} {'n':>5} " + f"{'p50':>8} {'p90':>8} {'p95':>8} {'max':>8}") + for key in sorted(groups, key=lambda k: (k[:5], phase_sort_key(k[5]))): + vals = sorted(groups[key]) + source, op, sandbox_class, kind, scope, name = key + row = { + "layer": source, "op": op, "class": sandbox_class, "kind": kind, + "scope": scope, "phase": name, + "count": len(vals), + "p50_ms": percentile(vals, 50) * 1000, + "p90_ms": percentile(vals, 90) * 1000, + "p95_ms": percentile(vals, 95) * 1000, + "max_ms": max(vals) * 1000, + } + rows.append(row) + writer(f"{source:7} {op:11} {sandbox_class:8} {kind:7} {scope:15} {name:16} {len(vals):5d} " + f"{fmt_s(percentile(vals, 50))} {fmt_s(percentile(vals, 90))} " + f"{fmt_s(percentile(vals, 95))} {fmt_s(max(vals))}") + failed = sum(1 for b in breakdowns if b.failed) + if failed: + writer(f"(excluded {failed} failed operation records)") + return rows + + +def ateom_window(op: Breakdown) -> tuple[float, float] | None: + """When, relative to the atelet record's time, the ateom record of the + same operation was written. ateom writes it as its RPC returns, i.e. at + the end of the atelet ateom_* phase: for a checkpoint that is before + persist runs, for a restore it is the last phase. An operation that never + reached the ateom call has no window and nothing to pair with.""" + total = op.phases.get(TOTAL) + if op.ts is None or total is None or INNER_PHASE[op.op] not in op.phases: + return None + if op.op == "checkpoint": + return (op.ts - total - JOIN_SLACK_S, + op.ts - op.phases.get("persist", 0.0) + JOIN_SLACK_S) + return (op.ts - op.phases[INNER_PHASE[op.op]] - JOIN_SLACK_S, op.ts + JOIN_SLACK_S) + + +def pair_records(breakdowns: list[Breakdown]) -> dict[int, Breakdown]: + """Match each atelet record to the ateom record of the same operation: + same actor and op, written inside the atelet ateom_* phase's window, and + not already claimed by another operation. Returns id(atelet) -> ateom. + + One cycle emits exactly one of each. An ateom record outside the window + belongs to another cycle whose partner is missing (pod gone, log rotated, + window cut); claiming it would print a gap that never happened, so an + operation without a match prints without one. Matching in time order + with a consumed set keeps rapid cycles of one actor from sharing or + swapping records. A paired ateom record also inherits the snapshot kind + its layer does not log, so the two layers' percentiles split alike.""" + by_actor: dict[tuple, list[Breakdown]] = defaultdict(list) + for b in breakdowns: + if b.source == "ateom" and b.ts is not None: + by_actor[(b.actor_uid, b.op)].append(b) + for v in by_actor.values(): + v.sort(key=lambda b: b.ts) + + pairs: dict[int, Breakdown] = {} + consumed: set[int] = set() + atelet = sorted((b for b in breakdowns if b.source == "atelet" and b.ts is not None), + key=lambda b: b.ts) + for op in atelet: + window = ateom_window(op) + if window is None: + continue + lo, hi = window + candidates = [a for a in by_actor.get((op.actor_uid, op.op), []) + if id(a) not in consumed and lo <= a.ts <= hi] + if not candidates: + continue + inner = candidates[-1] + consumed.add(id(inner)) + pairs[id(op)] = inner + if not inner.kind: + inner.kind = op.kind + return pairs + + +def report_waterfalls(breakdowns: list[Breakdown], writer, slowest: int, + pairs: dict[int, Breakdown] | None = None) -> None: + """The slowest operations: the ateom record nested under the atelet + ateom_* phase, and the gap between that phase and the ateom total (RPC + and queueing between the layers).""" + if pairs is None: + pairs = pair_records(breakdowns) + atelet = [b for b in breakdowns if b.source == "atelet" and not b.failed] + atelet.sort(key=lambda b: b.phases.get(TOTAL, 0), reverse=True) + + writer("") + writer(f"== Slowest {slowest} operations (waterfall, ms) ==") + for b in atelet[:slowest]: + inner = pairs.get(id(b)) + total = b.phases.get(TOTAL, 0) + writer(f"{b.op} actor={b.actor_uid} template={b.template} " + f"class={b.sandbox_class or '-'} scope={b.scope or '-'} kind={b.kind or '-'} " + f"total={total * 1000:.1f}") + for name in sorted(b.phases, key=phase_sort_key): + if name == TOTAL: + continue + writer(f" atelet {name:15} {fmt_s(b.phases[name])}") + if name == INNER_PHASE.get(b.op) and inner: + for iname in sorted(inner.phases, key=phase_sort_key): + if iname == TOTAL: + continue + writer(f" ateom {iname:13} {fmt_s(inner.phases[iname])}") + if TOTAL in inner.phases: + gap = b.phases[name] - inner.phases[TOTAL] + writer(f" (gap) {'rpc/queueing':13} {fmt_s(gap)}") + + +def write_csv(dest: Path, name: str, rows: list[dict]) -> None: + if not rows: + return + dest.mkdir(parents=True, exist_ok=True) + path = dest / name + with open(path, "w", newline="", encoding="utf-8") as f: + w = csv.DictWriter(f, fieldnames=list(rows[0].keys())) + w.writeheader() + w.writerows(rows) + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("logs", nargs="+", + help="kubectl logs dumps (JSON lines) or Cloud Logging exports " + "(JSON array, as gcloud logging read --format json writes); " + "'-' for stdin") + ap.add_argument("--csv", type=Path, default=None, + help="also write phase_percentiles.csv here") + ap.add_argument("--slowest", type=int, default=3, + help="number of slowest operations to print as waterfalls") + args = ap.parse_args() + + parsed = parse_files(args.logs) + print(f"parsed {parsed.lines_matched} timing breakdown records " + f"out of {parsed.lines_seen} JSON log entries") + if parsed.bad_values: + print(f"(skipped {parsed.bad_values} non-numeric duration values)") + if not parsed.breakdowns: + print("no matching records; are these the atelet and worker pod logs?", file=sys.stderr) + return 1 + + # Pair before the percentiles: a paired ateom record takes its kind from + # the atelet record, so both layers' rows split the same way. + pairs = pair_records(parsed.breakdowns) + print() + phase_rows = report_phases(parsed.breakdowns, print) + report_waterfalls(parsed.breakdowns, print, args.slowest, pairs) + + if args.csv: + write_csv(args.csv, "phase_percentiles.csv", phase_rows) + print(f"\nCSV written to {args.csv}/") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/benchmarking/analysis/test_phase_report.py b/benchmarking/analysis/test_phase_report.py new file mode 100644 index 0000000000..45e61169d6 --- /dev/null +++ b/benchmarking/analysis/test_phase_report.py @@ -0,0 +1,300 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Parser and aggregation tests for phase_report. No cluster needed: + + python3 -m unittest discover -s benchmarking/analysis +""" + +import contextlib +import io +import json +import os +import sys +import tempfile +import unittest + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +import phase_report # noqa: E402 + + +def parse(*records): + out = phase_report.Parsed() + for rec in records: + phase_report.parse_line(json.loads(json.dumps(rec)), out) + return out + + +def render(fn, *args, **kwargs): + buf = io.StringIO() + fn(*args, writer=lambda line: buf.write(line + "\n"), **kwargs) + return buf.getvalue() + + +ATELET_RESTORE = { + "time": "2026-09-23T10:00:05.500000000Z", "level": "INFO", + "msg": "Restore timing breakdown", + "ate.actor.uid": "uid-1", "ate.template.name": "swebench-astropy-7336", + "ate.snapshot.scope": "full", "ate.snapshot.kind": "latest", + "ate.actor.restore.duration.download": 2.4, + "ate.actor.restore.duration.ateom_restore": 1.1, + "ate.actor.restore.duration.total": 3.9, +} + +ATEOM_RESTORE = { + "time": "2026-09-23T10:00:05.400000000Z", "level": "INFO", + "msg": "Restore timing breakdown", + "ate.actor.uid": "uid-1", "ate.template.name": "swebench-astropy-7336", + "ate.snapshot.scope": "full", + "ateom.actor.restore.duration.vm_restore": 0.8, + "ateom.actor.restore.duration.wakeup_probe": 0.2, + "ateom.actor.restore.duration.total": 1.05, +} + +ATELET_CHECKPOINT = { + "time": "2026-09-23T10:01:00.000000000Z", "level": "INFO", + "msg": "Checkpoint timing breakdown", + "ate.actor.uid": "uid-1", "ate.template.name": "swebench-astropy-7336", + "ate.snapshot.scope": "full", "ate.snapshot.kind": "latest", + "ate.actor.checkpoint.duration.sandbox_assets": 0.01, + "ate.actor.checkpoint.duration.ateom_checkpoint": 1.14, + "ate.actor.checkpoint.duration.persist": 4.48, + "ate.actor.checkpoint.duration.total": 6.0, +} + +ATEOM_CHECKPOINT = { + "time": "2026-09-23T10:00:55.000000000Z", "level": "INFO", + "msg": "Checkpoint timing breakdown", + "ate.actor.uid": "uid-1", "ate.template.name": "swebench-astropy-7336", + "ate.snapshot.scope": "full", + "ateom.actor.checkpoint.duration.prep": 0.04, + "ateom.actor.checkpoint.duration.pause": 0.01, + "ateom.actor.checkpoint.duration.snapshot": 0.5, + "ateom.actor.checkpoint.duration.rootfs_upper": 0.9, + "ateom.actor.checkpoint.duration.teardown": 0.2, + "ateom.actor.checkpoint.duration.total": 1.2, +} + +# The same actor's ateom restore record from an hour earlier: a different +# cycle, whose own atelet partner is missing from the dump. +ATEOM_RESTORE_STALE = dict(ATEOM_RESTORE, time="2026-09-23T09:00:05.400000000Z") + +# One entry as `gcloud logging read --format json` writes it, trimmed to the +# fields that matter: the record under jsonPayload, slog's time promoted to +# the entry's timestamp, dotted keys kept as they are. +CLOUD_LOGGING_ENTRY = { + "insertId": "m9xljnzmzppup1vj", + "jsonPayload": { + "ate.actor.checkpoint.duration.ateom_checkpoint": 1.024788329, + "ate.actor.checkpoint.duration.persist": 4.18451256, + "ate.actor.checkpoint.duration.sandbox_assets": 2.4358e-05, + "ate.actor.checkpoint.duration.total": 5.385861508, + "ate.actor.name": "sb-d790f7ed", "ate.actor.uid": "cc7a2ac7", + "ate.atespace": "benchmark", "ate.sandbox.class": "microvm", + "ate.snapshot.kind": "latest", "ate.snapshot.scope": "full", + "ate.template.atespace": "benchmark-workloads", "ate.template.name": "glutton", + "level": "INFO", "msg": "Checkpoint timing breakdown", + }, + "logName": "projects/p/logs/stdout", + "resource": {"labels": {"container_name": "atelet", "namespace_name": "ate-system"}, + "type": "k8s_container"}, + "severity": "INFO", + "timestamp": "2026-09-28T17:49:47.665836237Z", +} + +FAILED = { + "time": "2026-09-23T10:02:00Z", "level": "INFO", + "msg": "Restore timing breakdown", + "ate.actor.uid": "uid-2", "ate.snapshot.scope": "full", + "error.type": "DeadlineExceeded", + "ate.actor.restore.duration.download": 30.0, + "ate.actor.restore.duration.total": 30.0, +} + + +class ParseTest(unittest.TestCase): + def test_parses_both_layers_of_the_same_msg(self): + out = parse(ATELET_RESTORE, ATEOM_RESTORE) + self.assertEqual([(b.source, b.op) for b in out.breakdowns], + [("atelet", "restore"), ("ateom", "restore")]) + self.assertEqual(out.breakdowns[0].phases["download"], 2.4) + self.assertEqual(out.breakdowns[1].phases["vm_restore"], 0.8) + self.assertEqual(out.breakdowns[1].actor_uid, "uid-1") + + def test_nanosecond_timestamps_parse(self): + out = parse(ATELET_RESTORE, FAILED) + self.assertIsNotNone(out.breakdowns[0].ts) + self.assertIsNotNone(out.breakdowns[1].ts) + self.assertAlmostEqual(out.breakdowns[0].ts % 1, 0.5, places=6) + self.assertIsNone(phase_report.parse_time("yesterday")) + + def test_ateom_checkpoint_parses(self): + b = parse(ATEOM_CHECKPOINT).breakdowns[0] + self.assertEqual((b.source, b.op), ("ateom", "checkpoint")) + self.assertEqual(b.phases["rootfs_upper"], 0.9) + + def test_cloud_logging_entry_parses_with_the_entry_timestamp(self): + b = parse(CLOUD_LOGGING_ENTRY).breakdowns[0] + self.assertEqual((b.source, b.op), ("atelet", "checkpoint")) + self.assertEqual(b.time, "2026-09-28T17:49:47.665836237Z") + self.assertIsNotNone(b.ts) + self.assertEqual((b.sandbox_class, b.kind, b.scope), ("microvm", "latest", "full")) + self.assertAlmostEqual(b.phases["persist"], 4.18451256) + + def test_cloud_logging_export_file_is_a_json_array(self): + with tempfile.TemporaryDirectory() as d: + path = os.path.join(d, "export.json") + second = dict(CLOUD_LOGGING_ENTRY, timestamp="2026-09-28T17:50:47.000000000Z") + with open(path, "w") as f: + json.dump([CLOUD_LOGGING_ENTRY, second], f, indent=2) + out = phase_report.parse_files([path]) + self.assertEqual((out.lines_seen, len(out.breakdowns)), (2, 2)) + + def test_ateom_records_default_to_the_microvm_class(self): + out = parse(ATEOM_RESTORE, ATELET_RESTORE) + self.assertEqual(out.breakdowns[0].sandbox_class, "microvm") + self.assertEqual(out.breakdowns[1].sandbox_class, "") # not on this fixture + + def test_non_numeric_duration_is_skipped_not_fatal(self): + rec = dict(ATELET_RESTORE, **{"ate.actor.restore.duration.download": None, + "ate.actor.restore.duration.oci_unpack": "fast"}) + out = parse(rec) + self.assertEqual(out.bad_values, 2) + self.assertEqual(out.breakdowns[0].phases["total"], 3.9) + self.assertNotIn("download", out.breakdowns[0].phases) + + def test_malformed_export_warns_instead_of_vanishing(self): + with tempfile.TemporaryDirectory() as d: + path = os.path.join(d, "export.json") + with open(path, "w") as f: + f.write('[{"jsonPayload": {"msg": "Checkpoint timing breakdown"') # truncated + err = io.StringIO() + with contextlib.redirect_stderr(err): + out = phase_report.parse_files([path]) + self.assertEqual(out.breakdowns, []) + self.assertIn("export.json", err.getvalue()) + self.assertIn("not a JSON array", err.getvalue()) + + def test_bracket_prefixed_dump_is_read_line_by_line(self): + # A dump whose lines start with "[INFO]" also starts with "[", but + # it is not an export; its records must not be lost. + with tempfile.TemporaryDirectory() as d: + path = os.path.join(d, "pod.log") + with open(path, "w") as f: + f.write("[INFO] " + json.dumps(ATELET_RESTORE) + "\n" + "[2026-09-23 10:00:05] " + json.dumps(ATEOM_RESTORE) + "\n") + err = io.StringIO() + with contextlib.redirect_stderr(err): + out = phase_report.parse_files([path]) + self.assertEqual(len(out.breakdowns), 2) + self.assertIn("reading it line by line", err.getvalue()) + + def test_prefixed_kubectl_lines_still_parse(self): + with tempfile.TemporaryDirectory() as d: + path = os.path.join(d, "pod.log") + with open(path, "w") as f: + f.write("pod/atelet-abc " + json.dumps(ATELET_RESTORE) + "\nnot json\n{\"msg\": \"other\"}\n") + out = phase_report.parse_files([path]) + self.assertEqual(len(out.breakdowns), 1) + self.assertEqual(out.lines_seen, 2) + + +class UnattributedTest(unittest.TestCase): + def test_atelet_checkpoint_residual_is_total_minus_sequential_phases(self): + b = parse(ATELET_CHECKPOINT).breakdowns[0] + self.assertAlmostEqual(b.phases["unattributed"], 6.0 - (0.01 + 1.14 + 4.48)) + + def test_ateom_checkpoint_counts_the_concurrent_captures_once(self): + b = parse(ATEOM_CHECKPOINT).breakdowns[0] + # prep + pause + max(snapshot, rootfs_upper) + teardown; the two + # captures overlapped, so only the slower one is spent wall time. + self.assertAlmostEqual(b.phases["unattributed"], 1.2 - (0.04 + 0.01 + 0.9 + 0.2)) + + def test_restore_layers_get_no_residual(self): + out = parse(ATELET_RESTORE, ATEOM_RESTORE) + for b in out.breakdowns: + self.assertNotIn("unattributed", b.phases) + + def test_negative_residual_is_not_clamped(self): + # Cannot happen on a well-formed record; if it does, it must show. + rec = dict(ATELET_CHECKPOINT, **{"ate.actor.checkpoint.duration.total": 1.0}) + self.assertAlmostEqual(parse(rec).breakdowns[0].phases["unattributed"], 1.0 - (0.01 + 1.14 + 4.48)) + + +class ReportTest(unittest.TestCase): + def test_failed_records_are_flagged_and_excluded_from_percentiles(self): + out = parse(ATELET_RESTORE, FAILED) + self.assertEqual([b.failed for b in out.breakdowns], [False, True]) + rows = phase_report.report_phases(out.breakdowns, lambda _: None) + downloads = [r for r in rows if r["phase"] == "download"] + self.assertEqual(len(downloads), 1) + self.assertEqual(downloads[0]["count"], 1) # the failed 30s never entered + + def test_percentiles_split_by_sandbox_class_and_kind(self): + golden = dict(ATELET_RESTORE, **{"time": "2026-09-23T09:59:00.000000000Z", + "ate.snapshot.kind": "golden", "ate.sandbox.class": "gvisor", + "ate.actor.restore.duration.download": 9.0, + "ate.actor.restore.duration.total": 10.0}) + latest = dict(ATELET_RESTORE, **{"ate.sandbox.class": "gvisor"}) + rows = phase_report.report_phases(parse(golden, latest).breakdowns, lambda _: None) + downloads = {(r["class"], r["kind"]): r["p50_ms"] for r in rows if r["phase"] == "download"} + self.assertEqual(downloads, {("gvisor", "golden"): 9000.0, ("gvisor", "latest"): 2400.0}) + + def test_paired_ateom_record_inherits_the_atelet_kind(self): + out = parse(ATEOM_RESTORE, ATELET_RESTORE) + pairs = phase_report.pair_records(out.breakdowns) + self.assertEqual(len(pairs), 1) + self.assertEqual(out.breakdowns[0].kind, "latest") + rows = phase_report.report_phases(out.breakdowns, lambda _: None) + self.assertEqual({r["kind"] for r in rows if r["layer"] == "ateom"}, {"latest"}) + + def test_checkpoint_pairing_window_ends_where_persist_starts(self): + # The ateom record is written when ateom_checkpoint ends, before the + # 4.48 s persist; a record from inside the persist window belongs to + # a later cycle of a rapidly cycling actor. + during_persist = dict(ATEOM_CHECKPOINT, time="2026-09-23T10:00:59.000000000Z") + out = parse(ATEOM_CHECKPOINT, during_persist, ATELET_CHECKPOINT) + pairs = phase_report.pair_records(out.breakdowns) + atelet = next(b for b in out.breakdowns if b.source == "atelet") + self.assertIs(pairs[id(atelet)], out.breakdowns[0]) + + def test_an_ateom_record_pairs_with_at_most_one_operation(self): + second = dict(ATELET_RESTORE, time="2026-09-23T10:00:05.900000000Z") + out = parse(ATEOM_RESTORE, ATELET_RESTORE, second) + pairs = phase_report.pair_records(out.breakdowns) + self.assertEqual(len(pairs), 1) + first = next(b for b in out.breakdowns if b.source == "atelet") + self.assertIn(id(first), pairs) + + def test_waterfall_ignores_an_ateom_record_from_another_cycle(self): + out = parse(ATEOM_RESTORE_STALE, ATELET_RESTORE) + text = render(phase_report.report_waterfalls, out.breakdowns, slowest=1) + self.assertIn("atelet ateom_restore", text) + self.assertNotIn("ateom vm_restore", text) + self.assertNotIn("(gap)", text) + + def test_waterfall_nests_ateom_and_gap_under_atelet(self): + out = parse(ATEOM_RESTORE, ATELET_RESTORE) + text = render(phase_report.report_waterfalls, out.breakdowns, slowest=1) + lines = [line.strip() for line in text.splitlines()] + self.assertTrue(any(line.startswith("atelet ateom_restore") for line in lines), text) + self.assertTrue(any(line.startswith("ateom vm_restore") for line in lines), text) + gap = [line for line in lines if line.startswith("(gap)")] + self.assertEqual(len(gap), 1, text) + self.assertIn("50.0", gap[0]) # 1.1s atelet ateom_restore - 1.05s ateom total + + +if __name__ == "__main__": + unittest.main() diff --git a/benchmarking/automation/README.md b/benchmarking/automation/README.md index 43270d147e..a8f8aa6003 100644 --- a/benchmarking/automation/README.md +++ b/benchmarking/automation/README.md @@ -23,7 +23,7 @@ the router capacity benchmark — see `ateArgs`) + `benchmarking/workloads/deploy.sh --deploy --sandbox-class ` (these build & push substrate / workload images via `ko` as part of their deploy steps — there's no separate - `make build-images` step). For a `microvm` test the orchestrator also + image build step). For a `microvm` test the orchestrator also runs `hack/install-microvm-deps.sh --install` between the two, which stages kata + cloud-hypervisor + virtiofsd assets to the cluster's object store bucket and applies the cluster-wide `microvm` SandboxConfig. @@ -45,7 +45,7 @@ the router capacity benchmark — see ## Choosing a sandbox class Each entry in `tests.yaml` may set `sandboxClass: gvisor | microvm` (default -`gvisor`). This controls both `spec.sandboxClass` on the benchmark WorkerPool +`gvisor`). This controls both `spec.sandboxClasses[0].name` on the benchmark WorkerPool and its `workerImage` (`ateom-gvisor` vs `ateom-microvm`). For `microvm` tests the target cluster must have KVM-capable nodes and the diff --git a/benchmarking/workloads/manifests/glutton-durdir-data-template.yaml.tmpl b/benchmarking/workloads/manifests/glutton-durdir-data-template.yaml.tmpl index 4e2d4b053e..5d984e4daf 100644 --- a/benchmarking/workloads/manifests/glutton-durdir-data-template.yaml.tmpl +++ b/benchmarking/workloads/manifests/glutton-durdir-data-template.yaml.tmpl @@ -44,7 +44,7 @@ volumes: - name: data durableDir: {} snapshotConfig: - onCommit: SNAPSHOT_CONTENT_SCOPE_DATA + preferredFidelity: SNAPSHOT_FIDELITY_VOLUMES storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/glutton-durdir-data/ sandboxConfig: sandboxClass: ${SANDBOX_CLASS_ENUM} diff --git a/benchmarking/workloads/manifests/glutton-durdir-full-template.yaml.tmpl b/benchmarking/workloads/manifests/glutton-durdir-full-template.yaml.tmpl index 4d174b896a..10c7272666 100644 --- a/benchmarking/workloads/manifests/glutton-durdir-full-template.yaml.tmpl +++ b/benchmarking/workloads/manifests/glutton-durdir-full-template.yaml.tmpl @@ -41,7 +41,7 @@ volumes: - name: data durableDir: {} snapshotConfig: - onCommit: SNAPSHOT_CONTENT_SCOPE_FULL + preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/glutton-durdir-full/ sandboxConfig: sandboxClass: ${SANDBOX_CLASS_ENUM} diff --git a/benchmarking/workloads/manifests/glutton-template.yaml.tmpl b/benchmarking/workloads/manifests/glutton-template.yaml.tmpl index e57c1dd838..bfa9469172 100644 --- a/benchmarking/workloads/manifests/glutton-template.yaml.tmpl +++ b/benchmarking/workloads/manifests/glutton-template.yaml.tmpl @@ -61,7 +61,7 @@ resources: - name: memory quantity: ${ACTOR_MEMORY} snapshotConfig: - onCommit: SNAPSHOT_CONTENT_SCOPE_FULL + preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/glutton/ sandboxConfig: sandboxClass: ${SANDBOX_CLASS_ENUM} diff --git a/benchmarking/workloads/manifests/kernelmem-template.yaml.tmpl b/benchmarking/workloads/manifests/kernelmem-template.yaml.tmpl index 38a5905700..7ffa43a637 100644 --- a/benchmarking/workloads/manifests/kernelmem-template.yaml.tmpl +++ b/benchmarking/workloads/manifests/kernelmem-template.yaml.tmpl @@ -31,7 +31,7 @@ containers: image: containerstack/alpine-stress:latest command: ["sh", "-c", "find /; for i in $(seq 50000); do sleep 10000 & done; sleep 10000"] snapshotConfig: - onCommit: SNAPSHOT_CONTENT_SCOPE_FULL + preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/kernelmem/ sandboxConfig: sandboxClass: ${SANDBOX_CLASS_ENUM} diff --git a/benchmarking/workloads/manifests/sleep-template.yaml.tmpl b/benchmarking/workloads/manifests/sleep-template.yaml.tmpl index 787fff639f..9549ca8672 100644 --- a/benchmarking/workloads/manifests/sleep-template.yaml.tmpl +++ b/benchmarking/workloads/manifests/sleep-template.yaml.tmpl @@ -35,7 +35,7 @@ resources: - name: memory quantity: ${ACTOR_MEMORY} snapshotConfig: - onCommit: SNAPSHOT_CONTENT_SCOPE_FULL + preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/sleep/ sandboxConfig: sandboxClass: ${SANDBOX_CLASS_ENUM} diff --git a/benchmarking/workloads/manifests/swebench-astropy-7336-template.yaml.tmpl b/benchmarking/workloads/manifests/swebench-astropy-7336-template.yaml.tmpl index f092ea551b..8d8fd737c9 100644 --- a/benchmarking/workloads/manifests/swebench-astropy-7336-template.yaml.tmpl +++ b/benchmarking/workloads/manifests/swebench-astropy-7336-template.yaml.tmpl @@ -50,5 +50,5 @@ sandboxConfig: sandboxClass: ${SANDBOX_CLASS_ENUM} configName: ${SANDBOX_CONFIG_NAME} snapshotConfig: - onCommit: SNAPSHOT_CONTENT_SCOPE_FULL + preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/swebench-astropy-7336/ diff --git a/benchmarking/workloads/manifests/usermem-template.yaml.tmpl b/benchmarking/workloads/manifests/usermem-template.yaml.tmpl index db8df41b3e..21eab89cb3 100644 --- a/benchmarking/workloads/manifests/usermem-template.yaml.tmpl +++ b/benchmarking/workloads/manifests/usermem-template.yaml.tmpl @@ -30,7 +30,7 @@ containers: image: containerstack/alpine-stress:latest command: ["stress", "--vm", "1", "--vm-bytes", "1G", "--vm-keep", "--vm-hang", "0"] snapshotConfig: - onCommit: SNAPSHOT_CONTENT_SCOPE_FULL + preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/usermem/ sandboxConfig: sandboxClass: ${SANDBOX_CLASS_ENUM} diff --git a/benchmarking/workloads/manifests/workloads.yaml.tmpl b/benchmarking/workloads/manifests/workloads.yaml.tmpl index 4309442441..022014bd10 100644 --- a/benchmarking/workloads/manifests/workloads.yaml.tmpl +++ b/benchmarking/workloads/manifests/workloads.yaml.tmpl @@ -33,7 +33,8 @@ metadata: workload: benchmark-ateom spec: replicas: ${WORKER_COUNT} - sandboxClass: ${SANDBOX_CLASS} + sandboxClasses: + - name: ${SANDBOX_CLASS} workerImage: ko://github.com/agent-substrate/substrate/cmd/ateom-${SANDBOX_CLASS} template: # Lets gvisor and microvm workers share a node tainted ate.dev/sandboxClass=shared. diff --git a/charts/substrate-crds/templates/ate.dev_sandboxconfigs.yaml b/charts/substrate-crds/templates/ate.dev_sandboxconfigs.yaml index ba5cda8547..1f52428e43 100644 --- a/charts/substrate-crds/templates/ate.dev_sandboxconfigs.yaml +++ b/charts/substrate-crds/templates/ate.dev_sandboxconfigs.yaml @@ -33,6 +33,9 @@ spec: - jsonPath: .spec.sandboxClass name: Class type: string + - jsonPath: .spec.defaultVersion + name: Default + type: string - jsonPath: .metadata.creationTimestamp name: Age type: date @@ -40,10 +43,10 @@ spec: schema: openAPIV3Schema: description: |- - SandboxConfig is cluster-scoped configuration describing the sandbox binaries - for a sandbox runtime family. It is referenced by an ActorTemplate's - sandbox_config.config_name (required) and decouples - sandbox binary selection from the workload definition. + SandboxConfig is cluster-scoped configuration describing the versions of the + sandbox binaries for a sandbox runtime family. It is referenced by an + ActorTemplate's sandbox_config.config_name (required) and decouples sandbox + binary selection from the workload definition. properties: apiVersion: description: |- @@ -65,60 +68,13 @@ spec: spec: description: spec defines the desired state of SandboxConfig properties: - assets: - additionalProperties: - additionalProperties: - description: |- - AssetFile is one content-addressed file that atelet fetches for a sandbox - runtime (e.g. the gVisor runsc binary, or a micro-VM kernel/firmware/config). - properties: - sha256: - description: |- - SHA256 is the lower-case hex SHA256 of the asset. It both names the cached - file (preventing collisions) and verifies the download's integrity. - pattern: ^[a-f0-9]{64}$ - type: string - url: - description: |- - URL is where to download the asset from (e.g. a gs:// URL). It may be - fetched anonymously or with credentials depending on atelet's - configuration. - minLength: 1 - type: string - required: - - sha256 - - url - type: object - type: object + defaultVersion: description: |- - Assets is the set of files atelet fetches for this runtime, keyed first by - architecture (GOARCH, e.g. "amd64", "arm64") and then by asset name. The - asset names are interpreted by the sandbox backend: gVisor expects a - "gvisor" asset (the release's gvisor.tar.zstd, which atelet extracts so - the gvisor-bin/ helpers sit next to runsc; a legacy bare-binary "runsc" - asset is still accepted); a micro-VM backend expects several (e.g. - "cloud-hypervisor", "kata-kernel", "kata-image"). The schema is - intentionally generic; per-class requirements are enforced by a - ValidatingAdmissionPolicy. - type: object - pauseImage: - description: |- - PauseImage is the container image used as the root sandbox container. - It holds the sandbox's namespaces and runs no workload code, so it is an - implementation detail of the sandbox rather than something actor authors - choose. Required for gvisor; not allowed for microvm, which runs no pause - container. It is captured in the snapshot manifest alongside the sandbox - binaries, so a restore always re-creates the sandbox from the same image - the snapshot was taken with. - - Typically, set it to [1] for on-gcp, and [2] for off-gcp - - - [1] gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da - - [2] registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4 + defaultVersion names the entry in versions that new sandboxes boot with. + It must name an existing, non-Disabled version. + maxLength: 63 + minLength: 1 type: string - x-kubernetes-validations: - - message: All images must include a digest - rule: self.contains('@') sandboxClass: default: gvisor description: |- @@ -129,13 +85,112 @@ spec: - gvisor - microvm type: string + versions: + description: versions are the sandbox versions this config offers, + keyed by name. + items: + description: |- + SandboxVersionConfig is one version of a SandboxConfig: the exact pause image + and asset set a sandbox boots with. + properties: + assets: + additionalProperties: + additionalProperties: + description: |- + AssetFile is one content-addressed file that atelet fetches for a sandbox + runtime (e.g. the gVisor runsc binary, or a micro-VM kernel/firmware/config). + properties: + sha256: + description: |- + SHA256 is the lower-case hex SHA256 of the asset. It both names the cached + file (preventing collisions) and verifies the download's integrity. + pattern: ^[a-f0-9]{64}$ + type: string + url: + description: |- + URL is where to download the asset from (e.g. a gs:// URL). It may be + fetched anonymously or with credentials depending on atelet's + configuration. + minLength: 1 + type: string + required: + - sha256 + - url + type: object + type: object + description: |- + Assets is the set of files atelet fetches for this runtime, keyed first by + architecture (GOARCH, e.g. "amd64", "arm64") and then by asset name. The + asset names are interpreted by the sandbox backend: gVisor expects a + "gvisor" asset (the release's gvisor.tar.zstd, which atelet extracts so + the gvisor-bin/ helpers sit next to runsc; a legacy bare-binary "runsc" + asset is still accepted); a micro-VM backend expects several (e.g. + "cloud-hypervisor", "kata-kernel", "kata-image"). The schema is + intentionally generic; per-class requirements are enforced by a + ValidatingAdmissionPolicy. + type: object + name: + description: |- + name identifies this version within its SandboxConfig. It is a DNS label + (lower-case alphanumerics and '-'). + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + pauseImage: + description: |- + PauseImage is the container image used as the root sandbox container. + It holds the sandbox's namespaces and runs no workload code, so it is an + implementation detail of the sandbox rather than something actor authors + choose. Required for gvisor; not allowed for microvm, which runs no pause + container. It is captured in the snapshot manifest alongside the sandbox + binaries, so a restore always re-creates the sandbox from the same image + the snapshot was taken with. + + Typically, set it to [1] for on-gcp, and [2] for off-gcp + + - [1] gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da + - [2] registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4 + maxLength: 512 + minLength: 1 + type: string + x-kubernetes-validations: + - message: All images must include a digest + rule: self.contains('@') + state: + default: Enabled + description: |- + state is whether this version may be used: Enabled (default) or + Disabled. + enum: + - Enabled + - Disabled + type: string + required: + - name + type: object + maxItems: 16 + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map required: + - defaultVersion - sandboxClass + - versions type: object x-kubernetes-validations: - - message: pauseImage is required for gvisor and not allowed for other - sandbox classes - rule: 'self.sandboxClass == ''gvisor'' ? has(self.pauseImage) : !has(self.pauseImage)' + - message: defaultVersion must name an entry in versions + rule: self.versions.exists(v, v.name == self.defaultVersion) + - message: the version named by defaultVersion must not be Disabled + rule: '!self.versions.exists(v, v.name == self.defaultVersion && has(v.state) + && v.state == ''Disabled'')' + - message: pauseImage is required on every version for gvisor and not + allowed for other sandbox classes + rule: 'self.sandboxClass == ''gvisor'' ? self.versions.all(v, has(v.pauseImage) + && size(v.pauseImage) > 0) : self.versions.all(v, !has(v.pauseImage) + || size(v.pauseImage) == 0)' required: - spec type: object diff --git a/charts/substrate-crds/templates/ate.dev_workerpools.yaml b/charts/substrate-crds/templates/ate.dev_workerpools.yaml index 27eb185ce6..0fca6028df 100644 --- a/charts/substrate-crds/templates/ate.dev_workerpools.yaml +++ b/charts/substrate-crds/templates/ate.dev_workerpools.yaml @@ -72,20 +72,47 @@ spec: format: int32 minimum: 0 type: integer - sandboxClass: - default: gvisor + sandboxClasses: description: |- - SandboxClass selects the sandbox runtime family for this pool, which drives - the worker pod shape (KVM/vhost device mounts and node placement). The - concrete binary is still selected by WorkerImage. Defaults to gvisor. - The sandbox binaries themselves come from the SandboxConfig each - ActorTemplate names (required). + SandboxClasses lists the sandbox runtime families this pool runs. Exactly + one entry is allowed today. + items: + description: WorkerPoolSandboxClass is one sandbox runtime family + a WorkerPool runs. + properties: + configRef: + description: |- + ConfigRef names a cluster-scoped SandboxConfig for this sandbox class. + The referenced config's SandboxClass must match Name. Not consumed yet. + properties: + name: + description: Name is the SandboxConfig's metadata.name. + minLength: 1 + type: string + required: + - name + type: object + name: + description: |- + Name selects the sandbox runtime family, which drives the worker pod + shape (KVM/vhost device mounts and node placement). The concrete binary + is still selected by WorkerImage. The sandbox binaries themselves come + from the SandboxConfig each ActorTemplate names (required). - See Also: TODOs in ActorTemplate SandboxClass - enum: - - gvisor - - microvm - type: string + See Also: TODOs in ActorTemplate SandboxClass + enum: + - gvisor + - microvm + type: string + required: + - name + type: object + maxItems: 1 + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map template: description: Template holds optional metadata, scheduling, and resource settings for worker workloads. @@ -448,6 +475,7 @@ spec: type: string required: - replicas + - sandboxClasses - workerImage type: object status: diff --git a/charts/substrate/README.md b/charts/substrate/README.md index a983a1fc8f..00b4dbdbcf 100644 --- a/charts/substrate/README.md +++ b/charts/substrate/README.md @@ -43,7 +43,7 @@ helm template substrate ./charts/substrate ``` `manifests/ate-install/` in the repo is the rendered mTLS output and is -regenerated by `make helm-template`. The separate `substrate-crds` chart +regenerated by `hack/render-manifests.sh`. The separate `substrate-crds` chart mirrors `manifests/ate-install/generated/`. ## Values @@ -65,6 +65,9 @@ See `values.yaml` for the full set; the important keys: | `atelet.imageCredentialProviderConfigYAML` | `""` | Credential provider config as text, rendered into a ConfigMap, for nodes where that file cannot be authored or must differ from the kubelet's; mutually exclusive with the host path | | `atelet.imageCredentialProviderBinDir` | `""` | Host path to the kubelet credential provider binaries; both paths are mounted read-only | | `credentialProvider.namespacePolicies` | `[]` | Default-deny atespace-to-namespace grants; the chart includes get-only Secret RBAC for the provider | +| `ateApi.actorJWTIssuer` | `https://idp..svc` | Issuer placed in actor JWTs; set a stable URL reachable by relying parties | +| `snapshotPlugin.extraEnv` | `[]` | Object storage credentials/settings shared by both snapshot plugins and atelet | +| `snapshotPlugin.control.resources` / `snapshotPlugin.node.resources` | see `values.yaml` | Resources for the API and node snapshot sidecars | | `ateApi.extraArgs` | `[]` | Additional command-line arguments appended to the ateapi defaults | | `otel.endpoint` | `""` | Set to an OTLP endpoint to export traces, metrics, the actor lifecycle events and the router access log | | `otel.traces.enabled` | `true` | Set to `false` to export no traces from the router; the Go components do not honor this yet | @@ -75,6 +78,20 @@ See `values.yaml` for the full set; the important keys: | `otel.logs.enabled` | `true` | Enable OTLP actor events from ateapi and the ateoms, plus the router access log. Actor events go to stdout when OTLP logs are disabled | | `otel.logs.endpoint` | `""` | OTLP endpoint for logs, overriding `otel.endpoint` | +## Upgrading sandbox and snapshot configuration + +Upgrade `substrate-crds` before the application chart. Custom `SandboxConfig` +resources now put assets and `pauseImage` under `spec.versions[]`, with a named +`spec.defaultVersion`. WorkerPools use `spec.sandboxClasses[]`, and actor +templates continue to select the config with `sandboxConfig.configName`. + +Snapshot transfers now run in native sidecars in the API server and atelet +pods. Move external object storage environment settings from +`atelet.extraEnv` or the API environment ConfigMap to +`snapshotPlugin.extraEnv`, so both plugins and atelet asset downloads receive +them. `ATE_STORAGE_BACKEND` must not be set on the API server itself; select +it through `atelet.storageBackend`. RustFS is wired automatically. + ## PostgreSQL setup The chart does not deploy or initialize PostgreSQL. Prepare the database, diff --git a/charts/substrate/templates/_helpers.tpl b/charts/substrate/templates/_helpers.tpl index 7b07e5b2a1..1a9892a4e8 100644 --- a/charts/substrate/templates/_helpers.tpl +++ b/charts/substrate/templates/_helpers.tpl @@ -233,3 +233,24 @@ definition so the fallback cannot drift between pods. {{- define "substrate.imagePullPolicy" -}} {{- ((.Values.global).imagePullPolicy) | default "IfNotPresent" -}} {{- end -}} + +{{/* Object storage settings shared by snapshot plugins and atelet asset downloads. */}} +{{- define "substrate.storage.env" -}} +- name: ATE_STORAGE_BACKEND + value: {{ .Values.atelet.storageBackend | quote }} +{{- if .Values.rustfs.enabled }} +- name: AWS_REGION + value: us-east-1 +- name: AWS_ENDPOINT_URL + value: http://{{ include "substrate.fullname" (list "rustfs" .) }}.{{ .Release.Namespace }}.svc:9000 +- name: AWS_S3_USE_PATH_STYLE + value: "true" +- name: AWS_ACCESS_KEY_ID + value: {{ .Values.rustfs.accessKey | quote }} +- name: AWS_SECRET_ACCESS_KEY + value: {{ .Values.rustfs.secretKey | quote }} +{{- end }} +{{- with .Values.snapshotPlugin.extraEnv }} +{{ toYaml . }} +{{- end }} +{{- end }} diff --git a/charts/substrate/templates/ate-api-server-envvars.yaml b/charts/substrate/templates/ate-api-server-envvars.yaml index 382ce4b9c9..26f8f0edc8 100644 --- a/charts/substrate/templates/ate-api-server-envvars.yaml +++ b/charts/substrate/templates/ate-api-server-envvars.yaml @@ -27,3 +27,4 @@ metadata: namespace: {{ .Release.Namespace }} data: ATE_API_POSTGRES_SCHEMA: {{ .Values.postgres.schema | quote }} + ATE_API_ACTOR_JWT_ISSUER: {{ .Values.ateApi.actorJWTIssuer | default (printf "https://idp.%s.svc" .Release.Namespace) | quote }} diff --git a/charts/substrate/templates/ate-api-server.yaml b/charts/substrate/templates/ate-api-server.yaml index 6d9697eb09..b665e339da 100644 --- a/charts/substrate/templates/ate-api-server.yaml +++ b/charts/substrate/templates/ate-api-server.yaml @@ -81,12 +81,41 @@ spec: serviceAccountName: {{ include "substrate.fullname" (list "ate-api-server" .) }} terminationGracePeriodSeconds: 40 {{- with include "substrate.imagePullSecrets" . }}{{- . | nindent 6 }}{{- end }} + initContainers: + - name: snapshot-plugin + image: {{ include "substrate.componentImage" (list "snapshot-plugin" .) }} + restartPolicy: Always + args: + - control + - --socket=/run/snapshot-plugin/control.sock + startupProbe: + exec: + command: + - /ko-app/snapshot-plugin + - healthcheck + - --socket=/run/snapshot-plugin/control.sock + periodSeconds: 2 + failureThreshold: 30 + securityContext: + capabilities: + drop: [ALL] + resources: +{{ toYaml .Values.snapshotPlugin.control.resources | indent 10 }} + env: +{{- include "substrate.storage.env" . | nindent 8 }} +{{- with include "substrate.otel.env" . }} +{{- . | trim | nindent 8 }} +{{- end }} + volumeMounts: + - name: snapshot-plugin + mountPath: /run/snapshot-plugin containers: - name: ate-api-server image: {{ include "substrate.componentImage" (list "ateapi" .) }} resources: {{ toYaml .Values.ateApi.resources | indent 10 }} args: + - --snapshot-plugin-socket=/run/snapshot-plugin/control.sock - "--grpc-listen-addr=0.0.0.0:443" - "--grpc-server-cred-bundle=/run/servicedns.podcert.ate.dev/credential-bundle.pem" - "--authentication-config=/etc/ateapi/authentication/authentication.yaml" @@ -95,6 +124,7 @@ spec: - {{ printf "--postgres-read-write-role=%s" .Values.postgres.readWriteRole | quote }} - {{ printf "--postgres-owner-role=%s" .Values.postgres.ownerRole | quote }} - "--postgres-schema=@env" + - "--actor-jwt-issuer=@env" - "--actor-id-jwt-pool=/run/actor-id-jwt-pool/pool.json" - "--actor-id-ca-pool=/run/actor-id-ca-pool/pool.json" - "--default-egress-gateway-address={{ include "substrate.fullname" (list "atenet-egress" .) }}.{{ .Release.Namespace }}.svc:443" @@ -133,26 +163,14 @@ spec: key: {{ get $ownerConnectionStringSecretRef "key" | default "ownerConnectionString" | quote }} {{- with include "substrate.otel.env" . }} {{- . | trim | nindent 8 }} -{{- end }} - - name: ATE_STORAGE_BACKEND - value: {{ .Values.atelet.storageBackend | quote }} -{{- if .Values.rustfs.enabled }} - - name: AWS_REGION - value: us-east-1 - - name: AWS_ENDPOINT_URL - value: http://{{ include "substrate.fullname" (list "rustfs" .) }}.{{ .Release.Namespace }}.svc:9000 - - name: AWS_S3_USE_PATH_STYLE - value: "true" - - name: AWS_ACCESS_KEY_ID - value: {{ .Values.rustfs.accessKey | quote }} - - name: AWS_SECRET_ACCESS_KEY - value: {{ .Values.rustfs.secretKey | quote }} {{- end }} envFrom: - configMapRef: name: {{ .Values.ateApiServerEnvVarsConfigMap }} optional: true volumeMounts: + - name: snapshot-plugin + mountPath: /run/snapshot-plugin - { name: servicedns, mountPath: /run/servicedns.podcert.ate.dev } - { name: actor-id-jwt-pool, mountPath: /run/actor-id-jwt-pool } - { name: actor-id-ca-pool, mountPath: /run/actor-id-ca-pool, readOnly: true } @@ -179,6 +197,8 @@ spec: initialDelaySeconds: 10 periodSeconds: 10 volumes: + - name: snapshot-plugin + emptyDir: {} - name: servicedns projected: sources: diff --git a/charts/substrate/templates/atelet.yaml b/charts/substrate/templates/atelet.yaml index 56124947b5..aacbd4d470 100644 --- a/charts/substrate/templates/atelet.yaml +++ b/charts/substrate/templates/atelet.yaml @@ -135,10 +135,44 @@ spec: operator: Exists effect: NoSchedule {{- with include "substrate.imagePullSecrets" . }}{{- . | nindent 6 }}{{- end }} + initContainers: + - name: snapshot-plugin + image: {{ include "substrate.componentImage" (list "snapshot-plugin" .) }} + restartPolicy: Always + args: + - node + - --socket=/run/snapshot-plugin/node.sock + - --root=/var/lib/ate + startupProbe: + exec: + command: + - /ko-app/snapshot-plugin + - healthcheck + - --socket=/run/snapshot-plugin/node.sock + periodSeconds: 2 + failureThreshold: 30 + securityContext: + runAsUser: 0 + runAsGroup: 0 + capabilities: + drop: [ALL] + resources: +{{ toYaml .Values.snapshotPlugin.node.resources | indent 10 }} + env: +{{- include "substrate.storage.env" . | nindent 8 }} +{{- with include "substrate.otel.env" . }} +{{- . | trim | nindent 8 }} +{{- end }} + volumeMounts: + - name: run-ateom + mountPath: /var/lib/ate + - name: snapshot-plugin + mountPath: /run/snapshot-plugin containers: - name: atelet image: {{ include "substrate.componentImage" (list "atelet" .) }} args: + - --snapshot-plugin-socket=/run/snapshot-plugin/node.sock {{- if $credentialProviderConfig }} - --image-credential-provider-config=/run/image-credential-provider/config.yaml - --image-credential-provider-bin-dir=/run/image-credential-provider/bin @@ -181,20 +215,7 @@ spec: {{- with include "substrate.otel.env" . }} {{- . | trim | nindent 8 }} {{- end }} - - name: ATE_STORAGE_BACKEND - value: {{ .Values.atelet.storageBackend | quote }} -{{- if .Values.rustfs.enabled }} - - name: AWS_REGION - value: us-east-1 - - name: AWS_ENDPOINT_URL - value: http://{{ include "substrate.fullname" (list "rustfs" .) }}.{{ .Release.Namespace }}.svc:9000 - - name: AWS_S3_USE_PATH_STYLE - value: "true" - - name: AWS_ACCESS_KEY_ID - value: {{ .Values.rustfs.accessKey | quote }} - - name: AWS_SECRET_ACCESS_KEY - value: {{ .Values.rustfs.secretKey | quote }} -{{- end }} +{{- include "substrate.storage.env" . | nindent 8 }} {{- with .Values.atelet.extraEnv }} {{ toYaml . | indent 8 }} {{- end }} @@ -207,6 +228,8 @@ spec: hostPort: 9090 protocol: TCP volumeMounts: + - name: snapshot-plugin + mountPath: /run/snapshot-plugin - name: run-ateom mountPath: /var/lib/ate - name: podidentity @@ -234,6 +257,8 @@ spec: readOnly: true {{- end }} volumes: + - name: snapshot-plugin + emptyDir: {} - name: run-ateom hostPath: path: /var/lib/ate diff --git a/charts/substrate/templates/atenet-egress.yaml b/charts/substrate/templates/atenet-egress.yaml index 8266364641..d62a62207e 100644 --- a/charts/substrate/templates/atenet-egress.yaml +++ b/charts/substrate/templates/atenet-egress.yaml @@ -29,6 +29,8 @@ data: config.yaml: | # yaml-language-server: $schema=https://agentgateway.dev/schema/config frontendPolicies: + connect: + mode: tunnel accessLog: add: substrate.connect.authority: source.connectHeaders["host"] @@ -43,16 +45,19 @@ data: key: /run/podidentity.podcert.ate.dev/credential-bundle.pem root: /run/servicedns.podcert.ate.dev/trust-bundle.pem - binds: - - port: 8443 - tunnelProtocol: connect - listeners: - - protocol: HTTPS + # Bind the CONNECT front door to the pod IP so tunnel re-entry selects + # the wildcard internal listener instead of this gateway again. + gateways: + egress: + port: 8443 + bindAddress: $POD_IP + protocol: HTTPS tls: cert: /run/servicedns.podcert.ate.dev/credential-bundle.pem key: /run/servicedns.podcert.ate.dev/credential-bundle.pem root: /run/actor-id-ca-certs/ca.crt - routes: [] + + binds: - mode: internal protocol: AUTO listeners: @@ -65,7 +70,8 @@ data: - backends: - dynamic: {} policies: - backendTLS: {} + backendTLS: + root: /etc/ssl/certs/ca-certificates.crt policies: substrateEgress: host: {{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443 @@ -143,6 +149,10 @@ spec: - -f - /etc/agentgateway/config.yaml env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP - name: RUST_LOG value: debug ports: diff --git a/charts/substrate/templates/atenet-router.yaml b/charts/substrate/templates/atenet-router.yaml index e438c510f5..2bda63273f 100644 --- a/charts/substrate/templates/atenet-router.yaml +++ b/charts/substrate/templates/atenet-router.yaml @@ -95,6 +95,17 @@ data: - path: pathPrefix: / policies: &substrate-ingress + # Inspect every value before forwarding; substrateIngress reads only + # the first actor header. Apply this to HTTP, gRPC, and CONNECT traffic. + directResponse: + conditional: + - condition: >- + ('ate-target-actor' in source.connectHeaders && + size(source.connectHeaders.raw()['ate-target-actor']) != 1) || + ('ate-target-actor' in request.headers && + size(request.headers.raw()['ate-target-actor']) != 1) + status: 404 + body: Invalid ate-target-actor header substrateIngress: host: {{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443 # AgentGateway only uses atunnel's CONNECT listener. diff --git a/charts/substrate/templates/sandboxconfig-gvisor.yaml b/charts/substrate/templates/sandboxconfig-gvisor.yaml index 286584eb01..52f52ef313 100644 --- a/charts/substrate/templates/sandboxconfig-gvisor.yaml +++ b/charts/substrate/templates/sandboxconfig-gvisor.yaml @@ -16,20 +16,32 @@ limitations under the License. # Cluster-wide SandboxConfig for the gVisor (runsc) sandbox class, shipped with # the platform so gVisor ActorTemplates have a config to name via -# sandboxConfig.configName. +# sandboxConfig.configName. atelet fetches the gVisor release tarball +# (gvisor.tar.zstd: runsc plus the gvisor-bin/ helpers runsc requires next to +# it) matching the worker node's architecture and extracts it locally. To pin a +# different release, add a version below and point defaultVersion at it, or +# create another SandboxConfig and name it from the ActorTemplate. apiVersion: ate.dev/v1alpha1 kind: SandboxConfig metadata: name: gvisor-default + annotations: + sandboxconfig.ate.dev/is-class-default: "true" spec: sandboxClass: gvisor - pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" - assets: - amd64: - gvisor: - url: "gs://gvisor/releases/nightly/2026-09-02/x86_64/gvisor.tar.zstd" - sha256: "d547d81401461fd1c679c5c4fa0a6c2b8ef7dc3c22ce23c9e25dcc4c69cfd06f" - arm64: - gvisor: - url: "gs://gvisor/releases/nightly/2026-09-02/aarch64/gvisor.tar.zstd" - sha256: "a64916f9813ce7e4841a30480a599337f7dda07b421c6bf0123db2212aa7d1df" + defaultVersion: v1 + versions: + - name: v1 + state: Enabled + # The root sandbox container's image. On GCP, prefer the in-project mirror + # gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da. + pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" + assets: + amd64: + gvisor: + url: "gs://gvisor/releases/nightly/2026-09-02/x86_64/gvisor.tar.zstd" + sha256: "d547d81401461fd1c679c5c4fa0a6c2b8ef7dc3c22ce23c9e25dcc4c69cfd06f" + arm64: + gvisor: + url: "gs://gvisor/releases/nightly/2026-09-02/aarch64/gvisor.tar.zstd" + sha256: "a64916f9813ce7e4841a30480a599337f7dda07b421c6bf0123db2212aa7d1df" diff --git a/charts/substrate/templates/sandboxconfig-validation.yaml b/charts/substrate/templates/sandboxconfig-validation.yaml index baa8768453..3ddbbbe2a2 100644 --- a/charts/substrate/templates/sandboxconfig-validation.yaml +++ b/charts/substrate/templates/sandboxconfig-validation.yaml @@ -16,7 +16,8 @@ limitations under the License. # Per-sandbox-class asset requirements for SandboxConfig. The CRD schema is # generic (any arch -> any asset name -> {url, sha256}); this policy enforces the -# requirements a given sandbox class actually needs, fail-closed at apply time. +# requirements a given sandbox class actually needs on every version under +# spec.versions, fail-closed at apply time. # (url/sha256 being required and well-formed is enforced by the CRD schema.) apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingAdmissionPolicy @@ -31,22 +32,26 @@ spec: operations: ["CREATE", "UPDATE"] resources: ["sandboxconfigs"] validations: - # gVisor needs a release tarball (or legacy runsc binary) for every architecture. + # gVisor needs a "gvisor" release-tarball asset (gvisor.tar.zstd: runsc plus + # its gvisor-bin/ helpers) for every architecture it advertises. A legacy + # bare-binary "runsc" asset is still accepted for releases that predate the + # tarball. - expression: >- object.spec.sandboxClass != 'gvisor' || - (has(object.spec.assets) && size(object.spec.assets) > 0 && - object.spec.assets.all(arch, - 'gvisor' in object.spec.assets[arch] || 'runsc' in object.spec.assets[arch])) - message: "a gvisor SandboxConfig must define a 'gvisor' (release tarball) or legacy 'runsc' asset for every architecture under spec.assets" + object.spec.versions.all(v, + has(v.assets) && size(v.assets) > 0 && + v.assets.all(arch, 'gvisor' in v.assets[arch] || 'runsc' in v.assets[arch])) + message: "a gvisor SandboxConfig must define a 'gvisor' (release tarball) or legacy 'runsc' asset for every architecture of every version under spec.versions[].assets" # The micro-VM (cloud-hypervisor) runtime needs its asset set for every # architecture it advertises. - expression: >- object.spec.sandboxClass != 'microvm' || - (has(object.spec.assets) && size(object.spec.assets) > 0 && - object.spec.assets.all(arch, - ['cloud-hypervisor', 'virtiofsd', 'kata-kernel', 'kata-image'] - .all(name, name in object.spec.assets[arch]))) - message: "a microvm SandboxConfig must define cloud-hypervisor, virtiofsd, kata-kernel, and kata-image assets for every architecture under spec.assets" + object.spec.versions.all(v, + has(v.assets) && size(v.assets) > 0 && + v.assets.all(arch, + ['cloud-hypervisor', 'virtiofsd', 'kata-kernel', 'kata-image'] + .all(name, name in v.assets[arch]))) + message: "a microvm SandboxConfig must define cloud-hypervisor, virtiofsd, kata-kernel, and kata-image assets for every architecture of every version under spec.versions[].assets" --- apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingAdmissionPolicyBinding diff --git a/charts/substrate/values.yaml b/charts/substrate/values.yaml index 961932eff2..a55c9c99a1 100644 --- a/charts/substrate/values.yaml +++ b/charts/substrate/values.yaml @@ -90,8 +90,21 @@ atelet: cpu: "2" memory: 1Gi +# Object storage credentials/settings for both snapshot plugins and atelet asset downloads. +snapshotPlugin: + extraEnv: [] + control: + resources: {} + node: + resources: + requests: + cpu: 50m + memory: 128Mi + # Additional arguments appended to the ateapi defaults. ateApi: + # Defaults to the in-cluster IdP URL; set a stable issuer for external consumers. + actorJWTIssuer: "" extraArgs: [] resources: requests: diff --git a/cmd/ate-setup/README.md b/cmd/ate-setup/README.md index 72316e675e..9d99f27b31 100644 --- a/cmd/ate-setup/README.md +++ b/cmd/ate-setup/README.md @@ -4,7 +4,7 @@ Installs and tears down Agent Substrate on a Kubernetes cluster. ``` go run ./cmd/ate-setup [global flags] [flags] -make build-ate-setup # builds bin/ate-setup +go build -o bin/ate-setup ./cmd/ate-setup ``` `ate-setup` is the installer. `hack/install-ate.sh` is a shim over it, kept so diff --git a/cmd/ate-setup/cli-diff.md b/cmd/ate-setup/cli-diff.md index 69eeffe46d..da775db851 100644 --- a/cmd/ate-setup/cli-diff.md +++ b/cmd/ate-setup/cli-diff.md @@ -41,7 +41,7 @@ no `KUBECTL_CONTEXT`, a `gcloud container clusters get-credentials` per action rather than one for the whole line. `go` is therefore required to run the installer. In exchange, `kubectl`, `jq`, -`openssl`, `sed`, `base64`, and `make` are not; see the external-binaries table +`openssl`, `sed`, and `base64` are not; see the external-binaries table in [`differences.md`](differences.md#external-binaries) for what is still needed. diff --git a/cmd/ate-setup/commands.md b/cmd/ate-setup/commands.md index 160dc5d6b1..5131b60c4c 100644 --- a/cmd/ate-setup/commands.md +++ b/cmd/ate-setup/commands.md @@ -28,6 +28,7 @@ a pre-scan pass, so they may appear anywhere on its command line. | `--rollout-timeout DURATION` | `--rollout-timeout DURATION` | Readiness timeout for workloads (default `60s`). Unlike the shell flag it also governs the podcertificate-controller and CSI waits, which stay at their 120s default until it is passed | | `--podcert-workers-per-signer N` | `--podcert-workers-per-signer N` | Concurrent workers per podcertificate-controller signer | | `--cluster-size size0\|size10` | `--cluster-size size0\|size10` | Footprint profile (default `size0`). `size10` assumes a dedicated PostgreSQL node: it resizes the bundled StatefulSet and its `postgresql.conf`, pins the apiserver's connection pool, and raises the podcertificate-controller's API rate limits. `ATE_INSTALL_CLUSTER_SIZE` when the flag is absent | +| `--ateapi-postgres-storage-class NAME` | `ATE_API_POSTGRES_STORAGE_CLASS=NAME` | StorageClass for the bundled PostgreSQL volume instead of the cluster default, such as `dynamic-rwo` on GKE so C4 and N4 nodes can attach it. An existing install must delete the postgres StatefulSet and its claim first | | `--cordon-control-plane` | `--cordon-control-plane` | Keep the control plane off the worker nodes. Assumes a small shared pool labeled and tainted `ate.dev/workloadType=ate-control-plane:NoSchedule`, across which each workload's replicas are spread, and a one-node pool labeled and tainted `ate.dev/workloadType=ate-postgres:NoSchedule` for postgres alone. `ATE_INSTALL_CORDON_CONTROL_PLANE=true` when the flag is absent | | `--experimental-additional-egress-extproc-service NS/SVC:PORT` | `--experimental-additional-egress-extproc-service NS/SVC:PORT` | External processor authorization filter | | `--credential-provider JSON` | `--credential-provider JSON`, or `ATE_CREDENTIAL_PROVIDER` | Required by `deploy ate-system` and `deploy atenet`. A JSON object: `{"name":"k8s.io"}` deploys and uses the bundled Kubernetes Secrets provider, with a NetworkPolicy that admits only the egress gateway; `{"enabled":false}` turns egress credential injection off; `{"name":"","address":":"}` uses a provider you deploy yourself. See [`docs/egress-credential-injection.md`](../../docs/egress-credential-injection.md) | @@ -62,9 +63,9 @@ toolchain, and write access to a registry. `REPO` has to hold every component image the manifests reference, all under the same tag, which is how a release publishes them. A release that adds a component has to publish it alongside the others before a pre-built install can use it. -`make build-release-images KO_DOCKER_REPO=REPO VERSION=TAG` publishes the full -set, including `envoy-dataplane`, which is built from a Dockerfile with `docker -buildx` rather than with `ko`. A build from source builds that image itself, so +`VERSION=TAG ate-setup publish release-images --ko-docker-repo REPO` publishes +the full set, including `envoy-dataplane`, which is built from a Dockerfile with +`docker buildx` rather than with `ko`. A build from source builds that image itself, so it needs `docker` as well as `ko`. Each reference is then pinned to the digest its tag names, which takes one HEAD request per image, so the installer needs read access to `REPO` and not only the @@ -104,9 +105,15 @@ a running cluster. | `ate-setup` | `hack/install-ate.sh` | |---|---| | `publish worker-images` | (no shell equivalent) | +| `publish release-images` | (no shell equivalent) | -Builds and pushes the ateom worker images for the checked-out build and prints -their refs; a WorkerPool points `spec.workerImage` to a build to use the ateom. +`publish worker-images` builds and pushes the ateom worker images for the +checked-out build and prints their refs; a WorkerPool points `spec.workerImage` +to a build to use the ateom. + +`publish release-images` builds and pushes every image a pre-built install +needs, all tagged with the build version; see [Installing a +release](#installing-a-release). ## Delete diff --git a/cmd/ate-setup/differences.md b/cmd/ate-setup/differences.md index 07104b6d6d..6e92de84d5 100644 --- a/cmd/ate-setup/differences.md +++ b/cmd/ate-setup/differences.md @@ -43,9 +43,9 @@ These were treated as contracts and reproduced exactly: - **Rendered bytes.** `authentication.yaml` is trimmed of its trailing newline because the shell built it inside `$(...)`, which strips them. Installing over a shell-installed cluster must not rewrite the ConfigMap. -- **ko's ldflags.** `make ldflags` emitted `-X=.Version=$(git +- **ko's ldflags.** The shell scripts stamped `-X=.Version=$(git describe --tags --always --dirty)`; `ko.Runner.ldflags` computes the identical - string without depending on make. + string. ## Execution model diff --git a/cmd/ate-setup/internal/cmd/publish.go b/cmd/ate-setup/internal/cmd/publish.go index 96525ace90..48d7718440 100644 --- a/cmd/ate-setup/internal/cmd/publish.go +++ b/cmd/ate-setup/internal/cmd/publish.go @@ -38,7 +38,24 @@ ateom ref.`, }, } +var publishReleaseImagesCmd = &cobra.Command{ + Use: "release-images", + Short: "Build and push every image a pre-built install needs, tagged with the build version", + Long: `Build and push every component image, including the Dockerfile-built +envoy-dataplane, to KO_DOCKER_REPO, all tagged with the build version (VERSION, +else git describe), and print their pushed references. + +The result is what "deploy --image-repo REPO --image-tag TAG" installs: + + VERSION=TAG ate-setup publish release-images --ko-docker-repo REPO`, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + return env.PublishReleaseImages(cmd.Context(), cmd.OutOrStdout()) + }, +} + func init() { rootCmd.AddCommand(publishCmd) publishCmd.AddCommand(publishWorkerImagesCmd) + publishCmd.AddCommand(publishReleaseImagesCmd) } diff --git a/cmd/ate-setup/internal/config/config.go b/cmd/ate-setup/internal/config/config.go index 7e7042398a..8050a58aea 100644 --- a/cmd/ate-setup/internal/config/config.go +++ b/cmd/ate-setup/internal/config/config.go @@ -121,6 +121,10 @@ type Config struct { // pool (ACTOR_JWT_ALGORITHM): ES256 or RS256. ActorJWTAlgorithm string + // ActorJWTIssuer is the iss claim of actor JWTs (ACTOR_JWT_ISSUER). It + // defaults to the in-cluster URL of the install's ate-idp-server Service. + ActorJWTIssuer string + // BucketName is the snapshot bucket demos are templated with. BucketName string @@ -160,6 +164,9 @@ type Config struct { // published as the postgres-server-ca Secret, which ate-api-server mounts // at /run/postgres-server-ca/server-ca.pem for sslmode=verify-ca DSNs. PostgresServerCAFile string + // PostgresStorageClass is the StorageClass of the bundled PostgreSQL volume + // (ATE_API_POSTGRES_STORAGE_CLASS). Empty leaves the cluster default. + PostgresStorageClass string // CloudSQL points the apiserver at a Cloud SQL instance through the Auth // Proxy sidecar instead of a directly reachable PostgreSQL. CloudSQL CloudSQLConfig diff --git a/cmd/ate-setup/internal/config/config_test.go b/cmd/ate-setup/internal/config/config_test.go index edba7d2de8..240446ecfe 100644 --- a/cmd/ate-setup/internal/config/config_test.go +++ b/cmd/ate-setup/internal/config/config_test.go @@ -39,6 +39,7 @@ func loadEnv(t *testing.T) { t.Setenv("NO_DEV_ENV", "1") for _, name := range []string{ "ACTOR_JWT_ALGORITHM", + "ACTOR_JWT_ISSUER", "ANTHROPIC_API_KEY", "ATE_ADDITIONAL_EGRESS_EXTPROC_SERVICE", "ATE_API_POSTGRES_CLOUDSQL_GSA", @@ -435,6 +436,47 @@ func TestLoadActorJWTAlgorithm(t *testing.T) { } } +func TestLoadActorJWTIssuer(t *testing.T) { + for _, tt := range []struct { + name string + env string + namespace string + want string + wantErr bool + }{ + {name: "unset", want: "https://idp.ate-system.svc"}, + {name: "unset in a relocated install", namespace: "team-a", want: "https://idp.team-a.svc"}, + {name: "set is used as given", env: "https://idp.example.com/prod/", want: "https://idp.example.com/prod/"}, + {name: "not https", env: "http://idp.example.com", wantErr: true}, + } { + t.Run(tt.name, func(t *testing.T) { + loadEnv(t) + if tt.env != "" { + t.Setenv("ACTOR_JWT_ISSUER", tt.env) + } + if tt.namespace != "" { + t.Setenv("ATE_NAMESPACE", tt.namespace) + } else { + unsetEnv(t, "ATE_NAMESPACE") + } + + cfg, err := Load(Options{}) + if tt.wantErr { + if err == nil { + t.Fatalf("Load() with ACTOR_JWT_ISSUER=%q returned nil error", tt.env) + } + return + } + if err != nil { + t.Fatalf("Load() error = %v", err) + } + if cfg.ActorJWTIssuer != tt.want { + t.Errorf("ActorJWTIssuer = %q, want %q", cfg.ActorJWTIssuer, tt.want) + } + }) + } +} + // The endpoint has to reach both the Go steps and the shell scripts ate-setup // still delegates to, or the two halves of an install export different // collectors. diff --git a/cmd/ate-setup/internal/config/coverage_test.go b/cmd/ate-setup/internal/config/coverage_test.go index 9b89535f7a..287c192027 100644 --- a/cmd/ate-setup/internal/config/coverage_test.go +++ b/cmd/ate-setup/internal/config/coverage_test.go @@ -98,6 +98,8 @@ func valueFor(key string) string { return `{"name":"k8s.io"}` case "actorJWT.algorithm": return "ES256" + case "actorJWT.issuer": + return "https://idp.example.com" case "csi.setup": return "none" } diff --git a/cmd/ate-setup/internal/config/load.go b/cmd/ate-setup/internal/config/load.go index 0d6bec4461..81fe2aa2dc 100644 --- a/cmd/ate-setup/internal/config/load.go +++ b/cmd/ate-setup/internal/config/load.go @@ -24,6 +24,8 @@ import ( "github.com/spf13/pflag" "github.com/agent-substrate/substrate/cmd/ate-setup/internal/images" + "github.com/agent-substrate/substrate/internal/installdefaults" + "github.com/agent-substrate/substrate/internal/oidcdiscovery" ) // ConfigPathEnv names the configuration document when --config is not given. @@ -196,6 +198,10 @@ func buildConfig(root string, env map[string]string, r *Resolved) (*Config, erro ownerConnectionString = readWriteConnectionString } instance, _ := r.Value("ateapi.postgres.cloudsql.instance") + actorJWTIssuer := r.String("actorJWT.issuer") + if actorJWTIssuer == "" { + actorJWTIssuer = installdefaults.ActorJWTIssuer(r.String("namespace")) + } cfg := &Config{ Root: root, @@ -211,6 +217,7 @@ func buildConfig(root string, env map[string]string, r *Resolved) (*Config, erro KODockerRepo: r.String("ko.dockerRepo"), KODefaultPlatforms: r.String("ko.defaultPlatforms"), ActorJWTAlgorithm: r.String("actorJWT.algorithm"), + ActorJWTIssuer: actorJWTIssuer, DockerBuildFlags: strings.Fields(r.String("docker.buildFlags")), Images: images.Source{ Repo: strings.TrimSuffix(r.String("images.repo"), "/"), @@ -220,6 +227,7 @@ func buildConfig(root string, env map[string]string, r *Resolved) (*Config, erro PostgresSchema: r.String("ateapi.postgres.schema"), PostgresPoolMaxConns: r.String("ateapi.postgres.poolMaxConns"), PostgresServerCAFile: r.String("ateapi.postgres.serverCAFile"), + PostgresStorageClass: r.String("ateapi.postgres.storageClass"), PostgresReadWriteConnectionString: readWriteConnectionString, PostgresOwnerConnectionString: ownerConnectionString, @@ -366,6 +374,10 @@ func validateResolved(cfg *Config, r *Resolved) error { alg, _ := r.Value("actorJWT.algorithm") return &InvalidError{Value: alg, Want: "ES256 or RS256"} } + if err := oidcdiscovery.ValidateIssuer(cfg.ActorJWTIssuer); err != nil { + issuer, _ := r.Value("actorJWT.issuer") + return fmt.Errorf("%s: %w (from %s)", issuer.Setting.Key, err, issuer.From.Describe(issuer.Setting)) + } if cfg.AdditionalEgressExtprocService != "" { extproc, _ := r.Value("atenet.egress.additionalExtprocService") if err := validateExtprocService(extproc); err != nil { diff --git a/cmd/ate-setup/internal/config/setting.go b/cmd/ate-setup/internal/config/setting.go index e52d783a4e..0f803f2a11 100644 --- a/cmd/ate-setup/internal/config/setting.go +++ b/cmd/ate-setup/internal/config/setting.go @@ -122,6 +122,12 @@ var Registry = []Setting{ Kind: KindString, Default: "ES256", Usage: "Signing algorithm of the key in a new actor JWT pool: ES256 or RS256", }, + { + // The default depends on the namespace, so buildConfig applies it. + Key: "actorJWT.issuer", Env: "ACTOR_JWT_ISSUER", Flag: "actor-jwt-issuer", Kind: KindString, + Usage: "iss claim of actor JWTs, verbatim. Relying parties fetch /.well-known/openid-configuration " + + "(default https://idp..svc)", + }, { // Whitespace-separated, as the environment variable is. The registry // has no list kind, so the value is carried as written and split @@ -211,6 +217,12 @@ var Registry = []Setting{ Flag: "ateapi-postgres-server-ca-file", Kind: KindString, Usage: "Local PEM file holding an external PostgreSQL server CA", }, + { + Key: "ateapi.postgres.storageClass", Env: "ATE_API_POSTGRES_STORAGE_CLASS", + Flag: "ateapi-postgres-storage-class", Kind: KindString, + Usage: "StorageClass for the bundled PostgreSQL volume, instead of the cluster default. " + + "An existing install must delete the postgres StatefulSet and its claim first", + }, { Key: "ateapi.postgres.cloudsql.instance", Env: "ATE_API_POSTGRES_CLOUDSQL_INSTANCE", Flag: "ateapi-postgres-cloudsql-instance", Kind: KindString, diff --git a/cmd/ate-setup/internal/images/dockerfile.go b/cmd/ate-setup/internal/images/dockerfile.go index b59c933568..5b0549d126 100644 --- a/cmd/ate-setup/internal/images/dockerfile.go +++ b/cmd/ate-setup/internal/images/dockerfile.go @@ -18,21 +18,54 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" + "io/fs" "os" "os/exec" + "path/filepath" "strings" "time" + + "sigs.k8s.io/yaml" ) -// dockerfilePlatforms is the buildx --platform value: the install's -// KO_DEFAULTPLATFORMS, so the image lands on the same nodes as the ko images, -// or linux/amd64 when unset. -func dockerfilePlatforms(koDefaultPlatforms string) string { - if koDefaultPlatforms == "" { - return "linux/amd64" +// dockerfilePlatforms is the buildx --platform value, chosen the way ko +// chooses its own so the image runs on the same nodes as the ko images: +// KO_DEFAULTPLATFORMS, else defaultPlatforms from the ko config, else +// linux/amd64. +func dockerfilePlatforms(rootDir, koDefaultPlatforms string) (string, error) { + if koDefaultPlatforms != "" { + return koDefaultPlatforms, nil + } + path := filepath.Join(rootDir, ".ko.yaml") + // ko runs from rootDir, so a relative KO_CONFIG_PATH is relative to it. + if p := os.Getenv("KO_CONFIG_PATH"); p != "" { + if !filepath.IsAbs(p) { + p = filepath.Join(rootDir, p) + } + path = p + if info, err := os.Stat(p); err == nil && info.IsDir() { + path = filepath.Join(p, ".ko.yaml") + } + } + raw, err := os.ReadFile(path) + if errors.Is(err, fs.ErrNotExist) { + return "linux/amd64", nil + } + if err != nil { + return "", fmt.Errorf("while reading the ko config: %w", err) + } + var koConfig struct { + DefaultPlatforms []string `json:"defaultPlatforms"` } - return koDefaultPlatforms + if err := yaml.Unmarshal(raw, &koConfig); err != nil { + return "", fmt.Errorf("while parsing %s: %w", path, err) + } + if len(koConfig.DefaultPlatforms) == 0 { + return "linux/amd64", nil + } + return strings.Join(koConfig.DefaultPlatforms, ","), nil } // dockerBuildArgs returns the docker arguments that build contextPath for @@ -51,11 +84,21 @@ func dockerBuildArgs(platforms string, extraFlags []string, tag, contextPath str // push to; the returned reference always uses the digest, so a stale tag can // never be resolved by accident. func BuildDockerfileImage(ctx context.Context, rootDir, dockerRepo, imageName, contextPath, koDefaultPlatforms string, extraFlags []string) (string, error) { + return PublishDockerfileImage(ctx, rootDir, dockerRepo, imageName, fmt.Sprintf("build-%d", time.Now().Unix()), contextPath, koDefaultPlatforms, extraFlags) +} + +// PublishDockerfileImage is BuildDockerfileImage with the tag chosen by the +// caller, for a release that publishes every image under one tag. +func PublishDockerfileImage(ctx context.Context, rootDir, dockerRepo, imageName, tag, contextPath, koDefaultPlatforms string, extraFlags []string) (string, error) { repo := strings.TrimSuffix(dockerRepo, "/") + "/" + imageName - stageTag := fmt.Sprintf("%s:build-%d", repo, time.Now().Unix()) + stageTag := repo + ":" + tag + platforms, err := dockerfilePlatforms(rootDir, koDefaultPlatforms) + if err != nil { + return "", err + } build := exec.CommandContext(ctx, "docker", - dockerBuildArgs(dockerfilePlatforms(koDefaultPlatforms), extraFlags, stageTag, contextPath)...) + dockerBuildArgs(platforms, extraFlags, stageTag, contextPath)...) build.Dir = rootDir // The shell version sent build output to stderr so it could capture the // image reference on stdout; keeping that split makes the two behave the diff --git a/cmd/ate-setup/internal/images/dockerfile_test.go b/cmd/ate-setup/internal/images/dockerfile_test.go index c018c3dac7..97fea6f08f 100644 --- a/cmd/ate-setup/internal/images/dockerfile_test.go +++ b/cmd/ate-setup/internal/images/dockerfile_test.go @@ -15,6 +15,9 @@ package images import ( + "cmp" + "os" + "path/filepath" "slices" "testing" ) @@ -31,3 +34,46 @@ func TestDockerBuildArgs(t *testing.T) { t.Errorf("dockerBuildArgs() without flags = %q, want %q", got, want) } } + +// ko builds for .ko.yaml's defaultPlatforms unless KO_DEFAULTPLATFORMS says +// otherwise, and a Dockerfile image installed beside the ko images has to +// cover the same platforms or it cannot run on some of their nodes. +func TestDockerfilePlatforms(t *testing.T) { + const twoPlatforms = "defaultPlatforms:\n - linux/amd64\n - linux/arm64\n" + for _, tc := range []struct { + name string + koYAML string // empty: no ko config + koYAMLFile string // where it is written, relative to the root; default .ko.yaml + koConfigPath string + koPlatforms string + want string + }{ + {name: "explicit platforms win", koYAML: twoPlatforms, koPlatforms: "linux/s390x", want: "linux/s390x"}, + {name: "ko config", koYAML: twoPlatforms, want: "linux/amd64,linux/arm64"}, + {name: "ko config without platforms", koYAML: "defaultBaseImage: example.com/base\n", want: "linux/amd64"}, + {name: "no ko config", want: "linux/amd64"}, + {name: "KO_CONFIG_PATH file", koYAML: twoPlatforms, koYAMLFile: "conf/ko.yaml", koConfigPath: "conf/ko.yaml", want: "linux/amd64,linux/arm64"}, + {name: "KO_CONFIG_PATH directory", koYAML: twoPlatforms, koYAMLFile: "conf/.ko.yaml", koConfigPath: "conf", want: "linux/amd64,linux/arm64"}, + } { + t.Run(tc.name, func(t *testing.T) { + root := t.TempDir() + t.Setenv("KO_CONFIG_PATH", tc.koConfigPath) + if tc.koYAML != "" { + file := filepath.Join(root, cmp.Or(tc.koYAMLFile, ".ko.yaml")) + if err := os.MkdirAll(filepath.Dir(file), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(file, []byte(tc.koYAML), 0o644); err != nil { + t.Fatal(err) + } + } + got, err := dockerfilePlatforms(root, tc.koPlatforms) + if err != nil { + t.Fatalf("dockerfilePlatforms() error = %v", err) + } + if got != tc.want { + t.Errorf("dockerfilePlatforms() = %q, want %q", got, tc.want) + } + }) + } +} diff --git a/cmd/ate-setup/internal/images/images.go b/cmd/ate-setup/internal/images/images.go index 461ebef1b9..dd4e97f6eb 100644 --- a/cmd/ate-setup/internal/images/images.go +++ b/cmd/ate-setup/internal/images/images.go @@ -48,6 +48,7 @@ var Components = []string{ "cmd/ateom-microvm", "cmd/credential-provider/kubernetes-secrets", "cmd/podcertcontroller", + "cmd/snapshot-plugin", "demos/counter", "demos/egress", "demos/multi-template/fspersist", @@ -68,8 +69,7 @@ func KoReference(pkg string) string { return "ko://" + ModulePath + "/" + pkg } // well, which changes nothing while every package here already is. // // This only describes reality because every ko invocation asks for that -// naming — internal/ko passes the flag, and the Makefile passes it as -// KO_NAMING. ko's own default appends an md5 of the full import path, which +// naming: internal/ko passes the flag. ko's own default appends an md5 of the full import path, which // nothing here can reproduce, so a build that loses the flag publishes images // no --image-repo install can find. func ImageName(pkg string) string { return path.Base(pkg) } diff --git a/cmd/ate-setup/internal/images/images_test.go b/cmd/ate-setup/internal/images/images_test.go index 9006a03191..523e01950f 100644 --- a/cmd/ate-setup/internal/images/images_test.go +++ b/cmd/ate-setup/internal/images/images_test.go @@ -134,36 +134,3 @@ func TestImageNamesAreUnique(t *testing.T) { byName[name] = pkg } } - -// ImageName is only right if the images were published under that naming, and -// the Makefile is what publishes them. Losing the flag there breaks nothing -// visible -- an install from source never looks an image up by name -- and -// surfaces much later as an --image-repo install that cannot find a single -// component. -func TestMakefilePublishesWithBaseImportPaths(t *testing.T) { - root, err := config.RepoRoot() - if err != nil { - t.Fatalf("resolving repo root: %v", err) - } - makefile, err := os.ReadFile(filepath.Join(root, "Makefile")) - if err != nil { - t.Fatalf("reading the Makefile: %v", err) - } - - var koBuilds int - for line := range strings.Lines(string(makefile)) { - if !strings.Contains(line, "$(KO) build") { - continue - } - koBuilds++ - if !strings.Contains(line, "$(KO_NAMING)") { - t.Errorf("Makefile line %q builds images without $(KO_NAMING), so they publish under ko's md5 naming", strings.TrimSpace(line)) - } - } - if koBuilds == 0 { - t.Error("found no `$(KO) build` line in the Makefile; this test no longer checks anything") - } - if !strings.Contains(string(makefile), "KO_NAMING := --base-import-paths") { - t.Error("the Makefile no longer defines KO_NAMING as --base-import-paths") - } -} diff --git a/cmd/ate-setup/internal/ko/ko.go b/cmd/ate-setup/internal/ko/ko.go index a176226340..44d8d82852 100644 --- a/cmd/ate-setup/internal/ko/ko.go +++ b/cmd/ate-setup/internal/ko/ko.go @@ -37,7 +37,6 @@ import ( ) // versionPkg is the package whose Version variable receives the build stamp. -// It matches VERSION_PKG in the Makefile. const versionPkg = "github.com/agent-substrate/substrate/internal/version" // baseImportPaths publishes each image as the last element of its import path, @@ -46,8 +45,7 @@ const versionPkg = "github.com/agent-substrate/substrate/internal/version" // ko's default instead appends an md5 of the full import path, which nothing // outside ko can predict. images.ImageName is the other half of this: it is // how `--image-repo` addresses a published image, and it can only be right if -// the images were published under this naming. The Makefile passes the same -// flag, as KO_NAMING. +// the images were published under this naming. const baseImportPaths = "--base-import-paths" // Runner invokes ko with a fixed repository root and environment. @@ -139,6 +137,26 @@ func (r *Runner) Build(ctx context.Context, importPath string) (string, error) { return lines[len(lines)-1], nil } +// BuildTagged builds and publishes the images for importPaths in one ko +// invocation, tagging each with tag, and returns their pushed references. +func (r *Runner) BuildTagged(ctx context.Context, tag string, importPaths ...string) ([]string, error) { + defer log.Elapsed(time.Now(), "ko build --tags="+tag) + cmd := exec.CommandContext(ctx, r.binary, append(r.args("build", importPaths...), "--tags="+tag)...) + cmd.Dir = r.Root + cmd.Env = append(os.Environ(), r.Env...) + var stdout bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = r.Stderr + if err := cmd.Run(); err != nil { + return nil, fmt.Errorf("while running ko build --tags=%s: %w", tag, err) + } + refs := strings.Fields(stdout.String()) + if len(refs) != len(importPaths) { + return nil, fmt.Errorf("ko build printed %d image references for %d packages", len(refs), len(importPaths)) + } + return refs, nil +} + // ResolvePath resolves a manifest file or directory. func (r *Runner) ResolvePath(ctx context.Context, path string) ([]byte, error) { return r.Resolve(ctx, path, nil) @@ -160,15 +178,13 @@ func (r *Runner) args(subcommand string, target ...string) []string { return args } -// ldflags returns the version stamp ko should bake into binaries. The shell -// scripts shelled out to `make ldflags` for this; computing it here keeps the -// value identical without depending on make. +// ldflags returns the version stamp ko should bake into binaries. func (r *Runner) ldflags() []string { return []string{fmt.Sprintf("-X=%s.Version=%s", versionPkg, BuildVersion(r.Root))} } -// BuildVersion mirrors the Makefile's VERSION: `git describe`, or "dev" when -// git has nothing to say. It is what ko stamps into the binaries. +// BuildVersion is the VERSION environment variable, else `git describe`, or +// "dev" when git has nothing to say. It is what ko stamps into the binaries. func BuildVersion(root string) string { if v := os.Getenv("VERSION"); v != "" { return v diff --git a/cmd/ate-setup/internal/ko/ko_test.go b/cmd/ate-setup/internal/ko/ko_test.go index 55e427d451..a1bbfa9e6a 100644 --- a/cmd/ate-setup/internal/ko/ko_test.go +++ b/cmd/ate-setup/internal/ko/ko_test.go @@ -15,7 +15,10 @@ package ko import ( + "os" + "path/filepath" "slices" + "strings" "testing" ) @@ -66,7 +69,7 @@ func TestBuildVersionPrefersTheEnvironment(t *testing.T) { } } -// A source tarball has no git metadata, and the Makefile falls back to "dev" +// A source tarball has no git metadata, so BuildVersion falls back to "dev" // there rather than stamping an empty version. func TestBuildVersionFallsBackToDev(t *testing.T) { t.Setenv("VERSION", "") @@ -74,3 +77,57 @@ func TestBuildVersionFallsBackToDev(t *testing.T) { t.Errorf("BuildVersion() = %q, want dev", got) } } + +// fakeKo writes a ko stand-in that records its arguments, one per line, and +// prints stdout. +func fakeKo(t *testing.T, stdout string) (binary, argsFile string) { + t.Helper() + dir := t.TempDir() + argsFile = filepath.Join(dir, "args") + binary = filepath.Join(dir, "ko") + script := "#!/bin/sh\nprintf '%s\\n' \"$@\" > " + argsFile + "\nprintf '" + stdout + "'\n" + if err := os.WriteFile(binary, []byte(script), 0o755); err != nil { + t.Fatal(err) + } + return binary, argsFile +} + +// A release is only installable if every image carries the tag and the +// --base-import-paths name that --image-repo looks it up by. +func TestBuildTagged(t *testing.T) { + t.Setenv("VERSION", "v1.2.3") + binary, argsFile := fakeKo(t, "repo/ateapi:v1.2.3@sha256:a\\nrepo/atelet:v1.2.3@sha256:b\\n") + r := &Runner{Root: t.TempDir(), Stderr: os.Stderr, binary: binary} + + refs, err := r.BuildTagged(t.Context(), "v1.2.3", "./cmd/ateapi", "./cmd/atelet") + if err != nil { + t.Fatalf("BuildTagged() error = %v", err) + } + if want := []string{"repo/ateapi:v1.2.3@sha256:a", "repo/atelet:v1.2.3@sha256:b"}; !slices.Equal(refs, want) { + t.Errorf("BuildTagged() = %q, want %q", refs, want) + } + raw, err := os.ReadFile(argsFile) + if err != nil { + t.Fatal(err) + } + args := strings.Fields(string(raw)) + if got := args[:3]; !slices.Equal(got, []string{"build", "./cmd/ateapi", "./cmd/atelet"}) { + t.Errorf("args[:3] = %v, want [build ./cmd/ateapi ./cmd/atelet]", got) + } + for _, want := range []string{"--tags=v1.2.3", "--base-import-paths"} { + if !slices.Contains(args, want) { + t.Errorf("args = %v, want %s", args, want) + } + } +} + +// A ref missing from ko's output would leave a release short an image, so it +// fails the build rather than printing an incomplete list. +func TestBuildTaggedRejectsMissingRefs(t *testing.T) { + binary, _ := fakeKo(t, "repo/ateapi:v1@sha256:a\\n") + r := &Runner{Root: t.TempDir(), Stderr: os.Stderr, binary: binary} + + if _, err := r.BuildTagged(t.Context(), "v1", "./cmd/ateapi", "./cmd/atelet"); err == nil { + t.Error("BuildTagged() error = nil, want one for 1 ref from 2 packages") + } +} diff --git a/cmd/ate-setup/internal/steps/apienv.go b/cmd/ate-setup/internal/steps/apienv.go index 209f2af5c9..2669a533d2 100644 --- a/cmd/ate-setup/internal/steps/apienv.go +++ b/cmd/ate-setup/internal/steps/apienv.go @@ -34,7 +34,8 @@ const envHashAnnotation = "ate.dev/env-hash" // CreateAPIServerEnvVars reconciles how ate-api-server reaches its PostgreSQL // store: both DSNs and the schema into the ate-api-server-secret-envvars Secret, // stable roles and Cloud SQL settings into the ConfigMap, -// and an external server CA into postgres-server-ca. +// and an external server CA into postgres-server-ca. The ConfigMap also +// carries the actor JWT issuer. // // ate-api-server.yaml pulls both in through optional envFrom sources and // resolves the PostgreSQL connection, role, and schema flags from @@ -112,6 +113,7 @@ func (e *Env) CreateAPIServerEnvVars(ctx context.Context) error { if poolMaxConns != "" { configVars["ATE_API_POSTGRES_POOL_MAX_CONNS"] = poolMaxConns } + configVars["ATE_API_ACTOR_JWT_ISSUER"] = e.Cfg.ActorJWTIssuer if err := e.Kube.ApplyConfigMap(ctx, e.Namespace(), ConfigMapAPIEnvVars, configVars); err != nil { return err } diff --git a/cmd/ate-setup/internal/steps/apienv_test.go b/cmd/ate-setup/internal/steps/apienv_test.go index 6d51453da5..f47616e465 100644 --- a/cmd/ate-setup/internal/steps/apienv_test.go +++ b/cmd/ate-setup/internal/steps/apienv_test.go @@ -177,6 +177,28 @@ func TestCreateAPIServerEnvVarsPoolSize(t *testing.T) { } } +func TestCreateAPIServerEnvVarsActorJWTIssuer(t *testing.T) { + cfg := config.Config{ + PostgresReadWriteConnectionString: "postgres://runtime@postgres/atepg", + ActorJWTIssuer: "https://idp.example.com/prod", + } + e := &Env{Cfg: &cfg, Kube: fakeKube(t, + &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: NamespaceAteSystem}}, + &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: ConfigMapAPIEnvVars, Namespace: NamespaceAteSystem}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: SecretAPIEnvVars, Namespace: NamespaceAteSystem}}, + )} + if err := e.CreateAPIServerEnvVars(t.Context()); err != nil { + t.Fatal(err) + } + cm, err := e.Kube.GetConfigMap(t.Context(), NamespaceAteSystem, ConfigMapAPIEnvVars) + if err != nil { + t.Fatal(err) + } + if got := cm.Data["ATE_API_ACTOR_JWT_ISSUER"]; got != cfg.ActorJWTIssuer { + t.Errorf("ATE_API_ACTOR_JWT_ISSUER = %q, want %q", got, cfg.ActorJWTIssuer) + } +} + func TestCreateAPIServerEnvVarsAdoptsPostgresIdentity(t *testing.T) { const recordedDSN = "user=svc@p.iam host=127.0.0.1 dbname=atepg" const explicitOwnerDSN = "user=new-owner@p.iam host=127.0.0.1 dbname=atepg" diff --git a/cmd/ate-setup/internal/steps/env.go b/cmd/ate-setup/internal/steps/env.go index ad569874c7..1e22e5b889 100644 --- a/cmd/ate-setup/internal/steps/env.go +++ b/cmd/ate-setup/internal/steps/env.go @@ -166,8 +166,8 @@ func (e *Env) koRunner() (*ko.Runner, error) { // KO_DOCKER_REPO. A pre-built install builds nothing: --image-repo installs // images someone else published, and needs neither docker nor a registry to // push to, so the image is pinned from REPO/name:TAG like every ko image. -// That only works if the release published it there, which the Makefile's -// build- target does. +// That only works if the release published it there, which +// `ate-setup publish release-images` does. func (e *Env) dockerfileImage(ctx context.Context, name, contextDir string) (string, error) { if !e.Cfg.Images.IsPrebuilt() { return images.BuildDockerfileImage(ctx, e.Cfg.Root, e.Cfg.KODockerRepo, name, e.Cfg.Path(contextDir), e.Cfg.KODefaultPlatforms, e.Cfg.DockerBuildFlags) @@ -183,7 +183,7 @@ func (e *Env) dockerfileImage(ctx context.Context, name, contextDir string) (str ref, err := prebuilt.Pin(ctx, name) if err != nil { return "", fmt.Errorf("%s is built from a Dockerfile, not with ko, so a release has to publish it "+ - "alongside the other images (make build-%s): %w", name, name, err) + "alongside the other images (ate-setup publish release-images): %w", name, err) } return ref, nil } diff --git a/cmd/ate-setup/internal/steps/overlay_test.go b/cmd/ate-setup/internal/steps/overlay_test.go index 440febd241..3cfd8610e7 100644 --- a/cmd/ate-setup/internal/steps/overlay_test.go +++ b/cmd/ate-setup/internal/steps/overlay_test.go @@ -651,9 +651,20 @@ func TestRenderAgentgatewayCredentialProvider(t *testing.T) { t.Fatal(err) } for _, protocol := range []string{"HTTP", "HTTPS"} { - policy, err := gateway.Pipe(kyaml.Lookup("binds", "[mode=internal]", "listeners", "[protocol="+protocol+"]", "routes", "0", "policies", "substrateEgress")) - if err != nil || policy == nil { - t.Fatalf("%s egress policy missing: %v", protocol, err) + routes, err := gateway.Pipe(kyaml.Lookup("binds", "[mode=internal]", "listeners", "[protocol="+protocol+"]", "routes")) + if err != nil || routes == nil { + t.Fatalf("%s routes missing: %v", protocol, err) + } + var policy *kyaml.RNode + for _, route := range routes.YNode().Content { + candidate, err := kyaml.NewRNode(route).Pipe(kyaml.Lookup("policies", "substrateEgress")) + if err == nil && candidate != nil { + policy = candidate + break + } + } + if policy == nil { + t.Fatalf("%s egress policy missing", protocol) } var got struct { Providers []map[string]any `yaml:"credentialProviders"` @@ -828,7 +839,7 @@ func TestRenderAtenetEgressManifestPrebuilt(t *testing.T) { } // A release that did not publish envoy-dataplane fails the install with a -// message naming the image and the target that publishes it, rather than a bare +// message naming the image and the command that publishes it, rather than a bare // registry error. func TestDockerfileImagePrebuiltNotPublished(t *testing.T) { src := images.Source{Repo: "example.com/substrate", Tag: "v1.2.3"} @@ -844,7 +855,7 @@ func TestDockerfileImagePrebuiltNotPublished(t *testing.T) { t.Fatal("dockerfileImage() error = nil, want one") } for _, want := range []string{ - "make build-envoy-dataplane", + "ate-setup publish release-images", "resolving example.com/substrate/envoy-dataplane:v1.2.3 to a digest: MANIFEST_UNKNOWN", } { if !strings.Contains(err.Error(), want) { diff --git a/cmd/ate-setup/internal/steps/postgres.go b/cmd/ate-setup/internal/steps/postgres.go index 33d7179326..dea4883edf 100644 --- a/cmd/ate-setup/internal/steps/postgres.go +++ b/cmd/ate-setup/internal/steps/postgres.go @@ -152,35 +152,82 @@ func (e *Env) postgresManifestPath() string { // applyPostgres renders and applies the bundled PostgreSQL StatefulSet at the // selected cluster size. -// -// The size10 changes are made to the rendered objects rather than patched onto -// the cluster after a base apply: one apply means one rollout, and the pod the -// rollout wait sees is the resized one. It also means a later server-side -// apply of the same objects cannot half-revert them, which a post-apply patch -// under a different field manager would be exposed to. func (e *Env) applyPostgres(ctx context.Context) error { if err := e.requirePostgresPool(ctx); err != nil { return err } - manifest, err := e.render(e.postgresManifestPath()) + objs, err := e.postgresObjects() if err != nil { return err } + return e.Kube.Apply(ctx, objs) +} + +// postgresObjects renders the bundled PostgreSQL objects with the configured +// StorageClass and cluster size applied. +// +// The changes are made to the rendered objects rather than patched onto the +// cluster after a base apply: one apply means one rollout, and the pod the +// rollout wait sees is the resized one. It also means a later server-side +// apply of the same objects cannot half-revert them, which a post-apply patch +// under a different field manager would be exposed to. +func (e *Env) postgresObjects() ([]*unstructured.Unstructured, error) { + manifest, err := e.render(e.postgresManifestPath()) + if err != nil { + return nil, err + } objs, err := kube.DecodeManifestBytes(manifest) if err != nil { - return err + return nil, err + } + if class := e.Cfg.PostgresStorageClass; class != "" { + log.Stepf("Using StorageClass %s for the bundled PostgreSQL volume", class) + if err := setPostgresStorageClass(objs, class); err != nil { + return nil, err + } } if e.Cfg.Size10() { log.Step("apply_postgres_size10_overrides") conf, err := os.ReadFile(e.Cfg.Manifest("postgres-size10", "postgres-config-patch.yaml")) if err != nil { - return fmt.Errorf("while reading the size10 postgres config: %w", err) + return nil, fmt.Errorf("while reading the size10 postgres config: %w", err) } if err := applyPostgresSize10Overrides(objs, conf); err != nil { - return err + return nil, err } } - return e.Kube.Apply(ctx, objs) + return objs, nil +} + +// setPostgresStorageClass sets the StorageClass of the StatefulSet's data +// volume claim. A claim without one gets the cluster default, which some +// machine series cannot attach. +func setPostgresStorageClass(objs []*unstructured.Unstructured, class string) error { + for _, obj := range objs { + if obj.GetKind() != "StatefulSet" || obj.GetName() != "postgres" { + continue + } + claims, _, err := unstructured.NestedSlice(obj.Object, "spec", "volumeClaimTemplates") + if err != nil { + return fmt.Errorf("while reading the volume claims of %s: %w", kube.Describe(obj), err) + } + for i, c := range claims { + claim, ok := c.(map[string]any) + if !ok { + continue + } + if name, _, _ := unstructured.NestedString(claim, "metadata", "name"); name != "data" { + continue + } + if err := unstructured.SetNestedField(claim, class, "spec", "storageClassName"); err != nil { + return fmt.Errorf("while setting the StorageClass on %s: %w", kube.Describe(obj), err) + } + claims[i] = claim + return unstructured.SetNestedSlice(obj.Object, claims, "spec", "volumeClaimTemplates") + } + return fmt.Errorf("%s has no data volume claim to set the StorageClass on", kube.Describe(obj)) + } + return fmt.Errorf("the postgres manifest has no statefulset/postgres to set the StorageClass on") } // requirePostgresPool refuses to apply the bundled PostgreSQL under diff --git a/cmd/ate-setup/internal/steps/postgres_test.go b/cmd/ate-setup/internal/steps/postgres_test.go index 6caf28ed8c..7b326f92fb 100644 --- a/cmd/ate-setup/internal/steps/postgres_test.go +++ b/cmd/ate-setup/internal/steps/postgres_test.go @@ -152,6 +152,92 @@ func TestApplyPostgresSize10OverridesRejectsMissingObjects(t *testing.T) { } } +// dataClaimStorageClass returns the StorageClass of the postgres StatefulSet's +// data claim, and whether it is set at all. +func dataClaimStorageClass(t *testing.T, objs []*unstructured.Unstructured) (string, bool) { + t.Helper() + ss := findObject(objs, "StatefulSet", "postgres") + if ss == nil { + t.Fatal("no statefulset/postgres") + } + claims, _, _ := unstructured.NestedSlice(ss.Object, "spec", "volumeClaimTemplates") + for _, c := range claims { + claim := c.(map[string]any) + if name, _, _ := unstructured.NestedString(claim, "metadata", "name"); name == "data" { + class, found, _ := unstructured.NestedString(claim, "spec", "storageClassName") + return class, found + } + } + t.Fatal("statefulset/postgres has no data claim") + return "", false +} + +// Runs over the real manifests, so a renamed claim fails here rather than on +// an install that asked for a StorageClass. +func TestSetPostgresStorageClass(t *testing.T) { + for _, kind := range []bool{false, true} { + objs := postgresObjects(t, kind) + if class, found := dataClaimStorageClass(t, objs); found { + t.Fatalf("kind=%v: manifest already names StorageClass %q", kind, class) + } + if err := setPostgresStorageClass(objs, "dynamic-rwo"); err != nil { + t.Fatalf("kind=%v: setPostgresStorageClass: %v", kind, err) + } + if class, _ := dataClaimStorageClass(t, objs); class != "dynamic-rwo" { + t.Errorf("kind=%v: StorageClass = %q, want dynamic-rwo", kind, class) + } + } +} + +func TestSetPostgresStorageClassRejectsMissingObjects(t *testing.T) { + objs := postgresObjects(t, false) + + var withoutStatefulSet []*unstructured.Unstructured + for _, obj := range objs { + if obj.GetKind() != "StatefulSet" { + withoutStatefulSet = append(withoutStatefulSet, obj) + } + } + if err := setPostgresStorageClass(withoutStatefulSet, "x"); err == nil { + t.Error("setPostgresStorageClass succeeded without a StatefulSet, want an error") + } + + ss := findObject(objs, "StatefulSet", "postgres") + unstructured.RemoveNestedField(ss.Object, "spec", "volumeClaimTemplates") + if err := setPostgresStorageClass(objs, "x"); err == nil { + t.Error("setPostgresStorageClass succeeded without a data claim, want an error") + } +} + +func TestPostgresObjectsStorageClass(t *testing.T) { + for _, tc := range []struct { + name string + class string + size10 bool + wantClass string + wantSet bool + }{ + {name: "unset keeps the cluster default"}, + {name: "set", class: "dynamic-rwo", wantClass: "dynamic-rwo", wantSet: true}, + {name: "set with size10", class: "dynamic-rwo", size10: true, wantClass: "dynamic-rwo", wantSet: true}, + } { + t.Run(tc.name, func(t *testing.T) { + cfg := &config.Config{Root: repoRoot(t), PostgresStorageClass: tc.class} + if tc.size10 { + cfg.ClusterSize = config.ClusterSizeSize10 + } + objs, err := (&Env{Cfg: cfg}).postgresObjects() + if err != nil { + t.Fatalf("postgresObjects: %v", err) + } + class, set := dataClaimStorageClass(t, objs) + if class != tc.wantClass || set != tc.wantSet { + t.Errorf("StorageClass = %q (set %v), want %q (set %v)", class, set, tc.wantClass, tc.wantSet) + } + }) + } +} + func TestPlanPostgres(t *testing.T) { for _, tc := range []struct { name string diff --git a/cmd/ate-setup/internal/steps/publish.go b/cmd/ate-setup/internal/steps/publish.go index 1760c54697..e43f207a6f 100644 --- a/cmd/ate-setup/internal/steps/publish.go +++ b/cmd/ate-setup/internal/steps/publish.go @@ -19,6 +19,7 @@ import ( "fmt" "io" + "github.com/agent-substrate/substrate/cmd/ate-setup/internal/images" "github.com/agent-substrate/substrate/cmd/ate-setup/internal/log" ) @@ -60,3 +61,46 @@ func (e *Env) PublishWorkerImages(ctx context.Context, w io.Writer) error { } return nil } + +// PublishReleaseImages builds and pushes every image a pre-built install +// needs, all tagged with the build version, and writes their pushed references +// to w. That is what `deploy --image-repo KO_DOCKER_REPO --image-tag VERSION` +// installs: the ko images in images.Components, plus envoy-dataplane, which is +// built from a Dockerfile. +func (e *Env) PublishReleaseImages(ctx context.Context, w io.Writer) error { + if e.Cfg.KODockerRepo == "" { + return fmt.Errorf("publishing release images needs a registry to push to; set KO_DOCKER_REPO or --ko-docker-repo") + } + tag, _, err := e.SubstrateVersion() + if err != nil { + return err + } + log.Stepf("publish_release_images (%s)", tag) + runner, err := e.koRunner() + if err != nil { + return err + } + pkgs := make([]string, 0, len(images.Components)) + for _, pkg := range images.Components { + pkgs = append(pkgs, "./"+pkg) + } + refs, err := runner.BuildTagged(ctx, tag, pkgs...) + if err != nil { + return err + } + envoy, err := images.PublishDockerfileImage(ctx, e.Cfg.Root, e.Cfg.KODockerRepo, envoyDataplaneImage, tag, + e.Cfg.Path(envoyDataplaneDockefile), e.Cfg.KODefaultPlatforms, e.Cfg.DockerBuildFlags) + if err != nil { + return err + } + refs = append(refs, envoy) + if _, err := fmt.Fprintf(w, "\nRelease images for %s:\n", tag); err != nil { + return err + } + for _, ref := range refs { + if _, err := fmt.Fprintln(w, ref); err != nil { + return err + } + } + return nil +} diff --git a/cmd/ate-setup/internal/steps/publish_test.go b/cmd/ate-setup/internal/steps/publish_test.go index 9cd51f6dde..99f4add104 100644 --- a/cmd/ate-setup/internal/steps/publish_test.go +++ b/cmd/ate-setup/internal/steps/publish_test.go @@ -15,6 +15,7 @@ package steps import ( + "io" "strings" "testing" @@ -41,3 +42,14 @@ func TestKoRunnerPrebuilt(t *testing.T) { t.Errorf("koRunner() error = %q; want it to name --image-repo", err) } } + +// Without a registry ko would fall back to its own default, so the release +// command refuses before building anything. +func TestPublishReleaseImagesNeedsARegistry(t *testing.T) { + e := &Env{Cfg: &config.Config{Root: t.TempDir()}} + + err := e.PublishReleaseImages(t.Context(), io.Discard) + if err == nil || !strings.Contains(err.Error(), "KO_DOCKER_REPO") { + t.Errorf("PublishReleaseImages() error = %v, want one naming KO_DOCKER_REPO", err) + } +} diff --git a/cmd/ateapi/helm_test.go b/cmd/ateapi/helm_test.go index 07c6bfd1e8..acf71ade00 100644 --- a/cmd/ateapi/helm_test.go +++ b/cmd/ateapi/helm_test.go @@ -19,7 +19,9 @@ import ( "github.com/agent-substrate/substrate/pkg/postgressetup" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" + "os" "os/exec" + "reflect" "slices" "strings" "testing" @@ -191,3 +193,129 @@ func TestHelmPostgresCertificates(t *testing.T) { }) } } + +// Snapshot transfers must reach the same backend from both plugin processes. +// The API rejects storage settings on its own container after the plugin split. +func TestHelmSnapshotPlugins(t *testing.T) { + for _, external := range []bool{false, true} { + t.Run(fmt.Sprintf("external=%t", external), func(t *testing.T) { + args := []string{"template", "test", "../../charts/substrate", "-n", "custom"} + issuer := "https://idp.custom.svc" + if external { + issuer = "https://actors.example.com" + args = append(args, "--set", "rustfs.enabled=false", "--set", "atelet.storageBackend=gcs", + "--set", "ateApi.actorJWTIssuer="+issuer, + "--set", "snapshotPlugin.extraEnv[0].name=GOOGLE_APPLICATION_CREDENTIALS", + "--set", "snapshotPlugin.extraEnv[0].value=/credentials/key.json") + } + out, err := exec.CommandContext(t.Context(), "helm", args...).CombinedOutput() + if err != nil { + t.Fatalf("render: %v\n%s", err, out) + } + found := map[string]bool{} + for _, doc := range strings.Split(string(out), "\n---\n") { + var deployment appsv1.Deployment + if err := yaml.Unmarshal([]byte(doc), &deployment); err != nil { + t.Fatal(err) + } + if deployment.Kind == "ConfigMap" && deployment.Name == "ate-api-server-envvars" { + var config corev1.ConfigMap + if err := yaml.Unmarshal([]byte(doc), &config); err != nil { + t.Fatal(err) + } + if got := config.Data["ATE_API_ACTOR_JWT_ISSUER"]; got != issuer { + t.Errorf("issuer = %q, want %q", got, issuer) + } + found["issuer"] = true + } + mode, app := "control", "ate-api-server" + if deployment.Kind == "DaemonSet" { + mode, app = "node", "atelet" + } else if deployment.Kind != "Deployment" || deployment.Name != "test-ate-api-server" { + continue + } + pod := deployment.Spec.Template.Spec + if len(pod.InitContainers) != 1 { + t.Fatalf("%s: expected one snapshot sidecar, got %v", mode, pod.InitContainers) + } + plugin := pod.InitContainers[0] + socketArg := "--socket=/run/snapshot-plugin/" + mode + ".sock" + if plugin.Name != "snapshot-plugin" || plugin.RestartPolicy == nil || *plugin.RestartPolicy != corev1.ContainerRestartPolicyAlways || !slices.Contains(plugin.Args, mode) || !slices.Contains(plugin.Args, socketArg) { + t.Fatalf("%s: invalid snapshot sidecar: %+v", mode, plugin) + } + if plugin.StartupProbe == nil || plugin.StartupProbe.Exec == nil || !slices.Contains(plugin.StartupProbe.Exec.Command, socketArg) { + t.Errorf("%s: startup probe does not check the plugin socket", mode) + } + env := map[string]string{} + for _, entry := range plugin.Env { + env[entry.Name] = entry.Value + } + if external { + if env["ATE_STORAGE_BACKEND"] != "gcs" || env["GOOGLE_APPLICATION_CREDENTIALS"] != "/credentials/key.json" || env["AWS_ENDPOINT_URL"] != "" { + t.Errorf("%s: external storage settings = %v", mode, env) + } + } else if env["ATE_STORAGE_BACKEND"] != "s3" || env["AWS_ENDPOINT_URL"] != "http://test-rustfs.custom.svc:9000" || env["AWS_ACCESS_KEY_ID"] == "" || env["AWS_SECRET_ACCESS_KEY"] == "" { + t.Errorf("%s: RustFS storage settings = %v", mode, env) + } + if !slices.ContainsFunc(pod.Volumes, func(v corev1.Volume) bool { return v.Name == "snapshot-plugin" && v.EmptyDir != nil }) { + t.Errorf("%s: missing shared socket volume", mode) + } + for _, container := range append(pod.Containers, plugin) { + if !slices.ContainsFunc(container.VolumeMounts, func(m corev1.VolumeMount) bool { + return m.Name == "snapshot-plugin" && m.MountPath == "/run/snapshot-plugin" + }) { + t.Errorf("%s: %s does not mount the shared socket", mode, container.Name) + } + if container.Name == app && !slices.Contains(container.Args, "--snapshot-plugin-socket=/run/snapshot-plugin/"+mode+".sock") { + t.Errorf("%s: application does not use the plugin socket", mode) + } + if container.Name == "ate-api-server" && slices.ContainsFunc(container.Env, func(e corev1.EnvVar) bool { return e.Name == "ATE_STORAGE_BACKEND" }) { + t.Error("API server still receives ATE_STORAGE_BACKEND") + } + } + if mode == "node" && (!slices.Contains(plugin.Args, "--root=/var/lib/ate") || !slices.ContainsFunc(plugin.VolumeMounts, func(m corev1.VolumeMount) bool { return m.Name == "run-ateom" && m.MountPath == "/var/lib/ate" })) { + t.Error("node plugin cannot access snapshot files") + } + found[mode] = true + } + for _, name := range []string{"issuer", "control", "node"} { + if !found[name] { + t.Errorf("missing %s configuration", name) + } + } + }) + } +} + +// These resources are excluded from the general chart render check, but must +// use the same version schema and validation as the upstream installation. +func TestHelmSandboxConfigParity(t *testing.T) { + for _, name := range []string{"sandboxconfig-gvisor.yaml", "sandboxconfig-validation.yaml"} { + t.Run(name, func(t *testing.T) { + out, err := exec.CommandContext(t.Context(), "helm", "template", "test", "../../charts/substrate", "--show-only", "templates/"+name).CombinedOutput() + if err != nil { + t.Fatalf("render: %v\n%s", err, out) + } + manifest, err := os.ReadFile("../../manifests/ate-install/" + name) + if err != nil { + t.Fatal(err) + } + parse := func(data string) []map[string]any { + var result []map[string]any + for _, doc := range strings.Split(data, "\n---\n") { + var resource map[string]any + if err := yaml.Unmarshal([]byte(doc), &resource); err != nil { + t.Fatal(err) + } + if len(resource) > 0 { + result = append(result, resource) + } + } + return result + } + if !reflect.DeepEqual(parse(string(out)), parse(string(manifest))) { + t.Error("chart sandbox configuration differs from the upstream manifest") + } + }) + } +} diff --git a/cmd/ateapi/internal/apivalidation/access_policy.go b/cmd/ateapi/internal/apivalidation/access_policy.go index e263ed157f..1f281ceb27 100644 --- a/cmd/ateapi/internal/apivalidation/access_policy.go +++ b/cmd/ateapi/internal/apivalidation/access_policy.go @@ -95,7 +95,7 @@ func ValidateDeleteAtespaceAccessPolicyRequest(ctx context.Context, req *ateapip } // ValidateCustom_AccessPolicy_Metadata requires an access policy to be the -// unatespaced singleton named "default". An empty name is left to defaulting. +// unatespaced singleton named "default". func ValidateCustom_AccessPolicy_Metadata(_ context.Context, _ operation.Operation, root *field.Path, meta, _ *ateapipb.ResourceMetadata) field.ErrorList { var errs field.ErrorList if meta.GetAtespace() != "" { diff --git a/cmd/ateapi/internal/apivalidation/access_policy_test.go b/cmd/ateapi/internal/apivalidation/access_policy_test.go index 71119037ee..9ac5dcad47 100644 --- a/cmd/ateapi/internal/apivalidation/access_policy_test.go +++ b/cmd/ateapi/internal/apivalidation/access_policy_test.go @@ -89,6 +89,12 @@ func TestValidateCreateGlobalAccessPolicyRequest(t *testing.T) { p.Metadata.Atespace = "team-a" })}, wantError: field.ErrorList{field.Forbidden(accessPolicyPath.Child("metadata", "atespace"), "")}, + }, { + name: "missing name", + req: &ateapipb.CreateGlobalAccessPolicyRequest{AccessPolicy: validAccessPolicy(func(p *ateapipb.AccessPolicy) { + p.Metadata.Name = "" + })}, + wantError: field.ErrorList{field.Required(accessPolicyPath.Child("metadata", "name"), "")}, }, { name: "name not default", req: &ateapipb.CreateGlobalAccessPolicyRequest{AccessPolicy: validAccessPolicy(func(p *ateapipb.AccessPolicy) { @@ -252,6 +258,18 @@ func TestValidateCreateAtespaceAccessPolicyRequest(t *testing.T) { name: "missing access_policy", req: &ateapipb.CreateAtespaceAccessPolicyRequest{Atespace: atespace}, wantError: field.ErrorList{field.Required(accessPolicyPath, "")}, + }, { + name: "missing name", + req: &ateapipb.CreateAtespaceAccessPolicyRequest{Atespace: atespace, AccessPolicy: validAccessPolicy(func(p *ateapipb.AccessPolicy) { + p.Metadata.Name = "" + })}, + wantError: field.ErrorList{field.Required(accessPolicyPath.Child("metadata", "name"), "")}, + }, { + name: "name not default", + req: &ateapipb.CreateAtespaceAccessPolicyRequest{Atespace: atespace, AccessPolicy: validAccessPolicy(func(p *ateapipb.AccessPolicy) { + p.Metadata.Name = "other" + })}, + wantError: field.ErrorList{field.Invalid(accessPolicyPath.Child("metadata", "name"), nil, "").WithOrigin("custom=default")}, }, { name: "unknown role", req: &ateapipb.CreateAtespaceAccessPolicyRequest{Atespace: atespace, AccessPolicy: validAccessPolicy(func(p *ateapipb.AccessPolicy) { diff --git a/cmd/ateapi/internal/apivalidation/actor_template.go b/cmd/ateapi/internal/apivalidation/actor_template.go index f25107ed96..3366dd272f 100644 --- a/cmd/ateapi/internal/apivalidation/actor_template.go +++ b/cmd/ateapi/internal/apivalidation/actor_template.go @@ -237,6 +237,15 @@ func ValidateCustom_SnapshotConfig_StorageLocation(_ context.Context, _ operatio return nil } +// ValidateCustom_SnapshotConfig_PreferredFidelity rejects ROOTFS until a +// sandbox runtime can capture root filesystem changes without memory. +func ValidateCustom_SnapshotConfig_PreferredFidelity(_ context.Context, _ operation.Operation, fldPath *field.Path, value, _ *ateapipb.SnapshotFidelity) field.ErrorList { + if *value == ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS { + return field.ErrorList{field.Invalid(fldPath, value.String(), "ROOTFS fidelity is not supported yet")} + } + return nil +} + // envVarNameRE constrains env var names to any printable ASCII character // except '='. var envVarNameRE = regexp.MustCompile(`^[ -<>-~]+$`) diff --git a/cmd/ateapi/internal/apivalidation/actor_template_test.go b/cmd/ateapi/internal/apivalidation/actor_template_test.go index 5a12b817d3..d04127cbaa 100644 --- a/cmd/ateapi/internal/apivalidation/actor_template_test.go +++ b/cmd/ateapi/internal/apivalidation/actor_template_test.go @@ -63,11 +63,19 @@ func TestValidateCreateActorTemplateRequest(t *testing.T) { })}, field.ErrorList{field.Invalid(field.NewPath("actor_template", "metadata", "name"), "Tmpl_A", "").WithOrigin("format=k8s-short-name")}, }, { - "valid data-scoped snapshots", + "valid volumes fidelity", &ateapipb.CreateActorTemplateRequest{ActorTemplate: validActorTemplate(func(tmpl *ateapipb.ActorTemplate) { - tmpl.SnapshotConfig.OnCommit = ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA + tmpl.SnapshotConfig.PreferredFidelity = ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES })}, nil, + }, { + // ROOTFS is in the enum for the API's sake but no runtime captures it + // yet, so templates may not ask for it. + "rootfs fidelity not supported", + &ateapipb.CreateActorTemplateRequest{ActorTemplate: validActorTemplate(func(tmpl *ateapipb.ActorTemplate) { + tmpl.SnapshotConfig.PreferredFidelity = ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS + })}, + field.ErrorList{field.Invalid(field.NewPath("actor_template", "snapshot_config", "preferred_fidelity"), "SNAPSHOT_FIDELITY_ROOTFS", "")}, }, { "invalid worker_selector label key", &ateapipb.CreateActorTemplateRequest{ActorTemplate: validActorTemplate(func(tmpl *ateapipb.ActorTemplate) { @@ -136,14 +144,14 @@ func TestValidateCreateActorTemplateRequest(t *testing.T) { })}, field.ErrorList{field.Invalid(field.NewPath("actor_template", "snapshot_config", "storage_location"), "gs://my-bucket/snapshots?versions=true", "")}, }, { - // on_commit has no default of its own at this layer, so leaving it + // preferred_fidelity has no default of its own at this layer, so leaving it // unset is a required violation. - "on_commit unset", + "preferred_fidelity unset", &ateapipb.CreateActorTemplateRequest{ActorTemplate: validActorTemplate(func(tmpl *ateapipb.ActorTemplate) { - tmpl.SnapshotConfig.OnCommit = ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_UNSPECIFIED + tmpl.SnapshotConfig.PreferredFidelity = ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED })}, field.ErrorList{ - field.Required(field.NewPath("actor_template", "snapshot_config", "on_commit"), ""), + field.Required(field.NewPath("actor_template", "snapshot_config", "preferred_fidelity"), ""), }, }, { "missing sandbox_config", @@ -337,23 +345,23 @@ func TestValidateActorTemplate(t *testing.T) { }, { name: "unspecified snapshot scope", mutate: func(tmpl *ateapipb.ActorTemplate) { - tmpl.SnapshotConfig.OnCommit = ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_UNSPECIFIED + tmpl.SnapshotConfig.PreferredFidelity = ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED }, want: field.ErrorList{ - field.Required(field.NewPath("snapshot_config", "on_commit"), ""), + field.Required(field.NewPath("snapshot_config", "preferred_fidelity"), ""), }, }, { - name: "on_commit outside the enum", + name: "preferred_fidelity outside the enum", mutate: func(tmpl *ateapipb.ActorTemplate) { - tmpl.SnapshotConfig.OnCommit = ateapipb.SnapshotContentScope(99) + tmpl.SnapshotConfig.PreferredFidelity = ateapipb.SnapshotFidelity(99) }, - want: field.ErrorList{field.Invalid(field.NewPath("snapshot_config", "on_commit"), nil, "").WithOrigin("maximum")}, + want: field.ErrorList{field.Invalid(field.NewPath("snapshot_config", "preferred_fidelity"), nil, "").WithOrigin("maximum")}, }, { - name: "negative on_commit", + name: "negative preferred_fidelity", mutate: func(tmpl *ateapipb.ActorTemplate) { - tmpl.SnapshotConfig.OnCommit = ateapipb.SnapshotContentScope(-1) + tmpl.SnapshotConfig.PreferredFidelity = ateapipb.SnapshotFidelity(-1) }, - want: field.ErrorList{field.Invalid(field.NewPath("snapshot_config", "on_commit"), nil, "").WithOrigin("minimum")}, + want: field.ErrorList{field.Invalid(field.NewPath("snapshot_config", "preferred_fidelity"), nil, "").WithOrigin("minimum")}, }, { name: "no containers", mutate: func(tmpl *ateapipb.ActorTemplate) { tmpl.Containers = nil }, @@ -982,8 +990,8 @@ func validActorTemplate(mutations ...func(*ateapipb.ActorTemplate)) *ateapipb.Ac Metadata: &ateapipb.ResourceMetadata{Atespace: "ns1", Name: "tmpl-a"}, Containers: []*ateapipb.Container{{Name: "main", Image: "example.com/app:v1@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}}, SnapshotConfig: &ateapipb.SnapshotConfig{ - StorageLocation: "gs://my-bucket/snapshots", - OnCommit: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, + StorageLocation: "gs://my-bucket/snapshots", + PreferredFidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, }, SandboxConfig: &ateapipb.SandboxConfig{SandboxClass: ateapipb.SandboxClass_SANDBOX_CLASS_GVISOR, ConfigName: "gvisor-default"}, } diff --git a/cmd/ateapi/internal/apivalidation/actor_test.go b/cmd/ateapi/internal/apivalidation/actor_test.go index b383608746..6b4cfcc627 100644 --- a/cmd/ateapi/internal/apivalidation/actor_test.go +++ b/cmd/ateapi/internal/apivalidation/actor_test.go @@ -435,126 +435,248 @@ func TestValidateActorUpdate(t *testing.T) { })), field.ErrorList{field.Invalid(field.NewPath("status", "assigned_node"), nil, "").WithOrigin("format=k8s-long-name")}, }, { - "valid actor.status.in_progress_snapshot_uri", + "valid actor.status.last_assigned_generation", validInput(), validOutput(withStatus(func(s *ateapipb.ActorStatus) { - s.InProgressSnapshotUri = "gs://private/atespaces/as/actors/" + someActorUID + "/snapshots/snap-1" + s.LastAssignedGeneration = 1 })), nil, }, { - "invalid actor.status.in_progress_snapshot_uri: too long", + "increasing actor.status.last_assigned_generation", + validInput(withStatus(func(s *ateapipb.ActorStatus) { + s.LastAssignedGeneration = 2 + })), + validOutput(withStatus(func(s *ateapipb.ActorStatus) { + s.LastAssignedGeneration = 3 + })), + nil, + }, { + "unchanged actor.status.last_assigned_generation", + validInput(withStatus(func(s *ateapipb.ActorStatus) { + s.LastAssignedGeneration = 3 + })), + validOutput(withStatus(func(s *ateapipb.ActorStatus) { + s.LastAssignedGeneration = 3 + })), + nil, + }, { + "decreasing actor.status.last_assigned_generation", + validInput(withStatus(func(s *ateapipb.ActorStatus) { + s.LastAssignedGeneration = 3 + })), + validOutput(withStatus(func(s *ateapipb.ActorStatus) { + s.LastAssignedGeneration = 1 + })), + field.ErrorList{field.Invalid(field.NewPath("status", "last_assigned_generation"), nil, "").WithOrigin("monotonic")}, + }, { + "unsetting actor.status.last_assigned_generation", + validInput(withStatus(func(s *ateapipb.ActorStatus) { + s.LastAssignedGeneration = 3 + })), + validOutput(withStatus(func(s *ateapipb.ActorStatus) { + s.LastAssignedGeneration = 0 + })), + field.ErrorList{field.Invalid(field.NewPath("status", "last_assigned_generation"), nil, "").WithOrigin("update")}, + }, { + "negative actor.status.last_assigned_generation", validInput(), validOutput(withStatus(func(s *ateapipb.ActorStatus) { - s.InProgressSnapshotUri = "gs://" + strings.Repeat("x", 2044) + s.LastAssignedGeneration = -1 })), - field.ErrorList{field.TooLong(field.NewPath("status", "in_progress_snapshot_uri"), nil, 2048).WithOrigin("maxLength")}, + field.ErrorList{field.Invalid(field.NewPath("status", "last_assigned_generation"), nil, "").WithOrigin("minimum")}, }, { - "valid actor.status.external_snapshot", + "valid actor.status.snapshots with durable object snapshot", validInput(), validOutput(withStatus(func(s *ateapipb.ActorStatus) { - s.ExternalSnapshot = &ateapipb.ExternalSnapshot{ - SnapshotUri: "gs://private/atespaces/as/actors/" + someActorUID + "/snapshots/snap-1", - ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, - } + s.Snapshots = []*ateapipb.Snapshot{validSnapshot()} })), nil, }, { - "valid actor.status.external_snapshot.actor_template_uid", + "valid actor.status.snapshots actor_template_uid", validInput(), validOutput(withStatus(func(s *ateapipb.ActorStatus) { - s.ExternalSnapshot = &ateapipb.ExternalSnapshot{SnapshotUri: "gs://private/atespaces/as/actors/" + someActorUID + "/snapshots/snap-1", ActorTemplateUid: "01234567-89ab-cdef-0123-456789abcdef"} + s.Snapshots = []*ateapipb.Snapshot{validSnapshot(func(snap *ateapipb.Snapshot) { + snap.ActorTemplateUid = "01234567-89ab-cdef-0123-456789abcdef" + })} })), nil, }, { // Each suspend restamps the UID of the template the snapshot was captured under. - "changing actor.status.external_snapshot.actor_template_uid is allowed", + "changing actor.status.snapshots.actor_template_uid is allowed", validInput(withStatus(func(s *ateapipb.ActorStatus) { - s.ExternalSnapshot = &ateapipb.ExternalSnapshot{SnapshotUri: "gs://private/atespaces/as/actors/" + someActorUID + "/snapshots/snap-1", ActorTemplateUid: "01234567-89ab-cdef-0123-456789abcdef"} + s.Snapshots = []*ateapipb.Snapshot{validSnapshot(func(snap *ateapipb.Snapshot) { + snap.ActorTemplateUid = "01234567-89ab-cdef-0123-456789abcdef" + })} })), validOutput(withStatus(func(s *ateapipb.ActorStatus) { - s.ExternalSnapshot = &ateapipb.ExternalSnapshot{SnapshotUri: "gs://private/atespaces/as/actors/" + someActorUID + "/snapshots/snap-1", ActorTemplateUid: "fedcba98-7654-3210-fedc-ba9876543210"} + s.Snapshots = []*ateapipb.Snapshot{validSnapshot(func(snap *ateapipb.Snapshot) { + snap.ActorTemplateUid = "fedcba98-7654-3210-fedc-ba9876543210" + })} })), nil, }, { - "invalid actor.status.external_snapshot.actor_template_uid", + "invalid actor.status.snapshots.actor_template_uid", validInput(), validOutput(withStatus(func(s *ateapipb.ActorStatus) { - s.ExternalSnapshot = &ateapipb.ExternalSnapshot{SnapshotUri: "gs://private/atespaces/as/actors/" + someActorUID + "/snapshots/snap-1", ActorTemplateUid: "not-a-uuid"} + s.Snapshots = []*ateapipb.Snapshot{validSnapshot(func(snap *ateapipb.Snapshot) { + snap.ActorTemplateUid = "not-a-uuid" + })} })), - field.ErrorList{field.Invalid(field.NewPath("status", "external_snapshot", "actor_template_uid"), nil, "").WithOrigin("format=k8s-uuid")}, + field.ErrorList{field.Invalid(field.NewPath("status", "snapshots").Index(0).Child("actor_template_uid"), nil, "").WithOrigin("format=k8s-uuid")}, }, { - "valid actor.status.local_snapshot.snapshot_name", + "valid actor.status.snapshots local snapshot_name", validInput(), validOutput(withStatus(func(s *ateapipb.ActorStatus) { - s.LocalSnapshot = &ateapipb.LocalSnapshot{SnapshotName: "snap-1"} + s.Snapshots = []*ateapipb.Snapshot{validSnapshot(func(snap *ateapipb.Snapshot) { + snap.Storage = []*ateapipb.SnapshotStorage{{ + Durability: ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL, + Status: ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED, + Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + Local: &ateapipb.LocalSnapshot{SnapshotName: "snap-1"}, + }} + })} })), nil, }, { - "invalid actor.status.local_snapshot.snapshot_name", + "invalid actor.status.snapshots local snapshot_name", validInput(), validOutput(withStatus(func(s *ateapipb.ActorStatus) { - s.LocalSnapshot = &ateapipb.LocalSnapshot{SnapshotName: "SNAP 1"} + s.Snapshots = []*ateapipb.Snapshot{validSnapshot(func(snap *ateapipb.Snapshot) { + snap.Storage = []*ateapipb.SnapshotStorage{{ + Durability: ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL, + Status: ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED, + Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + Local: &ateapipb.LocalSnapshot{SnapshotName: "SNAP 1"}, + }} + })} })), - field.ErrorList{field.Invalid(field.NewPath("status", "local_snapshot", "snapshot_name"), nil, "").WithOrigin("format=k8s-short-name")}, + field.ErrorList{field.Invalid(field.NewPath("status", "snapshots").Index(0).Child("storage").Index(0).Child("local", "snapshot_name"), nil, "").WithOrigin("format=k8s-short-name")}, }, { - "valid actor.status.local_snapshot.content_scope", + "duplicate actor.status.snapshots generation", validInput(), validOutput(withStatus(func(s *ateapipb.ActorStatus) { - s.LocalSnapshot = &ateapipb.LocalSnapshot{ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA} + s.Snapshots = []*ateapipb.Snapshot{ + validSnapshot(func(snap *ateapipb.Snapshot) { snap.Generation = 1 }), + validSnapshot(func(snap *ateapipb.Snapshot) { snap.Generation = 1 }), + } + })), + field.ErrorList{field.Duplicate(field.NewPath("status", "snapshots").Index(1), nil)}, + }, { + "duplicate actor.status.snapshots.storage durability", + validInput(), + validOutput(withStatus(func(s *ateapipb.ActorStatus) { + s.Snapshots = []*ateapipb.Snapshot{validSnapshot(func(snap *ateapipb.Snapshot) { + snap.Storage = []*ateapipb.SnapshotStorage{ + validSnapshotStorage(), + validSnapshotStorage(), + } + })} + })), + field.ErrorList{field.Duplicate(field.NewPath("status", "snapshots").Index(0).Child("storage").Index(1), nil)}, + }, { + "too many actor.status.snapshots.storage entries", + validInput(), + validOutput(withStatus(func(s *ateapipb.ActorStatus) { + s.Snapshots = []*ateapipb.Snapshot{validSnapshot(func(snap *ateapipb.Snapshot) { + snap.Storage = []*ateapipb.SnapshotStorage{ + validSnapshotStorage(func(ss *ateapipb.SnapshotStorage) { + ss.Durability = ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL + ss.Object = nil + ss.Local = &ateapipb.LocalSnapshot{SnapshotName: "snap-1"} + }), + validSnapshotStorage(func(ss *ateapipb.SnapshotStorage) { + ss.Durability = ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE + }), + validSnapshotStorage(func(ss *ateapipb.SnapshotStorage) { + ss.Durability = ateapipb.SnapshotDurability(3) + }), + } + })} + })), + field.ErrorList{field.TooMany(field.NewPath("status", "snapshots").Index(0).Child("storage"), 3, 2).WithOrigin("maxItems")}, + }, { + "invalid actor.status.snapshots.storage.durability", + validInput(), + validOutput(withStatus(func(s *ateapipb.ActorStatus) { + s.Snapshots = []*ateapipb.Snapshot{validSnapshot(func(snap *ateapipb.Snapshot) { + snap.Storage[0].Durability = ateapipb.SnapshotDurability(3) + })} + })), + field.ErrorList{field.Invalid(field.NewPath("status", "snapshots").Index(0).Child("storage").Index(0).Child("durability"), nil, "").WithOrigin("maximum")}, + }, { + "valid actor.status.snapshots.storage.fidelity", + validInput(), + validOutput(withStatus(func(s *ateapipb.ActorStatus) { + s.Snapshots = []*ateapipb.Snapshot{validSnapshot(func(snap *ateapipb.Snapshot) { + snap.Storage[0].Fidelity = ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES + })} })), nil, }, { - "negative actor.status.local_snapshot.content_scope", + "missing actor.status.snapshots.storage.fidelity", validInput(), validOutput(withStatus(func(s *ateapipb.ActorStatus) { - s.LocalSnapshot = &ateapipb.LocalSnapshot{ContentScope: ateapipb.SnapshotContentScope(-1)} + s.Snapshots = []*ateapipb.Snapshot{validSnapshot(func(snap *ateapipb.Snapshot) { + snap.Storage[0].Fidelity = ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED + })} })), - field.ErrorList{field.Invalid(field.NewPath("status", "local_snapshot", "content_scope"), nil, "").WithOrigin("minimum")}, + field.ErrorList{field.Required(field.NewPath("status", "snapshots").Index(0).Child("storage").Index(0).Child("fidelity"), "")}, }, { - "invalid actor.status.local_snapshot.content_scope", + "negative actor.status.snapshots.storage.fidelity", validInput(), validOutput(withStatus(func(s *ateapipb.ActorStatus) { - s.LocalSnapshot = &ateapipb.LocalSnapshot{ContentScope: ateapipb.SnapshotContentScope(3)} + s.Snapshots = []*ateapipb.Snapshot{validSnapshot(func(snap *ateapipb.Snapshot) { + snap.Storage[0].Fidelity = ateapipb.SnapshotFidelity(-1) + })} })), - field.ErrorList{field.Invalid(field.NewPath("status", "local_snapshot", "content_scope"), nil, "").WithOrigin("maximum")}, + field.ErrorList{field.Invalid(field.NewPath("status", "snapshots").Index(0).Child("storage").Index(0).Child("fidelity"), nil, "").WithOrigin("minimum")}, }, { - "too many actor_volumes", + "invalid actor.status.snapshots.storage.fidelity", + validInput(), + validOutput(withStatus(func(s *ateapipb.ActorStatus) { + s.Snapshots = []*ateapipb.Snapshot{validSnapshot(func(snap *ateapipb.Snapshot) { + snap.Storage[0].Fidelity = ateapipb.SnapshotFidelity(4) + })} + })), + field.ErrorList{field.Invalid(field.NewPath("status", "snapshots").Index(0).Child("storage").Index(0).Child("fidelity"), nil, "").WithOrigin("maximum")}, + }, { + "too many external_volumes", validInput(), validOutput(withStatus(func(s *ateapipb.ActorStatus) { vols := make([]*ateapipb.ExternalVolume, 33) for i := range vols { - vols[i] = &ateapipb.ExternalVolume{VolumeName: fmt.Sprintf("vol-%d", i), VolumeType: "substrate.io/mock"} + vols[i] = &ateapipb.ExternalVolume{Name: fmt.Sprintf("vol-%d", i), VolumeType: "substrate.io/mock"} } - s.ActorVolumes = vols + s.ExternalVolumes = vols })), - field.ErrorList{field.TooMany(field.NewPath("status", "actor_volumes"), 33, 32).WithOrigin("maxItems")}, + field.ErrorList{field.TooMany(field.NewPath("status", "external_volumes"), 33, 32).WithOrigin("maxItems")}, }, { // Set-once fields permit the nil->set transition, so a volume added // in an update validates like one added at creation. "adding a volume on update is allowed", validInput(withStatus()), validOutput(withStatus(func(s *ateapipb.ActorStatus) { - s.ActorVolumes = []*ateapipb.ExternalVolume{{VolumeName: "vol-a", VolumeType: "substrate.io/mock"}} + s.ExternalVolumes = []*ateapipb.ExternalVolume{{Name: "vol-a", VolumeType: "substrate.io/mock"}} })), nil, }, { - "duplicate actor_volumes volume_name", + "duplicate external_volumes name", validInput(withStatus()), validOutput(withStatus(func(s *ateapipb.ActorStatus) { - s.ActorVolumes = []*ateapipb.ExternalVolume{ - {VolumeName: "vol-a", VolumeType: "substrate.io/mock"}, - {VolumeName: "vol-a", VolumeType: "substrate.io/mock"}, + s.ExternalVolumes = []*ateapipb.ExternalVolume{ + {Name: "vol-a", VolumeType: "substrate.io/mock"}, + {Name: "vol-a", VolumeType: "substrate.io/mock"}, } })), - field.ErrorList{field.Duplicate(field.NewPath("status", "actor_volumes").Index(1), nil)}, + field.ErrorList{field.Duplicate(field.NewPath("status", "external_volumes").Index(1), nil)}, }, { "provisioning transition on an existing volume is valid", validInput(withStatus(func(s *ateapipb.ActorStatus) { - s.ActorVolumes = []*ateapipb.ExternalVolume{{VolumeName: "vol-a", VolumeType: "substrate.io/mock", Status: ateapipb.ExternalVolume_STATUS_PENDING}} + s.ExternalVolumes = []*ateapipb.ExternalVolume{{Name: "vol-a", VolumeType: "substrate.io/mock", Status: ateapipb.ExternalVolume_STATUS_PENDING}} })), validOutput(withStatus(func(s *ateapipb.ActorStatus) { - s.ActorVolumes = []*ateapipb.ExternalVolume{{ - VolumeName: "vol-a", + s.ExternalVolumes = []*ateapipb.ExternalVolume{{ + Name: "vol-a", VolumeType: "substrate.io/mock", StorageVolumeId: "csi-426d29b7", Status: ateapipb.ExternalVolume_STATUS_CREATED, @@ -562,11 +684,6 @@ func TestValidateActorUpdate(t *testing.T) { }} })), nil, - }, { - "invalid actor.status.in_progress_local_snapshot_name", - validInput(), - validOutput(withStatus(func(s *ateapipb.ActorStatus) { s.InProgressLocalSnapshotName = "BAD NAME" })), - field.ErrorList{field.Invalid(field.NewPath("status", "in_progress_local_snapshot_name"), nil, "").WithOrigin("format=k8s-short-name")}, }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -1016,7 +1133,7 @@ func TestValidateMintActorJWTRequest(t *testing.T) { withExpiration := func(seconds int64) *ateapipb.MintActorJWTRequest { return &ateapipb.MintActorJWTRequest{ Actor: &ateapipb.ObjectRef{Atespace: "ns1", Name: "id1"}, - Audience: []string{"https://example.com"}, + Audiences: []string{"https://example.com"}, ExpirationSeconds: seconds, } } diff --git a/cmd/ateapi/internal/apivalidation/common_test.go b/cmd/ateapi/internal/apivalidation/common_test.go index b7e5039d81..5df329e104 100644 --- a/cmd/ateapi/internal/apivalidation/common_test.go +++ b/cmd/ateapi/internal/apivalidation/common_test.go @@ -26,10 +26,6 @@ const ( ) const ( - // testStorageLocation is the snapshot_config.storage_location the tests - // build snapshot URIs under. - testStorageLocation = "gs://bucket/root" - // someActorUID stands in for the UID the store assigns an Actor, for tests // that need a well-formed snapshot URI but never exercise who owns it. Those // seed their Actor in a single call, before a real UID exists. diff --git a/cmd/ateapi/internal/apivalidation/tag_test.go b/cmd/ateapi/internal/apivalidation/tag_test.go index ec53badf30..23bbd43958 100644 --- a/cmd/ateapi/internal/apivalidation/tag_test.go +++ b/cmd/ateapi/internal/apivalidation/tag_test.go @@ -54,9 +54,8 @@ func TestValidateCreateTagRequest(t *testing.T) { req: &ateapipb.CreateTagRequest{ Tag: validTag(func(tag *ateapipb.Tag) { tag.Status = &ateapipb.TagStatus{ - Snapshot: validExternalSnapshot(), + Snapshot: validSnapshot(), ActorTemplateUid: someActorUID, - StorageLocation: testStorageLocation, } }), }, diff --git a/cmd/ateapi/internal/apivalidation/validation_test.go b/cmd/ateapi/internal/apivalidation/validation_test.go index 37d67bbee0..f5ed650b13 100644 --- a/cmd/ateapi/internal/apivalidation/validation_test.go +++ b/cmd/ateapi/internal/apivalidation/validation_test.go @@ -913,7 +913,7 @@ func TestValidateTrustBundleDataSource(t *testing.T) { func validExternalVolume(mutate ...func(*ateapipb.ExternalVolume)) *ateapipb.ExternalVolume { v := &ateapipb.ExternalVolume{ - VolumeName: "my-vol", + Name: "my-vol", StorageVolumeId: "valid-storage-id", VolumeType: "mock", Status: ateapipb.ExternalVolume_STATUS_CREATED, @@ -936,12 +936,12 @@ func TestValidateExternalVolume(t *testing.T) { obj: valid(), }, { name: "missing volume name", - obj: valid(func(v *ateapipb.ExternalVolume) { v.VolumeName = "" }), - want: field.ErrorList{field.Required(field.NewPath("volume_name"), "")}, + obj: valid(func(v *ateapipb.ExternalVolume) { v.Name = "" }), + want: field.ErrorList{field.Required(field.NewPath("name"), "")}, }, { name: "invalid volume name", - obj: valid(func(v *ateapipb.ExternalVolume) { v.VolumeName = "NOT A VOLUME" }), - want: field.ErrorList{field.Invalid(field.NewPath("volume_name"), nil, "").WithOrigin("format=k8s-short-name")}, + obj: valid(func(v *ateapipb.ExternalVolume) { v.Name = "NOT A VOLUME" }), + want: field.ErrorList{field.Invalid(field.NewPath("name"), nil, "").WithOrigin("format=k8s-short-name")}, }, { name: "valid external volume with empty storage volume id", obj: valid(func(v *ateapipb.ExternalVolume) { v.StorageVolumeId = "" }), @@ -1058,10 +1058,10 @@ func TestValidateExternalVolume_Update(t *testing.T) { oldObj: valid(), newObj: valid(), }, { - name: "volume_name changed is invalid", - oldObj: valid(func(v *ateapipb.ExternalVolume) { v.VolumeName = "vol1" }), - newObj: valid(func(v *ateapipb.ExternalVolume) { v.VolumeName = "vol2" }), - want: field.ErrorList{field.Invalid(field.NewPath("volume_name"), nil, "").WithOrigin("update")}, + name: "name changed is invalid", + oldObj: valid(func(v *ateapipb.ExternalVolume) { v.Name = "vol1" }), + newObj: valid(func(v *ateapipb.ExternalVolume) { v.Name = "vol2" }), + want: field.ErrorList{field.Invalid(field.NewPath("name"), nil, "").WithOrigin("update")}, }, { name: "storage_volume_id transition from empty to non-empty is valid", oldObj: valid(func(v *ateapipb.ExternalVolume) { v.StorageVolumeId = "" }), @@ -1142,10 +1142,34 @@ func TestValidateDeleteOptions(t *testing.T) { } } -func validExternalSnapshot(mutate ...func(*ateapipb.ExternalSnapshot)) *ateapipb.ExternalSnapshot { - s := &ateapipb.ExternalSnapshot{ - SnapshotUri: "gs://private/atespaces/as/actors/" + someActorUID + "/snapshots/snap-1", - ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, +func validObjectSnapshot(mutate ...func(*ateapipb.ObjectSnapshot)) *ateapipb.ObjectSnapshot { + s := &ateapipb.ObjectSnapshot{ + SnapshotUri: "gs://private/atespaces/as/actors/" + someActorUID + "/snapshots/snap-1", + } + for _, m := range mutate { + m(s) + } + return s +} + +func validSnapshotStorage(mutate ...func(*ateapipb.SnapshotStorage)) *ateapipb.SnapshotStorage { + ss := &ateapipb.SnapshotStorage{ + Durability: ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, + Status: ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED, + Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + Object: validObjectSnapshot(), + } + for _, m := range mutate { + m(ss) + } + return ss +} + +func validSnapshot(mutate ...func(*ateapipb.Snapshot)) *ateapipb.Snapshot { + s := &ateapipb.Snapshot{ + Generation: 1, + Owner: ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, + Storage: []*ateapipb.SnapshotStorage{validSnapshotStorage()}, } for _, m := range mutate { m(s) @@ -1153,24 +1177,31 @@ func validExternalSnapshot(mutate ...func(*ateapipb.ExternalSnapshot)) *ateapipb return s } -// badExternalSnapshot violates both of ExternalSnapshot's rules at once, so a -// caller can assert that a containing type reaches every field of it. -func badExternalSnapshot(mutate ...func(*ateapipb.ExternalSnapshot)) *ateapipb.ExternalSnapshot { - breakIt := func(s *ateapipb.ExternalSnapshot) { - s.SnapshotUri = "" - s.ContentScope = ateapipb.SnapshotContentScope(3) +// badSnapshot violates Snapshot's rules at once, so a caller can assert that a +// containing type reaches every field of it. +func badSnapshot(mutate ...func(*ateapipb.Snapshot)) *ateapipb.Snapshot { + breakIt := func(s *ateapipb.Snapshot) { + s.Storage = []*ateapipb.SnapshotStorage{{ + Durability: ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, + Status: ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED, + Fidelity: ateapipb.SnapshotFidelity(4), + Object: &ateapipb.ObjectSnapshot{SnapshotUri: ""}, + }} } - return validExternalSnapshot(append([]func(*ateapipb.ExternalSnapshot){breakIt}, mutate...)...) + return validSnapshot(append([]func(*ateapipb.Snapshot){breakIt}, mutate...)...) } -func TestValidateExternalSnapshot(t *testing.T) { - valid := validExternalSnapshot - uriPath := field.NewPath("snapshot_uri") - scopePath := field.NewPath("content_scope") +func TestValidateSnapshot(t *testing.T) { + valid := validSnapshot + genPath := field.NewPath("generation") + ownerPath := field.NewPath("owner") + storagePath := field.NewPath("storage") + fidelityPath := storagePath.Index(0).Child("fidelity") + uriPath := storagePath.Index(0).Child("object", "snapshot_uri") tests := []struct { name string - obj *ateapipb.ExternalSnapshot + obj *ateapipb.Snapshot want field.ErrorList }{ { @@ -1178,67 +1209,146 @@ func TestValidateExternalSnapshot(t *testing.T) { obj: valid(), }, { - name: "valid content_scope: data", - obj: valid(func(s *ateapipb.ExternalSnapshot) { - s.ContentScope = ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA + name: "missing generation", + obj: valid(func(s *ateapipb.Snapshot) { s.Generation = 0 }), + want: field.ErrorList{field.Required(genPath, "")}, + }, + { + name: "negative generation", + obj: valid(func(s *ateapipb.Snapshot) { s.Generation = -1 }), + want: field.ErrorList{field.Invalid(genPath, nil, "").WithOrigin("minimum")}, + }, + { + name: "missing owner", + obj: valid(func(s *ateapipb.Snapshot) { s.Owner = ateapipb.SnapshotOwner_SNAPSHOT_OWNER_UNSPECIFIED }), + want: field.ErrorList{field.Required(ownerPath, "")}, + }, + { + name: "owner above the enum", + obj: valid(func(s *ateapipb.Snapshot) { s.Owner = ateapipb.SnapshotOwner(3) }), + want: field.ErrorList{field.Invalid(ownerPath, nil, "").WithOrigin("maximum")}, + }, + { + name: "valid fidelity: volumes", + obj: valid(func(s *ateapipb.Snapshot) { + s.Storage[0].Fidelity = ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES }), }, { - // UNSPECIFIED reads as FULL, so optional lets the zero value skip - // the bounds rather than failing the minimum. - name: "valid content_scope: unspecified", - obj: valid(func(s *ateapipb.ExternalSnapshot) { - s.ContentScope = ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_UNSPECIFIED + name: "missing fidelity", + obj: valid(func(s *ateapipb.Snapshot) { + s.Storage[0].Fidelity = ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED }), + want: field.ErrorList{field.Required(fidelityPath, "")}, + }, + { + name: "missing storage", + obj: valid(func(s *ateapipb.Snapshot) { s.Storage = nil }), + want: field.ErrorList{field.Required(storagePath, "")}, + }, + { + name: "too many storage entries", + obj: valid(func(s *ateapipb.Snapshot) { + s.Storage = []*ateapipb.SnapshotStorage{ + validSnapshotStorage(func(ss *ateapipb.SnapshotStorage) { + ss.Durability = ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL + ss.Object = nil + ss.Local = &ateapipb.LocalSnapshot{SnapshotName: "snap-1"} + }), + validSnapshotStorage(func(ss *ateapipb.SnapshotStorage) { + ss.Durability = ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE + }), + validSnapshotStorage(func(ss *ateapipb.SnapshotStorage) { + ss.Durability = ateapipb.SnapshotDurability(3) + }), + } + }), + want: field.ErrorList{field.TooMany(storagePath, 3, 2).WithOrigin("maxItems")}, + }, + { + name: "missing storage durability", + obj: valid(func(s *ateapipb.Snapshot) { + s.Storage[0].Durability = ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_UNSPECIFIED + }), + want: field.ErrorList{field.Required(storagePath.Index(0).Child("durability"), "")}, + }, + { + name: "negative storage durability", + obj: valid(func(s *ateapipb.Snapshot) { + s.Storage[0].Durability = ateapipb.SnapshotDurability(-1) + }), + want: field.ErrorList{field.Invalid(storagePath.Index(0).Child("durability"), nil, "").WithOrigin("minimum")}, + }, + { + name: "storage durability above the enum", + obj: valid(func(s *ateapipb.Snapshot) { + s.Storage[0].Durability = ateapipb.SnapshotDurability(3) + }), + want: field.ErrorList{field.Invalid(storagePath.Index(0).Child("durability"), nil, "").WithOrigin("maximum")}, + }, + { + name: "storage union missing both local and object", + obj: valid(func(s *ateapipb.Snapshot) { + s.Storage[0].Object = nil + s.Storage[0].Local = nil + }), + want: field.ErrorList{field.Invalid(storagePath.Index(0), nil, "one of").WithOrigin("union")}, + }, + { + name: "storage union has both local and object", + obj: valid(func(s *ateapipb.Snapshot) { + s.Storage[0].Local = &ateapipb.LocalSnapshot{SnapshotName: "snap-1"} + }), + want: field.ErrorList{field.Invalid(storagePath.Index(0), nil, "one of").WithOrigin("union")}, }, { name: "missing snapshot_uri", - obj: valid(func(s *ateapipb.ExternalSnapshot) { s.SnapshotUri = "" }), + obj: valid(func(s *ateapipb.Snapshot) { s.Storage[0].Object.SnapshotUri = "" }), want: field.ErrorList{field.Required(uriPath, "")}, }, { name: "snapshot_uri too long", - obj: valid(func(s *ateapipb.ExternalSnapshot) { - s.SnapshotUri = "gs://" + strings.Repeat("x", 2044) + obj: valid(func(s *ateapipb.Snapshot) { + s.Storage[0].Object.SnapshotUri = "gs://" + strings.Repeat("x", 2044) }), want: field.ErrorList{field.TooLong(uriPath, nil, 2048).WithOrigin("maxLength")}, }, { - name: "content_scope above the enum", - obj: valid(func(s *ateapipb.ExternalSnapshot) { s.ContentScope = ateapipb.SnapshotContentScope(3) }), - want: field.ErrorList{field.Invalid(scopePath, nil, "").WithOrigin("maximum")}, + name: "fidelity above the enum", + obj: valid(func(s *ateapipb.Snapshot) { s.Storage[0].Fidelity = ateapipb.SnapshotFidelity(4) }), + want: field.ErrorList{field.Invalid(fidelityPath, nil, "").WithOrigin("maximum")}, }, { - name: "negative content_scope", - obj: valid(func(s *ateapipb.ExternalSnapshot) { s.ContentScope = ateapipb.SnapshotContentScope(-1) }), - want: field.ErrorList{field.Invalid(scopePath, nil, "").WithOrigin("minimum")}, + name: "negative fidelity", + obj: valid(func(s *ateapipb.Snapshot) { s.Storage[0].Fidelity = ateapipb.SnapshotFidelity(-1) }), + want: field.ErrorList{field.Invalid(fidelityPath, nil, "").WithOrigin("minimum")}, }, { name: "every field invalid", - obj: badExternalSnapshot(), + obj: badSnapshot(), want: field.ErrorList{ field.Required(uriPath, ""), - field.Invalid(scopePath, nil, "").WithOrigin("maximum"), + field.Invalid(fidelityPath, nil, "").WithOrigin("maximum"), }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { op := operation.Operation{Type: operation.Create} - assertValidateErr(t, Validate_ExternalSnapshot(context.Background(), op, nil, tt.obj, nil), tt.want) + assertValidateErr(t, Validate_Snapshot(context.Background(), op, nil, tt.obj, nil), tt.want) }) } } -func TestValidateExternalSnapshotUpdate(t *testing.T) { - valid := validExternalSnapshot - uriPath := field.NewPath("snapshot_uri") - scopePath := field.NewPath("content_scope") +func TestValidateSnapshotUpdate(t *testing.T) { + valid := validSnapshot + fidelityPath := field.NewPath("storage").Index(0).Child("fidelity") + uriPath := field.NewPath("storage").Index(0).Child("object", "snapshot_uri") tests := []struct { name string - oldObj *ateapipb.ExternalSnapshot - newObj *ateapipb.ExternalSnapshot + oldObj *ateapipb.Snapshot + newObj *ateapipb.Snapshot want field.ErrorList }{ { @@ -1251,84 +1361,92 @@ func TestValidateExternalSnapshotUpdate(t *testing.T) { // before these rules existed does not block updates to the rest of // the object. name: "unchanged invalid fields are not revalidated", - oldObj: badExternalSnapshot(), - newObj: badExternalSnapshot(), + oldObj: badSnapshot(), + newObj: badSnapshot(), }, { - name: "content_scope changed to a valid value", + name: "fidelity changed to a valid value", oldObj: valid(), - newObj: valid(func(s *ateapipb.ExternalSnapshot) { - s.ContentScope = ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA + newObj: valid(func(s *ateapipb.Snapshot) { + s.Storage[0].Fidelity = ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES }), }, { - name: "content_scope changed to a value outside the enum", + name: "fidelity cleared", oldObj: valid(), - newObj: valid(func(s *ateapipb.ExternalSnapshot) { s.ContentScope = ateapipb.SnapshotContentScope(3) }), - want: field.ErrorList{field.Invalid(scopePath, nil, "").WithOrigin("maximum")}, + newObj: valid(func(s *ateapipb.Snapshot) { + s.Storage[0].Fidelity = ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED + }), + want: field.ErrorList{field.Required(fidelityPath, "")}, + }, + { + name: "fidelity changed to a value outside the enum", + oldObj: valid(), + newObj: valid(func(s *ateapipb.Snapshot) { s.Storage[0].Fidelity = ateapipb.SnapshotFidelity(4) }), + want: field.ErrorList{field.Invalid(fidelityPath, nil, "").WithOrigin("maximum")}, }, { name: "snapshot_uri cleared", oldObj: valid(), - newObj: valid(func(s *ateapipb.ExternalSnapshot) { s.SnapshotUri = "" }), + newObj: valid(func(s *ateapipb.Snapshot) { s.Storage[0].Object.SnapshotUri = "" }), want: field.ErrorList{field.Required(uriPath, "")}, }, { // The other side of the ratchet: a row that predates these rules // can still be repaired, one field at a time. - name: "content_scope repaired", - oldObj: badExternalSnapshot(), - newObj: badExternalSnapshot(func(s *ateapipb.ExternalSnapshot) { - s.ContentScope = ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL + name: "fidelity repaired", + oldObj: badSnapshot(), + newObj: badSnapshot(func(s *ateapipb.Snapshot) { + s.Storage[0].Fidelity = ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY }), }, { name: "snapshot_uri changed to a value that is too long", oldObj: valid(), - newObj: valid(func(s *ateapipb.ExternalSnapshot) { - s.SnapshotUri = "gs://" + strings.Repeat("x", 2044) + newObj: valid(func(s *ateapipb.Snapshot) { + s.Storage[0].Object.SnapshotUri = "gs://" + strings.Repeat("x", 2044) }), want: field.ErrorList{field.TooLong(uriPath, nil, 2048).WithOrigin("maxLength")}, }, { name: "snapshot_uri repaired", - oldObj: badExternalSnapshot(), - newObj: badExternalSnapshot(func(s *ateapipb.ExternalSnapshot) { s.SnapshotUri = valid().SnapshotUri }), + oldObj: badSnapshot(), + newObj: badSnapshot(func(s *ateapipb.Snapshot) { s.Storage[0].Object.SnapshotUri = validObjectSnapshot().SnapshotUri }), }, { name: "every field repaired", - oldObj: badExternalSnapshot(), + oldObj: badSnapshot(), newObj: valid(), }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { op := operation.Operation{Type: operation.Update} - assertValidateErr(t, Validate_ExternalSnapshot(context.Background(), op, nil, tt.newObj, tt.oldObj), tt.want) + assertValidateErr(t, Validate_Snapshot(context.Background(), op, nil, tt.newObj, tt.oldObj), tt.want) }) } } -// TestValidateNestedExternalSnapshot checks that every type that -// holds ExternalSnapshot has to descend into it, and report under +// TestValidateNestedSnapshot checks that every type that +// holds Snapshot has to descend into it, and report under // the holder's own path. -func TestValidateNestedExternalSnapshot(t *testing.T) { +func TestValidateNestedSnapshot(t *testing.T) { tests := []struct { name string - // path is where the offending ExternalSnapshot sits in the holder. + // path is where the offending Snapshot sits in the holder. path *field.Path validate func(ctx context.Context) field.ErrorList }{ { - name: "actor.status.external_snapshot", - path: field.NewPath("status", "external_snapshot"), + name: "actor.status.snapshots", + path: field.NewPath("status", "snapshots").Index(0), validate: func(ctx context.Context) field.ErrorList { // The live path: the server validates the Actor it is about to // write, as an update against the stored one. op := operation.Operation{Type: operation.Update} oldVal := validActor(withActorStatus()) newVal := validActor(withActorStatus(func(s *ateapipb.ActorStatus) { - s.ExternalSnapshot = badExternalSnapshot() + s.Snapshots = []*ateapipb.Snapshot{badSnapshot()} })) return Validate_Actor(ctx, op, nil, newVal, oldVal) }, @@ -1339,7 +1457,7 @@ func TestValidateNestedExternalSnapshot(t *testing.T) { validate: func(ctx context.Context) field.ErrorList { op := operation.Operation{Type: operation.Create} obj := validTag(func(tag *ateapipb.Tag) { - tag.Status.Snapshot = badExternalSnapshot() + tag.Status.Snapshot = badSnapshot() }) return Validate_Tag(ctx, op, nil, obj, nil) }, @@ -1348,8 +1466,8 @@ func TestValidateNestedExternalSnapshot(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { want := field.ErrorList{ - field.Required(tt.path.Child("snapshot_uri"), ""), - field.Invalid(tt.path.Child("content_scope"), nil, "").WithOrigin("maximum"), + field.Required(tt.path.Child("storage").Index(0).Child("object", "snapshot_uri"), ""), + field.Invalid(tt.path.Child("storage").Index(0).Child("fidelity"), nil, "").WithOrigin("maximum"), } assertValidateErr(t, tt.validate(context.Background()), want) }) @@ -1360,9 +1478,8 @@ func validTag(mutate ...func(*ateapipb.Tag)) *ateapipb.Tag { tag := &ateapipb.Tag{ Metadata: validResourceMetadata(), Status: &ateapipb.TagStatus{ - Snapshot: validExternalSnapshot(), + Snapshot: validSnapshot(func(s *ateapipb.Snapshot) { s.Owner = ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG }), ActorTemplateUid: someActorUID, - StorageLocation: testStorageLocation, }, Scope: ateapipb.TagScope_TAG_SCOPE_ATESPACE, SourceActor: &ateapipb.ObjectRef{Atespace: "as", Name: "nm"}, @@ -1454,16 +1571,9 @@ func TestValidateTag(t *testing.T) { want: field.ErrorList{field.Invalid(statusPath.Child("actor_template_uid"), nil, "").WithOrigin("format=k8s-uuid")}, }, { - name: "missing status.storage_location", - obj: valid(func(tag *ateapipb.Tag) { tag.Status.StorageLocation = "" }), - want: field.ErrorList{field.Required(statusPath.Child("storage_location"), "")}, - }, - { - name: "status.storage_location too long", - obj: valid(func(tag *ateapipb.Tag) { - tag.Status.StorageLocation = "gs://" + strings.Repeat("x", 1020) - }), - want: field.ErrorList{field.TooLong(statusPath.Child("storage_location"), nil, 1024).WithOrigin("maxLength")}, + name: "missing status.snapshot", + obj: valid(func(tag *ateapipb.Tag) { tag.Status.Snapshot = nil }), + want: field.ErrorList{field.Required(statusPath.Child("snapshot"), "")}, }, } for _, tt := range tests { @@ -1551,13 +1661,13 @@ func TestValidateTagRequestPayloads(t *testing.T) { name: "create: invalid nested snapshot", validate: func(ctx context.Context, op operation.Operation) field.ErrorList { tag := validTag() - tag.Status.Snapshot = badExternalSnapshot() + tag.Status.Snapshot = badSnapshot() req := &ateapipb.CreateTagRequest{Tag: tag} return Validate_CreateTagRequest(ctx, op, nil, req, nil) }, want: field.ErrorList{ - field.Required(tagPath.Child("status", "snapshot", "snapshot_uri"), ""), - field.Invalid(tagPath.Child("status", "snapshot", "content_scope"), nil, "").WithOrigin("maximum"), + field.Required(tagPath.Child("status", "snapshot", "storage").Index(0).Child("object", "snapshot_uri"), ""), + field.Invalid(tagPath.Child("status", "snapshot", "storage").Index(0).Child("fidelity"), nil, "").WithOrigin("maximum"), }, }, { @@ -1581,7 +1691,7 @@ func TestValidateTagRequestPayloads(t *testing.T) { name: "update: nested snapshot is not descended into", validate: func(ctx context.Context, op operation.Operation) field.ErrorList { tag := validTag() - tag.Status.Snapshot = badExternalSnapshot() + tag.Status.Snapshot = badSnapshot() req := &ateapipb.UpdateTagRequest{Tag: tag} return Validate_UpdateTagRequest(ctx, op, nil, req, nil) }, diff --git a/cmd/ateapi/internal/apivalidation/worker_test.go b/cmd/ateapi/internal/apivalidation/worker_test.go index b867c7dda5..70667446d1 100644 --- a/cmd/ateapi/internal/apivalidation/worker_test.go +++ b/cmd/ateapi/internal/apivalidation/worker_test.go @@ -324,6 +324,7 @@ func TestValidateWorkerUpdate_RequireStatus(t *testing.T) { } func TestValidateRegisterWorkerRequest(t *testing.T) { + validRuntime := validSandboxRuntime valid := func(mutate ...func(*ateapipb.RegisterWorkerRequest)) *ateapipb.RegisterWorkerRequest { r := &ateapipb.RegisterWorkerRequest{ Worker: workerRef(apiWorkerName), @@ -333,13 +334,15 @@ func TestValidateRegisterWorkerRequest(t *testing.T) { Limits: []*ateapipb.Limits{{Name: "cpu", Quantity: "2"}, {Name: "memory", Quantity: "4Gi"}}, }, }, - Hardware: &ateapipb.HardwareIdentity{Attributes: map[string]string{"architecture": "amd64"}}, + DefaultRuntime: validRuntime(), } for _, m := range mutate { m(r) } return r } + defaultRuntimePath := field.NewPath("default_runtime") + restorablePath := field.NewPath("restorable_runtimes") tests := []struct { name string req *ateapipb.RegisterWorkerRequest @@ -347,6 +350,19 @@ func TestValidateRegisterWorkerRequest(t *testing.T) { }{{ name: "valid", req: valid(), + }, { + name: "valid with restorable runtimes", + req: valid(func(r *ateapipb.RegisterWorkerRequest) { + r.RestorableRuntimes = []*ateapipb.SandboxRuntime{{ + SandboxClass: "gvisor", + Version: &ateapipb.VersionedSandboxCompat{ + SchemaVersion: "v1", + Attributes: []*ateapipb.AttributeEntry{ + {Key: "architecture", Value: "amd64"}, + }, + }, + }} + }), }, { name: "valid empty capacity", req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.Capacity = &ateapipb.WorkerResources{} }), @@ -381,28 +397,96 @@ func TestValidateRegisterWorkerRequest(t *testing.T) { }), want: field.ErrorList{field.Invalid(field.NewPath("capacity", "resources", "limits").Index(0).Child("quantity"), nil, "")}, }, { - name: "missing hardware", - req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.Hardware = nil }), - want: field.ErrorList{field.Required(field.NewPath("hardware"), "")}, + name: "missing default_runtime", + req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.DefaultRuntime = nil }), + want: field.ErrorList{field.Required(defaultRuntimePath, "")}, }, { - name: "valid with empty hardware attributes", - req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.Hardware = &ateapipb.HardwareIdentity{} }), + name: "missing default_runtime.sandbox_class", + req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.DefaultRuntime.SandboxClass = "" }), + want: field.ErrorList{field.Required(defaultRuntimePath.Child("sandbox_class"), "")}, }, { - name: "hardware attribute key too long", + name: "default_runtime.sandbox_class too long", + req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.DefaultRuntime.SandboxClass = strings.Repeat("c", 64) }), + want: field.ErrorList{field.TooLong(defaultRuntimePath.Child("sandbox_class"), "", 63).WithOrigin("maxLength")}, + }, { + name: "missing default_runtime.version", + req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.DefaultRuntime.Version = nil }), + want: field.ErrorList{field.Required(defaultRuntimePath.Child("version"), "")}, + }, { + name: "missing default_runtime.version.schema_version", + req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.DefaultRuntime.Version.SchemaVersion = "" }), + want: field.ErrorList{field.Required(defaultRuntimePath.Child("version", "schema_version"), "")}, + }, { + name: "default_runtime.version.schema_version too long", req: valid(func(r *ateapipb.RegisterWorkerRequest) { - r.Hardware = &ateapipb.HardwareIdentity{ - Attributes: map[string]string{strings.Repeat("k", 129): "v"}, - } + r.DefaultRuntime.Version.SchemaVersion = strings.Repeat("v", 65) }), - want: field.ErrorList{field.TooLong(field.NewPath("hardware", "attributes"), "", 128).WithOrigin("maxLength")}, + want: field.ErrorList{field.TooLong(defaultRuntimePath.Child("version", "schema_version"), "", 64).WithOrigin("maxLength")}, }, { - name: "hardware attribute value too long", + name: "missing default_runtime.version.attributes", + req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.DefaultRuntime.Version.Attributes = nil }), + want: field.ErrorList{field.Required(defaultRuntimePath.Child("version", "attributes"), "")}, + }, { + name: "duplicate attribute key", req: valid(func(r *ateapipb.RegisterWorkerRequest) { - r.Hardware = &ateapipb.HardwareIdentity{ - Attributes: map[string]string{"k": strings.Repeat("v", 257)}, + r.DefaultRuntime.Version.Attributes = []*ateapipb.AttributeEntry{ + {Key: "architecture", Value: "amd64"}, + {Key: "architecture", Value: "arm64"}, } }), - want: field.ErrorList{field.TooLong(field.NewPath("hardware", "attributes").Key("k"), "", 256).WithOrigin("maxLength")}, + want: field.ErrorList{field.Duplicate(defaultRuntimePath.Child("version", "attributes").Index(1), nil)}, + }, { + name: "missing attribute key", + req: valid(func(r *ateapipb.RegisterWorkerRequest) { + r.DefaultRuntime.Version.Attributes = []*ateapipb.AttributeEntry{{Value: "amd64"}} + }), + want: field.ErrorList{field.Required(defaultRuntimePath.Child("version", "attributes").Index(0).Child("key"), "")}, + }, { + name: "attribute key too long", + req: valid(func(r *ateapipb.RegisterWorkerRequest) { + r.DefaultRuntime.Version.Attributes = []*ateapipb.AttributeEntry{{Key: strings.Repeat("k", 129), Value: "v"}} + }), + want: field.ErrorList{field.TooLong(defaultRuntimePath.Child("version", "attributes").Index(0).Child("key"), "", 128).WithOrigin("maxLength")}, + }, { + name: "missing attribute value", + req: valid(func(r *ateapipb.RegisterWorkerRequest) { + r.DefaultRuntime.Version.Attributes = []*ateapipb.AttributeEntry{{Key: "k"}} + }), + want: field.ErrorList{field.Required(defaultRuntimePath.Child("version", "attributes").Index(0).Child("value"), "")}, + }, { + name: "attribute value too long", + req: valid(func(r *ateapipb.RegisterWorkerRequest) { + r.DefaultRuntime.Version.Attributes = []*ateapipb.AttributeEntry{{Key: "k", Value: strings.Repeat("v", 257)}} + }), + want: field.ErrorList{field.TooLong(defaultRuntimePath.Child("version", "attributes").Index(0).Child("value"), "", 256).WithOrigin("maxLength")}, + }, { + name: "attributes at the bound", + req: valid(func(r *ateapipb.RegisterWorkerRequest) { + r.DefaultRuntime.Version.Attributes = attributeEntries(32) + }), + }, { + name: "too many attributes", + req: valid(func(r *ateapipb.RegisterWorkerRequest) { + r.DefaultRuntime.Version.Attributes = attributeEntries(33) + }), + want: field.ErrorList{field.TooMany(defaultRuntimePath.Child("version", "attributes"), 33, 32).WithOrigin("maxItems")}, + }, { + name: "nil restorable_runtimes entry", + req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.RestorableRuntimes = []*ateapipb.SandboxRuntime{nil} }), + want: field.ErrorList{field.Required(restorablePath.Index(0), "")}, + }, { + name: "invalid restorable_runtimes entry", + req: valid(func(r *ateapipb.RegisterWorkerRequest) { + r.RestorableRuntimes = []*ateapipb.SandboxRuntime{{SandboxClass: "gvisor"}} + }), + want: field.ErrorList{field.Required(restorablePath.Index(0).Child("version"), "")}, + }, { + name: "restorable_runtimes at the bound", + req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.RestorableRuntimes = sandboxRuntimes(32) }), + }, { + name: "too many restorable_runtimes", + req: valid(func(r *ateapipb.RegisterWorkerRequest) { r.RestorableRuntimes = sandboxRuntimes(33) }), + want: field.ErrorList{field.TooMany(restorablePath, 33, 32).WithOrigin("maxItems")}, }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -411,6 +495,64 @@ func TestValidateRegisterWorkerRequest(t *testing.T) { } } +// TestValidateWorkerUpdate_SandboxRuntimes pins the status-side checks on the +// runtimes RegisterWorker records. They reach the store as status, which no +// request validator sees, so ValidateWorkerUpdate is the only check on them. +func TestValidateWorkerUpdate_SandboxRuntimes(t *testing.T) { + statusPath := field.NewPath("worker", "status") + oldVal := validWorker(apiWorkerName, withWorkerStatus()) + tests := []struct { + name string + mutate func(s *ateapipb.WorkerStatus) + want field.ErrorList + }{{ + name: "valid runtimes", + mutate: func(s *ateapipb.WorkerStatus) { + s.DefaultRuntime = validSandboxRuntime() + s.RestorableRuntimes = sandboxRuntimes(32) + }, + }, { + name: "missing default_runtime.sandbox_class", + mutate: func(s *ateapipb.WorkerStatus) { + s.DefaultRuntime = validSandboxRuntime() + s.DefaultRuntime.SandboxClass = "" + }, + want: field.ErrorList{field.Required(statusPath.Child("default_runtime", "sandbox_class"), "")}, + }, { + name: "missing default_runtime.version", + mutate: func(s *ateapipb.WorkerStatus) { + s.DefaultRuntime = &ateapipb.SandboxRuntime{SandboxClass: "gvisor"} + }, + want: field.ErrorList{field.Required(statusPath.Child("default_runtime", "version"), "")}, + }, { + name: "too many default_runtime attributes", + mutate: func(s *ateapipb.WorkerStatus) { + s.DefaultRuntime = validSandboxRuntime() + s.DefaultRuntime.Version.Attributes = attributeEntries(33) + }, + want: field.ErrorList{field.TooMany(statusPath.Child("default_runtime", "version", "attributes"), 33, 32).WithOrigin("maxItems")}, + }, { + name: "invalid restorable_runtimes entry", + mutate: func(s *ateapipb.WorkerStatus) { + s.RestorableRuntimes = []*ateapipb.SandboxRuntime{{SandboxClass: "gvisor"}} + }, + want: field.ErrorList{field.Required(statusPath.Child("restorable_runtimes").Index(0).Child("version"), "")}, + }, { + name: "too many restorable_runtimes", + mutate: func(s *ateapipb.WorkerStatus) { + s.RestorableRuntimes = sandboxRuntimes(33) + }, + want: field.ErrorList{field.TooMany(statusPath.Child("restorable_runtimes"), 33, 32).WithOrigin("maxItems")}, + }} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + newVal := proto.Clone(oldVal).(*ateapipb.Worker) + tt.mutate(newVal.Status) + assertValidateErr(t, ValidateWorkerUpdate(context.Background(), field.NewPath("worker"), newVal, oldVal, true), tt.want) + }) + } +} + func TestValidateRequestActorSuspendRequest(t *testing.T) { valid := func(mutate ...func(*ateapipb.RequestActorSuspendRequest)) *ateapipb.RequestActorSuspendRequest { r := &ateapipb.RequestActorSuspendRequest{ @@ -523,3 +665,33 @@ func withWorkerStatus(mods ...func(*ateapipb.WorkerStatus)) func(*ateapipb.Worke func workerRef(name string) *ateapipb.ObjectRef { return &ateapipb.ObjectRef{Name: name} } + +// validSandboxRuntime returns a SandboxRuntime in the shape RegisterWorker +// accepts: a class and a v1 identity with one attribute. +func validSandboxRuntime() *ateapipb.SandboxRuntime { + return &ateapipb.SandboxRuntime{ + SandboxClass: "gvisor", + Version: &ateapipb.VersionedSandboxCompat{ + SchemaVersion: "v1", + Attributes: []*ateapipb.AttributeEntry{{Key: "architecture", Value: "amd64"}}, + }, + } +} + +// attributeEntries returns n valid attributes with distinct keys. +func attributeEntries(n int) []*ateapipb.AttributeEntry { + attrs := make([]*ateapipb.AttributeEntry, n) + for i := range attrs { + attrs[i] = &ateapipb.AttributeEntry{Key: fmt.Sprintf("k%d", i), Value: "v"} + } + return attrs +} + +// sandboxRuntimes returns n valid runtimes. +func sandboxRuntimes(n int) []*ateapipb.SandboxRuntime { + rts := make([]*ateapipb.SandboxRuntime, n) + for i := range rts { + rts[i] = validSandboxRuntime() + } + return rts +} diff --git a/cmd/ateapi/internal/apivalidation/zz_generated.validation.go b/cmd/ateapi/internal/apivalidation/zz_generated.validation.go index a502173329..f754fa9d6c 100644 --- a/cmd/ateapi/internal/apivalidation/zz_generated.validation.go +++ b/cmd/ateapi/internal/apivalidation/zz_generated.validation.go @@ -706,41 +706,56 @@ func Validate_ActorStatus( errs = append(errs, fn(fldPath.Child("worker_assignment"), obj.WorkerAssignment, oldVal, oldObj != nil)...) } - { // field ateapipb.ActorStatus.InProgressSnapshotUri + { // field ateapipb.ActorStatus.ExternalVolumes fn := func( fldPath *field.Path, - obj, oldObj *string, + obj, oldObj []*ateapipb.ExternalVolume, oldValueCorrelated bool) (errs field.ErrorList) { // don't revalidate unchanged data if oldValueCorrelated && op.Type == operation.Update { - if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + if ateDeepEqual(obj, oldObj) { return nil } } // call field-attached validations earlyReturn := false - if e := validate.OptionalValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + if e := validate.PtrSliceNoNils[ateapipb.ExternalVolume](ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.MaxItems(ctx, op, fldPath, obj, oldObj, 32).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.OptionalSlice(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { earlyReturn = true } if earlyReturn { return // do not proceed } - if e := validate.MaxLength(ctx, op, fldPath, obj, oldObj, 2048); len(e) != 0 { + // lists with map semantics require unique keys + if e := validate.PtrSliceUnique(ctx, op, fldPath, obj, oldObj, + func(a *ateapipb.ExternalVolume, b *ateapipb.ExternalVolume) bool { return a.Name == b.Name }); len(e) != 0 { + errs = append(errs, e...) + } + // iterate the list and call the type's validation function + if e := validate.EachPtrSliceVal(ctx, op, fldPath, obj, oldObj, + func(a *ateapipb.ExternalVolume, b *ateapipb.ExternalVolume) bool { return a.Name == b.Name }, ateDeepEqual, Validate_ExternalVolume); len(e) != 0 { errs = append(errs, e...) } return } oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.ActorStatus) *string { - return &oldObj.InProgressSnapshotUri + func(oldObj *ateapipb.ActorStatus) []*ateapipb.ExternalVolume { + return oldObj.ExternalVolumes }) - errs = append(errs, fn(fldPath.Child("in_progress_snapshot_uri"), &obj.InProgressSnapshotUri, oldVal, oldObj != nil)...) + errs = append(errs, fn(fldPath.Child("external_volumes"), obj.ExternalVolumes, oldVal, oldObj != nil)...) } - { // field ateapipb.ActorStatus.ExternalSnapshot + { // field ateapipb.ActorStatus.Crash fn := func( fldPath *field.Path, - obj, oldObj *ateapipb.ExternalSnapshot, + obj, oldObj *ateapipb.ActorCrash, oldValueCorrelated bool) (errs field.ErrorList) { // don't revalidate unchanged data if oldValueCorrelated && op.Type == operation.Update { @@ -757,50 +772,51 @@ func Validate_ActorStatus( return // do not proceed } // call the type's validation function - errs = append(errs, Validate_ExternalSnapshot(ctx, op, fldPath, obj, oldObj)...) + errs = append(errs, Validate_ActorCrash(ctx, op, fldPath, obj, oldObj)...) return } oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.ActorStatus) *ateapipb.ExternalSnapshot { - return oldObj.ExternalSnapshot + func(oldObj *ateapipb.ActorStatus) *ateapipb.ActorCrash { + return oldObj.Crash }) - errs = append(errs, fn(fldPath.Child("external_snapshot"), obj.ExternalSnapshot, oldVal, oldObj != nil)...) + errs = append(errs, fn(fldPath.Child("crash"), obj.Crash, oldVal, oldObj != nil)...) } - { // field ateapipb.ActorStatus.LocalSnapshot + { // field ateapipb.ActorStatus.AssignedNode fn := func( fldPath *field.Path, - obj, oldObj *ateapipb.LocalSnapshot, + obj, oldObj *string, oldValueCorrelated bool) (errs field.ErrorList) { // don't revalidate unchanged data if oldValueCorrelated && op.Type == operation.Update { - if ateDeepEqual(obj, oldObj) { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { return nil } } // call field-attached validations earlyReturn := false - if e := validate.OptionalPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + if e := validate.OptionalValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { earlyReturn = true } if earlyReturn { return // do not proceed } - // call the type's validation function - errs = append(errs, Validate_LocalSnapshot(ctx, op, fldPath, obj, oldObj)...) + if e := validate.LongName(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + errs = append(errs, e...) + } return } oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.ActorStatus) *ateapipb.LocalSnapshot { - return oldObj.LocalSnapshot + func(oldObj *ateapipb.ActorStatus) *string { + return &oldObj.AssignedNode }) - errs = append(errs, fn(fldPath.Child("local_snapshot"), obj.LocalSnapshot, oldVal, oldObj != nil)...) + errs = append(errs, fn(fldPath.Child("assigned_node"), &obj.AssignedNode, oldVal, oldObj != nil)...) } - { // field ateapipb.ActorStatus.ActorVolumes + { // field ateapipb.ActorStatus.Snapshots fn := func( fldPath *field.Path, - obj, oldObj []*ateapipb.ExternalVolume, + obj, oldObj []*ateapipb.Snapshot, oldValueCorrelated bool) (errs field.ErrorList) { // don't revalidate unchanged data if oldValueCorrelated && op.Type == operation.Update { @@ -810,11 +826,11 @@ func Validate_ActorStatus( } // call field-attached validations earlyReturn := false - if e := validate.PtrSliceNoNils[ateapipb.ExternalVolume](ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + if e := validate.PtrSliceNoNils[ateapipb.Snapshot](ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { errs = append(errs, e...) earlyReturn = true } - if e := validate.MaxItems(ctx, op, fldPath, obj, oldObj, 32).MarkShortCircuit(); len(e) != 0 { + if e := validate.MaxItems(ctx, op, fldPath, obj, oldObj, 16).MarkShortCircuit(); len(e) != 0 { errs = append(errs, e...) earlyReturn = true } @@ -826,27 +842,27 @@ func Validate_ActorStatus( } // lists with map semantics require unique keys if e := validate.PtrSliceUnique(ctx, op, fldPath, obj, oldObj, - func(a *ateapipb.ExternalVolume, b *ateapipb.ExternalVolume) bool { return a.VolumeName == b.VolumeName }); len(e) != 0 { + func(a *ateapipb.Snapshot, b *ateapipb.Snapshot) bool { return a.Generation == b.Generation }); len(e) != 0 { errs = append(errs, e...) } // iterate the list and call the type's validation function if e := validate.EachPtrSliceVal(ctx, op, fldPath, obj, oldObj, - func(a *ateapipb.ExternalVolume, b *ateapipb.ExternalVolume) bool { return a.VolumeName == b.VolumeName }, ateDeepEqual, Validate_ExternalVolume); len(e) != 0 { + func(a *ateapipb.Snapshot, b *ateapipb.Snapshot) bool { return a.Generation == b.Generation }, ateDeepEqual, Validate_Snapshot); len(e) != 0 { errs = append(errs, e...) } return } oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.ActorStatus) []*ateapipb.ExternalVolume { - return oldObj.ActorVolumes + func(oldObj *ateapipb.ActorStatus) []*ateapipb.Snapshot { + return oldObj.Snapshots }) - errs = append(errs, fn(fldPath.Child("actor_volumes"), obj.ActorVolumes, oldVal, oldObj != nil)...) + errs = append(errs, fn(fldPath.Child("snapshots"), obj.Snapshots, oldVal, oldObj != nil)...) } - { // field ateapipb.ActorStatus.InProgressLocalSnapshotName + { // field ateapipb.ActorStatus.LastAssignedGeneration fn := func( fldPath *field.Path, - obj, oldObj *string, + obj, oldObj *int32, oldValueCorrelated bool) (errs field.ErrorList) { // don't revalidate unchanged data if oldValueCorrelated && op.Type == operation.Update { @@ -859,80 +875,27 @@ func Validate_ActorStatus( if e := validate.OptionalValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { earlyReturn = true } - if earlyReturn { - return // do not proceed - } - if e := validate.ShortName(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + if e := validate.UpdateValue(ctx, op, fldPath, obj, oldObj, + func(a int32, b int32) bool { return a == b }, validate.NoUnset).MarkShortCircuit(); len(e) != 0 { errs = append(errs, e...) - } - return - } - oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.ActorStatus) *string { - return &oldObj.InProgressLocalSnapshotName - }) - errs = append(errs, fn(fldPath.Child("in_progress_local_snapshot_name"), &obj.InProgressLocalSnapshotName, oldVal, oldObj != nil)...) - } - - { // field ateapipb.ActorStatus.Crash - fn := func( - fldPath *field.Path, - obj, oldObj *ateapipb.ActorCrash, - oldValueCorrelated bool) (errs field.ErrorList) { - // don't revalidate unchanged data - if oldValueCorrelated && op.Type == operation.Update { - if ateDeepEqual(obj, oldObj) { - return nil - } - } - // call field-attached validations - earlyReturn := false - if e := validate.OptionalPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { earlyReturn = true } if earlyReturn { return // do not proceed } - // call the type's validation function - errs = append(errs, Validate_ActorCrash(ctx, op, fldPath, obj, oldObj)...) - return - } - oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.ActorStatus) *ateapipb.ActorCrash { - return oldObj.Crash - }) - errs = append(errs, fn(fldPath.Child("crash"), obj.Crash, oldVal, oldObj != nil)...) - } - - { // field ateapipb.ActorStatus.AssignedNode - fn := func( - fldPath *field.Path, - obj, oldObj *string, - oldValueCorrelated bool) (errs field.ErrorList) { - // don't revalidate unchanged data - if oldValueCorrelated && op.Type == operation.Update { - if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { - return nil - } - } - // call field-attached validations - earlyReturn := false - if e := validate.OptionalValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { - earlyReturn = true - } - if earlyReturn { - return // do not proceed + if e := validate.Monotonic(ctx, op, fldPath, obj, oldObj).MarkAlpha(); len(e) != 0 { + errs = append(errs, e...) } - if e := validate.LongName(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + if e := validate.Minimum(ctx, op, fldPath, obj, oldObj, 0); len(e) != 0 { errs = append(errs, e...) } return } oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.ActorStatus) *string { - return &oldObj.AssignedNode + func(oldObj *ateapipb.ActorStatus) *int32 { + return &oldObj.LastAssignedGeneration }) - errs = append(errs, fn(fldPath.Child("assigned_node"), &obj.AssignedNode, oldVal, oldObj != nil)...) + errs = append(errs, fn(fldPath.Child("last_assigned_generation"), &obj.LastAssignedGeneration, oldVal, oldObj != nil)...) } return errs @@ -1344,6 +1307,79 @@ func Validate_Atespace( return errs } +// Validate_AttributeEntry validates an instance of AttributeEntry according +// to declarative validation rules in the API schema. +func Validate_AttributeEntry( + ctx context.Context, op operation.Operation, fldPath *field.Path, + obj, oldObj *ateapipb.AttributeEntry) (errs field.ErrorList) { + + { // field ateapipb.AttributeEntry.Key + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.MaxLength(ctx, op, fldPath, obj, oldObj, 128); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.AttributeEntry) *string { + return &oldObj.Key + }) + errs = append(errs, fn(fldPath.Child("key"), &obj.Key, oldVal, oldObj != nil)...) + } + + { // field ateapipb.AttributeEntry.Value + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.MaxLength(ctx, op, fldPath, obj, oldObj, 256); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.AttributeEntry) *string { + return &oldObj.Value + }) + errs = append(errs, fn(fldPath.Child("value"), &obj.Value, oldVal, oldObj != nil)...) + } + + return errs +} + // Validate_Binding validates an instance of Binding according // to declarative validation rules in the API schema. func Validate_Binding( @@ -3587,13 +3623,13 @@ func Validate_EnvVar( return errs } -// Validate_ExternalSnapshot validates an instance of ExternalSnapshot according +// Validate_ExternalVolume validates an instance of ExternalVolume according // to declarative validation rules in the API schema. -func Validate_ExternalSnapshot( +func Validate_ExternalVolume( ctx context.Context, op operation.Operation, fldPath *field.Path, - obj, oldObj *ateapipb.ExternalSnapshot) (errs field.ErrorList) { + obj, oldObj *ateapipb.ExternalVolume) (errs field.ErrorList) { - { // field ateapipb.ExternalSnapshot.SnapshotUri + { // field ateapipb.ExternalVolume.Name fn := func( fldPath *field.Path, obj, oldObj *string, @@ -3610,56 +3646,27 @@ func Validate_ExternalSnapshot( errs = append(errs, e...) earlyReturn = true } - if earlyReturn { - return // do not proceed - } - if e := validate.MaxLength(ctx, op, fldPath, obj, oldObj, 2048); len(e) != 0 { + if e := validate.UpdateValue(ctx, op, fldPath, obj, oldObj, + func(a string, b string) bool { return a == b }, validate.NoUnset, validate.NoModify).MarkShortCircuit(); len(e) != 0 { errs = append(errs, e...) - } - return - } - oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.ExternalSnapshot) *string { - return &oldObj.SnapshotUri - }) - errs = append(errs, fn(fldPath.Child("snapshot_uri"), &obj.SnapshotUri, oldVal, oldObj != nil)...) - } - - { // field ateapipb.ExternalSnapshot.ContentScope - fn := func( - fldPath *field.Path, - obj, oldObj *ateapipb.SnapshotContentScope, - oldValueCorrelated bool) (errs field.ErrorList) { - // don't revalidate unchanged data - if oldValueCorrelated && op.Type == operation.Update { - if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { - return nil - } - } - // call field-attached validations - earlyReturn := false - if e := validate.OptionalValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { earlyReturn = true } if earlyReturn { return // do not proceed } - if e := validate.Maximum(ctx, op, fldPath, obj, oldObj, 2); len(e) != 0 { - errs = append(errs, e...) - } - if e := validate.Minimum(ctx, op, fldPath, obj, oldObj, 1); len(e) != 0 { + if e := validate.ShortName(ctx, op, fldPath, obj, oldObj); len(e) != 0 { errs = append(errs, e...) } return } oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.ExternalSnapshot) *ateapipb.SnapshotContentScope { - return &oldObj.ContentScope + func(oldObj *ateapipb.ExternalVolume) *string { + return &oldObj.Name }) - errs = append(errs, fn(fldPath.Child("content_scope"), &obj.ContentScope, oldVal, oldObj != nil)...) + errs = append(errs, fn(fldPath.Child("name"), &obj.Name, oldVal, oldObj != nil)...) } - { // field ateapipb.ExternalSnapshot.ActorTemplateUid + { // field ateapipb.ExternalVolume.StorageVolumeId fn := func( fldPath *field.Path, obj, oldObj *string, @@ -3675,31 +3682,31 @@ func Validate_ExternalSnapshot( if e := validate.OptionalValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { earlyReturn = true } + if e := validate.UpdateValue(ctx, op, fldPath, obj, oldObj, + func(a string, b string) bool { return a == b }, validate.NoUnset, validate.NoModify).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } if earlyReturn { return // do not proceed } - if e := validate.UUID(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + // custom validation + if e := ValidateCustom_ExternalVolume_StorageVolumeId(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + errs = append(errs, e...) + } + if e := validate.MaxLength(ctx, op, fldPath, obj, oldObj, 256); len(e) != 0 { errs = append(errs, e...) } return } oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.ExternalSnapshot) *string { - return &oldObj.ActorTemplateUid + func(oldObj *ateapipb.ExternalVolume) *string { + return &oldObj.StorageVolumeId }) - errs = append(errs, fn(fldPath.Child("actor_template_uid"), &obj.ActorTemplateUid, oldVal, oldObj != nil)...) + errs = append(errs, fn(fldPath.Child("storage_volume_id"), &obj.StorageVolumeId, oldVal, oldObj != nil)...) } - return errs -} - -// Validate_ExternalVolume validates an instance of ExternalVolume according -// to declarative validation rules in the API schema. -func Validate_ExternalVolume( - ctx context.Context, op operation.Operation, fldPath *field.Path, - obj, oldObj *ateapipb.ExternalVolume) (errs field.ErrorList) { - - { // field ateapipb.ExternalVolume.VolumeName + { // field ateapipb.ExternalVolume.VolumeType fn := func( fldPath *field.Path, obj, oldObj *string, @@ -3724,103 +3731,26 @@ func Validate_ExternalVolume( if earlyReturn { return // do not proceed } - if e := validate.ShortName(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + // custom validation + if e := ValidateCustom_ExternalVolume_VolumeType(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + errs = append(errs, e...) + } + if e := validate.MaxLength(ctx, op, fldPath, obj, oldObj, 253); len(e) != 0 { errs = append(errs, e...) } return } oldVal := safe.Field(oldObj, func(oldObj *ateapipb.ExternalVolume) *string { - return &oldObj.VolumeName + return &oldObj.VolumeType }) - errs = append(errs, fn(fldPath.Child("volume_name"), &obj.VolumeName, oldVal, oldObj != nil)...) + errs = append(errs, fn(fldPath.Child("volume_type"), &obj.VolumeType, oldVal, oldObj != nil)...) } - { // field ateapipb.ExternalVolume.StorageVolumeId + { // field ateapipb.ExternalVolume.Status fn := func( fldPath *field.Path, - obj, oldObj *string, - oldValueCorrelated bool) (errs field.ErrorList) { - // don't revalidate unchanged data - if oldValueCorrelated && op.Type == operation.Update { - if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { - return nil - } - } - // call field-attached validations - earlyReturn := false - if e := validate.OptionalValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { - earlyReturn = true - } - if e := validate.UpdateValue(ctx, op, fldPath, obj, oldObj, - func(a string, b string) bool { return a == b }, validate.NoUnset, validate.NoModify).MarkShortCircuit(); len(e) != 0 { - errs = append(errs, e...) - earlyReturn = true - } - if earlyReturn { - return // do not proceed - } - // custom validation - if e := ValidateCustom_ExternalVolume_StorageVolumeId(ctx, op, fldPath, obj, oldObj); len(e) != 0 { - errs = append(errs, e...) - } - if e := validate.MaxLength(ctx, op, fldPath, obj, oldObj, 256); len(e) != 0 { - errs = append(errs, e...) - } - return - } - oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.ExternalVolume) *string { - return &oldObj.StorageVolumeId - }) - errs = append(errs, fn(fldPath.Child("storage_volume_id"), &obj.StorageVolumeId, oldVal, oldObj != nil)...) - } - - { // field ateapipb.ExternalVolume.VolumeType - fn := func( - fldPath *field.Path, - obj, oldObj *string, - oldValueCorrelated bool) (errs field.ErrorList) { - // don't revalidate unchanged data - if oldValueCorrelated && op.Type == operation.Update { - if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { - return nil - } - } - // call field-attached validations - earlyReturn := false - if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { - errs = append(errs, e...) - earlyReturn = true - } - if e := validate.UpdateValue(ctx, op, fldPath, obj, oldObj, - func(a string, b string) bool { return a == b }, validate.NoUnset, validate.NoModify).MarkShortCircuit(); len(e) != 0 { - errs = append(errs, e...) - earlyReturn = true - } - if earlyReturn { - return // do not proceed - } - // custom validation - if e := ValidateCustom_ExternalVolume_VolumeType(ctx, op, fldPath, obj, oldObj); len(e) != 0 { - errs = append(errs, e...) - } - if e := validate.MaxLength(ctx, op, fldPath, obj, oldObj, 253); len(e) != 0 { - errs = append(errs, e...) - } - return - } - oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.ExternalVolume) *string { - return &oldObj.VolumeType - }) - errs = append(errs, fn(fldPath.Child("volume_type"), &obj.VolumeType, oldVal, oldObj != nil)...) - } - - { // field ateapipb.ExternalVolume.Status - fn := func( - fldPath *field.Path, - obj, oldObj *ateapipb.ExternalVolume_Status, + obj, oldObj *ateapipb.ExternalVolume_Status, oldValueCorrelated bool) (errs field.ErrorList) { // don't revalidate unchanged data if oldValueCorrelated && op.Type == operation.Update { @@ -4785,59 +4715,6 @@ func Validate_HTTPSRule( return errs } -// Validate_HardwareIdentity validates an instance of HardwareIdentity according -// to declarative validation rules in the API schema. -func Validate_HardwareIdentity( - ctx context.Context, op operation.Operation, fldPath *field.Path, - obj, oldObj *ateapipb.HardwareIdentity) (errs field.ErrorList) { - - { // field ateapipb.HardwareIdentity.Attributes - fn := func( - fldPath *field.Path, - obj, oldObj map[string]string, - oldValueCorrelated bool) (errs field.ErrorList) { - // don't revalidate unchanged data - if oldValueCorrelated && op.Type == operation.Update { - if ateDeepEqual(obj, oldObj) { - return nil - } - } - // call field-attached validations - earlyReturn := false - if e := validate.MaxProperties(ctx, op, fldPath, obj, oldObj, 32).MarkShortCircuit(); len(e) != 0 { - errs = append(errs, e...) - earlyReturn = true - } - if e := validate.OptionalMap(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { - earlyReturn = true - } - if earlyReturn { - return // do not proceed - } - if e := validate.EachMapKey(ctx, op, fldPath, obj, oldObj, - func(ctx context.Context, op operation.Operation, fldPath *field.Path, obj, oldObj *string) field.ErrorList { - return validate.MaxLength(ctx, op, fldPath, obj, oldObj, 128) - }); len(e) != 0 { - errs = append(errs, e...) - } - if e := validate.EachMapVal(ctx, op, fldPath, obj, oldObj, validate.DirectEqual, - func(ctx context.Context, op operation.Operation, fldPath *field.Path, obj, oldObj *string) field.ErrorList { - return validate.MaxLength(ctx, op, fldPath, obj, oldObj, 256) - }); len(e) != 0 { - errs = append(errs, e...) - } - return - } - oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.HardwareIdentity) map[string]string { - return oldObj.Attributes - }) - errs = append(errs, fn(fldPath.Child("attributes"), obj.Attributes, oldVal, oldObj != nil)...) - } - - return errs -} - // Validate_HttpRuleEffects validates an instance of HttpRuleEffects according // to declarative validation rules in the API schema. func Validate_HttpRuleEffects( @@ -5609,7 +5486,8 @@ func Validate_LocalSnapshot( } // call field-attached validations earlyReturn := false - if e := validate.OptionalValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) earlyReturn = true } if earlyReturn { @@ -5627,40 +5505,6 @@ func Validate_LocalSnapshot( errs = append(errs, fn(fldPath.Child("snapshot_name"), &obj.SnapshotName, oldVal, oldObj != nil)...) } - { // field ateapipb.LocalSnapshot.ContentScope - fn := func( - fldPath *field.Path, - obj, oldObj *ateapipb.SnapshotContentScope, - oldValueCorrelated bool) (errs field.ErrorList) { - // don't revalidate unchanged data - if oldValueCorrelated && op.Type == operation.Update { - if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { - return nil - } - } - // call field-attached validations - earlyReturn := false - if e := validate.OptionalValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { - earlyReturn = true - } - if earlyReturn { - return // do not proceed - } - if e := validate.Maximum(ctx, op, fldPath, obj, oldObj, 2); len(e) != 0 { - errs = append(errs, e...) - } - if e := validate.Minimum(ctx, op, fldPath, obj, oldObj, 1); len(e) != 0 { - errs = append(errs, e...) - } - return - } - oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.LocalSnapshot) *ateapipb.SnapshotContentScope { - return &oldObj.ContentScope - }) - errs = append(errs, fn(fldPath.Child("content_scope"), &obj.ContentScope, oldVal, oldObj != nil)...) - } - return errs } @@ -5805,7 +5649,7 @@ func Validate_MintActorJWTRequest( errs = append(errs, fn(fldPath.Child("actor"), obj.Actor, oldVal, oldObj != nil)...) } - { // field ateapipb.MintActorJWTRequest.Audience + { // field ateapipb.MintActorJWTRequest.Audiences fn := func( fldPath *field.Path, obj, oldObj []string, @@ -5843,9 +5687,9 @@ func Validate_MintActorJWTRequest( } oldVal := safe.Field(oldObj, func(oldObj *ateapipb.MintActorJWTRequest) []string { - return oldObj.Audience + return oldObj.Audiences }) - errs = append(errs, fn(fldPath.Child("audience"), obj.Audience, oldVal, oldObj != nil)...) + errs = append(errs, fn(fldPath.Child("audiences"), obj.Audiences, oldVal, oldObj != nil)...) } { // field ateapipb.MintActorJWTRequest.ExpirationSeconds @@ -6077,6 +5921,47 @@ func Validate_ObjectRef( return errs } +// Validate_ObjectSnapshot validates an instance of ObjectSnapshot according +// to declarative validation rules in the API schema. +func Validate_ObjectSnapshot( + ctx context.Context, op operation.Operation, fldPath *field.Path, + obj, oldObj *ateapipb.ObjectSnapshot) (errs field.ErrorList) { + + { // field ateapipb.ObjectSnapshot.SnapshotUri + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.MaxLength(ctx, op, fldPath, obj, oldObj, 2048); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.ObjectSnapshot) *string { + return &oldObj.SnapshotUri + }) + errs = append(errs, fn(fldPath.Child("snapshot_uri"), &obj.SnapshotUri, oldVal, oldObj != nil)...) + } + + return errs +} + // Validate_PauseActorRequest validates an instance of PauseActorRequest according // to declarative validation rules in the API schema. func Validate_PauseActorRequest( @@ -6322,10 +6207,10 @@ func Validate_RegisterWorkerRequest( errs = append(errs, fn(fldPath.Child("capacity"), obj.Capacity, oldVal, oldObj != nil)...) } - { // field ateapipb.RegisterWorkerRequest.Hardware + { // field ateapipb.RegisterWorkerRequest.DefaultRuntime fn := func( fldPath *field.Path, - obj, oldObj *ateapipb.HardwareIdentity, + obj, oldObj *ateapipb.SandboxRuntime, oldValueCorrelated bool) (errs field.ErrorList) { // don't revalidate unchanged data if oldValueCorrelated && op.Type == operation.Update { @@ -6343,14 +6228,54 @@ func Validate_RegisterWorkerRequest( return // do not proceed } // call the type's validation function - errs = append(errs, Validate_HardwareIdentity(ctx, op, fldPath, obj, oldObj)...) + errs = append(errs, Validate_SandboxRuntime(ctx, op, fldPath, obj, oldObj)...) + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.RegisterWorkerRequest) *ateapipb.SandboxRuntime { + return oldObj.DefaultRuntime + }) + errs = append(errs, fn(fldPath.Child("default_runtime"), obj.DefaultRuntime, oldVal, oldObj != nil)...) + } + + { // field ateapipb.RegisterWorkerRequest.RestorableRuntimes + fn := func( + fldPath *field.Path, + obj, oldObj []*ateapipb.SandboxRuntime, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.PtrSliceNoNils[ateapipb.SandboxRuntime](ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.MaxItems(ctx, op, fldPath, obj, oldObj, 32).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.OptionalSlice(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // iterate the list and call the type's validation function + if e := validate.EachPtrSliceVal(ctx, op, fldPath, obj, oldObj, nil, nil, Validate_SandboxRuntime); len(e) != 0 { + errs = append(errs, e...) + } return } oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.RegisterWorkerRequest) *ateapipb.HardwareIdentity { - return oldObj.Hardware + func(oldObj *ateapipb.RegisterWorkerRequest) []*ateapipb.SandboxRuntime { + return oldObj.RestorableRuntimes }) - errs = append(errs, fn(fldPath.Child("hardware"), obj.Hardware, oldVal, oldObj != nil)...) + errs = append(errs, fn(fldPath.Child("restorable_runtimes"), obj.RestorableRuntimes, oldVal, oldObj != nil)...) } return errs @@ -6957,117 +6882,484 @@ func Validate_SandboxConfig( return errs } -// Validate_SecurityContext validates an instance of SecurityContext according +// Validate_SandboxRuntime validates an instance of SandboxRuntime according // to declarative validation rules in the API schema. -func Validate_SecurityContext( +func Validate_SandboxRuntime( ctx context.Context, op operation.Operation, fldPath *field.Path, - obj, oldObj *ateapipb.SecurityContext) (errs field.ErrorList) { + obj, oldObj *ateapipb.SandboxRuntime) (errs field.ErrorList) { - { // field ateapipb.SecurityContext.Capabilities + { // field ateapipb.SandboxRuntime.SandboxClass fn := func( fldPath *field.Path, - obj, oldObj *ateapipb.Capabilities, + obj, oldObj *string, oldValueCorrelated bool) (errs field.ErrorList) { // don't revalidate unchanged data if oldValueCorrelated && op.Type == operation.Update { - if ateDeepEqual(obj, oldObj) { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { return nil } } // call field-attached validations earlyReturn := false - if e := validate.OptionalPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) earlyReturn = true } if earlyReturn { return // do not proceed } - // call the type's validation function - errs = append(errs, Validate_Capabilities(ctx, op, fldPath, obj, oldObj)...) + if e := validate.MaxLength(ctx, op, fldPath, obj, oldObj, 63); len(e) != 0 { + errs = append(errs, e...) + } return } oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.SecurityContext) *ateapipb.Capabilities { - return oldObj.Capabilities + func(oldObj *ateapipb.SandboxRuntime) *string { + return &oldObj.SandboxClass }) - errs = append(errs, fn(fldPath.Child("capabilities"), obj.Capabilities, oldVal, oldObj != nil)...) + errs = append(errs, fn(fldPath.Child("sandbox_class"), &obj.SandboxClass, oldVal, oldObj != nil)...) } - return errs -} - -var unionMembershipFor_github_com_agent_substrate_substrate_pkg_proto_ateapipb_Selector_ = validate.NewUnionMembership(validate.NewUnionMember("match_labels")) - + { // field ateapipb.SandboxRuntime.Version + fn := func( + fldPath *field.Path, + obj, oldObj *ateapipb.VersionedSandboxCompat, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // call the type's validation function + errs = append(errs, Validate_VersionedSandboxCompat(ctx, op, fldPath, obj, oldObj)...) + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.SandboxRuntime) *ateapipb.VersionedSandboxCompat { + return oldObj.Version + }) + errs = append(errs, fn(fldPath.Child("version"), obj.Version, oldVal, oldObj != nil)...) + } + + return errs +} + +// Validate_SecurityContext validates an instance of SecurityContext according +// to declarative validation rules in the API schema. +func Validate_SecurityContext( + ctx context.Context, op operation.Operation, fldPath *field.Path, + obj, oldObj *ateapipb.SecurityContext) (errs field.ErrorList) { + + { // field ateapipb.SecurityContext.Capabilities + fn := func( + fldPath *field.Path, + obj, oldObj *ateapipb.Capabilities, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.OptionalPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // call the type's validation function + errs = append(errs, Validate_Capabilities(ctx, op, fldPath, obj, oldObj)...) + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.SecurityContext) *ateapipb.Capabilities { + return oldObj.Capabilities + }) + errs = append(errs, fn(fldPath.Child("capabilities"), obj.Capabilities, oldVal, oldObj != nil)...) + } + + return errs +} + +var unionMembershipFor_github_com_agent_substrate_substrate_pkg_proto_ateapipb_Selector_ = validate.NewUnionMembership(validate.NewUnionMember("match_labels")) + // Validate_Selector validates an instance of Selector according // to declarative validation rules in the API schema. -func Validate_Selector( +func Validate_Selector( + ctx context.Context, op operation.Operation, fldPath *field.Path, + obj, oldObj *ateapipb.Selector) (errs field.ErrorList) { + + if e := validate.Union(ctx, op, fldPath, obj, oldObj, unionMembershipFor_github_com_agent_substrate_substrate_pkg_proto_ateapipb_Selector_, + func(obj *ateapipb.Selector) bool { + if obj == nil { + return false + } + return len(obj.MatchLabels) != 0 + }); len(e) != 0 { + errs = append(errs, e...) + } + + { // field ateapipb.Selector.MatchLabels + fn := func( + fldPath *field.Path, + obj, oldObj map[string]string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.MaxProperties(ctx, op, fldPath, obj, oldObj, 10).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.OptionalMap(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.EachMapKey(ctx, op, fldPath, obj, oldObj, validate.LabelKey); len(e) != 0 { + errs = append(errs, e...) + } + if e := validate.MinProperties(ctx, op, fldPath, obj, oldObj, 1); len(e) != 0 { + errs = append(errs, e...) + } + if e := validate.EachMapVal(ctx, op, fldPath, obj, oldObj, validate.DirectEqual, validate.LabelValue); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.Selector) map[string]string { + return oldObj.MatchLabels + }) + errs = append(errs, fn(fldPath.Child("match_labels"), obj.MatchLabels, oldVal, oldObj != nil)...) + } + + return errs +} + +// Validate_Snapshot validates an instance of Snapshot according +// to declarative validation rules in the API schema. +func Validate_Snapshot( + ctx context.Context, op operation.Operation, fldPath *field.Path, + obj, oldObj *ateapipb.Snapshot) (errs field.ErrorList) { + + { // field ateapipb.Snapshot.Generation + fn := func( + fldPath *field.Path, + obj, oldObj *int32, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.Minimum(ctx, op, fldPath, obj, oldObj, 1); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.Snapshot) *int32 { + return &oldObj.Generation + }) + errs = append(errs, fn(fldPath.Child("generation"), &obj.Generation, oldVal, oldObj != nil)...) + } + + { // field ateapipb.Snapshot.Owner + fn := func( + fldPath *field.Path, + obj, oldObj *ateapipb.SnapshotOwner, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.Maximum(ctx, op, fldPath, obj, oldObj, 2); len(e) != 0 { + errs = append(errs, e...) + } + if e := validate.Minimum(ctx, op, fldPath, obj, oldObj, 1); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.Snapshot) *ateapipb.SnapshotOwner { + return &oldObj.Owner + }) + errs = append(errs, fn(fldPath.Child("owner"), &obj.Owner, oldVal, oldObj != nil)...) + } + + { // field ateapipb.Snapshot.ActorTemplateUid + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.OptionalValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.UUID(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.Snapshot) *string { + return &oldObj.ActorTemplateUid + }) + errs = append(errs, fn(fldPath.Child("actor_template_uid"), &obj.ActorTemplateUid, oldVal, oldObj != nil)...) + } + + { // field ateapipb.Snapshot.Storage + fn := func( + fldPath *field.Path, + obj, oldObj []*ateapipb.SnapshotStorage, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.PtrSliceNoNils[ateapipb.SnapshotStorage](ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.MaxItems(ctx, op, fldPath, obj, oldObj, 2).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.RequiredSlice(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // lists with map semantics require unique keys + if e := validate.PtrSliceUnique(ctx, op, fldPath, obj, oldObj, + func(a *ateapipb.SnapshotStorage, b *ateapipb.SnapshotStorage) bool { + return a.Durability == b.Durability + }); len(e) != 0 { + errs = append(errs, e...) + } + // iterate the list and call the type's validation function + if e := validate.EachPtrSliceVal(ctx, op, fldPath, obj, oldObj, + func(a *ateapipb.SnapshotStorage, b *ateapipb.SnapshotStorage) bool { + return a.Durability == b.Durability + }, ateDeepEqual, Validate_SnapshotStorage); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.Snapshot) []*ateapipb.SnapshotStorage { + return oldObj.Storage + }) + errs = append(errs, fn(fldPath.Child("storage"), obj.Storage, oldVal, oldObj != nil)...) + } + + return errs +} + +// Validate_SnapshotConfig validates an instance of SnapshotConfig according +// to declarative validation rules in the API schema. +func Validate_SnapshotConfig( + ctx context.Context, op operation.Operation, fldPath *field.Path, + obj, oldObj *ateapipb.SnapshotConfig) (errs field.ErrorList) { + + { // field ateapipb.SnapshotConfig.StorageLocation + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // custom validation + if e := ValidateCustom_SnapshotConfig_StorageLocation(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + errs = append(errs, e...) + } + if e := validate.MaxLength(ctx, op, fldPath, obj, oldObj, 1024); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.SnapshotConfig) *string { + return &oldObj.StorageLocation + }) + errs = append(errs, fn(fldPath.Child("storage_location"), &obj.StorageLocation, oldVal, oldObj != nil)...) + } + + { // field ateapipb.SnapshotConfig.PreferredFidelity + fn := func( + fldPath *field.Path, + obj, oldObj *ateapipb.SnapshotFidelity, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // custom validation + if e := ValidateCustom_SnapshotConfig_PreferredFidelity(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + errs = append(errs, e...) + } + if e := validate.Maximum(ctx, op, fldPath, obj, oldObj, 3); len(e) != 0 { + errs = append(errs, e...) + } + if e := validate.Minimum(ctx, op, fldPath, obj, oldObj, 1); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.SnapshotConfig) *ateapipb.SnapshotFidelity { + return &oldObj.PreferredFidelity + }) + errs = append(errs, fn(fldPath.Child("preferred_fidelity"), &obj.PreferredFidelity, oldVal, oldObj != nil)...) + } + + return errs +} + +var unionMembershipFor_github_com_agent_substrate_substrate_pkg_proto_ateapipb_SnapshotStorage_ = validate.NewUnionMembership(validate.NewUnionMember("local"), validate.NewUnionMember("object")) + +// Validate_SnapshotStorage validates an instance of SnapshotStorage according +// to declarative validation rules in the API schema. +func Validate_SnapshotStorage( ctx context.Context, op operation.Operation, fldPath *field.Path, - obj, oldObj *ateapipb.Selector) (errs field.ErrorList) { + obj, oldObj *ateapipb.SnapshotStorage) (errs field.ErrorList) { - if e := validate.Union(ctx, op, fldPath, obj, oldObj, unionMembershipFor_github_com_agent_substrate_substrate_pkg_proto_ateapipb_Selector_, - func(obj *ateapipb.Selector) bool { + if e := validate.Union(ctx, op, fldPath, obj, oldObj, unionMembershipFor_github_com_agent_substrate_substrate_pkg_proto_ateapipb_SnapshotStorage_, + func(obj *ateapipb.SnapshotStorage) bool { if obj == nil { return false } - return len(obj.MatchLabels) != 0 + return obj.Local != nil + }, + func(obj *ateapipb.SnapshotStorage) bool { + if obj == nil { + return false + } + return obj.Object != nil }); len(e) != 0 { errs = append(errs, e...) } - { // field ateapipb.Selector.MatchLabels + { // field ateapipb.SnapshotStorage.Durability fn := func( fldPath *field.Path, - obj, oldObj map[string]string, + obj, oldObj *ateapipb.SnapshotDurability, oldValueCorrelated bool) (errs field.ErrorList) { // don't revalidate unchanged data if oldValueCorrelated && op.Type == operation.Update { - if ateDeepEqual(obj, oldObj) { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { return nil } } // call field-attached validations earlyReturn := false - if e := validate.MaxProperties(ctx, op, fldPath, obj, oldObj, 10).MarkShortCircuit(); len(e) != 0 { + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { errs = append(errs, e...) earlyReturn = true } - if e := validate.OptionalMap(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { - earlyReturn = true - } if earlyReturn { return // do not proceed } - if e := validate.EachMapKey(ctx, op, fldPath, obj, oldObj, validate.LabelKey); len(e) != 0 { - errs = append(errs, e...) - } - if e := validate.MinProperties(ctx, op, fldPath, obj, oldObj, 1); len(e) != 0 { + if e := validate.Maximum(ctx, op, fldPath, obj, oldObj, 2); len(e) != 0 { errs = append(errs, e...) } - if e := validate.EachMapVal(ctx, op, fldPath, obj, oldObj, validate.DirectEqual, validate.LabelValue); len(e) != 0 { + if e := validate.Minimum(ctx, op, fldPath, obj, oldObj, 1); len(e) != 0 { errs = append(errs, e...) } return } oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.Selector) map[string]string { - return oldObj.MatchLabels + func(oldObj *ateapipb.SnapshotStorage) *ateapipb.SnapshotDurability { + return &oldObj.Durability }) - errs = append(errs, fn(fldPath.Child("match_labels"), obj.MatchLabels, oldVal, oldObj != nil)...) + errs = append(errs, fn(fldPath.Child("durability"), &obj.Durability, oldVal, oldObj != nil)...) } - return errs -} - -// Validate_SnapshotConfig validates an instance of SnapshotConfig according -// to declarative validation rules in the API schema. -func Validate_SnapshotConfig( - ctx context.Context, op operation.Operation, fldPath *field.Path, - obj, oldObj *ateapipb.SnapshotConfig) (errs field.ErrorList) { - - { // field ateapipb.SnapshotConfig.OnCommit + { // field ateapipb.SnapshotStorage.Status fn := func( fldPath *field.Path, - obj, oldObj *ateapipb.SnapshotContentScope, + obj, oldObj *ateapipb.SnapshotStorageStatus, oldValueCorrelated bool) (errs field.ErrorList) { // don't revalidate unchanged data if oldValueCorrelated && op.Type == operation.Update { @@ -7093,16 +7385,16 @@ func Validate_SnapshotConfig( return } oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.SnapshotConfig) *ateapipb.SnapshotContentScope { - return &oldObj.OnCommit + func(oldObj *ateapipb.SnapshotStorage) *ateapipb.SnapshotStorageStatus { + return &oldObj.Status }) - errs = append(errs, fn(fldPath.Child("on_commit"), &obj.OnCommit, oldVal, oldObj != nil)...) + errs = append(errs, fn(fldPath.Child("status"), &obj.Status, oldVal, oldObj != nil)...) } - { // field ateapipb.SnapshotConfig.StorageLocation + { // field ateapipb.SnapshotStorage.Fidelity fn := func( fldPath *field.Path, - obj, oldObj *string, + obj, oldObj *ateapipb.SnapshotFidelity, oldValueCorrelated bool) (errs field.ErrorList) { // don't revalidate unchanged data if oldValueCorrelated && op.Type == operation.Update { @@ -7119,20 +7411,79 @@ func Validate_SnapshotConfig( if earlyReturn { return // do not proceed } - // custom validation - if e := ValidateCustom_SnapshotConfig_StorageLocation(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + if e := validate.Maximum(ctx, op, fldPath, obj, oldObj, 3); len(e) != 0 { errs = append(errs, e...) } - if e := validate.MaxLength(ctx, op, fldPath, obj, oldObj, 1024); len(e) != 0 { + if e := validate.Minimum(ctx, op, fldPath, obj, oldObj, 1); len(e) != 0 { errs = append(errs, e...) } return } oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.SnapshotConfig) *string { - return &oldObj.StorageLocation + func(oldObj *ateapipb.SnapshotStorage) *ateapipb.SnapshotFidelity { + return &oldObj.Fidelity }) - errs = append(errs, fn(fldPath.Child("storage_location"), &obj.StorageLocation, oldVal, oldObj != nil)...) + errs = append(errs, fn(fldPath.Child("fidelity"), &obj.Fidelity, oldVal, oldObj != nil)...) + } + + { // field ateapipb.SnapshotStorage.Local + fn := func( + fldPath *field.Path, + obj, oldObj *ateapipb.LocalSnapshot, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.OptionalPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // call the type's validation function + errs = append(errs, Validate_LocalSnapshot(ctx, op, fldPath, obj, oldObj)...) + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.SnapshotStorage) *ateapipb.LocalSnapshot { + return oldObj.Local + }) + errs = append(errs, fn(fldPath.Child("local"), obj.Local, oldVal, oldObj != nil)...) + } + + { // field ateapipb.SnapshotStorage.Object + fn := func( + fldPath *field.Path, + obj, oldObj *ateapipb.ObjectSnapshot, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.OptionalPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // call the type's validation function + errs = append(errs, Validate_ObjectSnapshot(ctx, op, fldPath, obj, oldObj)...) + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.SnapshotStorage) *ateapipb.ObjectSnapshot { + return oldObj.Object + }) + errs = append(errs, fn(fldPath.Child("object"), obj.Object, oldVal, oldObj != nil)...) } return errs @@ -7597,7 +7948,7 @@ func Validate_TagStatus( { // field ateapipb.TagStatus.Snapshot fn := func( fldPath *field.Path, - obj, oldObj *ateapipb.ExternalSnapshot, + obj, oldObj *ateapipb.Snapshot, oldValueCorrelated bool) (errs field.ErrorList) { // don't revalidate unchanged data if oldValueCorrelated && op.Type == operation.Update { @@ -7607,18 +7958,19 @@ func Validate_TagStatus( } // call field-attached validations earlyReturn := false - if e := validate.OptionalPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + if e := validate.RequiredPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) earlyReturn = true } if earlyReturn { return // do not proceed } // call the type's validation function - errs = append(errs, Validate_ExternalSnapshot(ctx, op, fldPath, obj, oldObj)...) + errs = append(errs, Validate_Snapshot(ctx, op, fldPath, obj, oldObj)...) return } oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.TagStatus) *ateapipb.ExternalSnapshot { + func(oldObj *ateapipb.TagStatus) *ateapipb.Snapshot { return oldObj.Snapshot }) errs = append(errs, fn(fldPath.Child("snapshot"), obj.Snapshot, oldVal, oldObj != nil)...) @@ -7656,38 +8008,6 @@ func Validate_TagStatus( errs = append(errs, fn(fldPath.Child("actor_template_uid"), &obj.ActorTemplateUid, oldVal, oldObj != nil)...) } - { // field ateapipb.TagStatus.StorageLocation - fn := func( - fldPath *field.Path, - obj, oldObj *string, - oldValueCorrelated bool) (errs field.ErrorList) { - // don't revalidate unchanged data - if oldValueCorrelated && op.Type == operation.Update { - if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { - return nil - } - } - // call field-attached validations - earlyReturn := false - if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { - errs = append(errs, e...) - earlyReturn = true - } - if earlyReturn { - return // do not proceed - } - if e := validate.MaxLength(ctx, op, fldPath, obj, oldObj, 1024); len(e) != 0 { - errs = append(errs, e...) - } - return - } - oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.TagStatus) *string { - return &oldObj.StorageLocation - }) - errs = append(errs, fn(fldPath.Child("storage_location"), &obj.StorageLocation, oldVal, oldObj != nil)...) - } - return errs } @@ -8171,6 +8491,94 @@ func Validate_UpdateWorkerRequest( return errs } +// Validate_VersionedSandboxCompat validates an instance of VersionedSandboxCompat according +// to declarative validation rules in the API schema. +func Validate_VersionedSandboxCompat( + ctx context.Context, op operation.Operation, fldPath *field.Path, + obj, oldObj *ateapipb.VersionedSandboxCompat) (errs field.ErrorList) { + + { // field ateapipb.VersionedSandboxCompat.SchemaVersion + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.MaxLength(ctx, op, fldPath, obj, oldObj, 64); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.VersionedSandboxCompat) *string { + return &oldObj.SchemaVersion + }) + errs = append(errs, fn(fldPath.Child("schema_version"), &obj.SchemaVersion, oldVal, oldObj != nil)...) + } + + { // field ateapipb.VersionedSandboxCompat.Attributes + fn := func( + fldPath *field.Path, + obj, oldObj []*ateapipb.AttributeEntry, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.PtrSliceNoNils[ateapipb.AttributeEntry](ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.MaxItems(ctx, op, fldPath, obj, oldObj, 32).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.RequiredSlice(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // lists with map semantics require unique keys + if e := validate.PtrSliceUnique(ctx, op, fldPath, obj, oldObj, + func(a *ateapipb.AttributeEntry, b *ateapipb.AttributeEntry) bool { return a.Key == b.Key }); len(e) != 0 { + errs = append(errs, e...) + } + // iterate the list and call the type's validation function + if e := validate.EachPtrSliceVal(ctx, op, fldPath, obj, oldObj, + func(a *ateapipb.AttributeEntry, b *ateapipb.AttributeEntry) bool { return a.Key == b.Key }, ateDeepEqual, Validate_AttributeEntry); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.VersionedSandboxCompat) []*ateapipb.AttributeEntry { + return oldObj.Attributes + }) + errs = append(errs, fn(fldPath.Child("attributes"), obj.Attributes, oldVal, oldObj != nil)...) + } + + return errs +} + var unionMembershipFor_github_com_agent_substrate_substrate_pkg_proto_ateapipb_Volume_ = validate.NewUnionMembership(validate.NewUnionMember("durable_dir"), validate.NewUnionMember("external_volume_template"), validate.NewUnionMember("system_info"), validate.NewUnionMember("image")) // Validate_Volume validates an instance of Volume according @@ -9353,10 +9761,10 @@ func Validate_WorkerStatus( errs = append(errs, fn(fldPath.Child("observed_epoch"), &obj.ObservedEpoch, oldVal, oldObj != nil)...) } - { // field ateapipb.WorkerStatus.Hardware + { // field ateapipb.WorkerStatus.DefaultRuntime fn := func( fldPath *field.Path, - obj, oldObj *ateapipb.HardwareIdentity, + obj, oldObj *ateapipb.SandboxRuntime, oldValueCorrelated bool) (errs field.ErrorList) { // don't revalidate unchanged data if oldValueCorrelated && op.Type == operation.Update { @@ -9373,14 +9781,54 @@ func Validate_WorkerStatus( return // do not proceed } // call the type's validation function - errs = append(errs, Validate_HardwareIdentity(ctx, op, fldPath, obj, oldObj)...) + errs = append(errs, Validate_SandboxRuntime(ctx, op, fldPath, obj, oldObj)...) + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.WorkerStatus) *ateapipb.SandboxRuntime { + return oldObj.DefaultRuntime + }) + errs = append(errs, fn(fldPath.Child("default_runtime"), obj.DefaultRuntime, oldVal, oldObj != nil)...) + } + + { // field ateapipb.WorkerStatus.RestorableRuntimes + fn := func( + fldPath *field.Path, + obj, oldObj []*ateapipb.SandboxRuntime, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.PtrSliceNoNils[ateapipb.SandboxRuntime](ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.MaxItems(ctx, op, fldPath, obj, oldObj, 32).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.OptionalSlice(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // iterate the list and call the type's validation function + if e := validate.EachPtrSliceVal(ctx, op, fldPath, obj, oldObj, nil, nil, Validate_SandboxRuntime); len(e) != 0 { + errs = append(errs, e...) + } return } oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.WorkerStatus) *ateapipb.HardwareIdentity { - return oldObj.Hardware + func(oldObj *ateapipb.WorkerStatus) []*ateapipb.SandboxRuntime { + return oldObj.RestorableRuntimes }) - errs = append(errs, fn(fldPath.Child("hardware"), obj.Hardware, oldVal, oldObj != nil)...) + errs = append(errs, fn(fldPath.Child("restorable_runtimes"), obj.RestorableRuntimes, oldVal, oldObj != nil)...) } return errs diff --git a/cmd/ateapi/internal/controlapi/actor.go b/cmd/ateapi/internal/controlapi/actor.go index 77bba00444..c6ca082f38 100644 --- a/cmd/ateapi/internal/controlapi/actor.go +++ b/cmd/ateapi/internal/controlapi/actor.go @@ -106,7 +106,7 @@ func (s *ServiceImpl) CreateActor(ctx context.Context, inActor *ateapipb.Actor) return nil, err } if inActor.GetSourceTag() == nil { - if err := validateGoldenSnapshotScope(sourceTag.GetStatus().GetSnapshot()); err != nil { + if err := validateGoldenSnapshotFidelity(sourceTag.GetStatus().GetSnapshot()); err != nil { return nil, err } } @@ -124,24 +124,23 @@ func (s *ServiceImpl) CreateActor(ctx context.Context, inActor *ateapipb.Actor) // Verify that the result is properly valid before storing it. outActor := proto.CloneOf(inActor) outActor.Status = &ateapipb.ActorStatus{ - State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, - ActorVolumes: initVols, + State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, + ExternalVolumes: initVols, } if sourceTag != nil { - // The Actor starts out borrowing the tag's external snapshot rather than + // The Actor starts out borrowing the tag's snapshot rather than // copying it. The snapshot URI is under the tag's prefix, not the Actor's, which // is what keeps the Actor from collecting those objects. Its first // suspend writes a snapshot under its own prefix and takes over from // there. - outActor.Status.ExternalSnapshot = proto.CloneOf(sourceTag.GetStatus().GetSnapshot()) + snap := proto.CloneOf(sourceTag.GetStatus().GetSnapshot()) + snap.Generation = 1 + snap.Owner = ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG // The Actor is born with guest state, so stamp the template that state - // was built on now rather than at the first resume. The Tag records it - // beside its snapshot rather than on it, so the clone above does not - // carry it. Left empty, a repoint before that first resume reads as "no - // guest state" instead of "replaced template", and the resume restores - // the old template's memory and rootfs in full instead of the volume - // data alone. - outActor.Status.ExternalSnapshot.ActorTemplateUid = sourceTag.GetStatus().GetActorTemplateUid() + // was built on now rather than at the first resume. + snap.ActorTemplateUid = sourceTag.GetStatus().GetActorTemplateUid() + outActor.Status.Snapshots = []*ateapipb.Snapshot{snap} + outActor.Status.LastAssignedGeneration = 1 } if errs := apivalidation.ValidateActorUpdate(ctx, field.NewPath("actor"), outActor, inActor, true); len(errs) > 0 { return nil, toGRPCInternalError(errs) @@ -187,10 +186,10 @@ func (s *ServiceImpl) resolveTagSource(ctx context.Context, actorAtespace string return nil, apierror.FailedPrecondition("source Tag has an invalid scope") } // A tag might have an empty Snapshot URI if the tag creation failed or is ongoing. - if tag.GetStatus().GetSnapshot().GetSnapshotUri() == "" { + if tagDurableSnapshotURI(tag, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) == "" { return nil, apierror.FailedPrecondition("source Tag is still being created or failed creation") } - // TODO: Permit compatible DATA snapshots when runtimes can extract portable data. + // TODO: Permit compatible VOLUMES snapshots when runtimes can extract portable data. if tag.GetStatus().GetActorTemplateUid() != template.GetMetadata().GetUid() { return nil, apierror.FailedPrecondition("source Tag must be taken from an actor with ActorTemplate uid %q", tag.GetStatus().GetActorTemplateUid()) } @@ -395,7 +394,8 @@ func validateTemplateVolumesUnchanged(oldTemplate, newTemplate *ateapipb.ActorTe // Deleting an actor collects everything under its external snapshot prefix. If // the location prefix ever changes, we risk leaking the snapshots under the old prefix. func validateSnapshotLocationUnchanged(actor *ateapipb.Actor, newTemplate *ateapipb.ActorTemplate) error { - currentSnapshotURI := actor.GetStatus().GetExternalSnapshot().GetSnapshotUri() + _, st := findLatestSnapshotStorage(actor.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + currentSnapshotURI := st.GetObject().GetSnapshotUri() if currentSnapshotURI == "" { return nil } @@ -562,7 +562,7 @@ func (s *RPCService) MintActorJWT(ctx context.Context, req *ateapipb.MintActorJW } // We only issue tokens with audience bindings. - if len(req.GetAudience()) == 0 { + if len(req.GetAudiences()) == 0 { return nil, fmt.Errorf("at least one audience must be requested") } @@ -573,7 +573,7 @@ func (s *RPCService) MintActorJWT(ctx context.Context, req *ateapipb.MintActorJW actorClaims := &actoridjwt.Claims{ Issuer: s.actorJWTIssuer, Subject: fmt.Sprintf("actor/%s/%s", dbActor.GetMetadata().GetAtespace(), dbActor.GetMetadata().GetName()), - Audiences: req.GetAudience(), + Audiences: req.GetAudiences(), Expiration: expiresAt, NotBefore: now.Add(-5 * time.Minute), IssuedAt: now, diff --git a/cmd/ateapi/internal/controlapi/actor_template_test.go b/cmd/ateapi/internal/controlapi/actor_template_test.go index 2889e13d67..07937d0abe 100644 --- a/cmd/ateapi/internal/controlapi/actor_template_test.go +++ b/cmd/ateapi/internal/controlapi/actor_template_test.go @@ -24,6 +24,7 @@ import ( "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest" "github.com/agent-substrate/substrate/internal/apierror" + "github.com/agent-substrate/substrate/internal/objectstoreplugin/objectstoreplugintest" "github.com/agent-substrate/substrate/internal/resources" atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" listersv1alpha1 "github.com/agent-substrate/substrate/pkg/client/listers/api/v1alpha1" @@ -43,8 +44,8 @@ func validActorTemplate(mutations ...func(*ateapipb.ActorTemplate)) *ateapipb.Ac Metadata: &ateapipb.ResourceMetadata{Atespace: "ns1", Name: "tmpl-a"}, Containers: []*ateapipb.Container{{Name: "main", Image: "example.com/app:v1@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}}, SnapshotConfig: &ateapipb.SnapshotConfig{ - StorageLocation: "gs://my-bucket/snapshots", - OnCommit: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, + StorageLocation: "gs://my-bucket/snapshots", + PreferredFidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, }, SandboxConfig: &ateapipb.SandboxConfig{SandboxClass: ateapipb.SandboxClass_SANDBOX_CLASS_GVISOR, ConfigName: "gvisor-default"}, } @@ -61,9 +62,13 @@ func gvisorDefaultLister(t *testing.T) listersv1alpha1.SandboxConfigLister { return sandboxConfigListerFor(t, []*atev1alpha1.SandboxConfig{{ ObjectMeta: metav1.ObjectMeta{Name: "gvisor-default"}, Spec: atev1alpha1.SandboxConfigSpec{ - SandboxClass: atev1alpha1.SandboxClassGvisor, - PauseImage: "registry.k8s.io/pause@sha256:x", - Assets: testAssets(), + SandboxClass: atev1alpha1.SandboxClassGvisor, + DefaultVersion: "v1", + Versions: []atev1alpha1.SandboxVersionConfig{{ + Name: "v1", + PauseImage: "registry.k8s.io/pause@sha256:x", + Assets: testAssets(), + }}, }, }}) } @@ -225,10 +230,9 @@ func TestDeleteActorTemplate(t *testing.T) { actorURI := mustActorSnapshotURI(t, tmpl, actor, "snapshot") objects.PutSnapshot(t, actorURI, "manifest.json") actor = mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapipb.ActorStatus) { - s.ExternalSnapshot = &ateapipb.ExternalSnapshot{ - SnapshotUri: actorURI.String(), - ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, - ActorTemplateUid: tmpl.GetMetadata().GetUid(), + s.LastAssignedGeneration = 1 + s.Snapshots = []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, tmpl.GetMetadata().GetUid(), actorURI.String(), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), } }) var tag *ateapipb.Tag @@ -244,7 +248,7 @@ func TestDeleteActorTemplate(t *testing.T) { tagRef := resources.TagRefFromTag(tag) tagURI := mustReservedTagSnapshotURI(t, tag) objects.PutSnapshot(t, tagURI, "manifest.json") - svc := &RPCService{impl: newServiceImpl(persistence, nil), actorWorkflow: workflow, objectStore: objects} + svc := &RPCService{impl: newServiceImpl(persistence, nil), actorWorkflow: workflow, snapshotPlugin: objectstoreplugintest.ControlClient(objects)} // The handler must request AnyState to clean up an active golden actor. mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapipb.ActorStatus) { s.State = ateapipb.ActorState_ACTOR_STATE_RUNNING @@ -286,7 +290,7 @@ func TestDeleteActorTemplate(t *testing.T) { req := &ateapipb.DeleteActorTemplateRequest{ActorTemplate: templateRef.ToObjectRef()} deleted, err := svc.DeleteActorTemplate(ctx, req) if tt.failPrefix != "" { - if !errors.Is(err, errObjectStore) { + if !isObjectStoreErr(err) { t.Fatalf("DeleteActorTemplate = %v, want object storage error", err) } if _, err := persistence.GetActorTemplate(ctx, templateRef); err != nil { diff --git a/cmd/ateapi/internal/controlapi/actor_test.go b/cmd/ateapi/internal/controlapi/actor_test.go index 12e91b3ea5..de1e64f5f1 100644 --- a/cmd/ateapi/internal/controlapi/actor_test.go +++ b/cmd/ateapi/internal/controlapi/actor_test.go @@ -509,7 +509,10 @@ func TestUpdateActor_RepointTemplateStorageLocation(t *testing.T) { if tt.snapshot != nil { uri := tt.snapshot(t, actor) actor = mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapipb.ActorStatus) { - s.ExternalSnapshot = &ateapipb.ExternalSnapshot{SnapshotUri: uri} + s.LastAssignedGeneration = 1 + s.Snapshots = []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", uri, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + } }) } @@ -814,7 +817,7 @@ func TestCreateActor_GoldenTagDefault(t *testing.T) { Scope: ateapipb.TagScope_TAG_SCOPE_PUBLISHED, Status: &ateapipb.TagStatus{ ActorTemplateUid: tmpl.GetMetadata().GetUid(), - Snapshot: &ateapipb.ExternalSnapshot{SnapshotUri: "gs://bucket/atespaces/ate-golden/tags/" + someActorUID, ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL}, + Snapshot: newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, tmpl.GetMetadata().GetUid(), "gs://bucket/atespaces/ate-golden/tags/"+someActorUID, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), }, } wantCode := codes.OK @@ -828,7 +831,7 @@ func TestCreateActor_GoldenTagDefault(t *testing.T) { tag.Status.ActorTemplateUid = "other" wantCode = codes.FailedPrecondition case "data scope": - tag.Status.Snapshot.ContentScope = ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA + tag.Status.Snapshot.Storage[0].Fidelity = ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES wantCode = codes.FailedPrecondition } if scenario != "missing" { @@ -845,7 +848,7 @@ func TestCreateActor_GoldenTagDefault(t *testing.T) { actor := &ateapipb.Actor{Metadata: &ateapipb.ResourceMetadata{Atespace: "team-a", Name: "actor"}, ActorTemplate: resources.ActorTemplateRefFromActorTemplate(tmpl).ToObjectRef()} if scenario == "explicit tag" { tag.Metadata.Name = "explicit" - tag.Status.Snapshot.SnapshotUri = "gs://bucket/atespaces/ate-golden/tags/explicit" + tag.Status.Snapshot = newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, tmpl.GetMetadata().GetUid(), "gs://bucket/atespaces/ate-golden/tags/explicit", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) if _, err := persistence.CreateTag(ctx, tag); err != nil { t.Fatal(err) } @@ -859,8 +862,9 @@ func TestCreateActor_GoldenTagDefault(t *testing.T) { if err != nil { return } - if got := created.GetStatus(); got.GetExternalSnapshot().GetSnapshotUri() != tag.GetStatus().GetSnapshot().GetSnapshotUri() || got.GetExternalSnapshot().GetActorTemplateUid() != tmpl.GetMetadata().GetUid() { - t.Fatalf("incorrect initial status: %v", got) + gotSnap, gotSt := findLatestSnapshotStorage(created.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + if gotSt.GetObject().GetSnapshotUri() != tagDurableSnapshotURI(tag, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) || gotSnap.GetActorTemplateUid() != tmpl.GetMetadata().GetUid() { + t.Fatalf("incorrect initial status: %v", created.GetStatus()) } if scenario == "own snapshot" { uri, err := resources.NewActorSnapshotURI(tmpl.GetSnapshotConfig().GetStorageLocation(), "team-a", created.GetMetadata().GetUid(), "snapshot") @@ -868,7 +872,10 @@ func TestCreateActor_GoldenTagDefault(t *testing.T) { t.Fatal(err) } if _, err := persistence.UpdateActor(ctx, resources.ActorRefFromActor(created), store.PreconditionFrom(created), func(db *ateapipb.Actor) error { - db.Status.ExternalSnapshot.SnapshotUri = uri.String() + db.Status.LastAssignedGeneration = 1 + db.Status.Snapshots = []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, tmpl.GetMetadata().GetUid(), uri.String(), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + } return nil }); err != nil { t.Fatal(err) diff --git a/cmd/ateapi/internal/controlapi/converter.go b/cmd/ateapi/internal/controlapi/converter.go index 5d1e80080b..f34b9aaeaf 100644 --- a/cmd/ateapi/internal/controlapi/converter.go +++ b/cmd/ateapi/internal/controlapi/converter.go @@ -20,11 +20,20 @@ import ( "github.com/agent-substrate/substrate/pkg/proto/ateapipb" ) -func actorSnapshotContentScopeToAtelet(in ateapipb.SnapshotContentScope) ateletpb.SnapshotScope { - if in == ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA { - return ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA +// fidelityToAtelet maps the public fidelity ladder onto the identical atelet +// wire enum, level by level. Anything else, including UNSPECIFIED, goes out +// as UNSPECIFIED, which atelet rejects, rather than being silently promoted. +func fidelityToAtelet(in ateapipb.SnapshotFidelity) ateletpb.SnapshotFidelity { + switch in { + case ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES: + return ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES + case ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS: + return ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS + case ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY: + return ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY + default: + return ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED } - return ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL } // sandboxClassString renders the proto enum in the CRD's lower-case string diff --git a/cmd/ateapi/internal/controlapi/converter_test.go b/cmd/ateapi/internal/controlapi/converter_test.go index 9ebb8dc368..4a0824230b 100644 --- a/cmd/ateapi/internal/controlapi/converter_test.go +++ b/cmd/ateapi/internal/controlapi/converter_test.go @@ -21,36 +21,46 @@ import ( "github.com/agent-substrate/substrate/pkg/proto/ateapipb" ) -// TestSnapshotScopeToAtelet covers the wire scope derivation for template -// content scopes: an unset scope falls back to Full. -func TestSnapshotScopeToAtelet(t *testing.T) { +// TestFidelityToAtelet pins the level-by-level mapping onto the atelet enum: +// nothing is promoted, and an unset fidelity stays unset so atelet rejects it. +func TestFidelityToAtelet(t *testing.T) { tests := []struct { name string - in ateapipb.SnapshotContentScope - expected ateletpb.SnapshotScope + in ateapipb.SnapshotFidelity + expected ateletpb.SnapshotFidelity }{ { - name: "Full scope", - in: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, - expected: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL, + name: "memory", + in: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + expected: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, }, { - name: "Data scope", - in: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA, - expected: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA, + name: "rootfs", + in: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS, + expected: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS, }, { - name: "Default scope (unspecified)", - in: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_UNSPECIFIED, - expected: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL, + name: "volumes", + in: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, + expected: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, + }, + { + name: "unspecified stays unspecified", + in: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED, + expected: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED, + }, + { + name: "value outside the enum", + in: ateapipb.SnapshotFidelity(99), + expected: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - result := actorSnapshotContentScopeToAtelet(tt.in) + result := fidelityToAtelet(tt.in) if result != tt.expected { - t.Errorf("actorSnapshotContentScopeToAtelet(%v) = %v, want %v", tt.in, result, tt.expected) + t.Errorf("fidelityToAtelet(%v) = %v, want %v", tt.in, result, tt.expected) } }) } diff --git a/cmd/ateapi/internal/controlapi/crash.go b/cmd/ateapi/internal/controlapi/crash.go index 0700de5710..d25745e6d1 100644 --- a/cmd/ateapi/internal/controlapi/crash.go +++ b/cmd/ateapi/internal/controlapi/crash.go @@ -93,6 +93,11 @@ func handleAteletError(ctx context.Context, st crashActorStore, actorRef resourc return fmt.Errorf("actor %s crashed: %w", actorRef, err) } + // The caller went away or its deadline passed: report that, not atelet's + // error. The actor keeps its assignment, so a retry can finish the operation. + if ctx.Err() != nil { + return fmt.Errorf("while calling atelet %s: %w: %w", rpc, err, ctx.Err()) + } if status.Code(err) == codes.Unavailable { return apierror.Unavailable("while calling atelet %s: %w", rpc, err) } @@ -135,11 +140,10 @@ func crashActor(ctx context.Context, st crashActorStore, actorRef resources.Acto toUpdate.Status.Crash = newActorCrash(opName, message) } - // InProgressSnapshotUri and InProgressLocalSnapshotName are kept so a - // later DeleteActor or RevertActor can delete what they name: each is - // the only pointer to it, so clearing them here would leak the objects - // for good; failed workflow steps must never promote either of them to an - // ExternalSnapshot or to LocalSnapshot. + // In-progress snapshot entries in Status.Snapshots are kept so a later + // DeleteActor or RevertActor can delete what they name: each is the only + // pointer to it, so clearing them here would leak the objects for good; + // failed workflow steps must never promote them to COMPLETED. toUpdate.Status.WorkerAssignment = nil return nil }) diff --git a/cmd/ateapi/internal/controlapi/crash_test.go b/cmd/ateapi/internal/controlapi/crash_test.go index 3c919f584b..7cf02bcece 100644 --- a/cmd/ateapi/internal/controlapi/crash_test.go +++ b/cmd/ateapi/internal/controlapi/crash_test.go @@ -63,7 +63,10 @@ func seedActor(t *testing.T, ctx context.Context, st store.Interface, actorRef r WorkerPodUid: "uid", WorkerPodIps: []string{"1.2.3.4"}, }, - InProgressSnapshotUri: "gs://bucket/atespaces/as/actors/uid/snapshots/reserved-snapshot", + LastAssignedGeneration: 1, + Snapshots: []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "gs://bucket/atespaces/as/actors/uid/snapshots/reserved-snapshot", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS), + }, }, }) } @@ -106,8 +109,11 @@ func seedUnboundActor(t *testing.T, ctx context.Context, st store.Interface, act storetest.MustCreateActor(t, ctx, st, &ateapipb.Actor{ Metadata: &ateapipb.ResourceMetadata{Name: actorRef.Name, Atespace: actorRef.Atespace}, Status: &ateapipb.ActorStatus{ - State: ateapipb.ActorState_ACTOR_STATE_RUNNING, - InProgressSnapshotUri: "gs://bucket/atespaces/as/actors/uid/snapshots/reserved-snapshot", + State: ateapipb.ActorState_ACTOR_STATE_RUNNING, + LastAssignedGeneration: 1, + Snapshots: []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "gs://bucket/atespaces/as/actors/uid/snapshots/reserved-snapshot", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS), + }, }, }) } @@ -124,8 +130,8 @@ func assertCrashed(t *testing.T, ctx context.Context, st store.Interface, actorR t.Errorf("status = %v, want %v", got.GetStatus().GetState(), ateapipb.ActorState_ACTOR_STATE_CRASHED) } // Keep the snapshot uri for debugging. - if got.GetStatus().GetInProgressSnapshotUri() == "" { - t.Error(`InProgressSnapshotUri = "", want preserved`) + if _, st := findLatestSnapshotStorage(got.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS); st == nil { + t.Error(`in-progress durable snapshot URI = "", want preserved`) } if got.GetStatus().GetWorkerAssignment() != nil { t.Errorf("WorkerAssignment = %v, want cleared", got.GetStatus().GetWorkerAssignment()) @@ -352,6 +358,8 @@ func TestAteletCrashMessage(t *testing.T) { func TestHandleAteletError(t *testing.T) { ended, cancel := context.WithCancel(context.Background()) cancel() + expired, cancelExpired := context.WithDeadline(context.Background(), time.Unix(0, 0)) + defer cancelExpired() tests := []struct { name string @@ -396,12 +404,28 @@ func TestHandleAteletError(t *testing.T) { wantCode: codes.Unavailable, wantState: ateapipb.ActorState_ACTOR_STATE_RUNNING, }, + { + name: "cancelled caller gets Canceled and leaves the actor as it was", + ctx: ended, + rpc: "Restore", + err: fmt.Errorf("while restoring actor: %w", status.Error(codes.Canceled, "connection closing")), + wantCode: codes.Canceled, + wantState: ateapipb.ActorState_ACTOR_STATE_RUNNING, + }, + { + name: "expired caller deadline gets DeadlineExceeded and leaves the actor as it was", + ctx: expired, + rpc: "Restore", + err: fmt.Errorf("while restoring actor: %w", status.Error(codes.DeadlineExceeded, "restore reply timed out")), + wantCode: codes.DeadlineExceeded, + wantState: ateapipb.ActorState_ACTOR_STATE_RUNNING, + }, { name: "ended workflow context leaves the actor as it was", ctx: ended, rpc: "Restore", err: status.Error(codes.Internal, "context canceled"), - wantCode: codes.Internal, + wantCode: codes.Canceled, wantState: ateapipb.ActorState_ACTOR_STATE_RUNNING, }, { @@ -452,6 +476,16 @@ func TestHandleAteletError(t *testing.T) { t.Errorf("apierror.Code(handleAteletError()) = %v, want %v (err: %v)", got, tt.wantCode, err) } + if !errors.Is(err, tt.err) { + t.Errorf("original atelet error was lost: %v", err) + } + if tt.wantCode == codes.Canceled && !errors.Is(err, context.Canceled) { + t.Errorf("error does not wrap context.Canceled: %v", err) + } + if tt.wantCode == codes.DeadlineExceeded && !errors.Is(err, context.DeadlineExceeded) { + t.Errorf("error does not wrap context.DeadlineExceeded: %v", err) + } + actor, err := st.GetActor(ctx, actorRef) if err != nil { t.Fatalf("GetActor: %v", err) diff --git a/cmd/ateapi/internal/controlapi/functionaltest/actor_template_test.go b/cmd/ateapi/internal/controlapi/functionaltest/actor_template_test.go index 4f0041532b..ee694af879 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/actor_template_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/actor_template_test.go @@ -62,8 +62,8 @@ func TestActorTemplateCRUD(t *testing.T) { Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "tmpl-a", Version: 1}, Containers: []*ateapipb.Container{{Name: "main", Image: "example.com/app:v1@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}}, SnapshotConfig: &ateapipb.SnapshotConfig{ - StorageLocation: "gs://my-bucket/snapshots", - OnCommit: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, + StorageLocation: "gs://my-bucket/snapshots", + PreferredFidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, }, SandboxConfig: &ateapipb.SandboxConfig{ SandboxClass: ateapipb.SandboxClass_SANDBOX_CLASS_GVISOR, @@ -174,7 +174,7 @@ func TestGoldenTagLifecycle(t *testing.T) { if err != nil { t.Fatal(err) } - uri := golden.GetStatus().GetSnapshot().GetSnapshotUri() + uri := tagSnapshotURI(golden) parsed, err := resources.ParseSnapshotURI(uri) if err != nil { t.Fatal(err) @@ -193,7 +193,7 @@ func TestGoldenTagLifecycle(t *testing.T) { if err != nil { t.Fatal(err) } - if late.GetStatus().GetExternalSnapshot().GetSnapshotUri() != uri || late.GetStatus().GetExternalSnapshot().GetActorTemplateUid() != tmpl.GetMetadata().GetUid() { + if durableSnapshotURI(late.GetStatus()) != uri || durableSnapshot(late.GetStatus()).GetActorTemplateUid() != tmpl.GetMetadata().GetUid() { t.Fatal("actor did not inherit golden tag snapshot and template UID") } waitForWorkerAvailable(t, tc, workerName) diff --git a/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go b/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go index 513f8a6a9c..4e56ab6816 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go @@ -78,8 +78,11 @@ func TestCreateActor_Success(t *testing.T) { Metadata: &ateapipb.ResourceMetadata{Name: "id1", Atespace: testAtespace, Version: 1}, ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"}, Status: &ateapipb.ActorStatus{ - State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, - ExternalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: goldenSnapshotURI(t), ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, ActorTemplateUid: tmpl.GetMetadata().GetUid()}, + State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, + LastAssignedGeneration: 1, + Snapshots: []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, tmpl.GetMetadata().GetUid(), goldenSnapshotURI(t)), + }, }, WorkerSelector: &ateapipb.Selector{MatchLabels: map[string]string{"tier": "free"}}, } @@ -138,12 +141,12 @@ func TestCreateActor_WithExternalVolumes(t *testing.T) { t.Fatalf("CreateActor failed: %v", err) } - if len(createResp.GetStatus().GetActorVolumes()) != 1 { - t.Fatalf("expected 1 volume in CreateActor response, got %d", len(createResp.GetStatus().GetActorVolumes())) + if len(createResp.GetStatus().GetExternalVolumes()) != 1 { + t.Fatalf("expected 1 volume in CreateActor response, got %d", len(createResp.GetStatus().GetExternalVolumes())) } - vol := createResp.GetStatus().GetActorVolumes()[0] - if vol.GetVolumeName() != "ext-vol-1" { - t.Errorf("volume name = %q, want %q", vol.GetVolumeName(), "ext-vol-1") + vol := createResp.GetStatus().GetExternalVolumes()[0] + if vol.GetName() != "ext-vol-1" { + t.Errorf("volume name = %q, want %q", vol.GetName(), "ext-vol-1") } if vol.GetStatus() != ateapipb.ExternalVolume_STATUS_PENDING { t.Errorf("volume status = %v, want %v", vol.GetStatus(), ateapipb.ExternalVolume_STATUS_PENDING) @@ -159,11 +162,11 @@ func TestCreateActor_WithExternalVolumes(t *testing.T) { if err != nil { t.Fatalf("GetActor failed: %v", err) } - if len(getResp.GetStatus().GetActorVolumes()) != 1 { - t.Fatalf("expected 1 volume in GetActor response, got %d", len(getResp.GetStatus().GetActorVolumes())) + if len(getResp.GetStatus().GetExternalVolumes()) != 1 { + t.Fatalf("expected 1 volume in GetActor response, got %d", len(getResp.GetStatus().GetExternalVolumes())) } - if getResp.GetStatus().GetActorVolumes()[0].GetStatus() != ateapipb.ExternalVolume_STATUS_PENDING { - t.Errorf("GetActor status = %v, want %v", getResp.GetStatus().GetActorVolumes()[0].GetStatus(), ateapipb.ExternalVolume_STATUS_PENDING) + if getResp.GetStatus().GetExternalVolumes()[0].GetStatus() != ateapipb.ExternalVolume_STATUS_PENDING { + t.Errorf("GetActor status = %v, want %v", getResp.GetStatus().GetExternalVolumes()[0].GetStatus(), ateapipb.ExternalVolume_STATUS_PENDING) } } @@ -294,7 +297,7 @@ func TestCreateActor_RejectsDifferentTemplateForDataSnapshot(t *testing.T) { createTemplateWithSelector(t, tc, "tmpl2", nil) seedTag(t, tc, "data-source", "data-snapshot", func(tag *ateapipb.Tag) { - tag.Status.Snapshot.ContentScope = ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA + tag.Status.Snapshot.Storage[0].Fidelity = ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES tag.Status.ActorTemplateUid = tmpl.GetMetadata().GetUid() }) @@ -364,11 +367,10 @@ func TestCreateActor_PendingTag(t *testing.T) { ctx := context.Background() tmpl := createTemplate(t, tc, ns) - // Simulates a tag creation that failed in while writing the snapshot to + // Simulates a tag creation that failed while writing the snapshot to // external storage. pending := seedTag(t, tc, "pending-source", "pending", func(tag *ateapipb.Tag) { - tag.Status.StorageLocation = testStorageLocation - tag.Status.Snapshot = nil + tag.Status.Snapshot.Storage[0].Status = ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS tag.Status.ActorTemplateUid = tmpl.GetMetadata().GetUid() }) tagRef := &ateapipb.ObjectRef{Atespace: testAtespace, Name: "pending"} @@ -382,17 +384,11 @@ func TestCreateActor_PendingTag(t *testing.T) { assertGrpcError(t, err, codes.FailedPrecondition, "source Tag is still being created or failed creation") // Finishing the tag creation, so now the tag is qualified to be a tag source. - snapshotURI, err := resources.NewTagSnapshotURI(pending.GetStatus().GetStorageLocation(), pending.GetMetadata().GetAtespace(), pending.GetMetadata().GetUid()) - if err != nil { - t.Fatalf("NewTagSnapshotURI: %v", err) - } + snapshotURI := pending.GetStatus().GetSnapshot().GetStorage()[0].GetObject().GetSnapshotUri() if _, err := tc.persistence.UpdateTag(ctx, resources.TagRefFromTag(pending), store.PreconditionFrom(pending), func(toUpdate *ateapipb.Tag) error { - toUpdate.Status.Snapshot = &ateapipb.ExternalSnapshot{ - SnapshotUri: snapshotURI.String(), - ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, - } + toUpdate.Status.Snapshot.Storage[0].Status = ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED return nil }); err != nil { t.Fatalf("finalizing the tag: %v", err) @@ -409,7 +405,7 @@ func TestCreateActor_PendingTag(t *testing.T) { } // The clone points at the tag's snapshot, under the tag's own prefix: the // tag still owns those objects. - if got := clone.GetStatus().GetExternalSnapshot().GetSnapshotUri(); got != snapshotURI.String() { + if got := durableSnapshotURI(clone.GetStatus()); got != snapshotURI { t.Errorf("clone external snapshot = %q, want the tag's %q", got, snapshotURI) } } @@ -699,8 +695,11 @@ func TestUpdateActor_Success(t *testing.T) { Metadata: &ateapipb.ResourceMetadata{Name: "id1", Atespace: testAtespace, Version: 2}, ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"}, Status: &ateapipb.ActorStatus{ - State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, - ExternalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: goldenSnapshotURI(t), ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, ActorTemplateUid: tmpl.GetMetadata().GetUid()}, + State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, + LastAssignedGeneration: 1, + Snapshots: []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, tmpl.GetMetadata().GetUid(), goldenSnapshotURI(t)), + }, }, WorkerSelector: &ateapipb.Selector{ MatchLabels: map[string]string{"tier": "paid"}, @@ -845,8 +844,11 @@ func TestUpdateActor(t *testing.T) { Metadata: &ateapipb.ResourceMetadata{Name: "id1", Atespace: testAtespace, Version: 2}, ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"}, Status: &ateapipb.ActorStatus{ - State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, - ExternalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: goldenSnapshotURI(t), ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, ActorTemplateUid: tmpl.GetMetadata().GetUid()}, + State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, + LastAssignedGeneration: 1, + Snapshots: []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, tmpl.GetMetadata().GetUid(), goldenSnapshotURI(t)), + }, }, WorkerSelector: &ateapipb.Selector{ MatchLabels: map[string]string{"tier": "paid"}, @@ -1117,16 +1119,13 @@ func TestDeleteActor_MissingSnapshotBucket(t *testing.T) { } if _, err := tc.persistence.UpdateActor(ctx, resources.ActorRefFromActor(actor), store.PreconditionFrom(actor), func(toUpdate *ateapipb.Actor) error { toUpdate.Status.State = state - toUpdate.Status.ExternalSnapshot = nil - if state == ateapipb.ActorState_ACTOR_STATE_SUSPENDED { - toUpdate.Status.ExternalSnapshot = &ateapipb.ExternalSnapshot{ - SnapshotUri: uri.String(), ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, - } - } else { + snap := newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", uri.String()) + if state != ateapipb.ActorState_ACTOR_STATE_SUSPENDED { // A failed upload leaves only the in-progress URI, even // when a prior deletion already marked the actor DELETING. - toUpdate.Status.InProgressSnapshotUri = uri.String() + snap.Storage[0].Status = ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS } + toUpdate.Status.Snapshots = []*ateapipb.Snapshot{snap} return nil }); err != nil { t.Fatalf("UpdateActor: %v", err) @@ -1242,8 +1241,8 @@ func TestDeleteActor_Crashed(t *testing.T) { } if !tc.fakeAtelet.TerminateCalled { t.Errorf("expected Terminate call to clean up assigned node for crashed actor") - } else if gotUID := tc.fakeAtelet.TerminateRequest.GetTargetAteomUid(); gotUID != "" { - t.Errorf("TerminateRequest.TargetAteomUid = %q, want empty for crashed actor", gotUID) + } else if gotUID := tc.fakeAtelet.TerminateRequest.GetWorkerPodUid(); gotUID != "" { + t.Errorf("TerminateRequest.WorkerPodUid = %q, want empty for crashed actor", gotUID) } _, err = tc.client.GetActor(ctx, &ateapipb.GetActorRequest{ @@ -1373,9 +1372,9 @@ func TestDeleteActor_MultipleVolumeDeletionFailures(t *testing.T) { ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"}, Status: &ateapipb.ActorStatus{ State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, - ActorVolumes: []*ateapipb.ExternalVolume{ - {VolumeName: "vol1", StorageVolumeId: "storage-vol-1", Status: ateapipb.ExternalVolume_STATUS_CREATED, VolumeType: "substrate.io/mock"}, - {VolumeName: "vol2", StorageVolumeId: "storage-vol-2", Status: ateapipb.ExternalVolume_STATUS_CREATED, VolumeType: "substrate.io/mock"}, + ExternalVolumes: []*ateapipb.ExternalVolume{ + {Name: "vol1", StorageVolumeId: "storage-vol-1", Status: ateapipb.ExternalVolume_STATUS_CREATED, VolumeType: "substrate.io/mock"}, + {Name: "vol2", StorageVolumeId: "storage-vol-2", Status: ateapipb.ExternalVolume_STATUS_CREATED, VolumeType: "substrate.io/mock"}, }, }, } @@ -1449,8 +1448,8 @@ func TestDeleteActor_VolumeDeletionFailure_RetrySuccess(t *testing.T) { ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"}, Status: &ateapipb.ActorStatus{ State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, - ActorVolumes: []*ateapipb.ExternalVolume{ - {VolumeName: "vol1", StorageVolumeId: "storage-vol-1", Status: ateapipb.ExternalVolume_STATUS_CREATED, VolumeType: "substrate.io/mock"}, + ExternalVolumes: []*ateapipb.ExternalVolume{ + {Name: "vol1", StorageVolumeId: "storage-vol-1", Status: ateapipb.ExternalVolume_STATUS_CREATED, VolumeType: "substrate.io/mock"}, }, }, } @@ -1476,8 +1475,8 @@ func TestDeleteActor_VolumeDeletionFailure_RetrySuccess(t *testing.T) { if getResp.GetStatus().GetState() != ateapipb.ActorState_ACTOR_STATE_DELETING { t.Errorf("actor state = %v, want ACTOR_STATE_DELETING", getResp.GetStatus().GetState()) } - if len(getResp.GetStatus().GetActorVolumes()) != 1 || getResp.GetStatus().GetActorVolumes()[0].GetStatus() != ateapipb.ExternalVolume_STATUS_DELETING { - t.Errorf("actor volume status = %v, want STATUS_DELETING", getResp.GetStatus().GetActorVolumes()) + if len(getResp.GetStatus().GetExternalVolumes()) != 1 || getResp.GetStatus().GetExternalVolumes()[0].GetStatus() != ateapipb.ExternalVolume_STATUS_DELETING { + t.Errorf("actor volume status = %v, want STATUS_DELETING", getResp.GetStatus().GetExternalVolumes()) } // 4. Recover the CSI plugin to simulate backend recovery. @@ -1609,8 +1608,8 @@ func TestActorLifecycle_WithExternalVolumes(t *testing.T) { if createResp.GetStatus().GetState() != ateapipb.ActorState_ACTOR_STATE_SUSPENDED { t.Fatalf("expected initial state ACTOR_STATE_SUSPENDED, got %v", createResp.GetStatus().GetState()) } - if len(createResp.GetStatus().GetActorVolumes()) != 1 || createResp.GetStatus().GetActorVolumes()[0].GetStatus() != ateapipb.ExternalVolume_STATUS_PENDING { - t.Fatalf("expected 1 pending volume after CreateActor, got %v", createResp.GetStatus().GetActorVolumes()) + if len(createResp.GetStatus().GetExternalVolumes()) != 1 || createResp.GetStatus().GetExternalVolumes()[0].GetStatus() != ateapipb.ExternalVolume_STATUS_PENDING { + t.Fatalf("expected 1 pending volume after CreateActor, got %v", createResp.GetStatus().GetExternalVolumes()) } // 2. ResumeActor @@ -1623,10 +1622,10 @@ func TestActorLifecycle_WithExternalVolumes(t *testing.T) { if resumeResp.GetActor().GetStatus().GetState() != ateapipb.ActorState_ACTOR_STATE_RUNNING { t.Fatalf("expected state ACTOR_STATE_RUNNING after resume, got %v", resumeResp.GetActor().GetStatus().GetState()) } - if len(resumeResp.GetActor().GetStatus().GetActorVolumes()) != 1 || resumeResp.GetActor().GetStatus().GetActorVolumes()[0].GetStatus() != ateapipb.ExternalVolume_STATUS_CREATED { - t.Fatalf("expected 1 created volume after ResumeActor, got %v", resumeResp.GetActor().GetStatus().GetActorVolumes()) + if len(resumeResp.GetActor().GetStatus().GetExternalVolumes()) != 1 || resumeResp.GetActor().GetStatus().GetExternalVolumes()[0].GetStatus() != ateapipb.ExternalVolume_STATUS_CREATED { + t.Fatalf("expected 1 created volume after ResumeActor, got %v", resumeResp.GetActor().GetStatus().GetExternalVolumes()) } - if resumeResp.GetActor().GetStatus().GetActorVolumes()[0].GetStorageVolumeId() == "" { + if resumeResp.GetActor().GetStatus().GetExternalVolumes()[0].GetStorageVolumeId() == "" { t.Fatalf("expected non-empty storageVolumeId after ResumeActor") } @@ -1779,12 +1778,12 @@ func TestResumeActor_VolumeCreationFailure(t *testing.T) { } // Verify that succ-vol1 was updated to CREATED with a storageVolumeId, and fail-vol2 is still PENDING - if len(getResp.GetStatus().GetActorVolumes()) != 2 { - t.Fatalf("expected 2 volumes on actor, got %d", len(getResp.GetStatus().GetActorVolumes())) + if len(getResp.GetStatus().GetExternalVolumes()) != 2 { + t.Fatalf("expected 2 volumes on actor, got %d", len(getResp.GetStatus().GetExternalVolumes())) } volsByName := make(map[string]*ateapipb.ExternalVolume) - for _, v := range getResp.GetStatus().GetActorVolumes() { - volsByName[v.GetVolumeName()] = v + for _, v := range getResp.GetStatus().GetExternalVolumes() { + volsByName[v.GetName()] = v } if v1, ok := volsByName["succ-vol1"]; !ok || v1.GetStatus() != ateapipb.ExternalVolume_STATUS_CREATED || v1.GetStorageVolumeId() == "" { t.Errorf("succ-vol1 unexpected state: %v", v1) @@ -1917,8 +1916,8 @@ func TestResumeActor_VolumeCreationRetrySuccess(t *testing.T) { } volsByName := make(map[string]*ateapipb.ExternalVolume) - for _, v := range getResp.GetStatus().GetActorVolumes() { - volsByName[v.GetVolumeName()] = v + for _, v := range getResp.GetStatus().GetExternalVolumes() { + volsByName[v.GetName()] = v } if v1, ok := volsByName["succ-vol1"]; !ok || v1.GetStatus() != ateapipb.ExternalVolume_STATUS_CREATED || v1.GetStorageVolumeId() == "" { t.Errorf("succ-vol1 unexpected state after first resume: %v", v1) @@ -1945,9 +1944,9 @@ func TestResumeActor_VolumeCreationRetrySuccess(t *testing.T) { if getResp.GetStatus().GetState() != ateapipb.ActorState_ACTOR_STATE_RUNNING { t.Errorf("actor state after second resume = %v, want %v", getResp.GetStatus().GetState(), ateapipb.ActorState_ACTOR_STATE_RUNNING) } - for _, v := range getResp.GetStatus().GetActorVolumes() { + for _, v := range getResp.GetStatus().GetExternalVolumes() { if v.GetStatus() != ateapipb.ExternalVolume_STATUS_CREATED || v.GetStorageVolumeId() == "" { - t.Errorf("volume %s unexpected state after second resume: %v", v.GetVolumeName(), v) + t.Errorf("volume %s unexpected state after second resume: %v", v.GetName(), v) } } @@ -2067,10 +2066,10 @@ func TestResumeActor_VolumeAttachFailureAndRetry(t *testing.T) { if getResp.GetStatus().GetWorkerAssignment() == nil || getResp.GetStatus().GetWorkerAssignment().GetWorkerPod() != "worker-1" { t.Errorf("worker assignment = %v, want worker-1", getResp.GetStatus().GetWorkerAssignment()) } - if len(getResp.GetStatus().GetActorVolumes()) != 1 { - t.Fatalf("expected 1 volume on actor, got %d", len(getResp.GetStatus().GetActorVolumes())) + if len(getResp.GetStatus().GetExternalVolumes()) != 1 { + t.Fatalf("expected 1 volume on actor, got %d", len(getResp.GetStatus().GetExternalVolumes())) } - vol := getResp.GetStatus().GetActorVolumes()[0] + vol := getResp.GetStatus().GetExternalVolumes()[0] if vol.GetStatus() != ateapipb.ExternalVolume_STATUS_CREATED || vol.GetStorageVolumeId() == "" { t.Errorf("vol1 unexpected state after failed attach: %v", vol) } @@ -2908,9 +2907,12 @@ func TestResumeActor(t *testing.T) { Metadata: &ateapipb.ResourceMetadata{Name: name, Atespace: testAtespace}, ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"}, Status: &ateapipb.ActorStatus{ - State: ateapipb.ActorState_ACTOR_STATE_RUNNING, - AssignedNode: "node1", - ExternalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: goldenSnapshotURI(t), ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, ActorTemplateUid: tmpl.GetMetadata().GetUid()}, + State: ateapipb.ActorState_ACTOR_STATE_RUNNING, + AssignedNode: "node1", + LastAssignedGeneration: 1, + Snapshots: []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, tmpl.GetMetadata().GetUid(), goldenSnapshotURI(t)), + }, WorkerAssignment: &ateapipb.WorkerAssignment{ Worker: &ateapipb.ObjectRef{Name: podUID}, WorkerNamespace: ns, @@ -3022,10 +3024,10 @@ func TestResumeActorPassesLiteralEnv(t *testing.T) { } } -// createDataCommitTemplate creates "tmpl1" like createTemplate, but with -// onCommit DATA, so a resumed-after-suspend actor restores from a DATA +// createDataFidelityTemplate creates "tmpl1" like createTemplate, but with +// preferredFidelity DATA, so a resumed-after-suspend actor restores from a DATA // snapshot while the template also has a golden snapshot. -func createDataCommitTemplate(t *testing.T, tc *testContext, ns string) *ateapipb.ActorTemplate { +func createDataFidelityTemplate(t *testing.T, tc *testContext, ns string) *ateapipb.ActorTemplate { t.Helper() ensureDefaultGvisorSandboxConfig(t, tc) createWorkerPool(t, tc, ns, "pool1", map[string]string{poolLabelKey: ns}) @@ -3037,8 +3039,8 @@ func createDataCommitTemplate(t *testing.T, tc *testContext, ns string) *ateapip Name: "tmpl1", }, SnapshotConfig: &ateapipb.SnapshotConfig{ - StorageLocation: testStorageLocation, - OnCommit: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA, + StorageLocation: testStorageLocation, + PreferredFidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, }, SandboxConfig: &ateapipb.SandboxConfig{ SandboxClass: ateapipb.SandboxClass_SANDBOX_CLASS_GVISOR, @@ -3066,8 +3068,7 @@ func createDataCommitTemplate(t *testing.T, tc *testContext, ns string) *ateapip SourceActor: &ateapipb.ObjectRef{Atespace: resources.GoldenActorAtespace, Name: created.GetMetadata().GetUid()}, Scope: ateapipb.TagScope_TAG_SCOPE_PUBLISHED, Status: &ateapipb.TagStatus{ - Snapshot: &ateapipb.ExternalSnapshot{SnapshotUri: goldenSnapshotURI(t), ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL}, - StorageLocation: testStorageLocation, + Snapshot: newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, created.GetMetadata().GetUid(), goldenSnapshotURI(t)), ActorTemplateUid: created.GetMetadata().GetUid(), }, }) @@ -3098,7 +3099,7 @@ func TestResumeActor_DataSnapshotIgnoresGolden(t *testing.T) { tc := setupTest(t, ns) defer tc.cleanup() - createDataCommitTemplate(t, tc, ns) + createDataFidelityTemplate(t, tc, ns) workerName := createWorkerPod(t, tc, ns, "worker-1", "node1", "pool1") const name = "id1" @@ -3111,7 +3112,7 @@ func TestResumeActor_DataSnapshotIgnoresGolden(t *testing.T) { } // First resume runs fresh from the golden; the suspend then commits a - // DATA snapshot per onCommit. + // DATA snapshot per preferredFidelity. if _, err := tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{Actor: actorRef}); err != nil { t.Fatalf("ResumeActor (first) failed: %v", err) } @@ -3120,7 +3121,7 @@ func TestResumeActor_DataSnapshotIgnoresGolden(t *testing.T) { t.Fatalf("SuspendActor failed: %v", err) } waitForWorkerAvailable(t, tc, workerName) - actorSnapshotURI := suspended.GetActor().GetStatus().GetExternalSnapshot().GetSnapshotUri() + actorSnapshotURI := durableSnapshotURI(suspended.GetActor().GetStatus()) if actorSnapshotURI == "" { t.Fatal("SuspendActor recorded no external snapshot") } @@ -3133,8 +3134,8 @@ func TestResumeActor_DataSnapshotIgnoresGolden(t *testing.T) { if restoreReq == nil { t.Fatal("second resume sent no Restore request to atelet") } - if got := restoreReq.GetScope(); got != ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA { - t.Fatalf("restore scope = %v, want SNAPSHOT_SCOPE_DATA", got) + if got := restoreReq.GetFidelity(); got != ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES { + t.Fatalf("restore scope = %v, want SNAPSHOT_FIDELITY_VOLUMES", got) } if got := restoreReq.GetExternalConfig().GetSnapshotUri(); got != actorSnapshotURI { t.Errorf("restore config snapshot uri = %q, want the actor's data snapshot %q", got, actorSnapshotURI) @@ -3173,7 +3174,7 @@ func TestSuspendActor_ReplacedSnapshotReleaseFailure(t *testing.T) { t.Fatalf("SuspendActor (first) failed: %v", err) } waitForWorkerAvailable(t, tc, workerName) - replacedURI := first.GetActor().GetStatus().GetExternalSnapshot().GetSnapshotUri() + replacedURI := durableSnapshotURI(first.GetActor().GetStatus()) assertSnapshotOwnedByActor(t, first.GetActor(), replacedURI) if _, err := tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{Actor: actorRef}); err != nil { @@ -3191,7 +3192,7 @@ func TestSuspendActor_ReplacedSnapshotReleaseFailure(t *testing.T) { if got := second.GetActor().GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_SUSPENDED { t.Errorf("state after the second suspend = %v, want SUSPENDED", got) } - freshURI := second.GetActor().GetStatus().GetExternalSnapshot().GetSnapshotUri() + freshURI := durableSnapshotURI(second.GetActor().GetStatus()) if freshURI == "" || freshURI == replacedURI { t.Errorf("external snapshot after the second suspend = %q, want a new one replacing %q", freshURI, replacedURI) } @@ -3213,7 +3214,7 @@ func TestSuspendActor_ReplacedSnapshotReleaseFailure(t *testing.T) { t.Fatalf("SuspendActor (third) failed: %v", err) } waitForWorkerAvailable(t, tc, workerName) - lastURI := third.GetActor().GetStatus().GetExternalSnapshot().GetSnapshotUri() + lastURI := durableSnapshotURI(third.GetActor().GetStatus()) assertSnapshotCollected(t, tc, freshURI) assertSnapshotPresent(t, tc, replacedURI) @@ -3400,60 +3401,87 @@ func TestResumeActor_AteletFailureCrashesActor(t *testing.T) { } } -// TestResumeActor_AteletUnavailableLeavesActorResuming: an Unavailable from -// atelet, such as atelet restarting, does not crash the actor. It stays -// RESUMING on its worker, and the next resume finishes. -func TestResumeActor_AteletUnavailableLeavesActorResuming(t *testing.T) { - ns := namespaceForTest("ns-resume-atelet-unavailable") - tc := setupTest(t, ns) - defer tc.cleanup() +// A transient atelet failure, or a caller whose deadline passes during the +// restore, leaves the actor RESUMING on its worker, and the next resume finishes. +// Only Unavailable and the caller's own deadline reach the caller with their +// code. Atelet's own Canceled or DeadlineExceeded reach it as Internal. +func TestResumeActor_AteletTransportErrorLeavesActorResuming(t *testing.T) { + for _, tt := range []struct { + name string + fail error + delay time.Duration + deadline time.Duration + wantCode codes.Code + }{ + {name: "Unavailable", fail: status.Error(codes.Unavailable, "restore reply lost"), wantCode: codes.Unavailable}, + {name: "atelet Canceled", fail: status.Error(codes.Canceled, "restore reply lost"), wantCode: codes.Internal}, + {name: "atelet DeadlineExceeded", fail: status.Error(codes.DeadlineExceeded, "restore reply lost"), wantCode: codes.Internal}, + {name: "caller deadline", delay: time.Second, deadline: 200 * time.Millisecond, wantCode: codes.DeadlineExceeded}, + } { + t.Run(tt.name, func(t *testing.T) { + ns := namespaceForTest("ns-resume-atelet-" + strings.ReplaceAll(strings.ToLower(tt.name), " ", "-")) + tc := setupTest(t, ns) + defer tc.cleanup() - createTemplate(t, tc, ns) - podUID := createWorkerPod(t, tc, ns, "worker-1", "node1", "pool1") + createTemplate(t, tc, ns) + podUID := createWorkerPod(t, tc, ns, "worker-1", "node1", "pool1") - name := "id1" - ref := &ateapipb.ObjectRef{Atespace: testAtespace, Name: name} - actorRef := resources.ActorRef{Atespace: testAtespace, Name: name} - if _, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: name}, - ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"}, - }}); err != nil { - t.Fatalf("CreateActor failed: %v", err) - } + name := "id1" + ref := &ateapipb.ObjectRef{Atespace: testAtespace, Name: name} + actorRef := resources.ActorRef{Atespace: testAtespace, Name: name} + if _, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: name}, + ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"}, + }}); err != nil { + t.Fatalf("CreateActor failed: %v", err) + } - tc.fakeAtelet.FailRestore = status.Error(codes.Unavailable, "connection refused") - _, err := tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{Actor: ref}) - if got := status.Code(err); got != codes.Unavailable { - t.Fatalf("ResumeActor status code = %v, want %v (err: %v)", got, codes.Unavailable, err) - } + tc.fakeAtelet.FailRestore = tt.fail + tc.fakeAtelet.RestoreDelay = tt.delay + ctx := context.Background() + if tt.deadline > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, tt.deadline) + defer cancel() + } + _, err := tc.client.ResumeActor(ctx, &ateapipb.ResumeActorRequest{Actor: ref}) + if got := status.Code(err); got != tt.wantCode { + t.Fatalf("ResumeActor status code = %v, want %v (err: %v)", got, tt.wantCode, err) + } - actor, err := tc.persistence.GetActor(context.Background(), actorRef) - if err != nil { - t.Fatalf("failed to get actor from store: %v", err) - } - if got := actor.GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_RESUMING { - t.Fatalf("state after Unavailable = %v, want RESUMING", got) - } - if crash := actor.GetStatus().GetCrash(); crash != nil { - t.Errorf("crash recorded after Unavailable: %v", crash) - } - if got := actor.GetStatus().GetWorkerAssignment().GetWorker().GetName(); got != podUID { - t.Errorf("assigned worker = %q, want %q kept for the retry", got, podUID) - } + actor, err := tc.persistence.GetActor(context.Background(), actorRef) + if err != nil { + t.Fatalf("failed to get actor from store: %v", err) + } + if got := actor.GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_RESUMING { + t.Fatalf("state after %s = %v, want RESUMING", tt.name, got) + } + if crash := actor.GetStatus().GetCrash(); crash != nil { + t.Errorf("crash recorded after %s: %v", tt.name, crash) + } + if got := actor.GetStatus().GetWorkerAssignment().GetWorker().GetName(); got != podUID { + t.Errorf("assigned worker = %q, want %q kept for the retry", got, podUID) + } - tc.fakeAtelet.FailRestore = nil - if _, err := tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{Actor: ref}); err != nil { - t.Fatalf("retried ResumeActor failed: %v", err) - } - actor, err = tc.persistence.GetActor(context.Background(), actorRef) - if err != nil { - t.Fatalf("failed to get actor from store: %v", err) - } - if got := actor.GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_RUNNING { - t.Errorf("state after retry = %v, want RUNNING", got) - } - if got := actor.GetStatus().GetWorkerAssignment().GetWorker().GetName(); got != podUID { - t.Errorf("assigned worker after retry = %q, want %q", got, podUID) + // The fake still holds its lock while a delayed restore sleeps. + tc.fakeAtelet.Lock.Lock() + tc.fakeAtelet.FailRestore = nil + tc.fakeAtelet.RestoreDelay = 0 + tc.fakeAtelet.Lock.Unlock() + if _, err := tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{Actor: ref}); err != nil { + t.Fatalf("retried ResumeActor failed: %v", err) + } + actor, err = tc.persistence.GetActor(context.Background(), actorRef) + if err != nil { + t.Fatalf("failed to get actor from store: %v", err) + } + if got := actor.GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_RUNNING { + t.Errorf("state after retry = %v, want RUNNING", got) + } + if got := actor.GetStatus().GetWorkerAssignment().GetWorker().GetName(); got != podUID { + t.Errorf("assigned worker after retry = %q, want %q", got, podUID) + } + }) } } @@ -3484,8 +3512,8 @@ func TestResumeActor_LocalRestoreFailureCrashesActor(t *testing.T) { tc.fakeAtelet.Reset() tc.fakeAtelet.FailRestore = status.Error(codes.Internal, "injected restore failure") - if _, err := tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{Actor: ref}); err == nil { - t.Fatal("ResumeActor succeeded despite failing restore") + if _, err := tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{Actor: ref}); status.Code(err) != codes.Internal { + t.Fatalf("ResumeActor status code = %v, want Internal (err: %v)", status.Code(err), err) } if got := tc.fakeAtelet.RestoreRequest.GetType(); got != ateletpb.CheckpointType_CHECKPOINT_TYPE_LOCAL { t.Fatalf("restore type = %v, want LOCAL", got) @@ -3581,7 +3609,7 @@ func TestSuspendActor(t *testing.T) { } sourceActor := suspended.GetActor() - snapshotURI := sourceActor.GetStatus().GetExternalSnapshot().GetSnapshotUri() + snapshotURI := durableSnapshotURI(sourceActor.GetStatus()) if snapshotURI == "" { t.Fatalf("SuspendActor wrote no external snapshot: %v", suspended) } @@ -3605,21 +3633,20 @@ func TestSuspendActor(t *testing.T) { // The tag owns a copy of its own, so the Actor's later suspends and its // deletion cannot collect the tag's snapshot copy. - tagSnapshotURI := tagged.GetStatus().GetSnapshot().GetSnapshotUri() - if tagSnapshotURI == snapshotURI || tagSnapshotURI == "" { - t.Fatalf("tag snapshot uri = %q, want an external snapshot of its own", tagSnapshotURI) + taggedSnapshotURI := tagSnapshotURI(tagged) + if taggedSnapshotURI == snapshotURI || taggedSnapshotURI == "" { + t.Fatalf("tag snapshot uri = %q, want an external snapshot of its own", taggedSnapshotURI) } assertSnapshotPresent(t, tc, snapshotURI) - assertSnapshotPresent(t, tc, tagSnapshotURI) + assertSnapshotPresent(t, tc, taggedSnapshotURI) wantTag := &ateapipb.Tag{ Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: tagName}, Scope: ateapipb.TagScope_TAG_SCOPE_ATESPACE, SourceActor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: name}, Status: &ateapipb.TagStatus{ - Snapshot: &ateapipb.ExternalSnapshot{SnapshotUri: tagSnapshotURI, ContentScope: sourceActor.GetStatus().GetExternalSnapshot().GetContentScope()}, + Snapshot: newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG, durableSnapshotStorage(sourceActor.GetStatus()).GetFidelity(), "", taggedSnapshotURI), ActorTemplateUid: tmpl.GetMetadata().GetUid(), - StorageLocation: tmpl.GetSnapshotConfig().GetStorageLocation(), }, } stored, err := tc.client.GetTag(context.Background(), &ateapipb.GetTagRequest{Tag: tagRef}) @@ -3654,8 +3681,8 @@ func TestSuspendActor(t *testing.T) { if err != nil || updated.GetScope() != ateapipb.TagScope_TAG_SCOPE_PUBLISHED { t.Fatalf("UpdateTag = (%v, %v), want published", updated, err) } - if updated.GetStatus().GetSnapshot().GetSnapshotUri() != tagSnapshotURI { - t.Errorf("tag snapshot uri after publication = %q, want %q", updated.GetStatus().GetSnapshot().GetSnapshotUri(), tagSnapshotURI) + if tagSnapshotURI(updated) != taggedSnapshotURI { + t.Errorf("tag snapshot uri after publication = %q, want %q", tagSnapshotURI(updated), taggedSnapshotURI) } if _, err := tc.client.CreateActor(context.Background(), crossAtespaceClone); err != nil { t.Fatalf("CreateActor from published tag failed: %v", err) @@ -3674,11 +3701,11 @@ func TestSuspendActor(t *testing.T) { if err != nil { t.Fatalf("CreateActor from tag failed: %v", err) } - if got := clone.GetStatus().GetExternalSnapshot().GetSnapshotUri(); got != tagSnapshotURI { - t.Errorf("clone snapshot uri = %q, want the tag's %q", got, tagSnapshotURI) + if got := durableSnapshotURI(clone.GetStatus()); got != taggedSnapshotURI { + t.Errorf("clone snapshot uri = %q, want the tag's %q", got, taggedSnapshotURI) } - if snapshotOwnedByActor(t, clone, tagSnapshotURI) { - t.Errorf("clone snapshot %s sits under the clone's own prefix, want it left under the tag's", tagSnapshotURI) + if snapshotOwnedByActor(t, clone, taggedSnapshotURI) { + t.Errorf("clone snapshot %s sits under the clone's own prefix, want it left under the tag's", taggedSnapshotURI) } if !proto.Equal(clone.GetSourceTag(), tagRef) { t.Errorf("clone source tag = %v, want %v", clone.GetSourceTag(), tagRef) @@ -3696,13 +3723,13 @@ func TestSuspendActor(t *testing.T) { if err != nil { t.Fatalf("SuspendActor clone failed: %v", err) } - cloneSnapshotURI := cloneSuspended.GetActor().GetStatus().GetExternalSnapshot().GetSnapshotUri() - if cloneSnapshotURI == tagSnapshotURI || cloneSnapshotURI == "" { + cloneSnapshotURI := durableSnapshotURI(cloneSuspended.GetActor().GetStatus()) + if cloneSnapshotURI == taggedSnapshotURI || cloneSnapshotURI == "" { t.Errorf("clone snapshot uri after suspension = %q, want an external snapshot of its own", cloneSnapshotURI) } assertSnapshotOwnedByActor(t, cloneSuspended.GetActor(), cloneSnapshotURI) // It stopped borrowing without releasing what it had borrowed. - assertSnapshotPresent(t, tc, tagSnapshotURI) + assertSnapshotPresent(t, tc, taggedSnapshotURI) assertSnapshotPresent(t, tc, cloneSnapshotURI) // The untagged suspend created no tag. listed, err = tc.client.ListTags(context.Background(), &ateapipb.ListTagsRequest{Atespace: testAtespace}) @@ -3720,11 +3747,10 @@ func TestSuspendActor(t *testing.T) { Metadata: &ateapipb.ResourceMetadata{Name: name, Atespace: testAtespace}, ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"}, Status: &ateapipb.ActorStatus{ - State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, - ExternalSnapshot: &ateapipb.ExternalSnapshot{ - SnapshotUri: snapshotURI, - ContentScope: sourceActor.GetStatus().GetExternalSnapshot().GetContentScope(), - ActorTemplateUid: tmpl.GetMetadata().GetUid(), + State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, + LastAssignedGeneration: 2, + Snapshots: []*ateapipb.Snapshot{ + newDurableSnapshot(2, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, durableSnapshotStorage(sourceActor.GetStatus()).GetFidelity(), tmpl.GetMetadata().GetUid(), snapshotURI), }, }, } @@ -3747,7 +3773,7 @@ func TestSuspendActor(t *testing.T) { } // The Actor took only what it owned with it. assertSnapshotCollected(t, tc, snapshotURI) - assertSnapshotPresent(t, tc, tagSnapshotURI) + assertSnapshotPresent(t, tc, taggedSnapshotURI) if deleted, err := tc.client.DeleteTag(context.Background(), &ateapipb.DeleteTagRequest{Tag: tagRef}); err != nil || deleted.GetMetadata().GetName() != tagRef.GetName() { t.Fatalf("DeleteTag = (%v, %v)", deleted, err) @@ -3757,7 +3783,7 @@ func TestSuspendActor(t *testing.T) { } // Deleting the tag is what ends its snapshot's life. The clone that // borrowed it has one of its own by now and is unaffected. - assertSnapshotCollected(t, tc, tagSnapshotURI) + assertSnapshotCollected(t, tc, taggedSnapshotURI) assertSnapshotPresent(t, tc, cloneSnapshotURI) } @@ -3775,19 +3801,19 @@ func TestResumeActor_RepointTemplateBeforeResume(t *testing.T) { // after it is created but before its first resume. moveActorToAnotherTemplate bool wantTemplate string - wantScope ateletpb.SnapshotScope + wantScope ateletpb.SnapshotFidelity }{ { name: "clone left on the tag's template", moveActorToAnotherTemplate: false, wantTemplate: "tmpl1", - wantScope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL, + wantScope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, }, { name: "clone repointed before its first resume", moveActorToAnotherTemplate: true, wantTemplate: "tmpl2", - wantScope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA, + wantScope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, }, } for _, tt := range tests { @@ -3870,13 +3896,13 @@ func TestResumeActor_RepointTemplateBeforeResume(t *testing.T) { if got := restoreReq.GetActorTemplateName(); got != tt.wantTemplate { t.Errorf("restore request to atelet had actor template = %q, want %q", got, tt.wantTemplate) } - if got := restoreReq.GetScope(); got != tt.wantScope { + if got := restoreReq.GetFidelity(); got != tt.wantScope { t.Errorf("restore request to atelet had scope = %v, want %v", got, tt.wantScope) } // Either way the restore reads the snapshot the clone borrowed // from the tag, not the template's golden image. - if got := restoreReq.GetExternalConfig().GetSnapshotUri(); got != cloneActor.GetStatus().GetExternalSnapshot().GetSnapshotUri() { - t.Errorf("restore request to atelet had snapshot uri = %q, want the clone's borrowed %q", got, cloneActor.GetStatus().GetExternalSnapshot().GetSnapshotUri()) + if got := restoreReq.GetExternalConfig().GetSnapshotUri(); got != durableSnapshotURI(cloneActor.GetStatus()) { + t.Errorf("restore request to atelet had snapshot uri = %q, want the clone's borrowed %q", got, durableSnapshotURI(cloneActor.GetStatus())) } }) } @@ -3932,7 +3958,7 @@ func TestResumeActor_PausedAfterRepointUsesLocalProvenance(t *testing.T) { if _, err := tc.client.ResumeActor(ctx, &ateapipb.ResumeActorRequest{Actor: actorRef}); err != nil { t.Fatalf("ResumeActor(v2 from v1 snapshot) failed: %v", err) } - if got := tc.fakeAtelet.lastRestoreRequest().GetScope(); got != ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA { + if got := tc.fakeAtelet.lastRestoreRequest().GetFidelity(); got != ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES { t.Fatalf("first resume on v2 had scope = %v, want DATA", got) } if _, err := tc.client.PauseActor(ctx, &ateapipb.PauseActorRequest{Actor: actorRef}); err != nil { @@ -3949,7 +3975,7 @@ func TestResumeActor_PausedAfterRepointUsesLocalProvenance(t *testing.T) { if got := restoreReq.GetType(); got != ateletpb.CheckpointType_CHECKPOINT_TYPE_LOCAL { t.Errorf("restore request type = %v, want LOCAL", got) } - if got := restoreReq.GetScope(); got != ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL { + if got := restoreReq.GetFidelity(); got != ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY { t.Errorf("restore request scope = %v, want FULL (local checkpoint was captured on v2)", got) } } @@ -4012,12 +4038,13 @@ func TestPauseActor(t *testing.T) { Metadata: &ateapipb.ResourceMetadata{Name: name, Atespace: testAtespace}, ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"}, Status: &ateapipb.ActorStatus{ - State: ateapipb.ActorState_ACTOR_STATE_PAUSED, - AssignedNode: "node1", - LocalSnapshot: &ateapipb.LocalSnapshot{ - ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, + State: ateapipb.ActorState_ACTOR_STATE_PAUSED, + AssignedNode: "node1", + LastAssignedGeneration: 2, + Snapshots: []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, tmpl.GetMetadata().GetUid(), goldenSnapshotURI(t)), + newLocalSnapshot(2, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, tmpl.GetMetadata().GetUid(), "ignored"), }, - ExternalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: goldenSnapshotURI(t), ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, ActorTemplateUid: tmpl.GetMetadata().GetUid()}, }, } @@ -4030,7 +4057,7 @@ func TestPauseActor(t *testing.T) { ); diff != "" { t.Errorf("GetActor response mismatch (-want +got):\n%s", diff) } - if getResp.GetStatus().GetLocalSnapshot().GetSnapshotName() == "" { + if _, ls := localSnapshot(getResp.GetStatus()); ls.GetSnapshotName() == "" { t.Error("LocalSnapshot.SnapshotName is empty, want the name the pause checkpointed under") } } @@ -4079,7 +4106,7 @@ func TestResumeActor_PausedLocalSnapshotMissing_Crashes(t *testing.T) { if getResp.GetStatus().GetState() != ateapipb.ActorState_ACTOR_STATE_PAUSED { t.Fatalf("actor state = %v, want ACTOR_STATE_PAUSED", getResp.GetStatus().GetState()) } - if getResp.GetStatus().GetLocalSnapshot() == nil { + if _, ls := localSnapshot(getResp.GetStatus()); ls == nil { t.Fatal("expected LocalSnapshot to be present on paused actor") } waitForWorkerAvailable(t, tc, workerName) @@ -4704,28 +4731,29 @@ func TestSuspendActor_FromPaused(t *testing.T) { t.Error("atelet Checkpoint called for a paused actor; there is no workload to checkpoint") } upload := tc.fakeAtelet.UploadRequest - if got, want := upload.GetLocalSnapshotName(), paused.GetStatus().GetLocalSnapshot().GetSnapshotName(); got != want { + _, pausedLS := localSnapshot(paused.GetStatus()) + if got, want := upload.GetLocalSnapshotName(), pausedLS.GetSnapshotName(); got != want { t.Errorf("upload local_snapshot_name = %q, want the pause snapshot %q", got, want) } if got, want := upload.GetAtespace(), testAtespace; got != want { t.Errorf("upload atespace = %q, want %q", got, want) } - if got := upload.GetDesiredScope(); got != ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL { - t.Errorf("upload desired_scope = %v, want FULL (template default)", got) + if got := upload.GetDesiredFidelity(); got != ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY { + t.Errorf("upload desired_fidelity = %v, want MEMORY (template default)", got) } actor := suspended.GetActor() if actor.GetStatus().GetState() != ateapipb.ActorState_ACTOR_STATE_SUSPENDED { t.Errorf("state = %v, want SUSPENDED", actor.GetStatus().GetState()) } - if actor.GetStatus().GetLocalSnapshot() != nil { - t.Errorf("LocalSnapshot = %v, want cleared (node pinning must not survive suspend)", actor.GetStatus().GetLocalSnapshot()) + if _, ls := localSnapshot(actor.GetStatus()); ls != nil { + t.Errorf("LocalSnapshot = %v, want cleared (node pinning must not survive suspend)", ls) } - if got, want := actor.GetStatus().GetExternalSnapshot().GetSnapshotUri(), upload.GetDestinationSnapshotUri(); got != want { + if got, want := durableSnapshotURI(actor.GetStatus()), upload.GetDestinationSnapshotUri(); got != want { t.Errorf("snapshot URI = %q, want the upload destination %q", got, want) } - if got := actor.GetStatus().GetExternalSnapshot().GetContentScope(); got != ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL { - t.Errorf("snapshot ContentScope = %v, want FULL", got) + if got := durableSnapshotStorage(actor.GetStatus()).GetFidelity(); got != ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY { + t.Errorf("snapshot Fidelity = %v, want FULL", got) } } @@ -4957,8 +4985,8 @@ func TestResumeActor_RelocatesAfterSuspendFromPaused(t *testing.T) { if got := suspended.GetActor().GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_SUSPENDED { t.Fatalf("state after suspend = %v, want SUSPENDED", got) } - if got := suspended.GetActor().GetStatus().GetLocalSnapshot(); got != nil { - t.Fatalf("LocalSnapshot = %v, want cleared so the actor can be scheduled anywhere", got) + if _, ls := localSnapshot(suspended.GetActor().GetStatus()); ls != nil { + t.Fatalf("LocalSnapshot = %v, want cleared so the actor can be scheduled anywhere", ls) } // Resume should succeed now and the actor scheduled on node2. @@ -5182,7 +5210,7 @@ func TestMintActorJWT_Success(t *testing.T) { Atespace: createResp.GetMetadata().GetAtespace(), Name: createResp.GetMetadata().GetName(), }, - Audience: []string{"foo"}, + Audiences: []string{"foo"}, ExpirationSeconds: 1800, }) if err != nil { @@ -5268,7 +5296,7 @@ func TestRevertActor(t *testing.T) { t.Fatalf("SuspendActor failed: %v", err) } waitForWorkerAvailable(t, tc, workerName) - snapshotURI := suspended.GetActor().GetStatus().GetExternalSnapshot().GetSnapshotUri() + snapshotURI := durableSnapshotURI(suspended.GetActor().GetStatus()) if snapshotURI == "" { t.Fatalf("SuspendActor wrote no external snapshot: %v", suspended) } @@ -5291,7 +5319,7 @@ func TestRevertActor(t *testing.T) { if got.GetState() != ateapipb.ActorState_ACTOR_STATE_SUSPENDED { t.Errorf("state = %v, want SUSPENDED", got.GetState()) } - if uri := got.GetExternalSnapshot().GetSnapshotUri(); uri != snapshotURI { + if uri := durableSnapshotURI(got); uri != snapshotURI { t.Errorf("external snapshot = %q, want it untouched at %q", uri, snapshotURI) } assertSnapshotPresent(t, tc, snapshotURI) @@ -5336,7 +5364,7 @@ func TestRevertActor_FromPaused(t *testing.T) { t.Fatalf("SuspendActor failed: %v", err) } waitForWorkerAvailable(t, tc, workerName) - snapshotURI := suspended.GetActor().GetStatus().GetExternalSnapshot().GetSnapshotUri() + snapshotURI := durableSnapshotURI(suspended.GetActor().GetStatus()) if snapshotURI == "" { t.Fatalf("SuspendActor wrote no external snapshot: %v", suspended) } @@ -5362,12 +5390,12 @@ func TestRevertActor_FromPaused(t *testing.T) { if got.GetState() != ateapipb.ActorState_ACTOR_STATE_SUSPENDED { t.Errorf("state = %v, want SUSPENDED", got.GetState()) } - if uri := got.GetExternalSnapshot().GetSnapshotUri(); uri != snapshotURI { + if uri := durableSnapshotURI(got); uri != snapshotURI { t.Errorf("external snapshot = %q, want it untouched at %q", uri, snapshotURI) } assertSnapshotPresent(t, tc, snapshotURI) - if got.GetLocalSnapshot() != nil { - t.Errorf("local snapshot info = %v, want nil", got.GetLocalSnapshot()) + if _, ls := localSnapshot(got); ls != nil { + t.Errorf("local snapshot info = %v, want nil", ls) } if got.GetWorkerAssignment() != nil { t.Errorf("worker assignment = %v, want nil", got.GetWorkerAssignment()) @@ -5377,8 +5405,8 @@ func TestRevertActor_FromPaused(t *testing.T) { } if !tc.fakeAtelet.TerminateCalled { t.Errorf("expected Terminate call to clean up assigned node for paused actor") - } else if gotUID := tc.fakeAtelet.TerminateRequest.GetTargetAteomUid(); gotUID != "" { - t.Errorf("TerminateRequest.TargetAteomUid = %q, want empty for paused actor", gotUID) + } else if gotUID := tc.fakeAtelet.TerminateRequest.GetWorkerPodUid(); gotUID != "" { + t.Errorf("TerminateRequest.WorkerPodUid = %q, want empty for paused actor", gotUID) } if tc.fakeAtelet.CheckpointCalled { t.Errorf("RevertActor checkpointed the workload, want the execution discarded") @@ -5414,7 +5442,7 @@ func TestRevertActor_FromCrashed(t *testing.T) { t.Fatalf("SuspendActor failed: %v", err) } waitForWorkerAvailable(t, tc, workerName) - snapshotURI := suspended.GetActor().GetStatus().GetExternalSnapshot().GetSnapshotUri() + snapshotURI := durableSnapshotURI(suspended.GetActor().GetStatus()) if snapshotURI == "" { t.Fatalf("SuspendActor wrote no external snapshot: %v", suspended) } @@ -5446,7 +5474,7 @@ func TestRevertActor_FromCrashed(t *testing.T) { if got.GetState() != ateapipb.ActorState_ACTOR_STATE_SUSPENDED { t.Errorf("state = %v, want SUSPENDED", got.GetState()) } - if uri := got.GetExternalSnapshot().GetSnapshotUri(); uri != snapshotURI { + if uri := durableSnapshotURI(got); uri != snapshotURI { t.Errorf("external snapshot = %q, want it untouched at %q", uri, snapshotURI) } assertSnapshotPresent(t, tc, snapshotURI) @@ -5458,8 +5486,8 @@ func TestRevertActor_FromCrashed(t *testing.T) { } if !tc.fakeAtelet.TerminateCalled { t.Errorf("expected Terminate call to clean up assigned node for crashed actor") - } else if gotUID := tc.fakeAtelet.TerminateRequest.GetTargetAteomUid(); gotUID != "" { - t.Errorf("TerminateRequest.TargetAteomUid = %q, want empty for crashed actor", gotUID) + } else if gotUID := tc.fakeAtelet.TerminateRequest.GetWorkerPodUid(); gotUID != "" { + t.Errorf("TerminateRequest.WorkerPodUid = %q, want empty for crashed actor", gotUID) } if tc.fakeAtelet.CheckpointCalled { t.Errorf("RevertActor checkpointed the workload, want the execution discarded") diff --git a/cmd/ateapi/internal/controlapi/functionaltest/common_test.go b/cmd/ateapi/internal/controlapi/functionaltest/common_test.go index c1908c7b27..67da93052f 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/common_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/common_test.go @@ -31,6 +31,7 @@ import ( "github.com/agent-substrate/substrate/internal/localca" "github.com/agent-substrate/substrate/internal/localjwtauthority" "github.com/agent-substrate/substrate/internal/objectstore/objectstoretest" + "github.com/agent-substrate/substrate/internal/objectstoreplugin/objectstoreplugintest" "github.com/agent-substrate/substrate/internal/resources" "github.com/agent-substrate/substrate/internal/volume" atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" @@ -221,7 +222,7 @@ func setupTestWithVolumePlugins(t *testing.T, ns string, plugins map[string]volu "", 30*time.Second, volPlugins, - objectStore, + objectstoreplugintest.ControlClient(objectStore), testActorJWTIssuer, actorJWTAuthorityPool, actorCAPool, @@ -457,8 +458,7 @@ func createTemplateWithContainersAndVolumes(t *testing.T, tc *testContext, ns st SourceActor: &ateapipb.ObjectRef{Atespace: resources.GoldenActorAtespace, Name: created.GetMetadata().GetUid()}, Scope: ateapipb.TagScope_TAG_SCOPE_PUBLISHED, Status: &ateapipb.TagStatus{ - Snapshot: &ateapipb.ExternalSnapshot{SnapshotUri: goldenSnapshotURI(t), ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL}, - StorageLocation: testStorageLocation, + Snapshot: newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, created.GetMetadata().GetUid(), goldenSnapshotURI(t)), ActorTemplateUid: created.GetMetadata().GetUid(), }, }) @@ -485,6 +485,86 @@ func createTemplateWithContainersAndVolumes(t *testing.T, tc *testContext, ns st return updated } +func newDurableSnapshot(gen int32, owner ateapipb.SnapshotOwner, fidelity ateapipb.SnapshotFidelity, templateUID, uri string) *ateapipb.Snapshot { + return &ateapipb.Snapshot{ + Generation: gen, + Owner: owner, + ActorTemplateUid: templateUID, + Storage: []*ateapipb.SnapshotStorage{{ + Durability: ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, + Status: ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED, + Fidelity: fidelity, + Object: &ateapipb.ObjectSnapshot{SnapshotUri: uri}, + }}, + } +} + +func newLocalSnapshot(gen int32, fidelity ateapipb.SnapshotFidelity, templateUID, name string) *ateapipb.Snapshot { + return &ateapipb.Snapshot{ + Generation: gen, + Owner: ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, + ActorTemplateUid: templateUID, + Storage: []*ateapipb.SnapshotStorage{{ + Durability: ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL, + Status: ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED, + Fidelity: fidelity, + Local: &ateapipb.LocalSnapshot{SnapshotName: name}, + }}, + } +} + +func durableSnapshot(status *ateapipb.ActorStatus) *ateapipb.Snapshot { + var best *ateapipb.Snapshot + for _, snap := range status.GetSnapshots() { + for _, st := range snap.GetStorage() { + if st.GetDurability() == ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE && + st.GetStatus() == ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED && + st.GetObject().GetSnapshotUri() != "" { + if best == nil || snap.GetGeneration() > best.GetGeneration() { + best = snap + } + } + } + } + return best +} + +func durableSnapshotStorage(status *ateapipb.ActorStatus) *ateapipb.SnapshotStorage { + snap := durableSnapshot(status) + for _, st := range snap.GetStorage() { + if st.GetDurability() == ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE { + return st + } + } + return nil +} + +func durableSnapshotURI(status *ateapipb.ActorStatus) string { + return durableSnapshotStorage(status).GetObject().GetSnapshotUri() +} + +func localSnapshot(status *ateapipb.ActorStatus) (*ateapipb.Snapshot, *ateapipb.LocalSnapshot) { + for _, snap := range status.GetSnapshots() { + for _, st := range snap.GetStorage() { + if st.GetDurability() == ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL && + st.GetStatus() == ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED && + st.GetLocal().GetSnapshotName() != "" { + return snap, st.GetLocal() + } + } + } + return nil, nil +} + +func tagSnapshotURI(tag *ateapipb.Tag) string { + for _, st := range tag.GetStatus().GetSnapshot().GetStorage() { + if st.GetDurability() == ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE { + return st.GetObject().GetSnapshotUri() + } + } + return "" +} + // testPauseImage is the pause image the default test SandboxConfig carries; // it is what a resolved WorkloadSpec's sandbox assets should name. const testPauseImage = "pause@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" @@ -503,18 +583,22 @@ func ensureGvisorSandboxConfig(t *testing.T, tc *testContext, name string) { sc := &atev1alpha1.SandboxConfig{ ObjectMeta: metav1.ObjectMeta{Name: name}, Spec: atev1alpha1.SandboxConfigSpec{ - SandboxClass: atev1alpha1.SandboxClassGvisor, - PauseImage: testPauseImage, - Assets: map[string]map[string]atev1alpha1.AssetFile{ - "amd64": {"runsc": { - URL: "gs://gvisor/releases/nightly/2026-05-19/x86_64/runsc", - SHA256: "a397be1abc2420d26bce6c70e6e2ff96c73aaaab929756c56f5e2089ea842b63", - }}, - "arm64": {"runsc": { - URL: "gs://gvisor/releases/nightly/2026-05-19/aarch64/runsc", - SHA256: "1ba2366ae2efceba166046f51a4104f9261c9cb72c6db8f5b3fe2dc57dea86b9", - }}, - }, + SandboxClass: atev1alpha1.SandboxClassGvisor, + DefaultVersion: "v1", + Versions: []atev1alpha1.SandboxVersionConfig{{ + Name: "v1", + PauseImage: testPauseImage, + Assets: map[string]map[string]atev1alpha1.AssetFile{ + "amd64": {"runsc": { + URL: "gs://gvisor/releases/nightly/2026-05-19/x86_64/runsc", + SHA256: "a397be1abc2420d26bce6c70e6e2ff96c73aaaab929756c56f5e2089ea842b63", + }}, + "arm64": {"runsc": { + URL: "gs://gvisor/releases/nightly/2026-05-19/aarch64/runsc", + SHA256: "1ba2366ae2efceba166046f51a4104f9261c9cb72c6db8f5b3fe2dc57dea86b9", + }}, + }, + }}, }, } if _, err := tc.substrateClient.ApiV1alpha1().SandboxConfigs().Create(context.Background(), sc, metav1.CreateOptions{}); err != nil && !apierrors.IsAlreadyExists(err) { @@ -537,8 +621,9 @@ func createWorkerPool(t *testing.T, tc *testContext, ns string, name string, lab Labels: labels, }, Spec: atev1alpha1.WorkerPoolSpec{ - Replicas: 1, - WorkerImage: "ateom@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + Replicas: 1, + WorkerImage: "ateom@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + SandboxClasses: []atev1alpha1.WorkerPoolSandboxClass{{Name: atev1alpha1.SandboxClassGvisor}}, }, } _, err := tc.substrateClient.ApiV1alpha1().WorkerPools(ns).Create(context.Background(), wp, metav1.CreateOptions{}) @@ -651,7 +736,7 @@ func createWorkerPod(t *testing.T, tc *testContext, ns string, name string, node WorkerPodUid: string(createdPod.UID), Ips: []string{"127.0.0.1"}, NodeName: nodeName, - SandboxClass: string(pool.Spec.SandboxClass), + SandboxClass: string(pool.Spec.DefaultSandboxClass()), Labels: pool.GetLabels(), // Capacity is not settable here: a Worker gets it from its own // ateom's report, which the reportWorkerCapacity below stands in diff --git a/cmd/ateapi/internal/controlapi/functionaltest/tag_test.go b/cmd/ateapi/internal/controlapi/functionaltest/tag_test.go index 0562bc91a6..205a022cb0 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/tag_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/tag_test.go @@ -47,43 +47,40 @@ func seedTag(t *testing.T, tc *testContext, actorName, tagName string, opts ...f t.Fatalf("NewActorSnapshotURI: %v", err) } actor, err = tc.persistence.UpdateActor(ctx, resources.ActorRefFromActor(actor), store.PreconditionFrom(actor), func(toUpdate *ateapipb.Actor) error { - toUpdate.Status.ExternalSnapshot = &ateapipb.ExternalSnapshot{SnapshotUri: actorSnapshotURI.String(), ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL} + toUpdate.Status.LastAssignedGeneration = 1 + toUpdate.Status.Snapshots = []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", actorSnapshotURI.String()), + } return nil }) if err != nil { t.Fatalf("recording the actor's external snapshot: %v", err) } + uri, err := resources.NewTagSnapshotURI(testStorageLocation, testAtespace, resources.NewSnapshotName()) + if err != nil { + t.Fatalf("NewTagSnapshotURI: %v", err) + } tag := &ateapipb.Tag{ Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: tagName}, Scope: ateapipb.TagScope_TAG_SCOPE_ATESPACE, SourceActor: resources.ActorRefFromActor(actor).ToObjectRef(), Status: &ateapipb.TagStatus{ - Snapshot: &ateapipb.ExternalSnapshot{ContentScope: actor.GetStatus().GetExternalSnapshot().GetContentScope()}, - StorageLocation: testStorageLocation, + Snapshot: &ateapipb.Snapshot{ + Generation: 1, + Owner: ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG, + Storage: []*ateapipb.SnapshotStorage{{ + Durability: ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, + Status: ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED, + Fidelity: durableSnapshotStorage(actor.GetStatus()).GetFidelity(), + Object: &ateapipb.ObjectSnapshot{SnapshotUri: uri.String()}, + }}, + }, }, } for _, opt := range opts { opt(tag) } - snapshot := tag.Status.Snapshot - tag.Status.Snapshot = nil - tag = storetest.MustCreateTag(t, ctx, tc.persistence, tag) - if snapshot == nil { - return tag - } - uri, err := resources.NewTagSnapshotURI(testStorageLocation, testAtespace, tag.GetMetadata().GetUid()) - if err != nil { - t.Fatalf("NewTagSnapshotURI: %v", err) - } - snapshot.SnapshotUri = uri.String() - tag, err = tc.persistence.UpdateTag(ctx, resources.TagRefFromTag(tag), store.PreconditionFrom(tag), func(toUpdate *ateapipb.Tag) error { - toUpdate.Status.Snapshot = snapshot - return nil - }) - if err != nil { - t.Fatalf("finalizing tag: %v", err) - } - return tag + return storetest.MustCreateTag(t, ctx, tc.persistence, tag) } // TestCreateTag_ReusedTagName tests that a tag does not move @@ -111,7 +108,7 @@ func TestCreateTag_ReusedTagName(t *testing.T) { if err != nil { t.Fatalf("CreateTag(actor-a) failed: %v", err) } - tagSnapshotURI := first.GetStatus().GetSnapshot().GetSnapshotUri() + tagSnapshotURI := tagSnapshotURI(first) // Tag owns a snapshot. So it must not be empty. wantObjects := snapshotObjectNames(t, tc, tagSnapshotURI) if len(wantObjects) == 0 { @@ -189,7 +186,7 @@ func suspendActorForTest(t *testing.T, tc *testContext, workerName, name string) if err != nil { t.Fatalf("SuspendActor(%s) failed: %v", name, err) } - uri := suspended.GetActor().GetStatus().GetExternalSnapshot().GetSnapshotUri() + uri := durableSnapshotURI(suspended.GetActor().GetStatus()) if uri == "" { t.Fatalf("SuspendActor(%s) wrote no external snapshot: %v", name, suspended) } diff --git a/cmd/ateapi/internal/controlapi/metrics.go b/cmd/ateapi/internal/controlapi/metrics.go index 291767fd74..8ace3106ea 100644 --- a/cmd/ateapi/internal/controlapi/metrics.go +++ b/cmd/ateapi/internal/controlapi/metrics.go @@ -87,7 +87,7 @@ func RegisterWorkerCount(meter metric.Meter, workers func() ([]*ateapipb.Worker, // list just means no seeding this cycle, not a broken observation. if pools, err := listPools(labels.Everything()); err == nil { for _, p := range pools { - class := string(p.Spec.SandboxClass) + class := string(p.Spec.DefaultSandboxClass()) if class == "" { class = string(atev1alpha1.SandboxClassGvisor) } @@ -202,7 +202,7 @@ func (i *Instruments) recordLifecycleOp(ctx context.Context, op string, start ti } // lifecycleOpAttrs builds the resume/suspend/pause dimensions from workflow -// state. Nil-safe, and omits the pool, snapshot-kind and snapshot-scope labels +// state. Nil-safe, and omits the pool, snapshot-kind and snapshot-fidelity labels // until they are known so a failure before the assign/restore steps never emits // an empty-string series. snapshotKind is empty for suspend/pause, which do not // restore; snapshotScope applies to all three and separates a full restore @@ -222,7 +222,7 @@ func lifecycleOpAttrs(actor *ateapipb.Actor, template *ateapipb.ActorTemplate, s attrs = append(attrs, ateattr.SnapshotKindKey.String(snapshotKind)) } if snapshotScope != "" { - attrs = append(attrs, ateattr.SnapshotScopeKey.String(snapshotScope)) + attrs = append(attrs, ateattr.SnapshotFidelityKey.String(snapshotScope)) } return attrs } diff --git a/cmd/ateapi/internal/controlapi/metrics_test.go b/cmd/ateapi/internal/controlapi/metrics_test.go index 8784b9c2ad..8f4b1f6a72 100644 --- a/cmd/ateapi/internal/controlapi/metrics_test.go +++ b/cmd/ateapi/internal/controlapi/metrics_test.go @@ -249,7 +249,7 @@ func TestLifecycleOpDurationShape(t *testing.T) { SandboxConfig: &ateapipb.SandboxConfig{SandboxClass: ateapipb.SandboxClass_SANDBOX_CLASS_GVISOR}, } inst.recordLifecycleOp(context.Background(), ateattr.OperationResume, time.Now(), nil, - lifecycleOpAttrs(actor, template, ateattr.SnapshotKindLatest, ateattr.SnapshotScopeData)...) + lifecycleOpAttrs(actor, template, ateattr.SnapshotKindLatest, ateattr.SnapshotFidelityVolumes)...) dp := singleHistogramDP(t, reader, lifecycleOpDurationMetric) assertAttrKeys(t, dp, @@ -260,7 +260,7 @@ func TestLifecycleOpDurationShape(t *testing.T) { ateattr.WorkerPoolNameKey, ateattr.SandboxClassKey, ateattr.SnapshotKindKey, - ateattr.SnapshotScopeKey, + ateattr.SnapshotFidelityKey, ) if op, _ := attrString(dp, ateattr.ActorOperationNameKey); op != ateattr.OperationResume { t.Errorf("operation = %q, want %q", op, ateattr.OperationResume) @@ -272,8 +272,8 @@ func TestLifecycleOpDurationShape(t *testing.T) { } // Kind and scope are independent: a data restore of the actor's // own latest snapshot must stay distinguishable from one of a local snapshot. - if scope, _ := attrString(dp, ateattr.SnapshotScopeKey); scope != ateattr.SnapshotScopeData { - t.Errorf("snapshot scope = %q, want %q", scope, ateattr.SnapshotScopeData) + if scope, _ := attrString(dp, ateattr.SnapshotFidelityKey); scope != ateattr.SnapshotFidelityVolumes { + t.Errorf("snapshot fidelity = %q, want %q", scope, ateattr.SnapshotFidelityVolumes) } if kind, _ := attrString(dp, ateattr.SnapshotKindKey); kind != ateattr.SnapshotKindLatest { t.Errorf("snapshot kind = %q, want %q", kind, ateattr.SnapshotKindLatest) @@ -288,7 +288,7 @@ func TestLifecycleOpAttrsOmitsUnknownScope(t *testing.T) { actor := &ateapipb.Actor{ActorTemplate: &ateapipb.ObjectRef{Atespace: "ate-agents", Name: "support-agent"}} for _, kv := range lifecycleOpAttrs(actor, nil, "", "") { switch kv.Key { - case ateattr.SnapshotScopeKey, ateattr.SnapshotKindKey, ateattr.WorkerPoolNamespaceKey, ateattr.WorkerPoolNameKey: + case ateattr.SnapshotFidelityKey, ateattr.SnapshotKindKey, ateattr.WorkerPoolNamespaceKey, ateattr.WorkerPoolNameKey: t.Errorf("attribute %s must be omitted while unknown, got %q", kv.Key, kv.Value.AsString()) } } @@ -400,7 +400,7 @@ func TestSchedulerAssignmentShapeAndOutcomes(t *testing.T) { func workerPool(namespace, name string, class atev1alpha1.SandboxClass) *atev1alpha1.WorkerPool { return &atev1alpha1.WorkerPool{ ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: name}, - Spec: atev1alpha1.WorkerPoolSpec{SandboxClass: class}, + Spec: atev1alpha1.WorkerPoolSpec{SandboxClasses: []atev1alpha1.WorkerPoolSandboxClass{{Name: class}}}, } } diff --git a/cmd/ateapi/internal/controlapi/sandbox_assets.go b/cmd/ateapi/internal/controlapi/sandbox_assets.go index 51160fcc54..b7aa641cce 100644 --- a/cmd/ateapi/internal/controlapi/sandbox_assets.go +++ b/cmd/ateapi/internal/controlapi/sandbox_assets.go @@ -49,7 +49,7 @@ func resolveTemplateSandboxConfig( // resolveSandboxAssets determines the sandbox binaries and pause image an actor // should boot with and projects them onto the ateletpb.SandboxAssets atelet -// fetches: the SandboxConfig the ActorTemplate names via +// fetches: the default version of the SandboxConfig the ActorTemplate names via // sandbox_config.config_name (required; enforced by CreateActorTemplate), // checked against the template's sandbox_class. func resolveSandboxAssets( @@ -67,18 +67,23 @@ func resolveSandboxAssets( if err != nil { return nil, err } - return sandboxAssetsProto(sc), nil + ver, ok := sc.Spec.DefaultVersionConfig() + if !ok { + return nil, apierror.FailedPrecondition( + "SandboxConfig %q has no enabled version %q named by its defaultVersion", sc.Name, sc.Spec.DefaultVersion) + } + return sandboxAssetsProto(sc, ver), nil } -// sandboxAssetsProto converts a resolved SandboxConfig into the proto atelet -// consumes. -func sandboxAssetsProto(sc *atev1alpha1.SandboxConfig) *ateletpb.SandboxAssets { +// sandboxAssetsProto converts a version of a resolved SandboxConfig into the +// proto atelet consumes. +func sandboxAssetsProto(sc *atev1alpha1.SandboxConfig, ver *atev1alpha1.SandboxVersionConfig) *ateletpb.SandboxAssets { out := &ateletpb.SandboxAssets{ SandboxClass: string(sc.Spec.SandboxClass), - PauseImage: sc.Spec.PauseImage, - Assets: make(map[string]*ateletpb.ArchAssets, len(sc.Spec.Assets)), + PauseImage: ver.PauseImage, + Assets: make(map[string]*ateletpb.ArchAssets, len(ver.Assets)), } - for arch, files := range sc.Spec.Assets { + for arch, files := range ver.Assets { archAssets := &ateletpb.ArchAssets{Files: make(map[string]*ateletpb.AssetFile, len(files))} for name, f := range files { archAssets.Files[name] = &ateletpb.AssetFile{Url: f.URL, Sha256: f.SHA256} diff --git a/cmd/ateapi/internal/controlapi/sandbox_assets_test.go b/cmd/ateapi/internal/controlapi/sandbox_assets_test.go index b037714c23..a9a98e3a8b 100644 --- a/cmd/ateapi/internal/controlapi/sandbox_assets_test.go +++ b/cmd/ateapi/internal/controlapi/sandbox_assets_test.go @@ -23,6 +23,7 @@ import ( "github.com/agent-substrate/substrate/pkg/proto/ateapipb" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/cache" + "k8s.io/utils/ptr" ) // sandboxConfigListerFor builds a lister over the given SandboxConfigs, using @@ -45,17 +46,46 @@ func testAssets() map[string]map[string]atev1alpha1.AssetFile { } // TestResolveSandboxAssets pins the template-side resolution: the config the -// template names is resolved (with its class checked), an empty name or an -// unrecognized class is an error, and the pause image travels with the -// sandbox binaries. +// template names is resolved (with its class checked) to its default version, +// a missing default version, an empty name or an unrecognized class is an +// error, and the pause image travels with the sandbox binaries. func TestResolveSandboxAssets(t *testing.T) { const namedPause = "gcr.io/gke-release/pause@sha256:named" namedConfig := &atev1alpha1.SandboxConfig{ ObjectMeta: metav1.ObjectMeta{Name: "gvisor-custom"}, Spec: atev1alpha1.SandboxConfigSpec{ - SandboxClass: atev1alpha1.SandboxClassGvisor, - PauseImage: namedPause, - Assets: testAssets(), + SandboxClass: atev1alpha1.SandboxClassGvisor, + DefaultVersion: "v2", + Versions: []atev1alpha1.SandboxVersionConfig{{ + Name: "v1", + PauseImage: "gcr.io/gke-release/pause@sha256:old", + Assets: testAssets(), + }, { + Name: "v2", + PauseImage: namedPause, + Assets: testAssets(), + }}, + }, + } + // A config whose defaultVersion names no entry; the CRD rejects this, but + // an object stored before the versions schema reads back this way. + noDefaultConfig := &atev1alpha1.SandboxConfig{ + ObjectMeta: metav1.ObjectMeta{Name: "gvisor-no-default"}, + Spec: atev1alpha1.SandboxConfigSpec{SandboxClass: atev1alpha1.SandboxClassGvisor}, + } + // A config whose defaultVersion names a Disabled entry; the CRD rejects + // this too, and it must not resolve to the disabled version. + disabledDefaultConfig := &atev1alpha1.SandboxConfig{ + ObjectMeta: metav1.ObjectMeta{Name: "gvisor-disabled-default"}, + Spec: atev1alpha1.SandboxConfigSpec{ + SandboxClass: atev1alpha1.SandboxClassGvisor, + DefaultVersion: "v1", + Versions: []atev1alpha1.SandboxVersionConfig{{ + Name: "v1", + State: ptr.To(atev1alpha1.SandboxVersionStateDisabled), + PauseImage: namedPause, + Assets: testAssets(), + }}, }, } @@ -71,6 +101,20 @@ func TestResolveSandboxAssets(t *testing.T) { ConfigName: "gvisor-custom", }, wantPauseImage: namedPause, + }, { + name: "named config without its default version", + sandbox: &ateapipb.SandboxConfig{ + SandboxClass: ateapipb.SandboxClass_SANDBOX_CLASS_GVISOR, + ConfigName: "gvisor-no-default", + }, + wantErr: `SandboxConfig "gvisor-no-default" has no enabled version ""`, + }, { + name: "named config whose default version is disabled", + sandbox: &ateapipb.SandboxConfig{ + SandboxClass: ateapipb.SandboxClass_SANDBOX_CLASS_GVISOR, + ConfigName: "gvisor-disabled-default", + }, + wantErr: `SandboxConfig "gvisor-disabled-default" has no enabled version "v1"`, }, { name: "named config class mismatch", sandbox: &ateapipb.SandboxConfig{ @@ -99,7 +143,7 @@ func TestResolveSandboxAssets(t *testing.T) { }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - configLister := sandboxConfigListerFor(t, []*atev1alpha1.SandboxConfig{namedConfig}) + configLister := sandboxConfigListerFor(t, []*atev1alpha1.SandboxConfig{namedConfig, noDefaultConfig, disabledDefaultConfig}) got, err := resolveSandboxAssets(configLister, tt.sandbox) if tt.wantErr != "" { diff --git a/cmd/ateapi/internal/controlapi/service.go b/cmd/ateapi/internal/controlapi/service.go index 567cd2f72d..409ea64298 100644 --- a/cmd/ateapi/internal/controlapi/service.go +++ b/cmd/ateapi/internal/controlapi/service.go @@ -23,12 +23,12 @@ import ( "github.com/agent-substrate/substrate/cmd/ateapi/internal/workercache" "github.com/agent-substrate/substrate/internal/localca" "github.com/agent-substrate/substrate/internal/localjwtauthority" - "github.com/agent-substrate/substrate/internal/objectstore" "github.com/agent-substrate/substrate/internal/resources" "github.com/agent-substrate/substrate/internal/volume" "github.com/agent-substrate/substrate/internal/volume/csi" listersv1alpha1 "github.com/agent-substrate/substrate/pkg/client/listers/api/v1alpha1" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + objectstorev1 "github.com/agent-substrate/substrate/pkg/proto/objectstorepb/v1" storagev1listers "k8s.io/client-go/listers/storage/v1" ) @@ -50,7 +50,7 @@ type RPCService struct { instruments *Instruments mu sync.RWMutex volumePlugins map[string]volume.VolumePluginControlPlane - objectStore objectstore.Store + snapshotPlugin objectstorev1.ControlProviderClient actorJWTIssuer string actorIDJWTPool localjwtauthority.Pool @@ -69,9 +69,9 @@ type VolumePluginRegistry interface { // // instruments may be nil; the record helpers no-op. // -// objectStore may be nil, which leaves external snapshots in place instead of -// copying and releasing them. Only tests that never reach those steps pass nil; -// ate-api always builds one. +// snapshotPlugin may be nil, which leaves external snapshots in place instead +// of copying and releasing them. Only tests that never reach those steps pass +// nil; ate-api always builds one. // // actorJWTIssuer is copied verbatim into the iss claim of every actor JWT. // actorWorkflowDeadline bounds each Resume/Suspend workflow end-to-end. @@ -86,7 +86,7 @@ func NewRPCService( egressGatewayAddress string, actorWorkflowDeadline time.Duration, volumePlugins map[string]volume.VolumePluginControlPlane, - objectStore objectstore.Store, + snapshotPlugin objectstorev1.ControlProviderClient, actorJWTIssuer string, actorIDJWTPool localjwtauthority.Pool, actorIDCAPool localca.Pool, @@ -101,12 +101,12 @@ func NewRPCService( dialer: dialer, instruments: instruments, volumePlugins: volumePlugins, - objectStore: objectStore, + snapshotPlugin: snapshotPlugin, actorJWTIssuer: actorJWTIssuer, actorIDJWTPool: actorIDJWTPool, actorIDCAPool: actorIDCAPool, } - s.actorWorkflow = NewActorWorkflow(impl, workerCache, dialer, sandboxConfigLister, storageClassLister, instruments, egressGatewayAddress, s, actorWorkflowDeadline, objectStore) + s.actorWorkflow = NewActorWorkflow(impl, workerCache, dialer, sandboxConfigLister, storageClassLister, instruments, egressGatewayAddress, s, actorWorkflowDeadline, snapshotPlugin) s.workerWorkflow = NewWorkerWorkflow(impl) return s } diff --git a/cmd/ateapi/internal/controlapi/snapshot_helpers.go b/cmd/ateapi/internal/controlapi/snapshot_helpers.go new file mode 100644 index 0000000000..e493914b49 --- /dev/null +++ b/cmd/ateapi/internal/controlapi/snapshot_helpers.go @@ -0,0 +1,186 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlapi + +import ( + "math" + + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" +) + +// findSnapshotByGeneration returns the Snapshot in snapshots with the given +// generation, or nil if none exists. +func findSnapshotByGeneration(snapshots []*ateapipb.Snapshot, gen int32) *ateapipb.Snapshot { + for _, snap := range snapshots { + if snap.GetGeneration() == gen { + return snap + } + } + return nil +} + +// snapshotAtLatestGeneration returns the Snapshot at status.last_assigned_generation, or +// nil if none exists. +func snapshotAtLatestGeneration(status *ateapipb.ActorStatus) *ateapipb.Snapshot { + if status == nil { + return nil + } + return findSnapshotByGeneration(status.GetSnapshots(), status.GetLastAssignedGeneration()) +} + +// findSnapshotStorage returns the SnapshotStorage entry on snap with the given +// durability, or nil if none exists. +func findSnapshotStorage(snap *ateapipb.Snapshot, durability ateapipb.SnapshotDurability) *ateapipb.SnapshotStorage { + for _, st := range snap.GetStorage() { + if st.GetDurability() == durability { + return st + } + } + return nil +} + +// setSnapshotStorage replaces an existing SnapshotStorage entry on snap with +// the same durability, or appends entry if none exists. +func setSnapshotStorage(snap *ateapipb.Snapshot, entry *ateapipb.SnapshotStorage) { + for i, st := range snap.Storage { + if st.GetDurability() == entry.GetDurability() { + snap.Storage[i] = entry + return + } + } + snap.Storage = append(snap.Storage, entry) +} + +// removeSnapshotStorage removes any SnapshotStorage entry on snap with the +// given durability. +func removeSnapshotStorage(snap *ateapipb.Snapshot, durability ateapipb.SnapshotDurability) { + if snap == nil { + return + } + filtered := snap.Storage[:0] + for _, st := range snap.Storage { + if st.GetDurability() != durability { + filtered = append(filtered, st) + } + } + snap.Storage = filtered +} + +// findLatestSnapshotStorage returns the highest-generation Snapshot on status +// that holds a SnapshotStorage entry matching durability and storageStatus, +// along with that SnapshotStorage entry, or (nil, nil) if none exists. +func findLatestSnapshotStorage(status *ateapipb.ActorStatus, durability ateapipb.SnapshotDurability, storageStatus ateapipb.SnapshotStorageStatus) (*ateapipb.Snapshot, *ateapipb.SnapshotStorage) { + var bestSnap *ateapipb.Snapshot + var bestStorage *ateapipb.SnapshotStorage + for _, snap := range status.GetSnapshots() { + st := findSnapshotStorage(snap, durability) + if st.GetStatus() != storageStatus { + continue + } + if bestSnap == nil || snap.GetGeneration() >= bestSnap.GetGeneration() { + bestSnap = snap + bestStorage = st + } + } + return bestSnap, bestStorage +} + +// removeSnapshotStorageEntries removes SnapshotStorage entries matching +// durability (and *storageStatus, if non-nil) from status.Snapshots, dropping +// any Snapshot left with no storage entries. +// +// Called from ensureWorkerReleased during DeleteActor, ensureRevertedFinalized +// during revert, and ensureSuspendedFinalized during suspend. +func removeSnapshotStorageEntries(status *ateapipb.ActorStatus, durability ateapipb.SnapshotDurability, storageStatus *ateapipb.SnapshotStorageStatus) { + removeOlderSnapshotStorageEntries(status, durability, storageStatus, math.MaxInt32) +} + +// removeOlderSnapshotStorageEntries removes SnapshotStorage entries matching +// durability (and *storageStatus, if non-nil) from Snapshots in +// status.Snapshots with generation older than keepGen, dropping any Snapshot +// left with no storage entries. +// +// status.LastAssignedGeneration is not decreased so that we can keep track of +// stale snapshots that need to be cleaned up and discarded generation numbers +// are never reused. +// +// Called from ensurePausedFinalized on pause success and +// ensureSuspendedFinalized during suspend. +func removeOlderSnapshotStorageEntries(status *ateapipb.ActorStatus, durability ateapipb.SnapshotDurability, storageStatus *ateapipb.SnapshotStorageStatus, keepGen int32) { + if status == nil { + return + } + kept := status.Snapshots[:0] + for _, snap := range status.Snapshots { + if snap.GetGeneration() < keepGen { + storage := snap.Storage[:0] + for _, st := range snap.Storage { + if st.GetDurability() == durability && + (storageStatus == nil || st.GetStatus() == *storageStatus) { + continue + } + storage = append(storage, st) + } + snap.Storage = storage + } + if len(snap.GetStorage()) > 0 { + kept = append(kept, snap) + } + } + status.Snapshots = kept +} + +// newDurableSnapshot constructs a Snapshot with a single DURABLE ObjectSnapshot +// storage entry. +func newDurableSnapshot(gen int32, owner ateapipb.SnapshotOwner, fidelity ateapipb.SnapshotFidelity, templateUID, uri string, storageStatus ateapipb.SnapshotStorageStatus) *ateapipb.Snapshot { + return &ateapipb.Snapshot{ + Generation: gen, + Owner: owner, + ActorTemplateUid: templateUID, + Storage: []*ateapipb.SnapshotStorage{{ + Durability: ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, + Status: storageStatus, + Fidelity: fidelity, + Object: &ateapipb.ObjectSnapshot{SnapshotUri: uri}, + }}, + } +} + +// newLocalSnapshot constructs an actor Snapshot with a single LOCAL +// LocalSnapshot storage entry. +func newLocalSnapshot(gen int32, fidelity ateapipb.SnapshotFidelity, templateUID, snapshotName string, storageStatus ateapipb.SnapshotStorageStatus) *ateapipb.Snapshot { + return &ateapipb.Snapshot{ + Generation: gen, + Owner: ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, + ActorTemplateUid: templateUID, + Storage: []*ateapipb.SnapshotStorage{{ + Durability: ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL, + Status: storageStatus, + Fidelity: fidelity, + Local: &ateapipb.LocalSnapshot{SnapshotName: snapshotName}, + }}, + } +} + +// tagDurableSnapshotURI returns the durable snapshot URI on tag matching +// storageStatus (or any status if storageStatus is UNSPECIFIED), or "" if none +// exists. +func tagDurableSnapshotURI(tag *ateapipb.Tag, storageStatus ateapipb.SnapshotStorageStatus) string { + st := findSnapshotStorage(tag.GetStatus().GetSnapshot(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE) + if storageStatus != ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_UNSPECIFIED && st.GetStatus() != storageStatus { + return "" + } + return st.GetObject().GetSnapshotUri() +} diff --git a/cmd/ateapi/internal/controlapi/snapshot_helpers_test.go b/cmd/ateapi/internal/controlapi/snapshot_helpers_test.go new file mode 100644 index 0000000000..0c00dc49ca --- /dev/null +++ b/cmd/ateapi/internal/controlapi/snapshot_helpers_test.go @@ -0,0 +1,230 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlapi + +import ( + "testing" + + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "github.com/google/go-cmp/cmp" + "google.golang.org/protobuf/testing/protocmp" +) + +func TestFindSnapshotByGenerationAndSnapshotAtLatestGeneration(t *testing.T) { + if got := snapshotAtLatestGeneration(nil); got != nil { + t.Errorf("snapshotAtLatestGeneration(nil) = %v, want nil", got) + } + if got := findSnapshotByGeneration(nil, 1); got != nil { + t.Errorf("findSnapshotByGeneration(nil, 1) = %v, want nil", got) + } + + s1 := newLocalSnapshot(1, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "local-1", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + s2 := newDurableSnapshot(2, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "gs://b/snap-2", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + status := &ateapipb.ActorStatus{ + LastAssignedGeneration: 2, + Snapshots: []*ateapipb.Snapshot{s1, s2}, + } + + if got := findSnapshotByGeneration(status.GetSnapshots(), 1); got != s1 { + t.Errorf("findSnapshotByGeneration(1) = %v, want %v", got, s1) + } + if got := findSnapshotByGeneration(status.GetSnapshots(), 99); got != nil { + t.Errorf("findSnapshotByGeneration(99) = %v, want nil", got) + } + if got := snapshotAtLatestGeneration(status); got != s2 { + t.Errorf("snapshotAtLatestGeneration() = %v, want %v", got, s2) + } +} + +func TestSnapshotStorageAccessors(t *testing.T) { + if got := findSnapshotStorage(nil, ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL); got != nil { + t.Errorf("findSnapshotStorage(nil) = %v, want nil", got) + } + // Should not panic on nil snapshot. + removeSnapshotStorage(nil, ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL) + + snap := newLocalSnapshot(1, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "local-in-progress", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS) + + // Replace existing LOCAL entry. + completedLocal := &ateapipb.SnapshotStorage{ + Durability: ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL, + Status: ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED, + Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + Local: &ateapipb.LocalSnapshot{SnapshotName: "local-completed"}, + } + setSnapshotStorage(snap, completedLocal) + if len(snap.GetStorage()) != 1 { + t.Fatalf("len(snap.Storage) = %d, want 1", len(snap.GetStorage())) + } + if got := findSnapshotStorage(snap, ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL); got != completedLocal { + t.Errorf("findSnapshotStorage(LOCAL) = %v, want %v", got, completedLocal) + } + + // Append DURABLE entry alongside LOCAL. + durableEntry := &ateapipb.SnapshotStorage{ + Durability: ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, + Status: ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED, + Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + Object: &ateapipb.ObjectSnapshot{SnapshotUri: "gs://b/snap-1"}, + } + setSnapshotStorage(snap, durableEntry) + if len(snap.GetStorage()) != 2 { + t.Fatalf("len(snap.Storage) = %d, want 2", len(snap.GetStorage())) + } + if got := findSnapshotStorage(snap, ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE); got != durableEntry { + t.Errorf("findSnapshotStorage(DURABLE) = %v, want %v", got, durableEntry) + } + + // Remove LOCAL and verify DURABLE remains. + removeSnapshotStorage(snap, ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL) + if got := findSnapshotStorage(snap, ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL); got != nil { + t.Errorf("findSnapshotStorage(LOCAL) after remove = %v, want nil", got) + } + if got := findSnapshotStorage(snap, ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE); got != durableEntry { + t.Errorf("findSnapshotStorage(DURABLE) after removing LOCAL = %v, want %v", got, durableEntry) + } +} + +func TestFindLatestSnapshotStorage(t *testing.T) { + if snap, st := findLatestSnapshotStorage(nil, ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED); snap != nil || st != nil { + t.Errorf("findLatestSnapshotStorage(nil) = (%v, %v), want (nil, nil)", snap, st) + } + + s1 := newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "gs://b/snap-1", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + s2 := newDurableSnapshot(2, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "gs://b/snap-2", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + s3 := newDurableSnapshot(3, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "gs://b/snap-3", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS) + s4 := newLocalSnapshot(4, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "local-4", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + + // Place s2 before s1 to verify generation comparison rather than slice order. + status := &ateapipb.ActorStatus{ + Snapshots: []*ateapipb.Snapshot{s2, s1, s3, s4}, + } + + gotSnap, gotSt := findLatestSnapshotStorage(status, ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + if gotSnap != s2 || gotSt.GetObject().GetSnapshotUri() != "gs://b/snap-2" { + t.Errorf("findLatestSnapshotStorage(DURABLE, COMPLETED) = (gen %d, %v), want (gen 2, gs://b/snap-2)", gotSnap.GetGeneration(), gotSt) + } + + gotSnap, gotSt = findLatestSnapshotStorage(status, ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS) + if gotSnap != s3 || gotSt.GetObject().GetSnapshotUri() != "gs://b/snap-3" { + t.Errorf("findLatestSnapshotStorage(DURABLE, IN_PROGRESS) = (gen %d, %v), want (gen 3, gs://b/snap-3)", gotSnap.GetGeneration(), gotSt) + } + + gotSnap, gotSt = findLatestSnapshotStorage(status, ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + if gotSnap != s4 || gotSt.GetLocal().GetSnapshotName() != "local-4" { + t.Errorf("findLatestSnapshotStorage(LOCAL, COMPLETED) = (gen %d, %v), want (gen 4, local-4)", gotSnap.GetGeneration(), gotSt) + } + + gotSnap, gotSt = findLatestSnapshotStorage(status, ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS) + if gotSnap != nil || gotSt != nil { + t.Errorf("findLatestSnapshotStorage(LOCAL, IN_PROGRESS) = (%v, %v), want (nil, nil)", gotSnap, gotSt) + } +} + +func TestRemoveSnapshotStorageEntries(t *testing.T) { + // Should not panic on nil status. + removeSnapshotStorageEntries(nil, ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL, nil) + removeOlderSnapshotStorageEntries(nil, ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL, nil, 3) + + t.Run("removes older local snapshots while preserving keepGen, newer generations, and durable storage", func(t *testing.T) { + bothGen1 := newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "gs://b/snap-1", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + setSnapshotStorage(bothGen1, &ateapipb.SnapshotStorage{ + Durability: ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL, + Status: ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED, + Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + Local: &ateapipb.LocalSnapshot{SnapshotName: "local-1"}, + }) + localGen2 := newLocalSnapshot(2, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "local-2", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + localGen3 := newLocalSnapshot(3, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "local-3", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + localGen4 := newLocalSnapshot(4, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "local-4", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS) + + status := &ateapipb.ActorStatus{ + LastAssignedGeneration: 4, + Snapshots: []*ateapipb.Snapshot{bothGen1, localGen2, localGen3, localGen4}, + } + + removeOlderSnapshotStorageEntries(status, ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL, nil, 3) + + want := []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "gs://b/snap-1", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + newLocalSnapshot(3, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "local-3", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + newLocalSnapshot(4, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "local-4", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS), + } + if diff := cmp.Diff(want, status.GetSnapshots(), protocmp.Transform()); diff != "" { + t.Errorf("status.Snapshots mismatch (-want +got):\n%s", diff) + } + if status.GetLastAssignedGeneration() != 4 { + t.Errorf("LastAssignedGeneration = %d, want 4", status.GetLastAssignedGeneration()) + } + }) + + t.Run("removes all matching snapshot storage entries across all generations", func(t *testing.T) { + status := &ateapipb.ActorStatus{ + Snapshots: []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "gs://b/snap-1", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + newLocalSnapshot(2, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "local-2", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + newLocalSnapshot(3, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "local-3", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS), + }, + } + + removeSnapshotStorageEntries(status, ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL, nil) + + want := []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "gs://b/snap-1", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + } + if diff := cmp.Diff(want, status.GetSnapshots(), protocmp.Transform()); diff != "" { + t.Errorf("status.Snapshots mismatch (-want +got):\n%s", diff) + } + }) + + t.Run("filters by specific storageStatus", func(t *testing.T) { + status := &ateapipb.ActorStatus{ + Snapshots: []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "gs://b/snap-1", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS), + newDurableSnapshot(2, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "gs://b/snap-2", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + }, + } + + removeSnapshotStorageEntries(status, ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, new(ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS)) + + want := []*ateapipb.Snapshot{ + newDurableSnapshot(2, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "gs://b/snap-2", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + } + if diff := cmp.Diff(want, status.GetSnapshots(), protocmp.Transform()); diff != "" { + t.Errorf("status.Snapshots mismatch (-want +got):\n%s", diff) + } + }) +} + +func TestTagDurableSnapshotURI(t *testing.T) { + if got := tagDurableSnapshotURI(nil, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_UNSPECIFIED); got != "" { + t.Errorf("tagDurableSnapshotURI(nil) = %q, want empty", got) + } + + inProgressTag := &ateapipb.Tag{ + Status: &ateapipb.TagStatus{ + Snapshot: newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-1", "gs://b/tag-1", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS), + }, + } + if got := tagDurableSnapshotURI(inProgressTag, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED); got != "" { + t.Errorf("tagDurableSnapshotURI(inProgressTag, COMPLETED) = %q, want empty", got) + } + if got := tagDurableSnapshotURI(inProgressTag, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS); got != "gs://b/tag-1" { + t.Errorf("tagDurableSnapshotURI(inProgressTag, IN_PROGRESS) = %q, want gs://b/tag-1", got) + } + if got := tagDurableSnapshotURI(inProgressTag, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_UNSPECIFIED); got != "gs://b/tag-1" { + t.Errorf("tagDurableSnapshotURI(inProgressTag, UNSPECIFIED) = %q, want gs://b/tag-1", got) + } +} diff --git a/cmd/ateapi/internal/controlapi/tag.go b/cmd/ateapi/internal/controlapi/tag.go index 7a3b3d6b7e..6c8789e7b8 100644 --- a/cmd/ateapi/internal/controlapi/tag.go +++ b/cmd/ateapi/internal/controlapi/tag.go @@ -123,7 +123,7 @@ func (s *RPCService) UpdateTag(ctx context.Context, req *ateapipb.UpdateTagReque // A tag whose create never finished names a partial copy. Publishing it // — or changing its scope at all — would hand out content that is still // being written, or may never be. - if toUpdate.GetStatus().GetSnapshot().GetSnapshotUri() == "" { + if tagDurableSnapshotURI(toUpdate, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) == "" { return errTagPending } // Metadata and status are server-owned fields. diff --git a/cmd/ateapi/internal/controlapi/tag_test.go b/cmd/ateapi/internal/controlapi/tag_test.go index 38002251c0..90b3779578 100644 --- a/cmd/ateapi/internal/controlapi/tag_test.go +++ b/cmd/ateapi/internal/controlapi/tag_test.go @@ -135,7 +135,7 @@ func TestUpdateTag(t *testing.T) { req: &ateapipb.Tag{ Scope: ateapipb.TagScope_TAG_SCOPE_PUBLISHED, Status: &ateapipb.TagStatus{ - Snapshot: &ateapipb.ExternalSnapshot{SnapshotUri: "gs://attacker/elsewhere", ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA}, + Snapshot: newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, "", "gs://attacker/elsewhere", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), ActorTemplateUid: "other-template-uid", }, }, @@ -249,7 +249,10 @@ func newTestSuspendedActor(t *testing.T, ctx context.Context, st store.Interface t.Fatalf("NewActorSnapshotURI: %v", err) } return mustUpdateActorStatus(t, ctx, st, actor, func(status *ateapipb.ActorStatus) { - status.ExternalSnapshot = &ateapipb.ExternalSnapshot{SnapshotUri: uri.String(), ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL} + status.LastAssignedGeneration = 1 + status.Snapshots = []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", uri.String(), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + } }) } @@ -263,27 +266,24 @@ func newTestTag(t *testing.T, name string, actor *ateapipb.Actor) *ateapipb.Tag if err != nil { t.Fatalf("NewTagSnapshotURI: %v", err) } + actorSnap, actorSt := findLatestSnapshotStorage(actor.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) return &ateapipb.Tag{ Metadata: &ateapipb.ResourceMetadata{Atespace: atespace, Name: name}, Scope: ateapipb.TagScope_TAG_SCOPE_ATESPACE, SourceActor: resources.ActorRefFromActor(actor).ToObjectRef(), Status: &ateapipb.TagStatus{ - Snapshot: &ateapipb.ExternalSnapshot{ - SnapshotUri: uri.String(), - ContentScope: actor.GetStatus().GetExternalSnapshot().GetContentScope(), - }, + Snapshot: newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG, actorSt.GetFidelity(), actorSnap.GetActorTemplateUid(), uri.String(), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), }, } } // newPendingTestTag builds the tag a create leaves behind when it dies between // reserving the name and finishing the copy: the row names the prefix it was -// writing into, and nothing else. +// writing into as IN_PROGRESS. func newPendingTestTag(t *testing.T, name string, actor *ateapipb.Actor) *ateapipb.Tag { t.Helper() tag := newTestTag(t, name, actor) - tag.Status.StorageLocation = testStorageLocation - tag.Status.Snapshot = nil + tag.Status.Snapshot.Storage[0].Status = ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS return tag } diff --git a/cmd/ateapi/internal/controlapi/template_reconciler.go b/cmd/ateapi/internal/controlapi/template_reconciler.go index 131bc80e96..03ef425127 100644 --- a/cmd/ateapi/internal/controlapi/template_reconciler.go +++ b/cmd/ateapi/internal/controlapi/template_reconciler.go @@ -224,7 +224,7 @@ func (r *ActorTemplateReconciler) reconcileOne(ctx context.Context, ref resource if tag.GetStatus().GetActorTemplateUid() != tmpl.GetMetadata().GetUid() || resources.ActorRefFromObjectRef(tag.GetSourceActor()) != resources.ActorRefFromObjectRef(goldenActorRef) { return 0, r.fail(ctx, tmpl, reasonGoldenTagConflict, "golden tag belongs to another actor or template") } - if tag.GetStatus().GetSnapshot().GetSnapshotUri() != "" { + if tagDurableSnapshotURI(tag, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) != "" { return 0, r.saveGoldenTag(ctx, tmpl, goldenActorRef) } // CreateTag cannot resume an incomplete copy. Delete it before retrying. @@ -284,7 +284,7 @@ func (r *ActorTemplateReconciler) reconcileOne(ctx context.Context, ref resource ateapipb.ActorState_ACTOR_STATE_SUSPENDED: // The golden actor was never resumed, or a previous resume didn't // finish; ResumeActor is reentrant from both. - if actor.GetStatus().GetExternalSnapshot().GetSnapshotUri() != "" { + if _, st := findLatestSnapshotStorage(actor.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED); st != nil { // Golden actors never start from a source snapshot, so an // existing snapshot means an earlier suspend completed // without being recorded. @@ -319,8 +319,7 @@ func (r *ActorTemplateReconciler) suspendActor(ctx context.Context, goldenRef *a // A crash during suspend is observed as CRASHED on the retry. return fmt.Errorf("while suspending golden actor: %w", err) } - suspended := resp.GetActor().GetStatus().GetExternalSnapshot() - if suspended.GetSnapshotUri() == "" { + if _, st := findLatestSnapshotStorage(resp.GetActor().GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED); st == nil { return fmt.Errorf("suspending golden actor produced no external snapshot") } return nil diff --git a/cmd/ateapi/internal/controlapi/template_reconciler_test.go b/cmd/ateapi/internal/controlapi/template_reconciler_test.go index 70010aed7d..7effb0c55b 100644 --- a/cmd/ateapi/internal/controlapi/template_reconciler_test.go +++ b/cmd/ateapi/internal/controlapi/template_reconciler_test.go @@ -182,6 +182,17 @@ func (c *fakeGoldenControl) CreateAtespace(_ context.Context, req *ateapipb.Crea return req.GetAtespace(), nil } +func (c *fakeGoldenControl) goldenActorStatus() *ateapipb.ActorStatus { + st := &ateapipb.ActorStatus{State: c.goldenState} + if c.goldenSnapshot != "" { + st.LastAssignedGeneration = 1 + st.Snapshots = []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, testTemplateUID, c.goldenSnapshot, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + } + } + return st +} + func (c *fakeGoldenControl) CreateActor(_ context.Context, req *ateapipb.CreateActorRequest) (*ateapipb.Actor, error) { c.mu.Lock() defer c.mu.Unlock() @@ -198,7 +209,7 @@ func (c *fakeGoldenControl) CreateActor(_ context.Context, req *ateapipb.CreateA // a status observing the initial SUSPENDED state. return &ateapipb.Actor{ Metadata: req.GetActor().GetMetadata(), - Status: &ateapipb.ActorStatus{State: c.goldenState, ExternalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: c.goldenSnapshot}}, + Status: c.goldenActorStatus(), }, nil } @@ -213,7 +224,7 @@ func (c *fakeGoldenControl) GetActor(_ context.Context, req *ateapipb.GetActorRe } return &ateapipb.Actor{ Metadata: &ateapipb.ResourceMetadata{Atespace: req.GetActor().GetAtespace(), Name: req.GetActor().GetName()}, - Status: &ateapipb.ActorStatus{State: c.goldenState, ExternalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: c.goldenSnapshot}}, + Status: c.goldenActorStatus(), }, nil } @@ -240,7 +251,7 @@ func (c *fakeGoldenControl) SuspendActor(_ context.Context, req *ateapipb.Suspen c.goldenSnapshot = c.snapshot } return &ateapipb.SuspendActorResponse{ - Actor: &ateapipb.Actor{Status: &ateapipb.ActorStatus{ExternalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: c.goldenSnapshot}}}, + Actor: &ateapipb.Actor{Status: c.goldenActorStatus()}, }, nil } @@ -261,7 +272,10 @@ func (c *fakeGoldenControl) CreateTag(_ context.Context, req *ateapipb.CreateTag return nil, c.tagErr } c.tag = proto.CloneOf(req.GetTag()) - c.tag.Status = &ateapipb.TagStatus{ActorTemplateUid: testTemplateUID, Snapshot: &ateapipb.ExternalSnapshot{SnapshotUri: c.goldenSnapshot}} + c.tag.Status = &ateapipb.TagStatus{ + ActorTemplateUid: testTemplateUID, + Snapshot: newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, testTemplateUID, c.goldenSnapshot, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + } return proto.CloneOf(c.tag), nil } @@ -606,7 +620,7 @@ func TestReconcileOne(t *testing.T) { // requests the happy path issues: the golden actor is named after the // template UID so recreated templates with the same name never collide, and // lives in the reserved ate-golden atespace so the suspend workflow commits -// it Full regardless of the template's onCommit scope. +// it Full regardless of the template's preferredFidelity scope. func TestReconcileOne_GoldenActorRequests(t *testing.T) { ctx := context.Background() st := newFakeTemplateStore(testTemplate()) @@ -856,7 +870,10 @@ func TestReconcileOne_GoldenTagRecovery(t *testing.T) { Metadata: &ateapipb.ResourceMetadata{Atespace: ref.Atespace, Name: ref.Name}, SourceActor: ref, Scope: ateapipb.TagScope_TAG_SCOPE_PUBLISHED, - Status: &ateapipb.TagStatus{ActorTemplateUid: testTemplateUID, Snapshot: &ateapipb.ExternalSnapshot{SnapshotUri: "gs://bucket/tag-snapshot"}}, + Status: &ateapipb.TagStatus{ + ActorTemplateUid: testTemplateUID, + Snapshot: newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, testTemplateUID, "gs://bucket/tag-snapshot", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + }, } incomplete := proto.CloneOf(completed) incomplete.Status.Snapshot = nil @@ -913,7 +930,7 @@ func TestReconcileOne_GoldenTagRecovery(t *testing.T) { if !proto.Equal(st.storedStatus(t, testTemplateRef).GetGoldenSnapshotStatus().GetGoldenTag(), ref) { t.Fatal("golden tag not recorded") } - if control.tag.GetStatus().GetSnapshot().GetSnapshotUri() == "" { + if tagDurableSnapshotURI(control.tag, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) == "" { t.Fatal("golden tag has no snapshot") } if len(control.createReqs) != 0 || len(control.resumeReqs) != 0 || len(control.suspendReqs) != 0 { diff --git a/cmd/ateapi/internal/controlapi/volumes.go b/cmd/ateapi/internal/controlapi/volumes.go index efd961eaeb..b2f8292599 100644 --- a/cmd/ateapi/internal/controlapi/volumes.go +++ b/cmd/ateapi/internal/controlapi/volumes.go @@ -47,7 +47,7 @@ func initialActorVolumes(ctx context.Context, scLister storagev1listers.StorageC } volumes = append(volumes, &ateapipb.ExternalVolume{ - VolumeName: vol.GetName(), + Name: vol.GetName(), VolumeType: sc.Provisioner, Status: ateapipb.ExternalVolume_STATUS_PENDING, }) @@ -75,7 +75,7 @@ func createActorVolumes(ctx context.Context, registry VolumePluginRegistry, scLi currentIdx = idx var specVol *ateapipb.Volume - volName := vol.GetVolumeName() + volName := vol.GetName() for _, tVol := range template.GetVolumes() { if tVol.GetName() == volName { specVol = tVol @@ -128,7 +128,7 @@ func createActorVolumes(ctx context.Context, registry VolumePluginRegistry, scLi } resultVolumes = append(resultVolumes, &ateapipb.ExternalVolume{ - VolumeName: volName, + Name: volName, StorageVolumeId: resp.VolumeID, VolumeType: sc.Provisioner, Status: ateapipb.ExternalVolume_STATUS_CREATED, @@ -150,7 +150,7 @@ func deleteActorVolumes(ctx context.Context, registry VolumePluginRegistry, acto // If the volume hasn't been successfully created yet, it's possible // that it doesn't have a storage volume ID. In that case, fallback // to the original requested volID. - volID = actorVolumeID(actorUID, vol.GetVolumeName()) + volID = actorVolumeID(actorUID, vol.GetName()) } // TODO: Standardize volume plugin lookup and error handling across control plane // and worker plane (e.g. via a shared helper). @@ -177,7 +177,7 @@ func getMountedActorVolumes(ctx context.Context, ref *ateapipb.ObjectRef, volume // Find the corresponding volume in the ActorTemplate to check if it's mounted var matchedTemplateVol *ateapipb.Volume for _, tVol := range template.GetVolumes() { - if vol.GetVolumeName() == tVol.GetName() { + if vol.GetName() == tVol.GetName() { matchedTemplateVol = tVol break } @@ -214,9 +214,9 @@ func detachActorVolumes(ctx context.Context, registry VolumePluginRegistry, acto // in the template's containers. If the template is missing/deleted, fall back to // attempting detachment for all external volumes recorded on the actor so we do // not orphan attached disks on the worker node. - volumesToDetach := actor.GetStatus().GetActorVolumes() + volumesToDetach := actor.GetStatus().GetExternalVolumes() if template != nil { - volumesToDetach = getMountedActorVolumes(ctx, ref, actor.GetStatus().GetActorVolumes(), template) + volumesToDetach = getMountedActorVolumes(ctx, ref, actor.GetStatus().GetExternalVolumes(), template) } // Collect errors for all volumes to detach, but continue processing so we attempt to detach all volumes. var errs []error @@ -224,7 +224,7 @@ func detachActorVolumes(ctx context.Context, registry VolumePluginRegistry, acto // StorageVolumeId is only populated once the volume is provisioned. // Skip volumes that were never created (e.g. failed during PENDING state). if vol.GetStorageVolumeId() == "" { - slog.WarnContext(ctx, "Volume has no storage volume ID, skipping detach", slog.String("volume_name", vol.GetVolumeName()), slog.String("actor_id", actor.GetMetadata().GetName())) + slog.WarnContext(ctx, "Volume has no storage volume ID, skipping detach", slog.String("volume_name", vol.GetName()), slog.String("actor_id", actor.GetMetadata().GetName())) continue } slog.InfoContext(ctx, "Detaching volume from node", slog.String("volume_id", vol.GetStorageVolumeId()), slog.String("node", node)) diff --git a/cmd/ateapi/internal/controlapi/volumes_test.go b/cmd/ateapi/internal/controlapi/volumes_test.go index 13ba49bfab..5fb207c11d 100644 --- a/cmd/ateapi/internal/controlapi/volumes_test.go +++ b/cmd/ateapi/internal/controlapi/volumes_test.go @@ -69,7 +69,7 @@ func TestActorVolumesStorageClassErrors(t *testing.T) { }}, } volumes := []*ateapipb.ExternalVolume{{ - VolumeName: "data-vol", + Name: "data-vol", VolumeType: "mock-standard", Status: ateapipb.ExternalVolume_STATUS_PENDING, }} @@ -143,12 +143,12 @@ func TestInitialActorVolumes_PendingState(t *testing.T) { want := []*ateapipb.ExternalVolume{ { - VolumeName: "data-vol-1", + Name: "data-vol-1", VolumeType: "mock-standard", Status: ateapipb.ExternalVolume_STATUS_PENDING, }, { - VolumeName: "data-vol-2", + Name: "data-vol-2", VolumeType: "mock-fast", Status: ateapipb.ExternalVolume_STATUS_PENDING, }, @@ -225,34 +225,34 @@ func TestCreateActorVolumes(t *testing.T) { tmpl: multiVolTmpl, inputVolumes: []*ateapipb.ExternalVolume{ { - VolumeName: "vol1", + Name: "vol1", VolumeType: "mock-standard", Status: ateapipb.ExternalVolume_STATUS_PENDING, }, { - VolumeName: "vol2", - Status: ateapipb.ExternalVolume_STATUS_DELETING, + Name: "vol2", + Status: ateapipb.ExternalVolume_STATUS_DELETING, }, { - VolumeName: "vol3", - Status: ateapipb.ExternalVolume_STATUS_PENDING, + Name: "vol3", + Status: ateapipb.ExternalVolume_STATUS_PENDING, }, }, wantErr: true, wantRes: []*ateapipb.ExternalVolume{ { - VolumeName: "vol1", + Name: "vol1", StorageVolumeId: "mock-vol-substrate-actor-uid-123-vol1", VolumeType: "mock-standard", Status: ateapipb.ExternalVolume_STATUS_CREATED, }, { - VolumeName: "vol2", - Status: ateapipb.ExternalVolume_STATUS_DELETING, + Name: "vol2", + Status: ateapipb.ExternalVolume_STATUS_DELETING, }, { - VolumeName: "vol3", - Status: ateapipb.ExternalVolume_STATUS_PENDING, + Name: "vol3", + Status: ateapipb.ExternalVolume_STATUS_PENDING, }, }, }, @@ -261,7 +261,7 @@ func TestCreateActorVolumes(t *testing.T) { tmpl: standardTmpl, inputVolumes: []*ateapipb.ExternalVolume{ { - VolumeName: "data-vol", + Name: "data-vol", StorageVolumeId: "existing-vol-id", Status: ateapipb.ExternalVolume_STATUS_CREATED, }, @@ -269,7 +269,7 @@ func TestCreateActorVolumes(t *testing.T) { wantErr: false, wantRes: []*ateapipb.ExternalVolume{ { - VolumeName: "data-vol", + Name: "data-vol", StorageVolumeId: "existing-vol-id", Status: ateapipb.ExternalVolume_STATUS_CREATED, }, @@ -280,15 +280,15 @@ func TestCreateActorVolumes(t *testing.T) { tmpl: standardTmpl, inputVolumes: []*ateapipb.ExternalVolume{ { - VolumeName: "data-vol", - Status: ateapipb.ExternalVolume_STATUS_UNSPECIFIED, + Name: "data-vol", + Status: ateapipb.ExternalVolume_STATUS_UNSPECIFIED, }, }, wantErr: true, wantRes: []*ateapipb.ExternalVolume{ { - VolumeName: "data-vol", - Status: ateapipb.ExternalVolume_STATUS_UNSPECIFIED, + Name: "data-vol", + Status: ateapipb.ExternalVolume_STATUS_UNSPECIFIED, }, }, }, @@ -297,15 +297,15 @@ func TestCreateActorVolumes(t *testing.T) { tmpl: &ateapipb.ActorTemplate{}, inputVolumes: []*ateapipb.ExternalVolume{ { - VolumeName: "missing-vol", - Status: ateapipb.ExternalVolume_STATUS_PENDING, + Name: "missing-vol", + Status: ateapipb.ExternalVolume_STATUS_PENDING, }, }, wantErr: true, wantRes: []*ateapipb.ExternalVolume{ { - VolumeName: "missing-vol", - Status: ateapipb.ExternalVolume_STATUS_PENDING, + Name: "missing-vol", + Status: ateapipb.ExternalVolume_STATUS_PENDING, }, }, }, @@ -314,7 +314,7 @@ func TestCreateActorVolumes(t *testing.T) { tmpl: standardTmpl, inputVolumes: []*ateapipb.ExternalVolume{ { - VolumeName: "data-vol", + Name: "data-vol", VolumeType: "mock-standard", Status: ateapipb.ExternalVolume_STATUS_PENDING, }, @@ -332,7 +332,7 @@ func TestCreateActorVolumes(t *testing.T) { wantErr: false, wantRes: []*ateapipb.ExternalVolume{ { - VolumeName: "data-vol", + Name: "data-vol", StorageVolumeId: "mock-vol-substrate-actor-uid-123-data-vol", VolumeType: "mock-standard", Status: ateapipb.ExternalVolume_STATUS_CREATED, @@ -403,7 +403,7 @@ func TestDeleteActorVolumes(t *testing.T) { name: "uses storage volume ID when present", actorUID: "uid-abc", volumes: []*ateapipb.ExternalVolume{ - {VolumeName: "vol1", StorageVolumeId: "storage-vol-123", VolumeType: "mock"}, + {Name: "vol1", StorageVolumeId: "storage-vol-123", VolumeType: "mock"}, }, wantDeleted: []string{"storage-vol-123"}, wantErr: false, @@ -412,7 +412,7 @@ func TestDeleteActorVolumes(t *testing.T) { name: "falls back to actorVolumeID when storage volume ID is empty regardless of status", actorUID: "uid-abc", volumes: []*ateapipb.ExternalVolume{ - {VolumeName: "vol1", StorageVolumeId: "", Status: ateapipb.ExternalVolume_STATUS_CREATED, VolumeType: "mock"}, + {Name: "vol1", StorageVolumeId: "", Status: ateapipb.ExternalVolume_STATUS_CREATED, VolumeType: "mock"}, }, wantDeleted: []string{"substrate-uid-abc-vol1"}, wantErr: false, @@ -491,9 +491,9 @@ func TestDetachActorVolumes(t *testing.T) { Metadata: &ateapipb.ResourceMetadata{Name: "actor-1", Atespace: "default"}, Status: &ateapipb.ActorStatus{ AssignedNode: "node-1", - ActorVolumes: []*ateapipb.ExternalVolume{ - {VolumeName: "vol1", StorageVolumeId: "storage-vol-1", VolumeType: "mock"}, - {VolumeName: "vol2", StorageVolumeId: "storage-vol-2", VolumeType: "mock"}, + ExternalVolumes: []*ateapipb.ExternalVolume{ + {Name: "vol1", StorageVolumeId: "storage-vol-1", VolumeType: "mock"}, + {Name: "vol2", StorageVolumeId: "storage-vol-2", VolumeType: "mock"}, }, }, }, @@ -522,9 +522,9 @@ func TestDetachActorVolumes(t *testing.T) { Metadata: &ateapipb.ResourceMetadata{Name: "actor-1", Atespace: "default"}, Status: &ateapipb.ActorStatus{ AssignedNode: "node-1", - ActorVolumes: []*ateapipb.ExternalVolume{ - {VolumeName: "mounted-vol", StorageVolumeId: "storage-vol-mounted", VolumeType: "mock"}, - {VolumeName: "unmounted-vol", StorageVolumeId: "storage-vol-unmounted", VolumeType: "mock"}, + ExternalVolumes: []*ateapipb.ExternalVolume{ + {Name: "mounted-vol", StorageVolumeId: "storage-vol-mounted", VolumeType: "mock"}, + {Name: "unmounted-vol", StorageVolumeId: "storage-vol-unmounted", VolumeType: "mock"}, }, }, }, @@ -551,9 +551,9 @@ func TestDetachActorVolumes(t *testing.T) { Metadata: &ateapipb.ResourceMetadata{Name: "actor-1", Atespace: "default"}, Status: &ateapipb.ActorStatus{ AssignedNode: "node-1", - ActorVolumes: []*ateapipb.ExternalVolume{ - {VolumeName: "vol1", StorageVolumeId: "", VolumeType: "mock"}, - {VolumeName: "vol2", StorageVolumeId: "storage-vol-2", VolumeType: "mock"}, + ExternalVolumes: []*ateapipb.ExternalVolume{ + {Name: "vol1", StorageVolumeId: "", VolumeType: "mock"}, + {Name: "vol2", StorageVolumeId: "storage-vol-2", VolumeType: "mock"}, }, }, }, @@ -581,9 +581,9 @@ func TestDetachActorVolumes(t *testing.T) { Metadata: &ateapipb.ResourceMetadata{Name: "actor-1", Atespace: "default"}, Status: &ateapipb.ActorStatus{ AssignedNode: "node-1", - ActorVolumes: []*ateapipb.ExternalVolume{ - {VolumeName: "vol1", StorageVolumeId: "storage-vol-1", VolumeType: "mock"}, - {VolumeName: "vol2", StorageVolumeId: "storage-vol-2", VolumeType: "mock"}, + ExternalVolumes: []*ateapipb.ExternalVolume{ + {Name: "vol1", StorageVolumeId: "storage-vol-1", VolumeType: "mock"}, + {Name: "vol2", StorageVolumeId: "storage-vol-2", VolumeType: "mock"}, }, }, }, @@ -599,8 +599,8 @@ func TestDetachActorVolumes(t *testing.T) { Metadata: &ateapipb.ResourceMetadata{Name: "actor-1", Atespace: "default"}, Status: &ateapipb.ActorStatus{ AssignedNode: "node-1", - ActorVolumes: []*ateapipb.ExternalVolume{ - {VolumeName: "vol1", StorageVolumeId: "storage-vol-1", VolumeType: "mock"}, + ExternalVolumes: []*ateapipb.ExternalVolume{ + {Name: "vol1", StorageVolumeId: "storage-vol-1", VolumeType: "mock"}, }, }, }, @@ -620,9 +620,9 @@ func TestDetachActorVolumes(t *testing.T) { Metadata: &ateapipb.ResourceMetadata{Name: "actor-1", Atespace: "default"}, Status: &ateapipb.ActorStatus{ AssignedNode: "node-1", - ActorVolumes: []*ateapipb.ExternalVolume{ - {VolumeName: "vol1", StorageVolumeId: "storage-vol-1", VolumeType: "mock"}, - {VolumeName: "vol2", StorageVolumeId: "storage-vol-2", VolumeType: "mock"}, + ExternalVolumes: []*ateapipb.ExternalVolume{ + {Name: "vol1", StorageVolumeId: "storage-vol-1", VolumeType: "mock"}, + {Name: "vol2", StorageVolumeId: "storage-vol-2", VolumeType: "mock"}, }, }, }, @@ -644,8 +644,8 @@ func TestDetachActorVolumes(t *testing.T) { Metadata: &ateapipb.ResourceMetadata{Name: "actor-1", Atespace: "default"}, Status: &ateapipb.ActorStatus{ AssignedNode: "node-1", - ActorVolumes: []*ateapipb.ExternalVolume{ - {VolumeName: "vol1", StorageVolumeId: "storage-vol-1", VolumeType: "unknown-plugin"}, + ExternalVolumes: []*ateapipb.ExternalVolume{ + {Name: "vol1", StorageVolumeId: "storage-vol-1", VolumeType: "unknown-plugin"}, }, }, }, @@ -658,8 +658,8 @@ func TestDetachActorVolumes(t *testing.T) { Metadata: &ateapipb.ResourceMetadata{Name: "actor-1", Atespace: "default"}, Status: &ateapipb.ActorStatus{ AssignedNode: "", - ActorVolumes: []*ateapipb.ExternalVolume{ - {VolumeName: "vol1", StorageVolumeId: "storage-vol-1", VolumeType: "mock"}, + ExternalVolumes: []*ateapipb.ExternalVolume{ + {Name: "vol1", StorageVolumeId: "storage-vol-1", VolumeType: "mock"}, }, }, }, diff --git a/cmd/ateapi/internal/controlapi/worker.go b/cmd/ateapi/internal/controlapi/worker.go index f66d6f5634..b67253ee3a 100644 --- a/cmd/ateapi/internal/controlapi/worker.go +++ b/cmd/ateapi/internal/controlapi/worker.go @@ -132,10 +132,11 @@ func (s *ServiceImpl) CreateWorker(ctx context.Context, inWorker *ateapipb.Worke ObservedEpoch: inWorker.GetEpoch(), } - // Capacity and hardware are left unset: a Worker holds nothing until its - // own ateom says what it has, through WorkerService.RegisterWorker. Nothing is placed - // on it in the meantime, which is the point -- the alternative is guessing - // on the Worker's behalf and placing against the guess. + // Capacity and runtimes are left unset: a Worker holds nothing and runs no + // sandbox until its own ateom says what it has, through + // WorkerService.RegisterWorker. Nothing is placed on it in the meantime, + // which is the point -- the alternative is guessing on the Worker's behalf + // and placing against the guess. // Verify that the result is properly valid before storing it. if errs := apivalidation.ValidateWorkerUpdate(ctx, field.NewPath("worker"), outWorker, inWorker, true); len(errs) > 0 { diff --git a/cmd/ateapi/internal/controlapi/workflow.go b/cmd/ateapi/internal/controlapi/workflow.go index 25c3100a23..6e21e849a5 100644 --- a/cmd/ateapi/internal/controlapi/workflow.go +++ b/cmd/ateapi/internal/controlapi/workflow.go @@ -27,10 +27,11 @@ import ( "github.com/agent-substrate/substrate/internal/actorevent" "github.com/agent-substrate/substrate/internal/apierror" "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/objectstore" + "github.com/agent-substrate/substrate/internal/objectstoreplugin" "github.com/agent-substrate/substrate/internal/resources" listersv1alpha1 "github.com/agent-substrate/substrate/pkg/client/listers/api/v1alpha1" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + objectstorev1 "github.com/agent-substrate/substrate/pkg/proto/objectstorepb/v1" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/codes" @@ -111,12 +112,15 @@ type ActorWorkflow struct { egressGatewayAddress string pluginRegistry VolumePluginRegistry workflowDeadline time.Duration - objectStore objectstore.Store + snapshotPlugin objectstorev1.ControlProviderClient } -// NewActorWorkflow creates a new ActorWorkflow. workflowDeadline bounds how -// long a single Resume/Suspend can run end-to-end; instruments and objectStore -// may be nil. +// NewActorWorkflow creates a new ActorWorkflow. workflowDeadline bounds +// each Resume/Suspend workflow end-to-end; instruments may be nil. +// +// snapshotPlugin may be nil, which leaves external snapshots in place instead +// of copying and releasing them. Only tests that never reach those steps pass +// nil; ate-api always builds one. func NewActorWorkflow( store actorWorkflowStore, workerCache *workercache.Cache, @@ -127,7 +131,7 @@ func NewActorWorkflow( egressGatewayAddress string, pluginRegistry VolumePluginRegistry, workflowDeadline time.Duration, - objectStore objectstore.Store, + snapshotPlugin objectstorev1.ControlProviderClient, ) *ActorWorkflow { return &ActorWorkflow{ store: store, @@ -140,10 +144,24 @@ func NewActorWorkflow( egressGatewayAddress: egressGatewayAddress, pluginRegistry: pluginRegistry, workflowDeadline: workflowDeadline, - objectStore: objectStore, + snapshotPlugin: snapshotPlugin, } } +// cleanupSnapshot deletes every object under prefix through the control +// snapshot plugin. +func (w *ActorWorkflow) cleanupSnapshot(ctx context.Context, prefix resources.StoragePrefix) error { + _, err := w.snapshotPlugin.CleanupSnapshot(ctx, &objectstorev1.CleanupSnapshotRequest{SnapshotUri: prefix.String()}) + return objectstoreplugin.CallError(err) +} + +// copySnapshot copies every object under src to dst through the control +// snapshot plugin. +func (w *ActorWorkflow) copySnapshot(ctx context.Context, src, dst resources.StoragePrefix) error { + _, err := w.snapshotPlugin.CopySnapshot(ctx, &objectstorev1.CopySnapshotRequest{SrcUri: src.String(), DstUri: dst.String()}) + return objectstoreplugin.CallError(err) +} + // actorWorkflowStore enumerates the exact storage methods needed by // ActorWorkflow and nothing more. type actorWorkflowStore interface { diff --git a/cmd/ateapi/internal/controlapi/workflow_delete.go b/cmd/ateapi/internal/controlapi/workflow_delete.go index 4a7e811ea5..5a4337fdc5 100644 --- a/cmd/ateapi/internal/controlapi/workflow_delete.go +++ b/cmd/ateapi/internal/controlapi/workflow_delete.go @@ -23,7 +23,6 @@ import ( "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" "github.com/agent-substrate/substrate/internal/apierror" "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/objectstore" "github.com/agent-substrate/substrate/internal/proto/ateletpb" "github.com/agent-substrate/substrate/internal/resources" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" @@ -132,9 +131,9 @@ func (w *ActorWorkflow) ensureAteletTerminated(ctx context.Context, actorRef res return nil } - targetAteomUID := "" + workerPodUID := "" if assignment := actor.GetStatus().GetWorkerAssignment(); assignment != nil { - targetAteomUID = assignment.GetWorkerPodUid() + workerPodUID = assignment.GetWorkerPodUid() if workerName := assignment.GetWorker().GetName(); workerName != "" { // Ask whether the worker still HOSTS this actor, not whether its one // assignment happens to be this actor: a worker hosting several is the @@ -147,7 +146,7 @@ func (w *ActorWorkflow) ensureAteletTerminated(ctx context.Context, actorRef res slog.InfoContext(ctx, "worker is no longer assigned to this actor, skipping ateom workload termination", slog.String("worker", workerName), slog.Any("actor", actorRef)) - targetAteomUID = "" + workerPodUID = "" } } } @@ -174,12 +173,12 @@ func (w *ActorWorkflow) ensureAteletTerminated(ctx context.Context, actorRef res slog.String("templateAtespace", actor.GetActorTemplate().GetAtespace()), slog.String("templateName", actor.GetActorTemplate().GetName())) workloadSpec = &ateletpb.WorkloadSpec{} - for _, vol := range actor.GetStatus().GetActorVolumes() { + for _, vol := range actor.GetStatus().GetExternalVolumes() { // StorageVolumeId is only populated once the volume is provisioned. // Skip volumes that were never created (e.g. failed during PENDING state). if vol.GetStorageVolumeId() != "" { workloadSpec.Volumes = append(workloadSpec.Volumes, &ateletpb.Volume{ - Name: vol.GetVolumeName(), + Name: vol.GetName(), Source: &ateletpb.Volume_External{ External: &ateletpb.ExternalVolumeSource{ StorageVolumeId: vol.GetStorageVolumeId(), @@ -193,7 +192,7 @@ func (w *ActorWorkflow) ensureAteletTerminated(ctx context.Context, actorRef res } req := &ateletpb.TerminateRequest{ - TargetAteomUid: targetAteomUID, + WorkerPodUid: workerPodUID, Atespace: actor.GetMetadata().GetAtespace(), ActorName: actor.GetMetadata().GetName(), ActorUid: actor.GetMetadata().GetUid(), @@ -290,7 +289,7 @@ func (w *ActorWorkflow) ensureWorkerReleased(ctx context.Context, actorRef resou updatedActor, err := w.store.UpdateActor(ctx, actorRef, store.PreconditionFrom(latestActor), func(dbActor *ateapipb.Actor) error { if dbActor.Status != nil { - dbActor.Status.LocalSnapshot = nil + removeSnapshotStorageEntries(dbActor.Status, ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL, nil) dbActor.Status.AssignedNode = "" dbActor.Status.WorkerAssignment = nil } @@ -334,7 +333,7 @@ func (w *ActorWorkflow) ensureMarkedDeleting(ctx context.Context, actorRef resou storedActor, err := w.store.UpdateActor(ctx, actorRef, store.PreconditionFrom(actor), func(toUpdate *ateapipb.Actor) error { toUpdate.Status.State = ateapipb.ActorState_ACTOR_STATE_DELETING - for _, vol := range toUpdate.GetStatus().GetActorVolumes() { + for _, vol := range toUpdate.GetStatus().GetExternalVolumes() { vol.Status = ateapipb.ExternalVolume_STATUS_DELETING } return nil @@ -360,7 +359,7 @@ func (w *ActorWorkflow) ensureVolumesDeleted(ctx context.Context, actor *ateapip return apierror.FailedPrecondition("DeleteVolumes prerequisite not met for Actor: %s (got: %v, want %s)", actor.GetMetadata().GetName(), st, ateapipb.ActorState_ACTOR_STATE_DELETING) } - if err := deleteActorVolumes(ctx, w.pluginRegistry, actor.GetMetadata().GetUid(), actor.GetStatus().GetActorVolumes()); err != nil { + if err := deleteActorVolumes(ctx, w.pluginRegistry, actor.GetMetadata().GetUid(), actor.GetStatus().GetExternalVolumes()); err != nil { return apierror.Internal("while deleting actor volumes: %v", err) } return nil @@ -376,7 +375,7 @@ func (w *ActorWorkflow) ensureExternalSnapshotsReleased(ctx context.Context, act ctx, done := stepSpan(ctx, "ReleaseExternalSnapshots") defer func() { err = done(err) }() - if w.objectStore == nil { + if w.snapshotPlugin == nil { markSkipped(ctx, "no object store configured") return nil } @@ -389,7 +388,7 @@ func (w *ActorWorkflow) ensureExternalSnapshotsReleased(ctx context.Context, act markSkipped(ctx, "the actor owns no external snapshot") return nil } - return objectstore.DeletePrefix(ctx, w.objectStore, prefix) + return w.cleanupSnapshot(ctx, prefix) } // actorSnapshotStoragePrefix returns the prefix holding every object the actor wrote: @@ -398,7 +397,8 @@ func (w *ActorWorkflow) ensureExternalSnapshotsReleased(ctx context.Context, act // wrote anything. func actorSnapshotStoragePrefix(actor *ateapipb.Actor) (resources.StoragePrefix, error) { actorOwner := actorSnapshotOwner(actor) - if snapshotURI := actor.GetStatus().GetExternalSnapshot().GetSnapshotUri(); snapshotURI != "" { + if _, st := findLatestSnapshotStorage(actor.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED); st != nil { + snapshotURI := st.GetObject().GetSnapshotUri() uri, err := resources.ParseSnapshotURI(snapshotURI) if err != nil { return resources.StoragePrefix{}, fmt.Errorf("while parsing the external snapshot %q: %w", snapshotURI, err) @@ -413,7 +413,8 @@ func actorSnapshotStoragePrefix(actor *ateapipb.Actor) (resources.StoragePrefix, // Nothing of the actor's own is recorded. Unless a suspend died partway, // nothing was ever written under its prefix: the in-progress URI is // recorded before atelet uploads the first object. - inProgress := actor.GetStatus().GetInProgressSnapshotUri() + _, inProgressSt := findLatestSnapshotStorage(actor.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS) + inProgress := inProgressSt.GetObject().GetSnapshotUri() if inProgress == "" { return resources.StoragePrefix{}, nil } diff --git a/cmd/ateapi/internal/controlapi/workflow_delete_test.go b/cmd/ateapi/internal/controlapi/workflow_delete_test.go index 8fc70e2fd0..da71ee17b7 100644 --- a/cmd/ateapi/internal/controlapi/workflow_delete_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_delete_test.go @@ -17,6 +17,7 @@ package controlapi import ( "context" "errors" + "strings" "testing" "time" @@ -24,6 +25,7 @@ import ( "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest" "github.com/agent-substrate/substrate/internal/apierror" "github.com/agent-substrate/substrate/internal/objectstore/objectstoretest" + "github.com/agent-substrate/substrate/internal/objectstoreplugin/objectstoreplugintest" "github.com/agent-substrate/substrate/internal/resources" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "google.golang.org/grpc/codes" @@ -262,8 +264,10 @@ func TestEnsureExternalSnapshotsReleased(t *testing.T) { // The actor's prefix is keyed on the UID the store just assigned, so // its snapshots can only be placed now. current := mustActorSnapshotURI(t, template, actor, "current") + currentType := ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR if tt.tagOwnedSnapshot { current = mustTagSnapshotURI(t, template, "team-a", "v1-snapshot") + currentType = ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG } objects.PutSnapshot(t, current, "manifest.json") inFlight := mustActorSnapshotURI(t, template, actor, inFlightSnapshotName) @@ -271,8 +275,11 @@ func TestEnsureExternalSnapshotsReleased(t *testing.T) { objects.PutSnapshot(t, inFlight, "manifest.json") } actor = mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapipb.ActorStatus) { - s.ExternalSnapshot = &ateapipb.ExternalSnapshot{SnapshotUri: current.String()} - s.InProgressSnapshotUri = inFlight.String() + s.LastAssignedGeneration = 2 + s.Snapshots = []*ateapipb.Snapshot{ + newDurableSnapshot(1, currentType, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", current.String(), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + newDurableSnapshot(2, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", inFlight.String(), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS), + } }) if err := w.ensureExternalSnapshotsReleased(ctx, actor); err != nil { @@ -317,7 +324,10 @@ func TestEnsureExternalSnapshotsReleased_CollectsStrandedSnapshots(t *testing.T) objects.PutSnapshot(t, uri, "manifest.json") } actor = mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapipb.ActorStatus) { - s.ExternalSnapshot = &ateapipb.ExternalSnapshot{SnapshotUri: currentSnapshot.String()} + s.LastAssignedGeneration = 1 + s.Snapshots = []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", currentSnapshot.String(), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + } }) if err := w.ensureExternalSnapshotsReleased(ctx, actor); err != nil { @@ -338,7 +348,7 @@ func TestDeleteActor_CollectsInFlightSnapshotWithoutTemplate(t *testing.T) { ctx := context.Background() persistence := newTestPersistence(t) objects := objectstoretest.New() - w := NewActorWorkflow(persistence, nil, nil, nil, nil, nil, "", nil, time.Minute, objects) + w := NewActorWorkflow(persistence, nil, nil, nil, nil, nil, "", nil, time.Minute, objectstoreplugintest.ControlClient(objects)) actorRef := resources.ActorRef{Atespace: "team-a", Name: "actor-1"} actor := storetest.MustCreateActor(t, ctx, persistence, &ateapipb.Actor{ @@ -354,7 +364,10 @@ func TestDeleteActor_CollectsInFlightSnapshotWithoutTemplate(t *testing.T) { }, actor, "abandoned") objects.PutSnapshot(t, inFlight, "manifest.json") mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapipb.ActorStatus) { - s.InProgressSnapshotUri = inFlight.String() + s.LastAssignedGeneration = 1 + s.Snapshots = []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", inFlight.String(), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS), + } }) if _, err := w.DeleteActor(ctx, actorRef, true, store.DeletePreconditions{}); err != nil { @@ -384,7 +397,10 @@ func TestEnsureExternalSnapshotsReleased_DeletePrefixFailure(t *testing.T) { current := mustActorSnapshotURI(t, template, actor, "current") objects.PutSnapshot(t, current, "manifest.json", "memory.zst") actor = mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapipb.ActorStatus) { - s.ExternalSnapshot = &ateapipb.ExternalSnapshot{SnapshotUri: current.String()} + s.LastAssignedGeneration = 1 + s.Snapshots = []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", current.String(), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + } }) errTransient := errors.New("simulated transient delete failure") @@ -393,8 +409,8 @@ func TestEnsureExternalSnapshotsReleased_DeletePrefixFailure(t *testing.T) { } err := w.ensureExternalSnapshotsReleased(ctx, actor) - if !errors.Is(err, errTransient) { - t.Fatalf("ensureExternalSnapshotsReleased error = %v, want error wrapping %v", err, errTransient) + if err == nil || !strings.Contains(err.Error(), errTransient.Error()) { + t.Fatalf("ensureExternalSnapshotsReleased error = %v, want it to report %v", err, errTransient) } // Objects should not have been deleted @@ -415,8 +431,8 @@ func TestEnsureExternalSnapshotsReleased_DeletePrefixFailure(t *testing.T) { // TestDeleteActor_CollectsSnapshotsAfterWorkerDelete verifies that // deleting an actor whose suspend a worker delete crashed mid-finalize deletes // every object that suspend wrote. When an actor crashes mid-suspend, only -// DeleteActor or RevertActor can delete the in-progress snapshot -// (in_progress_snapshot_uri): whatever they cannot name is leaked for good. +// DeleteActor or RevertActor can delete the in-progress snapshot: +// whatever they cannot name is leaked for good. func TestDeleteActor_CollectsSnapshotsAfterWorkerDelete(t *testing.T) { tests := []struct { name string @@ -478,18 +494,22 @@ func TestDeleteActor_CollectsSnapshotsAfterWorkerDelete(t *testing.T) { previous := mustActorSnapshotURI(t, template, actor, "old") objects.PutSnapshot(t, previous, "manifest.json") actor = mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapipb.ActorStatus) { - s.ExternalSnapshot = &ateapipb.ExternalSnapshot{SnapshotUri: previous.String()} + s.LastAssignedGeneration = 1 + s.Snapshots = []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", previous.String(), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + } }) } - actorWorkflow := NewActorWorkflow(persistence, nil, nil, nil, nil, nil, "", nil, time.Minute, objects) + actorWorkflow := NewActorWorkflow(persistence, nil, nil, nil, nil, nil, "", nil, time.Minute, objectstoreplugintest.ControlClient(objects)) // Suspend the actor as far as it gets: MarkSuspending mints the // in-progress URI, and the checkpoint writes under it actor, err := actorWorkflow.ensureMarkedSuspending(ctx, actorRef, actor, template) if err != nil { t.Fatalf("ensureMarkedSuspending: %v", err) } - fresh := mustParseSnapshotURI(t, actor.GetStatus().GetInProgressSnapshotUri()) + _, freshSt := findLatestSnapshotStorage(actor.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS) + fresh := mustParseSnapshotURI(t, freshSt.GetObject().GetSnapshotUri()) objects.PutSnapshot(t, fresh, "manifest.json") // The worker's pod goes away with the commit still outstanding, so diff --git a/cmd/ateapi/internal/controlapi/workflow_pause.go b/cmd/ateapi/internal/controlapi/workflow_pause.go index 32ec442af4..7e8a73180a 100644 --- a/cmd/ateapi/internal/controlapi/workflow_pause.go +++ b/cmd/ateapi/internal/controlapi/workflow_pause.go @@ -37,7 +37,7 @@ func (w *ActorWorkflow) PauseActor(ctx context.Context, actorRef resources.Actor start := time.Now() var actor *ateapipb.Actor var actorTemplate *ateapipb.ActorTemplate - var wireSnapshotScope string + var wireFidelity string // Set just before finalize; nil until then, so earlier exits label // themselves from the record they hold. var finalAttrs []attribute.KeyValue @@ -45,7 +45,7 @@ func (w *ActorWorkflow) PauseActor(ctx context.Context, actorRef resources.Actor defer func() { attrs := finalAttrs if attrs == nil { - attrs = lifecycleOpAttrs(actor, actorTemplate, "", wireSnapshotScope) + attrs = lifecycleOpAttrs(actor, actorTemplate, "", wireFidelity) } w.instruments.recordLifecycleOp(ctx, ateattr.OperationPause, start, err, attrs...) }() @@ -68,11 +68,11 @@ func (w *ActorWorkflow) PauseActor(ctx context.Context, actorRef resources.Actor return actor, nil } var marked *ateapipb.Actor - if marked, err = w.ensureMarkedPausing(leaseCtx, actorRef, actor); err != nil { + if marked, err = w.ensureMarkedPausing(leaseCtx, actorRef, actor, actorTemplate); err != nil { return nil, err } actor = marked - if wireSnapshotScope, err = w.ensureAteletPaused(leaseCtx, actorRef, actor, actorTemplate); err != nil { + if wireFidelity, err = w.ensureAteletPaused(leaseCtx, actorRef, actor, actorTemplate); err != nil { return nil, err } // TODO: There is no difference between suspend and pause for now, but we @@ -83,9 +83,9 @@ func (w *ActorWorkflow) PauseActor(ctx context.Context, actorRef resources.Actor } // FinalizePaused clears the WorkerAssignment the labels read, so snapshot // them here, as crash.go does for the crash counter. - finalAttrs = lifecycleOpAttrs(actor, actorTemplate, "", wireSnapshotScope) + finalAttrs = lifecycleOpAttrs(actor, actorTemplate, "", wireFidelity) var finalized *ateapipb.Actor - if finalized, err = w.ensurePausedFinalized(leaseCtx, actorRef, actorTemplate); err != nil { + if finalized, err = w.ensurePausedFinalized(leaseCtx, actorRef); err != nil { return nil, err } actor = finalized @@ -112,7 +112,7 @@ func (w *ActorWorkflow) loadActorForPause(ctx context.Context, actorRef resource // local snapshot name. Skips when a previous attempt already marked the // actor; the persisted name then stays authoritative for the rest of the // workflow. -func (w *ActorWorkflow) ensureMarkedPausing(ctx context.Context, actorRef resources.ActorRef, actor *ateapipb.Actor) (_ *ateapipb.Actor, err error) { +func (w *ActorWorkflow) ensureMarkedPausing(ctx context.Context, actorRef resources.ActorRef, actor *ateapipb.Actor, actorTemplate *ateapipb.ActorTemplate) (_ *ateapipb.Actor, err error) { ctx, done := stepSpan(ctx, "MarkPausing") defer func() { err = done(err) }() @@ -133,7 +133,17 @@ func (w *ActorWorkflow) ensureMarkedPausing(ctx context.Context, actorRef resour snapshotName := resources.NewSnapshotName() storedActor, err := w.store.UpdateActor(ctx, actorRef, store.PreconditionFrom(actor), func(toUpdate *ateapipb.Actor) error { toUpdate.Status.State = ateapipb.ActorState_ACTOR_STATE_PAUSING - toUpdate.Status.InProgressLocalSnapshotName = snapshotName + // Increment last_assigned_generation for the new Pause request. + gen := toUpdate.Status.LastAssignedGeneration + 1 + toUpdate.Status.LastAssignedGeneration = gen + toUpdate.Status.Snapshots = append(toUpdate.Status.Snapshots, + newLocalSnapshot( + gen, + actorTemplate.GetSnapshotConfig().GetPreferredFidelity(), + actorTemplate.GetMetadata().GetUid(), + snapshotName, + ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS, + )) return nil }) if err != nil { @@ -152,7 +162,7 @@ func (w *ActorWorkflow) ensureMarkedPausing(ctx context.Context, actorRef resour // the once-minted snapshot name, so a re-entered workflow re-sends the same // semantic request; once atelet's Checkpoint is idempotent on those keys this // step becomes fully reentrant with no changes here. -func (w *ActorWorkflow) ensureAteletPaused(ctx context.Context, actorRef resources.ActorRef, actor *ateapipb.Actor, actorTemplate *ateapipb.ActorTemplate) (wireSnapshotScope string, err error) { +func (w *ActorWorkflow) ensureAteletPaused(ctx context.Context, actorRef resources.ActorRef, actor *ateapipb.Actor, actorTemplate *ateapipb.ActorTemplate) (wireFidelity string, err error) { ctx, done := stepSpan(ctx, "CallAteletPause") defer func() { err = done(err) }() @@ -179,8 +189,9 @@ func (w *ActorWorkflow) ensureAteletPaused(ctx context.Context, actorRef resourc // Checkpoint does not carry the sandbox config: atelet uses the version the // actor is currently running (recorded on-node at Run/Restore) and pins it // into the snapshot manifest. + _, inProgressLocal := findLatestSnapshotStorage(actor.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS) req := &ateletpb.CheckpointRequest{ - TargetAteomUid: assignment.GetWorkerPodUid(), + WorkerPodUid: assignment.GetWorkerPodUid(), Atespace: actor.GetMetadata().GetAtespace(), ActorName: actor.GetMetadata().GetName(), ActorTemplateAtespace: actor.GetActorTemplate().GetAtespace(), @@ -189,18 +200,18 @@ func (w *ActorWorkflow) ensureAteletPaused(ctx context.Context, actorRef resourc Type: ateletpb.CheckpointType_CHECKPOINT_TYPE_LOCAL, Config: &ateletpb.CheckpointRequest_LocalConfig{ LocalConfig: &ateletpb.LocalCheckpointConfiguration{ - SnapshotName: actor.GetStatus().GetInProgressLocalSnapshotName(), + SnapshotName: inProgressLocal.GetLocal().GetSnapshotName(), }, }, - Scope: actorSnapshotContentScopeToAtelet(actorTemplate.GetSnapshotConfig().GetOnCommit()), + Fidelity: fidelityToAtelet(actorTemplate.GetSnapshotConfig().GetPreferredFidelity()), ActorUid: actor.GetMetadata().Uid, } - wireSnapshotScope = ateattr.SnapshotScopeValue(req.Scope) + wireFidelity = ateattr.SnapshotFidelityValue(req.Fidelity) if _, err = client.Checkpoint(ctx, req); err != nil { - return wireSnapshotScope, handleAteletError(ctx, w.store, actorRef, ateattr.OperationPause, "Checkpoint", false, err) + return wireFidelity, handleAteletError(ctx, w.store, actorRef, ateattr.OperationPause, "Checkpoint", false, err) } - return wireSnapshotScope, nil + return wireFidelity, nil } // ensurePausedFinalized releases the actor's worker (only when it is still @@ -210,7 +221,7 @@ func (w *ActorWorkflow) ensureAteletPaused(ctx context.Context, actorRef resourc // never be resumed. It re-reads the actor first so an out-of-band transition // (e.g. the syncer crashing the actor after its worker died) is not // overwritten: with no assignment left there is nothing to finalize. -func (w *ActorWorkflow) ensurePausedFinalized(ctx context.Context, actorRef resources.ActorRef, actorTemplate *ateapipb.ActorTemplate) (_ *ateapipb.Actor, err error) { +func (w *ActorWorkflow) ensurePausedFinalized(ctx context.Context, actorRef resources.ActorRef) (_ *ateapipb.Actor, err error) { ctx, done := stepSpan(ctx, "FinalizePaused") defer func() { err = done(err) }() @@ -260,7 +271,6 @@ func (w *ActorWorkflow) ensurePausedFinalized(ctx context.Context, actorRef reso newState = ateapipb.ActorState_ACTOR_STATE_CRASHED crashStatus = newActorCrash(ateattr.OperationPause, crashMessageLocalSnapshotNodeUnknown) } - contentScope := actorTemplate.GetSnapshotConfig().GetOnCommit() sandboxClass := "" if worker != nil { sandboxClass = worker.GetSandboxClass() @@ -274,14 +284,16 @@ func (w *ActorWorkflow) ensurePausedFinalized(ctx context.Context, actorRef reso if newState == ateapipb.ActorState_ACTOR_STATE_CRASHED && !wasAlreadyCrashed { toUpdate.Status.Crash = crashStatus } - // TODO(dberkov) - what if InProgressLocalSnapshotName is empty? That shouldn't be possible. - if toUpdate.GetStatus().GetInProgressLocalSnapshotName() != "" { - localSnapshot := &ateapipb.LocalSnapshot{ - SnapshotName: toUpdate.GetStatus().GetInProgressLocalSnapshotName(), - ContentScope: contentScope, + if newState != ateapipb.ActorState_ACTOR_STATE_CRASHED { + // Finalize the snapshot and move it from IN_PROGRESS to COMPLETED. + snap := snapshotAtLatestGeneration(toUpdate.Status) + if snap == nil { + return fmt.Errorf("actor %s has no latest snapshot", actorRef) + } + if inProgressSt := findSnapshotStorage(snap, ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL); inProgressSt.GetLocal().GetSnapshotName() != "" { + inProgressSt.Status = ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + removeOlderSnapshotStorageEntries(toUpdate.Status, ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL, nil, snap.GetGeneration()) } - toUpdate.Status.LocalSnapshot = localSnapshot - toUpdate.Status.InProgressLocalSnapshotName = "" } toUpdate.Status.WorkerAssignment = nil return nil diff --git a/cmd/ateapi/internal/controlapi/workflow_pause_test.go b/cmd/ateapi/internal/controlapi/workflow_pause_test.go index 9866baf230..d8725d34d7 100644 --- a/cmd/ateapi/internal/controlapi/workflow_pause_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_pause_test.go @@ -50,14 +50,17 @@ func TestEnsurePausedFinalized_WorkerGone(t *testing.T) { WorkerPool: "pool1", WorkerPod: "worker-pod-1", }, - InProgressLocalSnapshotName: "local-snap-1", + LastAssignedGeneration: 1, + Snapshots: []*ateapipb.Snapshot{ + newLocalSnapshot(1, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "local-snap-1", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS), + }, }, } storetest.MustCreateActor(t, ctx, st, actor) // Intentionally NOT creating the worker in store, simulates worker already gone. w := &ActorWorkflow{store: st} - finalized, err := w.ensurePausedFinalized(ctx, actorRef, &ateapipb.ActorTemplate{}) + finalized, err := w.ensurePausedFinalized(ctx, actorRef) if err != nil { t.Fatalf("ensurePausedFinalized: %v", err) } @@ -76,8 +79,8 @@ func TestEnsurePausedFinalized_WorkerGone(t *testing.T) { if got.GetStatus().GetAssignedNode() != "" { t.Errorf("AssignedNode = %q, want empty", got.GetStatus().GetAssignedNode()) } - if gotSnap := got.GetStatus().GetInProgressLocalSnapshotName(); gotSnap != "" { - t.Errorf("InProgressLocalSnapshotName = %q, want cleared on crash", gotSnap) + if _, gotSt := findLatestSnapshotStorage(got.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS); gotSt.GetLocal().GetSnapshotName() != "local-snap-1" { + t.Errorf("in-progress local snapshot name = %q, want %q preserved on crash", gotSt.GetLocal().GetSnapshotName(), "local-snap-1") } if finalized.GetStatus().GetWorkerAssignment() != nil { @@ -99,8 +102,8 @@ func TestEnsurePausedFinalized_WorkerGone(t *testing.T) { // TestEnsurePausedFinalized_AlreadyCrashed verifies that if the actor was // already crashed out-of-band when ensurePausedFinalized runs with no -// AssignedNode, its existing Crash status is preserved and no duplicate crash -// log record is emitted. +// AssignedNode, its existing Crash status and snapshots are preserved and no +// duplicate crash log record is emitted. func TestEnsurePausedFinalized_AlreadyCrashed(t *testing.T) { st, cleanup := storetest.SetupTestStore(t) defer cleanup() @@ -120,13 +123,16 @@ func TestEnsurePausedFinalized_AlreadyCrashed(t *testing.T) { WorkerPool: "pool1", WorkerPod: "worker-pod-1", }, - InProgressLocalSnapshotName: "local-snap-1", + LastAssignedGeneration: 1, + Snapshots: []*ateapipb.Snapshot{ + newLocalSnapshot(1, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "local-snap-1", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS), + }, }, } storetest.MustCreateActor(t, ctx, st, actor) w := &ActorWorkflow{store: st} - finalized, err := w.ensurePausedFinalized(ctx, actorRef, &ateapipb.ActorTemplate{}) + finalized, err := w.ensurePausedFinalized(ctx, actorRef) if err != nil { t.Fatalf("ensurePausedFinalized: %v", err) } @@ -136,6 +142,9 @@ func TestEnsurePausedFinalized_AlreadyCrashed(t *testing.T) { if got, want := finalized.GetStatus().GetCrash().GetMessage(), originalCrash.GetMessage(); got != want { t.Errorf("crash message = %q, want original %q preserved", got, want) } + if _, gotSt := findLatestSnapshotStorage(finalized.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS); gotSt.GetLocal().GetSnapshotName() != "local-snap-1" { + t.Errorf("in-progress local snapshot name = %q, want %q preserved", gotSt.GetLocal().GetSnapshotName(), "local-snap-1") + } if finalized.GetStatus().GetWorkerAssignment() != nil { t.Errorf("WorkerAssignment = %v, want nil", finalized.GetStatus().GetWorkerAssignment()) } @@ -144,18 +153,18 @@ func TestEnsurePausedFinalized_AlreadyCrashed(t *testing.T) { } } -// TestEnsurePausedFinalized_RecordsContentScope verifies pause finalization -// records the scope the pause checkpoint captured (the template's onCommit) -// in LocalSnapshot, so a later suspend or resume of the PAUSED actor knows -// what the local snapshot contains. -func TestEnsurePausedFinalized_RecordsContentScope(t *testing.T) { +// TestEnsurePausedFinalized_RecordsFidelity verifies pause marks the +// scope the pause checkpoint captures (the template's preferredFidelity) in +// SnapshotStorage and finalization promotes it to COMPLETED, so a later suspend +// or resume of the PAUSED actor knows what the local snapshot contains. +func TestEnsurePausedFinalized_RecordsFidelity(t *testing.T) { tests := []struct { name string - onCommit ateapipb.SnapshotContentScope - want ateapipb.SnapshotContentScope + fidelity ateapipb.SnapshotFidelity + want ateapipb.SnapshotFidelity }{ - {"data", ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA, ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA}, - {"full", ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL}, + {"data", ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES}, + {"full", ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { @@ -168,7 +177,7 @@ func TestEnsurePausedFinalized_RecordsContentScope(t *testing.T) { created := storetest.MustCreateActor(t, ctx, st, &ateapipb.Actor{ Metadata: &ateapipb.ResourceMetadata{Atespace: actorRef.Atespace, Name: actorRef.Name}, Status: &ateapipb.ActorStatus{ - State: ateapipb.ActorState_ACTOR_STATE_PAUSING, + State: ateapipb.ActorState_ACTOR_STATE_RUNNING, AssignedNode: "node1", WorkerAssignment: &ateapipb.WorkerAssignment{ Worker: &ateapipb.ObjectRef{Name: workerName}, @@ -177,7 +186,6 @@ func TestEnsurePausedFinalized_RecordsContentScope(t *testing.T) { WorkerPod: "worker-pod-1", WorkerPodUid: workerName, }, - InProgressLocalSnapshotName: "snap-prefix", }, }) if _, err := st.CreateWorker(ctx, &ateapipb.Worker{ @@ -198,9 +206,12 @@ func TestEnsurePausedFinalized_RecordsContentScope(t *testing.T) { w := &ActorWorkflow{store: st} tmpl := &ateapipb.ActorTemplate{ - SnapshotConfig: &ateapipb.SnapshotConfig{OnCommit: tc.onCommit}, + SnapshotConfig: &ateapipb.SnapshotConfig{PreferredFidelity: tc.fidelity}, } - got, err := w.ensurePausedFinalized(ctx, actorRef, tmpl) + if _, err := w.ensureMarkedPausing(ctx, actorRef, created, tmpl); err != nil { + t.Fatalf("ensureMarkedPausing: %v", err) + } + got, err := w.ensurePausedFinalized(ctx, actorRef) if err != nil { t.Fatalf("ensurePausedFinalized: %v", err) } @@ -208,8 +219,9 @@ func TestEnsurePausedFinalized_RecordsContentScope(t *testing.T) { if got.GetStatus().GetState() != ateapipb.ActorState_ACTOR_STATE_PAUSED { t.Fatalf("state = %v, want PAUSED", got.GetStatus().GetState()) } - if scope := got.GetStatus().GetLocalSnapshot().GetContentScope(); scope != tc.want { - t.Errorf("LocalSnapshot.ContentScope = %v, want %v", scope, tc.want) + _, localSt := findLatestSnapshotStorage(got.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + if scope := localSt.GetFidelity(); scope != tc.want { + t.Errorf("local SnapshotStorage.Fidelity = %v, want %v", scope, tc.want) } if got.GetStatus().GetAssignedNode() != "node1" { t.Errorf("AssignedNode = %q, want %q", got.GetStatus().GetAssignedNode(), "node1") @@ -303,7 +315,7 @@ func TestEnsureMarkedPausing_StateMatrix(t *testing.T) { Status: &ateapipb.ActorStatus{State: seedState}, }) - marked, err := w.ensureMarkedPausing(ctx, actorRef, actor) + marked, err := w.ensureMarkedPausing(ctx, actorRef, actor, &ateapipb.ActorTemplate{}) assertPrerequisiteResult(t, seedState, err, allowed[seedState]) if err == nil && marked.GetStatus().GetState() != ateapipb.ActorState_ACTOR_STATE_PAUSING { t.Errorf("state %v: ensureMarkedPausing returned actor in %v, want PAUSING", seedState, marked.GetStatus().GetState()) @@ -331,6 +343,12 @@ func TestEnsureAteletPaused_DialFailureLeavesActorRetryable(t *testing.T) { ctx := context.Background() persistence := newTestPersistence(t) + var snapshots []*ateapipb.Snapshot + if tt.prevSnapshot != "" { + snapshots = append(snapshots, newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", tt.prevSnapshot, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED)) + } + snapshots = append(snapshots, newLocalSnapshot(2, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "actor-1-never-written", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS)) + actor := &ateapipb.Actor{ Metadata: &ateapipb.ResourceMetadata{Atespace: "team-a", Name: "actor-1"}, Status: &ateapipb.ActorStatus{ @@ -341,8 +359,8 @@ func TestEnsureAteletPaused_DialFailureLeavesActorRetryable(t *testing.T) { WorkerPod: "pod-gone", NodeName: "node-gone", }, - InProgressLocalSnapshotName: "actor-1-never-written", - ExternalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: tt.prevSnapshot}, + LastAssignedGeneration: 2, + Snapshots: snapshots, }, } created := storetest.MustCreateActor(t, ctx, persistence, actor) @@ -359,10 +377,11 @@ func TestEnsureAteletPaused_DialFailureLeavesActorRetryable(t *testing.T) { if stored.GetStatus().GetState() != ateapipb.ActorState_ACTOR_STATE_PAUSING { t.Errorf("state = %v, want unchanged PAUSING", stored.GetStatus().GetState()) } - if got := stored.GetStatus().GetInProgressLocalSnapshotName(); got != "actor-1-never-written" { - t.Errorf("InProgressLocalSnapshotName = %q, want preserved for debugging", got) + if _, gotSt := findLatestSnapshotStorage(stored.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS); gotSt.GetLocal().GetSnapshotName() != "actor-1-never-written" { + t.Errorf("in-progress local snapshot name = %q, want preserved for debugging", gotSt.GetLocal().GetSnapshotName()) } - if got := stored.GetStatus().GetExternalSnapshot().GetSnapshotUri(); got != tt.prevSnapshot { + _, gotSt := findLatestSnapshotStorage(stored.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + if got := gotSt.GetObject().GetSnapshotUri(); got != tt.prevSnapshot { t.Errorf("SnapshotUri = %q, want %q", got, tt.prevSnapshot) } }) @@ -407,8 +426,9 @@ func TestEnsureMarkedPausing_GoldenAtespaceRejected(t *testing.T) { _, err := w.ensureMarkedPausing(context.Background(), resources.ActorRef{Atespace: resources.GoldenActorAtespace, Name: "golden-1"}, - &ateapipb.Actor{Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_RUNNING}}) + &ateapipb.Actor{Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_RUNNING}}, + &ateapipb.ActorTemplate{}) if got := apierror.Code(err); got != codes.FailedPrecondition { - t.Fatalf("status.Code = %v (err %v), want FailedPrecondition", got, err) + t.Fatalf("apierror.Code = %v (err %v), want FailedPrecondition", got, err) } } diff --git a/cmd/ateapi/internal/controlapi/workflow_reconcile_assignments_test.go b/cmd/ateapi/internal/controlapi/workflow_reconcile_assignments_test.go index 9adf4f12db..5a59ab01e0 100644 --- a/cmd/ateapi/internal/controlapi/workflow_reconcile_assignments_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_reconcile_assignments_test.go @@ -177,7 +177,10 @@ func TestReconcileAssignments_CrashesEarlierActors(t *testing.T) { svc, persistence := newWorkerAPIService(t) seedEpochWorker(t, ctx, persistence, 1, 1) actor := seedAPIActor(t, ctx, persistence, ateapipb.ActorState_ACTOR_STATE_RUNNING, func(a *ateapipb.Actor) { - a.Status.InProgressLocalSnapshotName = "partial-local-snapshot" + a.Status.LastAssignedGeneration = 1 + a.Status.Snapshots = []*ateapipb.Snapshot{ + newLocalSnapshot(1, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "partial-local-snapshot", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS), + } }) assignAPIWorker(t, ctx, persistence, apiWorkerName, actor.GetMetadata().GetUid()) mustRaiseEpoch(t, ctx, svc, persistence, 2) @@ -204,8 +207,8 @@ func TestReconcileAssignments_CrashesEarlierActors(t *testing.T) { if got.GetStatus().GetWorkerAssignment() != nil { t.Errorf("actor worker assignment = %v, want it cleared", got.GetStatus().GetWorkerAssignment()) } - if got.GetStatus().GetInProgressLocalSnapshotName() != "" { - t.Errorf("in-progress local checkpoint not cleared: %v", got.GetStatus()) + if _, gotSt := findLatestSnapshotStorage(got.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS); gotSt.GetLocal().GetSnapshotName() != "partial-local-snapshot" { + t.Errorf("in-progress local checkpoint = %q, want %q preserved", gotSt.GetLocal().GetSnapshotName(), "partial-local-snapshot") } if a := firstAssignment(t, persistence, apiWorkerName); a != nil { t.Errorf("worker still hosts %v, want the assignment released", a) diff --git a/cmd/ateapi/internal/controlapi/workflow_resume.go b/cmd/ateapi/internal/controlapi/workflow_resume.go index ee12bd5af1..b9bfbff09f 100644 --- a/cmd/ateapi/internal/controlapi/workflow_resume.go +++ b/cmd/ateapi/internal/controlapi/workflow_resume.go @@ -43,19 +43,19 @@ type resumeSnapshotSource struct { // Zero means cold boot from the spec (unless the actor holds a local // snapshot, which takes precedence at restore). SnapshotURI resources.SnapshotURI - Scope ateapipb.SnapshotContentScope + Fidelity ateapipb.SnapshotFidelity // TemplateReplaced is true when the external snapshot's recorded template // UID differs from the actor's current template. TemplateReplaced bool } // restoreTelemetry labels the restore operation for the resume lifecycle -// metric. WireSnapshotScope describes the restore requested, not the stored -// snapshot's scope: a full snapshot restored under a replaced template goes +// metric. WireFidelity describes the restore requested, not the stored +// snapshot's scope: a MEMORY snapshot restored under a replaced template goes // out as data. type restoreTelemetry struct { - SnapshotKind string - WireSnapshotScope string + SnapshotKind string + WireFidelity string } // ResumeActor executes the workflow to resume a suspended actor. Idempotent: @@ -77,7 +77,7 @@ func (w *ActorWorkflow) ResumeActor(ctx context.Context, actorRef resources.Acto return } w.instruments.recordLifecycleOp(ctx, ateattr.OperationResume, start, err, - lifecycleOpAttrs(actor, actorTemplate, tele.SnapshotKind, tele.WireSnapshotScope)...) + lifecycleOpAttrs(actor, actorTemplate, tele.SnapshotKind, tele.WireFidelity)...) }() // Routed requests call ResumeActor even when the actor is already running. @@ -132,22 +132,19 @@ func (w *ActorWorkflow) ResumeActor(ctx context.Context, actorRef resources.Acto return actor, true, nil } -// validateGoldenSnapshotScope rejects a golden snapshot that does not carry +// validateGoldenSnapshotFidelity rejects a golden snapshot that does not carry // the guest state (memory + fs delta) a restore needs. Golden actors always -// commit Full (commitSnapshotScope), so this only trips on golden snapshots -// taken before that rule existed — surface a clear error instead of shipping -// a restore request atelet would reject (or that would boot an empty guest). -func validateGoldenSnapshotScope(snapshot *ateapipb.ExternalSnapshot) error { - scope := snapshot.GetContentScope() - switch scope { - case ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_UNSPECIFIED, - ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL: - return nil - default: +// commit MEMORY (preferredFidelity), so this only trips on a golden snapshot +// whose record drifted — surface a clear error instead of shipping a restore +// request atelet would reject (or that would boot an empty guest). +func validateGoldenSnapshotFidelity(snapshot *ateapipb.Snapshot) error { + st := findSnapshotStorage(snapshot, ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE) + if fidelity := st.GetFidelity(); fidelity != ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY { return apierror.FailedPrecondition( - "ActorTemplate golden snapshot %q was taken with scope %s, not Full; regenerate the golden snapshot", - snapshot.GetSnapshotUri(), scope) + "ActorTemplate golden snapshot %q was taken with fidelity %s, not MEMORY; regenerate the golden snapshot", + st.GetObject().GetSnapshotUri(), fidelity) } + return nil } // loadActorForResume fetches the current actor record and its template, and @@ -176,12 +173,12 @@ func (w *ActorWorkflow) loadActorForResume(ctx context.Context, actorRef resourc if err != nil { return nil, nil, src, err } - if uri := actor.GetStatus().GetExternalSnapshot().GetSnapshotUri(); uri != "" { - if src.SnapshotURI, err = resources.ParseSnapshotURI(uri); err != nil { + if snap, st := findLatestSnapshotStorage(actor.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED); st != nil { + if src.SnapshotURI, err = resources.ParseSnapshotURI(st.GetObject().GetSnapshotUri()); err != nil { return nil, nil, src, apierror.DataLoss("Actor %s external snapshot: %v", actorRef, err) } - src.Scope = actor.GetStatus().GetExternalSnapshot().GetContentScope() - capturedUnder := actor.GetStatus().GetExternalSnapshot().GetActorTemplateUid() + src.Fidelity = st.GetFidelity() + capturedUnder := snap.GetActorTemplateUid() src.TemplateReplaced = capturedUnder != "" && capturedUnder != actorTemplate.GetMetadata().GetUid() } @@ -196,7 +193,7 @@ func (w *ActorWorkflow) ensureVolumesCreated(ctx context.Context, actorRef resou defer func() { err = done(err) }() pending := false - for _, vol := range actor.GetStatus().GetActorVolumes() { + for _, vol := range actor.GetStatus().GetExternalVolumes() { if vol.GetStatus() == ateapipb.ExternalVolume_STATUS_PENDING { pending = true break @@ -207,12 +204,12 @@ func (w *ActorWorkflow) ensureVolumesCreated(ctx context.Context, actorRef resou return actor, nil } - volumes, createErr := createActorVolumes(ctx, w.pluginRegistry, w.storageClassLister, actor.GetMetadata().GetUid(), actorTemplate, actor.GetStatus().GetActorVolumes()) + volumes, createErr := createActorVolumes(ctx, w.pluginRegistry, w.storageClassLister, actor.GetMetadata().GetUid(), actorTemplate, actor.GetStatus().GetExternalVolumes()) // createActorVolumes reports the state it got to even when it fails, so both // paths persist the same field. updatePrecondition := store.PreconditionFrom(actor) persistVolumes := func(toUpdate *ateapipb.Actor) error { - toUpdate.Status.ActorVolumes = volumes + toUpdate.Status.ExternalVolumes = volumes return nil } if createErr != nil { @@ -643,7 +640,7 @@ func (w *ActorWorkflow) ensureVolumesAttached(ctx context.Context, actor *ateapi ref := &ateapipb.ObjectRef{Atespace: actor.GetMetadata().GetAtespace(), Name: actor.GetMetadata().GetName()} volumePublishContexts := make(map[string]map[string]string) - for _, vol := range getMountedActorVolumes(ctx, ref, actor.GetStatus().GetActorVolumes(), actorTemplate) { + for _, vol := range getMountedActorVolumes(ctx, ref, actor.GetStatus().GetExternalVolumes(), actorTemplate) { slog.InfoContext(ctx, "Attaching volume to node", slog.String("volume_id", vol.GetStorageVolumeId()), slog.String("node", node)) plugin, err := w.pluginRegistry.GetPlugin(ctx, vol.GetVolumeType()) if err != nil { @@ -654,7 +651,7 @@ func (w *ActorWorkflow) ensureVolumesAttached(ctx context.Context, actor *ateapi return nil, fmt.Errorf("failed to attach volume %q to node %q: %w", vol.GetStorageVolumeId(), node, err) } if len(resp.PublishContext) > 0 { - volumePublishContexts[vol.GetVolumeName()] = resp.PublishContext + volumePublishContexts[vol.GetName()] = resp.PublishContext } } return volumePublishContexts, nil @@ -698,12 +695,12 @@ func (w *ActorWorkflow) ensureAteletRestored(ctx context.Context, actorRef resou return tele, fmt.Errorf("while resolving sandbox assets: %w", err) } - if local := actor.GetStatus().GetLocalSnapshot(); local != nil { + if _, localSt := findLatestSnapshotStorage(actor.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED); localSt != nil { slog.InfoContext(ctx, "Actor has snapshot; Restoring from snapshot") tele.SnapshotKind = ateattr.SnapshotKindLocal req := &ateletpb.RestoreRequest{ - TargetAteomUid: assignment.GetWorkerPodUid(), + WorkerPodUid: assignment.GetWorkerPodUid(), Atespace: actor.GetMetadata().GetAtespace(), ActorName: actor.GetMetadata().GetName(), ActorTemplateAtespace: actor.GetActorTemplate().GetAtespace(), @@ -717,10 +714,10 @@ func (w *ActorWorkflow) ensureAteletRestored(ctx context.Context, actorRef resou } req.Type = ateletpb.CheckpointType_CHECKPOINT_TYPE_LOCAL req.Config = &ateletpb.RestoreRequest_LocalConfig{ - LocalConfig: &ateletpb.LocalCheckpointConfiguration{SnapshotName: local.GetSnapshotName()}, + LocalConfig: &ateletpb.LocalCheckpointConfiguration{SnapshotName: localSt.GetLocal().GetSnapshotName()}, } - req.Scope = actorSnapshotContentScopeToAtelet(local.GetContentScope()) - tele.WireSnapshotScope = ateattr.SnapshotScopeValue(req.Scope) + req.Fidelity = fidelityToAtelet(localSt.GetFidelity()) + tele.WireFidelity = ateattr.SnapshotFidelityValue(req.Fidelity) if _, err = client.Restore(ctx, req); err != nil { return tele, handleAteletError(ctx, w.store, actorRef, ateattr.OperationResume, "Restore", false, err) @@ -729,13 +726,13 @@ func (w *ActorWorkflow) ensureAteletRestored(ctx context.Context, actorRef resou } else if !src.SnapshotURI.IsZero() { slog.InfoContext(ctx, "Actor has durable snapshot; Restoring from snapshot") tele.SnapshotKind = ateattr.SnapshotKindLatest - scope := actorSnapshotContentScopeToAtelet(src.Scope) + scope := fidelityToAtelet(src.Fidelity) if src.TemplateReplaced { - scope = ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA + scope = ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES } - tele.WireSnapshotScope = ateattr.SnapshotScopeValue(scope) + tele.WireFidelity = ateattr.SnapshotFidelityValue(scope) req := &ateletpb.RestoreRequest{ - TargetAteomUid: assignment.GetWorkerPodUid(), + WorkerPodUid: assignment.GetWorkerPodUid(), Atespace: actor.GetMetadata().GetAtespace(), ActorName: actor.GetMetadata().GetName(), ActorTemplateAtespace: actor.GetActorTemplate().GetAtespace(), @@ -747,7 +744,7 @@ func (w *ActorWorkflow) ensureAteletRestored(ctx context.Context, actorRef resou SnapshotUri: src.SnapshotURI.String(), }, }, - Scope: scope, + Fidelity: scope, SandboxAssets: sandboxAssets, ActorUid: actor.GetMetadata().Uid, EgressGateway: egressGateway, @@ -763,7 +760,7 @@ func (w *ActorWorkflow) ensureAteletRestored(ctx context.Context, actorRef resou tele.SnapshotKind = ateattr.SnapshotKindBoot req := &ateletpb.RunRequest{ - TargetAteomUid: assignment.GetWorkerPodUid(), + WorkerPodUid: assignment.GetWorkerPodUid(), Atespace: actor.GetMetadata().GetAtespace(), ActorName: actor.GetMetadata().GetName(), ActorTemplateAtespace: actor.GetActorTemplate().GetAtespace(), diff --git a/cmd/ateapi/internal/controlapi/workflow_resume_test.go b/cmd/ateapi/internal/controlapi/workflow_resume_test.go index 859a03862f..02a6030fa5 100644 --- a/cmd/ateapi/internal/controlapi/workflow_resume_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_resume_test.go @@ -969,10 +969,9 @@ func TestLoadActorForResume_TemplateReplaced(t *testing.T) { var seedOpts []func(*ateapipb.Actor) if !tt.noSnapshot { seedOpts = append(seedOpts, func(a *ateapipb.Actor) { - a.Status.ExternalSnapshot = &ateapipb.ExternalSnapshot{ - SnapshotUri: someActorSnapshotURI(t, testStorageLocation, actorRef.Atespace, "snap-1"), - ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, - ActorTemplateUid: resolve(tt.snapshotUID), + a.Status.LastAssignedGeneration = 1 + a.Status.Snapshots = []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, resolve(tt.snapshotUID), someActorSnapshotURI(t, testStorageLocation, actorRef.Atespace, "snap-1"), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), } }) } @@ -1110,9 +1109,13 @@ func newWireCaptureWorkflow(t *testing.T, persistence store.Interface) (*ActorWo lister := sandboxConfigListerFor(t, []*atev1alpha1.SandboxConfig{{ ObjectMeta: metav1.ObjectMeta{Name: "gvisor"}, Spec: atev1alpha1.SandboxConfigSpec{ - SandboxClass: atev1alpha1.SandboxClassGvisor, - PauseImage: "pause@sha256:abc", - Assets: testAssets(), + SandboxClass: atev1alpha1.SandboxClassGvisor, + DefaultVersion: "v1", + Versions: []atev1alpha1.SandboxVersionConfig{{ + Name: "v1", + PauseImage: "pause@sha256:abc", + Assets: testAssets(), + }}, }, }}) @@ -1122,7 +1125,7 @@ func newWireCaptureWorkflow(t *testing.T, persistence store.Interface) (*ActorWo // TestResumeActor_AteletWireRequest is the characteristic test for the // loadActorForResume + ensureAteletRestored seam: for every combination of // boot-source inputs it pins the exact request atelet receives — which RPC, -// req.Scope, and the snapshot the config names — and that a source-resolution +// req.Fidelity, and the snapshot the config names — and that a source-resolution // error never produces an atelet RPC. // // The rows are ordered strictly by input columns (local → external → tmplUID → @@ -1134,17 +1137,24 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { actorURI := someActorSnapshotURI(t, testStorageLocation, "team-a", "snap-1") goldenURI := someActorSnapshotURI(t, "gs://bucket/golden-root", "ate-golden", "golden-1") - fullScope := ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL - dataScope := ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA - unspecScope := ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_UNSPECIFIED + fullScope := ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY + dataScope := ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES + unspecScope := ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED + + extSnap := func(uri string, fidelity ateapipb.SnapshotFidelity) *ateapipb.Snapshot { + return newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, fidelity, "", uri, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + } + localSnap := func(name string, fidelity ateapipb.SnapshotFidelity) *ateapipb.Snapshot { + return newLocalSnapshot(1, fidelity, "", name, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + } // actorSeed is the actor status a row persists before resuming. type actorSeed struct { - // localSnapshot seeds Status.LocalSnapshot (the pause checkpoint); + // localSnapshot seeds a completed local snapshot (the pause checkpoint); // a non-nil value also parks the actor PAUSED instead of SUSPENDED. - localSnapshot *ateapipb.LocalSnapshot - // externalSnapshot seeds Status.ExternalSnapshot (the durable snapshot). - externalSnapshot *ateapipb.ExternalSnapshot + localSnapshot *ateapipb.Snapshot + // externalSnapshot seeds a completed durable snapshot. + externalSnapshot *ateapipb.Snapshot // tmplUID seeds the template UID the snapshot's guest state was built // on, stamped onto externalSnapshot: "current" stands for the created // template's store-assigned UID (unknown until runtime), "" leaves the @@ -1154,7 +1164,7 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { // templateSeed is the ActorTemplate configuration a row persists. type templateSeed struct { // golden seeds the template's golden tag snapshot. - golden *ateapipb.ExternalSnapshot + golden *ateapipb.Snapshot // configName is the SandboxConfig the template names; "" means the // "gvisor" config the workflow's lister serves. configName string @@ -1172,7 +1182,7 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { // a row also pins that the other config is absent. snapshotName string snapshotURI string - scope ateletpb.SnapshotScope + scope ateletpb.SnapshotFidelity } tests := []struct { @@ -1187,50 +1197,50 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { }, { name: "02 inherited golden snapshot restores in Full", - actor: actorSeed{externalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: goldenURI, ContentScope: fullScope}, tmplUID: "current"}, - tmpl: templateSeed{golden: &ateapipb.ExternalSnapshot{SnapshotUri: goldenURI, ContentScope: fullScope}}, + actor: actorSeed{externalSnapshot: extSnap(goldenURI, fullScope), tmplUID: "current"}, + tmpl: templateSeed{golden: extSnap(goldenURI, fullScope)}, want: restoreWant{ checkpointType: ateletpb.CheckpointType_CHECKPOINT_TYPE_EXTERNAL, snapshotURI: goldenURI, - scope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL, + scope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, }, }, { name: "03 late non-Full golden does not change a cold boot", - tmpl: templateSeed{golden: &ateapipb.ExternalSnapshot{SnapshotUri: goldenURI, ContentScope: dataScope}}, + tmpl: templateSeed{golden: extSnap(goldenURI, dataScope)}, want: restoreWant{run: true}, }, { name: "04 inherited golden snapshot rejects a malformed URI", - actor: actorSeed{externalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: malformedURI, ContentScope: fullScope}, tmplUID: "current"}, - tmpl: templateSeed{golden: &ateapipb.ExternalSnapshot{SnapshotUri: malformedURI, ContentScope: fullScope}}, + actor: actorSeed{externalSnapshot: extSnap(malformedURI, fullScope), tmplUID: "current"}, + tmpl: templateSeed{golden: extSnap(malformedURI, fullScope)}, want: restoreWant{code: codes.DataLoss}, }, { name: "05 template repoint with a late golden still cold-boots", actor: actorSeed{tmplUID: "old-template-uid"}, - tmpl: templateSeed{golden: &ateapipb.ExternalSnapshot{SnapshotUri: goldenURI, ContentScope: fullScope}}, + tmpl: templateSeed{golden: extSnap(goldenURI, fullScope)}, want: restoreWant{run: true}, }, { name: "06 Full durable snapshot restores itself in Full", - actor: actorSeed{externalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: actorURI, ContentScope: fullScope}}, + actor: actorSeed{externalSnapshot: extSnap(actorURI, fullScope)}, want: restoreWant{ checkpointType: ateletpb.CheckpointType_CHECKPOINT_TYPE_EXTERNAL, snapshotURI: actorURI, - scope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL, + scope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, }, }, { name: "07 durable snapshot built on the current template stays Full", actor: actorSeed{ - externalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: actorURI, ContentScope: fullScope}, + externalSnapshot: extSnap(actorURI, fullScope), tmplUID: "current", }, want: restoreWant{ checkpointType: ateletpb.CheckpointType_CHECKPOINT_TYPE_EXTERNAL, snapshotURI: actorURI, - scope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL, + scope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, }, }, { @@ -1239,61 +1249,62 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { // boots fresh and only the volume data carries over. name: "08 repointed actor's Full durable snapshot drops to Data", actor: actorSeed{ - externalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: actorURI, ContentScope: fullScope}, + externalSnapshot: extSnap(actorURI, fullScope), tmplUID: "mismatch", }, want: restoreWant{ checkpointType: ateletpb.CheckpointType_CHECKPOINT_TYPE_EXTERNAL, snapshotURI: actorURI, - scope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA, + scope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, }, }, { name: "09 Data durable snapshot restores as Data", - actor: actorSeed{externalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: actorURI, ContentScope: dataScope}}, + actor: actorSeed{externalSnapshot: extSnap(actorURI, dataScope)}, want: restoreWant{ checkpointType: ateletpb.CheckpointType_CHECKPOINT_TYPE_EXTERNAL, snapshotURI: actorURI, - scope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA, + scope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, }, }, { // The template's golden snapshot never supplies guest state for a // Data snapshot: it restores as plain Data. name: "10 Data durable snapshot ignores the template's golden", - actor: actorSeed{externalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: actorURI, ContentScope: dataScope}}, - tmpl: templateSeed{golden: &ateapipb.ExternalSnapshot{SnapshotUri: goldenURI, ContentScope: fullScope}}, + actor: actorSeed{externalSnapshot: extSnap(actorURI, dataScope)}, + tmpl: templateSeed{golden: extSnap(goldenURI, fullScope)}, want: restoreWant{ checkpointType: ateletpb.CheckpointType_CHECKPOINT_TYPE_EXTERNAL, snapshotURI: actorURI, - scope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA, + scope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, }, }, { - // Snapshots recorded before content_scope existed carry - // UNSPECIFIED; the conversion sends them out as Full. - name: "11 unspecified durable scope goes out as Full", - actor: actorSeed{externalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: actorURI, ContentScope: unspecScope}}, + // A stored snapshot always records its fidelity; one without is a + // drifted record, and ateapi forwards it unchanged for atelet's + // validation to reject rather than guessing MEMORY. + name: "11 unspecified durable fidelity goes out unspecified", + actor: actorSeed{externalSnapshot: extSnap(actorURI, unspecScope)}, want: restoreWant{ checkpointType: ateletpb.CheckpointType_CHECKPOINT_TYPE_EXTERNAL, snapshotURI: actorURI, - scope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL, + scope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED, }, }, { name: "12 malformed durable snapshot URI fails with DataLoss", - actor: actorSeed{externalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: malformedURI, ContentScope: fullScope}}, + actor: actorSeed{externalSnapshot: extSnap(malformedURI, fullScope)}, want: restoreWant{code: codes.DataLoss}, }, { name: "13 Full pause snapshot restores locally as Full", actor: actorSeed{ - localSnapshot: &ateapipb.LocalSnapshot{SnapshotName: localSnapshotName, ContentScope: fullScope}, + localSnapshot: localSnap(localSnapshotName, fullScope), }, want: restoreWant{ checkpointType: ateletpb.CheckpointType_CHECKPOINT_TYPE_LOCAL, snapshotName: localSnapshotName, - scope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL, + scope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, }, }, { @@ -1302,38 +1313,38 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { // resume-source resolution is being reworked. name: "14 local snapshot built on the current template stays Full", actor: actorSeed{ - localSnapshot: &ateapipb.LocalSnapshot{SnapshotName: localSnapshotName, ContentScope: fullScope}, + localSnapshot: localSnap(localSnapshotName, fullScope), tmplUID: "current", }, want: restoreWant{ checkpointType: ateletpb.CheckpointType_CHECKPOINT_TYPE_LOCAL, snapshotName: localSnapshotName, - scope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL, + scope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, }, }, { name: "15 Data pause snapshot restores locally as Data", actor: actorSeed{ - localSnapshot: &ateapipb.LocalSnapshot{SnapshotName: localSnapshotName, ContentScope: dataScope}, + localSnapshot: localSnap(localSnapshotName, dataScope), }, want: restoreWant{ checkpointType: ateletpb.CheckpointType_CHECKPOINT_TYPE_LOCAL, snapshotName: localSnapshotName, - scope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA, + scope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, }, }, { name: "16 Data pause snapshot ignores the template's golden", actor: actorSeed{ - localSnapshot: &ateapipb.LocalSnapshot{SnapshotName: localSnapshotName, ContentScope: dataScope}, + localSnapshot: localSnap(localSnapshotName, dataScope), }, tmpl: templateSeed{ - golden: &ateapipb.ExternalSnapshot{SnapshotUri: goldenURI, ContentScope: fullScope}, + golden: extSnap(goldenURI, fullScope), }, want: restoreWant{ checkpointType: ateletpb.CheckpointType_CHECKPOINT_TYPE_LOCAL, snapshotName: localSnapshotName, - scope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA, + scope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, }, }, { @@ -1341,13 +1352,13 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { // scope wins over the durable snapshot's. name: "17 local snapshot wins over a Full durable snapshot", actor: actorSeed{ - localSnapshot: &ateapipb.LocalSnapshot{SnapshotName: localSnapshotName, ContentScope: dataScope}, - externalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: actorURI, ContentScope: fullScope}, + localSnapshot: localSnap(localSnapshotName, dataScope), + externalSnapshot: extSnap(actorURI, fullScope), }, want: restoreWant{ checkpointType: ateletpb.CheckpointType_CHECKPOINT_TYPE_LOCAL, snapshotName: localSnapshotName, - scope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA, + scope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, }, }, { @@ -1357,27 +1368,27 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { // template. name: "18 local snapshot ignores an older external snapshot's template mismatch", actor: actorSeed{ - localSnapshot: &ateapipb.LocalSnapshot{SnapshotName: localSnapshotName, ContentScope: fullScope}, - externalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: actorURI, ContentScope: fullScope}, + localSnapshot: localSnap(localSnapshotName, fullScope), + externalSnapshot: extSnap(actorURI, fullScope), tmplUID: "mismatch", }, want: restoreWant{ checkpointType: ateletpb.CheckpointType_CHECKPOINT_TYPE_LOCAL, snapshotName: localSnapshotName, - scope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL, + scope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, }, }, { // Restores take their sandbox from the template, like cold boots, // so an unresolvable SandboxConfig stops them before atelet. name: "19 durable snapshot restore with a missing SandboxConfig is rejected", - actor: actorSeed{externalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: actorURI, ContentScope: fullScope}}, + actor: actorSeed{externalSnapshot: extSnap(actorURI, fullScope)}, tmpl: templateSeed{configName: "missing"}, want: restoreWant{code: codes.FailedPrecondition}, }, { name: "20 local snapshot restore with a missing SandboxConfig is rejected", - actor: actorSeed{localSnapshot: &ateapipb.LocalSnapshot{SnapshotName: localSnapshotName}}, + actor: actorSeed{localSnapshot: localSnap(localSnapshotName, unspecScope)}, tmpl: templateSeed{configName: "missing"}, want: restoreWant{code: codes.FailedPrecondition}, }, @@ -1385,11 +1396,10 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { // Every request atelet receives, Run or Restore, carries the sandbox the // template's SandboxConfig resolves to. - wantSandboxAssets := sandboxAssetsProto(&atev1alpha1.SandboxConfig{Spec: atev1alpha1.SandboxConfigSpec{ - SandboxClass: atev1alpha1.SandboxClassGvisor, - PauseImage: "pause@sha256:abc", - Assets: testAssets(), - }}) + wantSandboxAssets := sandboxAssetsProto( + &atev1alpha1.SandboxConfig{Spec: atev1alpha1.SandboxConfigSpec{SandboxClass: atev1alpha1.SandboxClassGvisor}}, + &atev1alpha1.SandboxVersionConfig{Name: "v1", PauseImage: "pause@sha256:abc", Assets: testAssets()}, + ) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -1444,21 +1454,31 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { actorRef := resources.ActorRef{Atespace: "team-a", Name: "id1"} seedWorkflowActor(t, ctx, persistence, actorRef, "ns", "tmpl1", actorState, func(a *ateapipb.Actor) { a.Status.WorkerAssignment = wireTestAssignment() - a.Status.LocalSnapshot = tt.actor.localSnapshot - if tt.actor.localSnapshot != nil { - a.Status.AssignedNode = "node-1" - } uid := tt.actor.tmplUID if uid == "current" { uid = createdTmpl.GetMetadata().GetUid() } + var gen int32 if tt.actor.externalSnapshot != nil { + gen++ ext := proto.CloneOf(tt.actor.externalSnapshot) + ext.Generation = gen if ext.ActorTemplateUid == "" { ext.ActorTemplateUid = uid } - a.Status.ExternalSnapshot = ext + a.Status.Snapshots = append(a.Status.Snapshots, ext) + } + if tt.actor.localSnapshot != nil { + gen++ + a.Status.AssignedNode = "node-1" + loc := proto.CloneOf(tt.actor.localSnapshot) + loc.Generation = gen + if loc.ActorTemplateUid == "" { + loc.ActorTemplateUid = uid + } + a.Status.Snapshots = append(a.Status.Snapshots, loc) } + a.Status.LastAssignedGeneration = gen }) actor, loadedTmpl, src, err := w.loadActorForResume(ctx, actorRef) @@ -1501,7 +1521,7 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { if got := restore.GetExternalConfig().GetSnapshotUri(); got != tt.want.snapshotURI { t.Errorf("ExternalConfig.SnapshotUri = %q, want %q", got, tt.want.snapshotURI) } - if got := restore.GetScope(); got != tt.want.scope { + if got := restore.GetFidelity(); got != tt.want.scope { t.Errorf("restore scope = %v, want %v", got, tt.want.scope) } }) @@ -1529,9 +1549,9 @@ func TestEnsureVolumesAttached_ReturnsPublishContext(t *testing.T) { Metadata: &ateapipb.ResourceMetadata{Atespace: "team-a", Name: "id1"}, Status: &ateapipb.ActorStatus{ State: ateapipb.ActorState_ACTOR_STATE_RESUMING, - ActorVolumes: []*ateapipb.ExternalVolume{ - {VolumeName: "mounted", StorageVolumeId: "storage-mounted", VolumeType: "mock", Status: ateapipb.ExternalVolume_STATUS_CREATED}, - {VolumeName: "unmounted", StorageVolumeId: "storage-unmounted", VolumeType: "mock", Status: ateapipb.ExternalVolume_STATUS_CREATED}, + ExternalVolumes: []*ateapipb.ExternalVolume{ + {Name: "mounted", StorageVolumeId: "storage-mounted", VolumeType: "mock", Status: ateapipb.ExternalVolume_STATUS_CREATED}, + {Name: "unmounted", StorageVolumeId: "storage-unmounted", VolumeType: "mock", Status: ateapipb.ExternalVolume_STATUS_CREATED}, }, }, }) diff --git a/cmd/ateapi/internal/controlapi/workflow_revert.go b/cmd/ateapi/internal/controlapi/workflow_revert.go index b8154eaeb7..2f4b3c6e2f 100644 --- a/cmd/ateapi/internal/controlapi/workflow_revert.go +++ b/cmd/ateapi/internal/controlapi/workflow_revert.go @@ -25,7 +25,6 @@ import ( "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" "github.com/agent-substrate/substrate/internal/apierror" "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/objectstore" "github.com/agent-substrate/substrate/internal/resources" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "go.opentelemetry.io/otel/attribute" @@ -200,9 +199,10 @@ func (w *ActorWorkflow) ensureInProgressSnapshotDiscarded(ctx context.Context, a ctx, done := stepSpan(ctx, "DiscardInProgressSnapshot") defer func() { err = done(err) }() - inProgress := actor.GetStatus().GetInProgressSnapshotUri() + _, inProgressSt := findLatestSnapshotStorage(actor.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS) + inProgress := inProgressSt.GetObject().GetSnapshotUri() switch { - case w.objectStore == nil: + case w.snapshotPlugin == nil: markSkipped(ctx, "no object store configured") return nil case inProgress == "": @@ -223,7 +223,7 @@ func (w *ActorWorkflow) ensureInProgressSnapshotDiscarded(ctx context.Context, a } // Only the abandoned snapshot goes, not the actor's whole prefix: the // external snapshot the revert returns the actor to lives under it too. - return objectstore.DeletePrefix(ctx, w.objectStore, uri.Prefix()) + return w.cleanupSnapshot(ctx, uri.Prefix()) } // ensureRevertedFinalized commits SUSPENDED and drops every pointer to the @@ -246,9 +246,13 @@ func (w *ActorWorkflow) ensureRevertedFinalized(ctx context.Context, actorRef re toUpdate.Status.State = ateapipb.ActorState_ACTOR_STATE_SUSPENDED toUpdate.Status.WorkerAssignment = nil toUpdate.Status.AssignedNode = "" - toUpdate.Status.InProgressSnapshotUri = "" - toUpdate.Status.InProgressLocalSnapshotName = "" - toUpdate.Status.LocalSnapshot = nil + // We revert back to the latest durable snapshot, so all local snapshots become invalid. + // We already cleaned up the node before this in ensureWorkerDiscarded. + // Even during revert, we do not decrease LastAssignedGeneration so that we + // can keep track of stale snapshots that need to be cleaned up. + removeSnapshotStorageEntries(toUpdate.Status, ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL, nil) + // We already cleaned up the inprogress durable snapshot in ensureInProgressSnapshotDiscarded + removeSnapshotStorageEntries(toUpdate.Status, ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, new(ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS)) toUpdate.Status.Crash = nil return nil }) diff --git a/cmd/ateapi/internal/controlapi/workflow_revert_test.go b/cmd/ateapi/internal/controlapi/workflow_revert_test.go index 15dcd49373..acec0029e5 100644 --- a/cmd/ateapi/internal/controlapi/workflow_revert_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_revert_test.go @@ -97,9 +97,12 @@ func TestRevertActor_ReturnsActorToItsSnapshot(t *testing.T) { // pause left behind, which it must not. const keptURI = "gs://snapshots/team-a/actors/keep/snapshot" mustUpdateActorStatus(t, ctx, st, actor, func(s *ateapipb.ActorStatus) { - s.ExternalSnapshot = &ateapipb.ExternalSnapshot{SnapshotUri: keptURI} - s.LocalSnapshot = &ateapipb.LocalSnapshot{SnapshotName: "local-1"} - s.InProgressLocalSnapshotName = "local-in-progress" + s.LastAssignedGeneration = 3 + s.Snapshots = []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", keptURI, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + newLocalSnapshot(2, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "local-1", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + newLocalSnapshot(3, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "local-in-progress", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS), + } }) reverted, err := w.RevertActor(ctx, actorRef) @@ -111,20 +114,30 @@ func TestRevertActor_ReturnsActorToItsSnapshot(t *testing.T) { if got := gotStatus.GetState(); got != ateapipb.ActorState_ACTOR_STATE_SUSPENDED { t.Errorf("state = %v, want SUSPENDED", got) } - if got := gotStatus.GetExternalSnapshot().GetSnapshotUri(); got != keptURI { + // LastAssignedGeneration is not decreased on revert so we can keep track + // of stale snapshots that need to be cleaned up and future snapshots + // never reuse generations 2 or 3 after revert discards them. + if got := gotStatus.GetLastAssignedGeneration(); got != 3 { + t.Errorf("last_assigned_generation = %d, want 3", got) + } + gotSnap, gotSt := findLatestSnapshotStorage(gotStatus, ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + if got := gotSnap.GetGeneration(); got != 1 { + t.Errorf("retained snapshot generation = %d, want 1", got) + } + if got := gotSt.GetObject().GetSnapshotUri(); got != keptURI { t.Errorf("external snapshot = %q, want it untouched at %q", got, keptURI) } // Resume prefers a local snapshot over the external one, so a // surviving LocalSnapshot would restore the execution this // revert just discarded. - if got := gotStatus.GetLocalSnapshot(); got != nil { + if _, got := findLatestSnapshotStorage(gotStatus, ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED); got != nil { t.Errorf("local snapshot info = %v, want nil", got) } - if got := gotStatus.GetInProgressSnapshotUri(); got != "" { - t.Errorf("in-progress snapshot uri = %q, want empty", got) + if _, got := findLatestSnapshotStorage(gotStatus, ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS); got != nil { + t.Errorf("in-progress durable snapshot = %v, want nil", got) } - if got := gotStatus.GetInProgressLocalSnapshotName(); got != "" { - t.Errorf("in-progress local snapshot name = %q, want empty", got) + if _, got := findLatestSnapshotStorage(gotStatus, ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS); got != nil { + t.Errorf("in-progress local snapshot = %v, want nil", got) } if got := gotStatus.GetAssignedNode(); got != "" { t.Errorf("assigned node = %q, want empty", got) @@ -132,6 +145,25 @@ func TestRevertActor_ReturnsActorToItsSnapshot(t *testing.T) { if got := gotStatus.GetWorkerAssignment(); got != nil { t.Errorf("worker assignment = %v, want nil", got) } + + // If the actor runs again and takes another snapshot, it must + // allocate generation 4 rather than reusing 2 or 3. + running := mustUpdateActorStatus(t, ctx, st, reverted, func(s *ateapipb.ActorStatus) { + s.State = ateapipb.ActorState_ACTOR_STATE_RUNNING + }) + tmpl := &ateapipb.ActorTemplate{ + SnapshotConfig: &ateapipb.SnapshotConfig{StorageLocation: "gs://snapshots/"}, + } + suspending, err := w.ensureMarkedSuspending(ctx, actorRef, running, tmpl) + if err != nil { + t.Fatalf("ensureMarkedSuspending after revert: %v", err) + } + if got := suspending.GetStatus().GetLastAssignedGeneration(); got != 4 { + t.Errorf("last_assigned_generation after post-revert suspend = %d, want 4", got) + } + if got := snapshotAtLatestGeneration(suspending.GetStatus()).GetGeneration(); got != 4 { + t.Errorf("new snapshot generation after post-revert suspend = %d, want 4", got) + } }) } } @@ -219,7 +251,8 @@ func TestRevertActor_NoSnapshotToRevertTo(t *testing.T) { if got := reverted.GetStatus().GetCrash(); got != nil { t.Errorf("Crash = %v, want cleared by the revert", got) } - if got := reverted.GetStatus().GetExternalSnapshot().GetSnapshotUri(); got != "" { + _, gotSt := findLatestSnapshotStorage(reverted.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + if got := gotSt.GetObject().GetSnapshotUri(); got != "" { t.Errorf("external snapshot = %q, want none", got) } } @@ -283,8 +316,10 @@ func TestEnsureInProgressSnapshotDiscarded(t *testing.T) { // The actor's prefix is keyed on the UID the store just assigned, // so its snapshots can only be placed now. current := mustActorSnapshotURI(t, template, actor, "current") + currentType := ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR if tt.tagOwnedSnapshot { current = mustTagSnapshotURI(t, template, "team-a", "v1-snapshot") + currentType = ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG } objects.PutSnapshot(t, current, "manifest.json") inFlight := mustActorSnapshotURI(t, template, actor, inFlightSnapshotName) @@ -295,9 +330,13 @@ func TestEnsureInProgressSnapshotDiscarded(t *testing.T) { objects.PutSnapshot(t, inFlight, "manifest.json") } actor = mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapipb.ActorStatus) { - s.ExternalSnapshot = &ateapipb.ExternalSnapshot{SnapshotUri: current.String()} + s.LastAssignedGeneration = 1 + s.Snapshots = []*ateapipb.Snapshot{ + newDurableSnapshot(1, currentType, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", current.String(), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + } if tt.inFlight != "" { - s.InProgressSnapshotUri = inFlight.String() + s.LastAssignedGeneration = 2 + s.Snapshots = append(s.Snapshots, newDurableSnapshot(2, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", inFlight.String(), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS)) } }) @@ -342,9 +381,12 @@ func TestEnsureRevertedFinalized_NoObjectStore(t *testing.T) { Metadata: &ateapipb.ResourceMetadata{Atespace: "team-a", Name: "actor-1"}, ActorTemplate: &ateapipb.ObjectRef{Atespace: "team-a", Name: "sub-tmpl"}, Status: &ateapipb.ActorStatus{ - State: ateapipb.ActorState_ACTOR_STATE_REVERTING, - AssignedNode: "node-1", - InProgressSnapshotUri: someActorSnapshotURI(t, testStorageLocation, "team-a", "abandoned"), + State: ateapipb.ActorState_ACTOR_STATE_REVERTING, + AssignedNode: "node-1", + LastAssignedGeneration: 1, + Snapshots: []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", someActorSnapshotURI(t, testStorageLocation, "team-a", "abandoned"), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS), + }, }, }) @@ -363,7 +405,7 @@ func TestEnsureRevertedFinalized_NoObjectStore(t *testing.T) { if got := finalized.GetStatus().GetAssignedNode(); got != "" { t.Errorf("assigned node = %q, want empty", got) } - if got := finalized.GetStatus().GetInProgressSnapshotUri(); got != "" { - t.Errorf("in-progress snapshot uri = %q, want empty", got) + if _, got := findLatestSnapshotStorage(finalized.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS); got != nil { + t.Errorf("in-progress durable snapshot = %v, want nil", got) } } diff --git a/cmd/ateapi/internal/controlapi/workflow_snapshot_plugin_test.go b/cmd/ateapi/internal/controlapi/workflow_snapshot_plugin_test.go new file mode 100644 index 0000000000..3344061492 --- /dev/null +++ b/cmd/ateapi/internal/controlapi/workflow_snapshot_plugin_test.go @@ -0,0 +1,72 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlapi + +import ( + "context" + "testing" + + "github.com/agent-substrate/substrate/internal/ateinterceptors" + "github.com/agent-substrate/substrate/internal/resources" + objectstorev1 "github.com/agent-substrate/substrate/pkg/proto/objectstorepb/v1" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// failingControlPlugin fails every call with err. +type failingControlPlugin struct { + err error +} + +func (p failingControlPlugin) CleanupSnapshot(context.Context, *objectstorev1.CleanupSnapshotRequest, ...grpc.CallOption) (*objectstorev1.CleanupSnapshotResponse, error) { + return nil, p.err +} + +func (p failingControlPlugin) CopySnapshot(context.Context, *objectstorev1.CopySnapshotRequest, ...grpc.CallOption) (*objectstorev1.CopySnapshotResponse, error) { + return nil, p.err +} + +// A control snapshot plugin that cannot be reached fails the request with +// Unavailable, which a client retries; other plugin errors stay Internal. +func TestSnapshotPluginErrorCodeToClient(t *testing.T) { + prefix, err := resources.ParseStoragePrefix("gs://bucket/root/atespaces/team-a/actors/uid1/snapshots/snap1") + if err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + pluginCode codes.Code + want codes.Code + }{ + {codes.Unavailable, codes.Unavailable}, + {codes.Internal, codes.Internal}, + {codes.FailedPrecondition, codes.Internal}, + } { + t.Run(tc.pluginCode.String(), func(t *testing.T) { + w := &ActorWorkflow{snapshotPlugin: failingControlPlugin{err: status.Error(tc.pluginCode, "plugin failed")}} + for name, call := range map[string]func(context.Context) error{ + "cleanup": func(ctx context.Context) error { return w.cleanupSnapshot(ctx, prefix) }, + "copy": func(ctx context.Context) error { return w.copySnapshot(ctx, prefix, prefix) }, + } { + // The public server interceptor decides the code the client sees. + _, err := ateinterceptors.ServerUnaryInterceptor(context.Background(), nil, &grpc.UnaryServerInfo{FullMethod: "/test"}, + func(ctx context.Context, _ any) (any, error) { return nil, call(ctx) }) + if got := status.Code(err); got != tc.want { + t.Errorf("%s: code = %s (%v), want %s", name, got, err, tc.want) + } + } + }) + } +} diff --git a/cmd/ateapi/internal/controlapi/workflow_suspend.go b/cmd/ateapi/internal/controlapi/workflow_suspend.go index e39a8ce4ee..28f3893cd7 100644 --- a/cmd/ateapi/internal/controlapi/workflow_suspend.go +++ b/cmd/ateapi/internal/controlapi/workflow_suspend.go @@ -24,12 +24,10 @@ import ( "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" "github.com/agent-substrate/substrate/internal/apierror" "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/objectstore" "github.com/agent-substrate/substrate/internal/proto/ateletpb" "github.com/agent-substrate/substrate/internal/resources" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "go.opentelemetry.io/otel/attribute" - "google.golang.org/protobuf/proto" ) // SuspendActor executes the workflow to suspend a running or paused actor: @@ -41,7 +39,7 @@ func (w *ActorWorkflow) SuspendActor(ctx context.Context, actorRef resources.Act start := time.Now() var actor *ateapipb.Actor var actorTemplate *ateapipb.ActorTemplate - var wireSnapshotScope string + var wireFidelity string // Set just before finalize; nil until then, so earlier exits label // themselves from the record they hold. var finalAttrs []attribute.KeyValue @@ -49,7 +47,7 @@ func (w *ActorWorkflow) SuspendActor(ctx context.Context, actorRef resources.Act defer func() { attrs := finalAttrs if attrs == nil { - attrs = lifecycleOpAttrs(actor, actorTemplate, "", wireSnapshotScope) + attrs = lifecycleOpAttrs(actor, actorTemplate, "", wireFidelity) } w.instruments.recordLifecycleOp(ctx, ateattr.OperationSuspend, start, err, attrs...) }() @@ -80,9 +78,9 @@ func (w *ActorWorkflow) SuspendActor(ctx context.Context, actorRef resources.Act } actor = marked if fromPaused { - wireSnapshotScope, err = w.ensurePausedSnapshotUploaded(leaseCtx, actorRef, actor, actorTemplate) + wireFidelity, err = w.ensurePausedSnapshotUploaded(leaseCtx, actorRef, actor, actorTemplate) } else { - wireSnapshotScope, err = w.ensureAteletSuspended(leaseCtx, actorRef, actor, actorTemplate) + wireFidelity, err = w.ensureAteletSuspended(leaseCtx, actorRef, actor, actorTemplate) } if err != nil { return nil, err @@ -92,9 +90,9 @@ func (w *ActorWorkflow) SuspendActor(ctx context.Context, actorRef resources.Act } // FinalizeSuspended clears the WorkerAssignment the labels read, so snapshot // them here, as crash.go does for the crash counter. - finalAttrs = lifecycleOpAttrs(actor, actorTemplate, "", wireSnapshotScope) + finalAttrs = lifecycleOpAttrs(actor, actorTemplate, "", wireFidelity) var finalized *ateapipb.Actor - if finalized, err = w.ensureSuspendedFinalized(leaseCtx, actorRef, actorTemplate); err != nil { + if finalized, err = w.ensureSuspendedFinalized(leaseCtx, actorRef); err != nil { return nil, err } actor = finalized @@ -133,6 +131,16 @@ func (w *ActorWorkflow) ensureMarkedSuspending(ctx context.Context, actorRef res if got := actor.GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_RUNNING && got != ateapipb.ActorState_ACTOR_STATE_PAUSED { return nil, apierror.FailedPrecondition("MarkSuspending prerequisite not met for Actor: %s (got: %v, want %s or %s)", actorRef, got, ateapipb.ActorState_ACTOR_STATE_RUNNING, ateapipb.ActorState_ACTOR_STATE_PAUSED) } + if actor.GetStatus().GetState() == ateapipb.ActorState_ACTOR_STATE_PAUSED { + localSnap, _ := findLatestSnapshotStorage(actor.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + if localSnap == nil { + if err := crashActor(ctx, w.store, actorRef, ateattr.OperationSuspend, crashMessageLocalSnapshotNodeUnknown); err != nil { + slog.ErrorContext(ctx, "Failed to crash actor", slog.String("err", err.Error())) + } + return nil, fmt.Errorf("actor is CRASHED because it was in PAUSED state with no completed local snapshot") + } + } + // Fail here rather than at checkpoint time if the template's location // cannot produce a usable URI: nothing has been written yet. uri, err := newInProgressSnapshotURI(actorTemplate, actor) @@ -140,8 +148,37 @@ func (w *ActorWorkflow) ensureMarkedSuspending(ctx context.Context, actorRef res return nil, err } storedActor, err := w.store.UpdateActor(ctx, actorRef, store.PreconditionFrom(actor), func(toUpdate *ateapipb.Actor) error { + wasPaused := toUpdate.Status.State == ateapipb.ActorState_ACTOR_STATE_PAUSED toUpdate.Status.State = ateapipb.ActorState_ACTOR_STATE_SUSPENDING - toUpdate.Status.InProgressSnapshotUri = uri.String() + if wasPaused { + // Add a new in progress durable snapshot to the current generation + snap, _ := findLatestSnapshotStorage(toUpdate.Status, ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + if snap == nil { + return fmt.Errorf("actor %s is PAUSED but has no completed local snapshot", actorRef) + } + snap.ActorTemplateUid = actorTemplate.GetMetadata().GetUid() + durableInProgress := &ateapipb.SnapshotStorage{ + Durability: ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, + Status: ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS, + Fidelity: preferredFidelity(actorRef.Atespace, actorTemplate), + Object: &ateapipb.ObjectSnapshot{SnapshotUri: uri.String()}, + } + setSnapshotStorage(snap, durableInProgress) + return nil + } + // Increment last_assigned_generation for the new Suspend request and create + // an in-progress snapshot for that generation. + gen := toUpdate.Status.LastAssignedGeneration + 1 + toUpdate.Status.LastAssignedGeneration = gen + toUpdate.Status.Snapshots = append(toUpdate.Status.Snapshots, + newDurableSnapshot( + gen, + ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, + preferredFidelity(actorRef.Atespace, actorTemplate), + actorTemplate.GetMetadata().GetUid(), + uri.String(), + ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS, + )) return nil }) if err != nil { @@ -154,15 +191,15 @@ func (w *ActorWorkflow) ensureMarkedSuspending(ctx context.Context, actorRef res return storedActor, nil } -// commitSnapshotScope returns the scope a commit (suspend) snapshot is taken -// with. Golden actors always commit Full regardless of the template's -// onCommit: new actors borrow the golden snapshot and resume it Full, so it -// must carry the guest memory and filesystem. -func commitSnapshotScope(atespace string, tmpl *ateapipb.ActorTemplate) ateapipb.SnapshotContentScope { +// preferredFidelity returns the fidelity a suspend snapshot is taken with. +// Golden actors always commit MEMORY regardless of the template's +// preferredFidelity: new actors borrow the golden snapshot and resume it +// with memory, so it must carry the guest memory and filesystem. +func preferredFidelity(atespace string, tmpl *ateapipb.ActorTemplate) ateapipb.SnapshotFidelity { if atespace == resources.GoldenActorAtespace { - return ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL + return ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY } - return tmpl.GetSnapshotConfig().GetOnCommit() + return tmpl.GetSnapshotConfig().GetPreferredFidelity() } // isPausedOriginSuspend reports whether the suspend must upload a PAUSED @@ -172,10 +209,15 @@ func commitSnapshotScope(atespace string, tmpl *ateapipb.ActorTemplate) ateapipb // nil worker assignment disambiguates — running-origin suspends keep their // assignment until finalize, paused actors never have one. func isPausedOriginSuspend(actor *ateapipb.Actor) bool { - return actor.GetStatus().GetState() == ateapipb.ActorState_ACTOR_STATE_PAUSED || - (actor.GetStatus().GetState() == ateapipb.ActorState_ACTOR_STATE_SUSPENDING && - actor.GetStatus().GetWorkerAssignment() == nil && - actor.GetStatus().GetLocalSnapshot() != nil) + if actor.GetStatus().GetState() == ateapipb.ActorState_ACTOR_STATE_PAUSED { + return true + } + if actor.GetStatus().GetState() == ateapipb.ActorState_ACTOR_STATE_SUSPENDING && + actor.GetStatus().GetWorkerAssignment() == nil { + _, localSt := findLatestSnapshotStorage(actor.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + return localSt != nil + } + return false } // ensureAteletSuspended checkpoints the workload to the actor's persisted @@ -184,7 +226,7 @@ func isPausedOriginSuspend(actor *ateapipb.Actor) bool { // once-minted snapshot location, so a re-entered workflow re-sends the same // semantic request; once atelet's Checkpoint is idempotent on those keys this // step becomes fully reentrant with no changes here. -func (w *ActorWorkflow) ensureAteletSuspended(ctx context.Context, actorRef resources.ActorRef, actor *ateapipb.Actor, actorTemplate *ateapipb.ActorTemplate) (wireSnapshotScope string, err error) { +func (w *ActorWorkflow) ensureAteletSuspended(ctx context.Context, actorRef resources.ActorRef, actor *ateapipb.Actor, actorTemplate *ateapipb.ActorTemplate) (wireFidelity string, err error) { ctx, done := stepSpan(ctx, "CallAteletSuspend") defer func() { err = done(err) }() @@ -211,8 +253,9 @@ func (w *ActorWorkflow) ensureAteletSuspended(ctx context.Context, actorRef reso // Checkpoint does not carry the sandbox config: atelet uses the version the // actor is currently running (recorded on-node at Run/Restore) and pins it // into the snapshot manifest. + _, inProgressDurable := findLatestSnapshotStorage(actor.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS) req := &ateletpb.CheckpointRequest{ - TargetAteomUid: assignment.GetWorkerPodUid(), + WorkerPodUid: assignment.GetWorkerPodUid(), Atespace: actor.GetMetadata().GetAtespace(), ActorName: actor.GetMetadata().GetName(), ActorTemplateAtespace: actor.GetActorTemplate().GetAtespace(), @@ -221,18 +264,18 @@ func (w *ActorWorkflow) ensureAteletSuspended(ctx context.Context, actorRef reso Type: ateletpb.CheckpointType_CHECKPOINT_TYPE_EXTERNAL, Config: &ateletpb.CheckpointRequest_ExternalConfig{ ExternalConfig: &ateletpb.ExternalCheckpointConfiguration{ - SnapshotUri: actor.GetStatus().GetInProgressSnapshotUri(), + SnapshotUri: inProgressDurable.GetObject().GetSnapshotUri(), }, }, - Scope: actorSnapshotContentScopeToAtelet(commitSnapshotScope(actor.GetMetadata().GetAtespace(), actorTemplate)), + Fidelity: fidelityToAtelet(preferredFidelity(actor.GetMetadata().GetAtespace(), actorTemplate)), ActorUid: actor.GetMetadata().Uid, } - wireSnapshotScope = ateattr.SnapshotScopeValue(req.Scope) + wireFidelity = ateattr.SnapshotFidelityValue(req.Fidelity) if _, err = client.Checkpoint(ctx, req); err != nil { - return wireSnapshotScope, handleAteletError(ctx, w.store, actorRef, ateattr.OperationSuspend, "Checkpoint", false, err) + return wireFidelity, handleAteletError(ctx, w.store, actorRef, ateattr.OperationSuspend, "Checkpoint", false, err) } - return wireSnapshotScope, nil + return wireFidelity, nil } // ensurePausedSnapshotUploaded suspends a PAUSED actor by telling the atelet @@ -241,13 +284,13 @@ func (w *ActorWorkflow) ensureAteletSuspended(ctx context.Context, actorRef reso // ateom to checkpoint. Retries re-send the same semantic request: the // destination is minted once and the upload overwrites deterministic object // names, with the remote manifest as the commit marker. -func (w *ActorWorkflow) ensurePausedSnapshotUploaded(ctx context.Context, actorRef resources.ActorRef, actor *ateapipb.Actor, actorTemplate *ateapipb.ActorTemplate) (wireSnapshotScope string, err error) { +func (w *ActorWorkflow) ensurePausedSnapshotUploaded(ctx context.Context, actorRef resources.ActorRef, actor *ateapipb.Actor, actorTemplate *ateapipb.ActorTemplate) (wireFidelity string, err error) { ctx, done := stepSpan(ctx, "UploadPausedCheckpoint") defer func() { err = done(err) }() - local := actor.GetStatus().GetLocalSnapshot() + _, localSt := findLatestSnapshotStorage(actor.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) nodeName := actor.GetStatus().GetAssignedNode() - if nodeName == "" { + if localSt == nil || nodeName == "" { // Without the node the snapshot can never be found (mirrors // FinalizePaused, which crashes rather than record an unknown node). if err := crashActor(ctx, w.store, actorRef, ateattr.OperationSuspend, crashMessageLocalSnapshotNodeUnknown); err != nil { @@ -265,24 +308,25 @@ func (w *ActorWorkflow) ensurePausedSnapshotUploaded(ctx context.Context, actorR } client := ateletpb.NewAteomHerderClient(ateletConn) + _, inProgressDurable := findLatestSnapshotStorage(actor.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS) req := &ateletpb.UploadPausedCheckpointRequest{ Atespace: actor.GetMetadata().GetAtespace(), ActorName: actor.GetMetadata().GetName(), ActorUid: actor.GetMetadata().GetUid(), ActorTemplateAtespace: actor.GetActorTemplate().GetAtespace(), ActorTemplateName: actor.GetActorTemplate().GetName(), - LocalSnapshotName: local.GetSnapshotName(), - DestinationSnapshotUri: actor.GetStatus().GetInProgressSnapshotUri(), + LocalSnapshotName: localSt.GetLocal().GetSnapshotName(), + DestinationSnapshotUri: inProgressDurable.GetObject().GetSnapshotUri(), // The commit scope, like a running-origin suspend; atelet converts // from the captured scope in the snapshot's manifest where possible. - DesiredScope: actorSnapshotContentScopeToAtelet(commitSnapshotScope(actor.GetMetadata().GetAtespace(), actorTemplate)), + DesiredFidelity: fidelityToAtelet(preferredFidelity(actor.GetMetadata().GetAtespace(), actorTemplate)), } - wireSnapshotScope = ateattr.SnapshotScopeValue(req.DesiredScope) + wireFidelity = ateattr.SnapshotFidelityValue(req.DesiredFidelity) if _, err = client.UploadPausedCheckpoint(ctx, req); err != nil { - return wireSnapshotScope, handleAteletError(ctx, w.store, actorRef, ateattr.OperationSuspend, "UploadPausedCheckpoint", false, err) + return wireFidelity, handleAteletError(ctx, w.store, actorRef, ateattr.OperationSuspend, "UploadPausedCheckpoint", false, err) } - return wireSnapshotScope, nil + return wireFidelity, nil } // newInProgressSnapshotURI is where the snapshot an actor is currently taking is @@ -310,12 +354,12 @@ func (w *ActorWorkflow) ensureVolumesDetached(ctx context.Context, actor *ateapi // ensureSuspendedFinalized releases the actor's worker (only when it is still // owned by this actor), records the in-progress snapshot as the actor's -// external snapshot, and commits SUSPENDED with the assignment cleared in a -// single update, then releases the external snapshot that update replaced. +// durable snapshot, and commits SUSPENDED with the assignment cleared in a +// single update, then releases the durable snapshot that update replaced. // It re-reads the actor first so an out-of-band transition (e.g. the syncer // crashing the actor after its worker died) is not overwritten: with no // assignment left there is nothing to finalize. -func (w *ActorWorkflow) ensureSuspendedFinalized(ctx context.Context, actorRef resources.ActorRef, actorTemplate *ateapipb.ActorTemplate) (_ *ateapipb.Actor, err error) { +func (w *ActorWorkflow) ensureSuspendedFinalized(ctx context.Context, actorRef resources.ActorRef) (_ *ateapipb.Actor, err error) { ctx, done := stepSpan(ctx, "FinalizeSuspended") defer func() { err = done(err) }() @@ -361,32 +405,29 @@ func (w *ActorWorkflow) ensureSuspendedFinalized(ctx context.Context, actorRef r } } - // 2. Finalize the actor: record its new external snapshot and mark it SUSPENDED. This + // 2. Finalize the actor: record its new durable snapshot and mark it SUSPENDED. This // must run even with no worker assignment (nothing to free), or the actor // would be left SUSPENDING forever with the workflow reporting success. - inProgressSnapshotURI := latestActor.GetStatus().GetInProgressSnapshotUri() - externalSnapshot := latestActor.GetStatus().GetExternalSnapshot() - if inProgressSnapshotURI != "" { - externalSnapshot = &ateapipb.ExternalSnapshot{ - SnapshotUri: inProgressSnapshotURI, - ContentScope: commitSnapshotScope(actorRef.Atespace, actorTemplate), - ActorTemplateUid: actorTemplate.GetMetadata().GetUid(), - } - } + _, prevSt := findLatestSnapshotStorage(latestActor.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) // 3. Commit the actor. t = time.Now() storedActor, err := w.store.UpdateActor(ctx, actorRef, store.PreconditionFrom(latestActor), func(toUpdate *ateapipb.Actor) error { toUpdate.Status.State = ateapipb.ActorState_ACTOR_STATE_SUSPENDED - if inProgressSnapshotURI != "" { - // The recorded URI is under the actor's own prefix, so the actor now - // owns its external snapshot rather than borrowing the tag's it may - // have been created from. - toUpdate.Status.ExternalSnapshot = proto.CloneOf(externalSnapshot) - toUpdate.Status.InProgressSnapshotUri = "" + snap := snapshotAtLatestGeneration(toUpdate.Status) + if snap == nil { + return fmt.Errorf("actor %s has no latest snapshot", actorRef) + } + if inProgressSt := findSnapshotStorage(snap, ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE); inProgressSt.GetObject().GetSnapshotUri() != "" { + inProgressSt.Status = ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED } + // Remove all local snapshots since they are superseded by the new durable snapshot. + // TODO: This will change when we merge pause and suspend and allow for both local + // and durable snapshots to be kept. + removeSnapshotStorageEntries(toUpdate.Status, ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL, nil) + // Remove all durable snapshots older than the new durable snapshot. + removeOlderSnapshotStorageEntries(toUpdate.Status, ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, new(ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), snap.GetGeneration()) toUpdate.Status.WorkerAssignment = nil - toUpdate.Status.LocalSnapshot = nil toUpdate.Status.AssignedNode = "" return nil }) @@ -405,12 +446,13 @@ func (w *ActorWorkflow) ensureSuspendedFinalized(ctx context.Context, actorRef r // snapshot's objects in storage until the actor is deleted, which removes // its whole prefix. t = time.Now() - releaseErr := w.releaseReplacedSnapshot(ctx, latestActor, externalSnapshot) + _, nextSt := findLatestSnapshotStorage(storedActor.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + releaseErr := w.releaseReplacedSnapshot(ctx, latestActor, nextSt) dReleaseSnapshot = time.Since(t) if releaseErr != nil { slog.WarnContext(ctx, "Failed to release the external snapshot a suspend replaced; its objects are left in storage until the actor is deleted", slog.Any("actor", actorRef), - slog.String("snapshot_uri", latestActor.GetStatus().GetExternalSnapshot().GetSnapshotUri()), + slog.String("snapshot_uri", prevSt.GetObject().GetSnapshotUri()), slog.String("err", releaseErr.Error())) } return storedActor, nil @@ -420,19 +462,20 @@ func (w *ActorWorkflow) ensureSuspendedFinalized(ctx context.Context, actorRef r // this suspend. actor is the record as it was before the suspend committed. // An actor that borrowed its current snapshot from a tag releases nothing — // the snapshot lives under the tag's prefix, and the tag outlives the actor. -func (w *ActorWorkflow) releaseReplacedSnapshot(ctx context.Context, actor *ateapipb.Actor, nextSnapshot *ateapipb.ExternalSnapshot) (err error) { +func (w *ActorWorkflow) releaseReplacedSnapshot(ctx context.Context, actor *ateapipb.Actor, nextSnapshot *ateapipb.SnapshotStorage) (err error) { ctx, done := stepSpan(ctx, "ReleaseReplacedSnapshot") defer func() { err = done(err) }() - previous := actor.GetStatus().GetExternalSnapshot().GetSnapshotUri() + _, prevSt := findLatestSnapshotStorage(actor.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + previous := prevSt.GetObject().GetSnapshotUri() switch { - case w.objectStore == nil: + case w.snapshotPlugin == nil: markSkipped(ctx, "no object store configured") return nil case previous == "": markSkipped(ctx, "the actor held no external snapshot") return nil - case previous == nextSnapshot.GetSnapshotUri(): + case previous == nextSnapshot.GetObject().GetSnapshotUri(): markSkipped(ctx, "the actor's external snapshot is unchanged") return nil } @@ -444,7 +487,7 @@ func (w *ActorWorkflow) releaseReplacedSnapshot(ctx context.Context, actor *atea markSkipped(ctx, "the replaced external snapshot is owned by another resource") return nil } - return objectstore.DeletePrefix(ctx, w.objectStore, uri.Prefix()) + return w.cleanupSnapshot(ctx, uri.Prefix()) } func actorSnapshotOwner(actor *ateapipb.Actor) resources.SnapshotOwner { diff --git a/cmd/ateapi/internal/controlapi/workflow_suspend_test.go b/cmd/ateapi/internal/controlapi/workflow_suspend_test.go index 8a1f7a1b9a..845283878b 100644 --- a/cmd/ateapi/internal/controlapi/workflow_suspend_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_suspend_test.go @@ -17,6 +17,7 @@ package controlapi import ( "context" "errors" + "strings" "testing" "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" @@ -24,6 +25,7 @@ import ( "github.com/agent-substrate/substrate/internal/apierror" "github.com/agent-substrate/substrate/internal/installdefaults" "github.com/agent-substrate/substrate/internal/objectstore/objectstoretest" + "github.com/agent-substrate/substrate/internal/objectstoreplugin/objectstoreplugintest" "github.com/agent-substrate/substrate/internal/resources" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "google.golang.org/grpc/codes" @@ -46,7 +48,8 @@ func TestEnsureMarkedSuspending_SnapshotURI(t *testing.T) { t.Fatalf("ensureMarkedSuspending: %v", err) } - uri := mustParseSnapshotURI(t, marked.GetStatus().GetInProgressSnapshotUri()) + _, inProgressSt := findLatestSnapshotStorage(marked.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS) + uri := mustParseSnapshotURI(t, inProgressSt.GetObject().GetSnapshotUri()) if !resources.IsValidResourceName(uri.Name()) { t.Errorf("in-progress snapshot name = %q, want a valid resource name", uri.Name()) } @@ -67,8 +70,11 @@ func TestEnsureMarkedSuspending_ReentryKeepsPersistedSnapshotLocation(t *testing actor := storetest.MustCreateActor(t, ctx, persistence, &ateapipb.Actor{ Metadata: &ateapipb.ResourceMetadata{Atespace: "team-a", Name: "actor-1"}, Status: &ateapipb.ActorStatus{ - State: ateapipb.ActorState_ACTOR_STATE_SUSPENDING, - InProgressSnapshotUri: firstAttempt, + State: ateapipb.ActorState_ACTOR_STATE_SUSPENDING, + LastAssignedGeneration: 1, + Snapshots: []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", firstAttempt, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS), + }, }, }) w := &ActorWorkflow{store: persistence} @@ -76,8 +82,8 @@ func TestEnsureMarkedSuspending_ReentryKeepsPersistedSnapshotLocation(t *testing if err != nil { t.Fatalf("ensureMarkedSuspending: %v", err) } - if got := marked.GetStatus().GetInProgressSnapshotUri(); got != firstAttempt { - t.Errorf("InProgressSnapshotUri = %q, want the first attempt's location", got) + if _, gotSt := findLatestSnapshotStorage(marked.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS); gotSt.GetObject().GetSnapshotUri() != firstAttempt { + t.Errorf("in-progress durable snapshot URI = %q, want the first attempt's location", gotSt.GetObject().GetSnapshotUri()) } } @@ -155,9 +161,16 @@ func TestEnsureMarkedSuspending_StateMatrix(t *testing.T) { w := &ActorWorkflow{store: persistence} actorRef := resources.ActorRef{Atespace: "team-a", Name: "id1"} + actorStatus := &ateapipb.ActorStatus{State: seedState} + if seedState == ateapipb.ActorState_ACTOR_STATE_PAUSED { + actorStatus.LastAssignedGeneration = 1 + actorStatus.Snapshots = []*ateapipb.Snapshot{ + newLocalSnapshot(1, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "snap-1", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + } + } actor := storetest.MustCreateActor(t, ctx, persistence, &ateapipb.Actor{ Metadata: &ateapipb.ResourceMetadata{Atespace: actorRef.Atespace, Name: actorRef.Name}, - Status: &ateapipb.ActorStatus{State: seedState}, + Status: actorStatus, }) tmpl := &ateapipb.ActorTemplate{ @@ -235,6 +248,12 @@ func TestEnsureAteletSuspended_DialFailureLeavesActorRetryable(t *testing.T) { t.Run(tt.name, func(t *testing.T) { ctx := context.Background() persistence := newTestPersistence(t) + var snapshots []*ateapipb.Snapshot + if tt.prevSnapshot != "" { + snapshots = append(snapshots, newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", tt.prevSnapshot, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED)) + } + snapshots = append(snapshots, newDurableSnapshot(2, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", neverWritten, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS)) + actor := &ateapipb.Actor{ Metadata: &ateapipb.ResourceMetadata{Atespace: "team-a", Name: "actor-1"}, Status: &ateapipb.ActorStatus{ @@ -245,8 +264,8 @@ func TestEnsureAteletSuspended_DialFailureLeavesActorRetryable(t *testing.T) { WorkerPod: "pod-gone", NodeName: "node-gone", }, - InProgressSnapshotUri: neverWritten, - ExternalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: tt.prevSnapshot}, + LastAssignedGeneration: 2, + Snapshots: snapshots, }, } created := storetest.MustCreateActor(t, ctx, persistence, actor) @@ -266,10 +285,11 @@ func TestEnsureAteletSuspended_DialFailureLeavesActorRetryable(t *testing.T) { if stored.GetStatus().GetState() != ateapipb.ActorState_ACTOR_STATE_SUSPENDING { t.Errorf("state = %v, want unchanged SUSPENDING", stored.GetStatus().GetState()) } - if got := stored.GetStatus().GetInProgressSnapshotUri(); got != neverWritten { - t.Errorf("InProgressSnapshotUri = %q, want preserved for debugging", got) + if _, gotSt := findLatestSnapshotStorage(stored.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS); gotSt.GetObject().GetSnapshotUri() != neverWritten { + t.Errorf("in-progress durable snapshot URI = %q, want preserved for debugging", gotSt.GetObject().GetSnapshotUri()) } - if got := stored.GetStatus().GetExternalSnapshot().GetSnapshotUri(); got != tt.prevSnapshot { + _, gotSt := findLatestSnapshotStorage(stored.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + if got := gotSt.GetObject().GetSnapshotUri(); got != tt.prevSnapshot { t.Errorf("SnapshotUri = %q, want %q", got, tt.prevSnapshot) } }) @@ -287,25 +307,26 @@ func TestEnsureSuspendedFinalized_NoAssignment(t *testing.T) { persistence := newTestPersistence(t) snapshotURI := someActorSnapshotURI(t, testStorageLocation, "team-a", "2026-01-01t00-00-00z-abc") + snap := newLocalSnapshot(1, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "tmpl-uid-1", "actor-1-pause-snapshot", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + snap.Storage = append(snap.Storage, &ateapipb.SnapshotStorage{ + Durability: ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, + Status: ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS, + Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + Object: &ateapipb.ObjectSnapshot{SnapshotUri: snapshotURI}, + }) actor := &ateapipb.Actor{ Metadata: &ateapipb.ResourceMetadata{Atespace: "team-a", Name: "actor-1"}, Status: &ateapipb.ActorStatus{ - State: ateapipb.ActorState_ACTOR_STATE_SUSPENDING, - AssignedNode: "node1", - InProgressSnapshotUri: snapshotURI, - LocalSnapshot: &ateapipb.LocalSnapshot{ - SnapshotName: "actor-1-pause-snapshot", - }, + State: ateapipb.ActorState_ACTOR_STATE_SUSPENDING, + AssignedNode: "node1", + LastAssignedGeneration: 1, + Snapshots: []*ateapipb.Snapshot{snap}, }, } storetest.MustCreateActor(t, ctx, persistence, actor) w := &ActorWorkflow{store: persistence} - tmpl := &ateapipb.ActorTemplate{ - Metadata: &ateapipb.ResourceMetadata{Atespace: "team-a", Name: "tmpl", Uid: "tmpl-uid-1"}, - SnapshotConfig: &ateapipb.SnapshotConfig{StorageLocation: testStorageLocation}, - } - stored, err := w.ensureSuspendedFinalized(ctx, resources.ActorRef{Atespace: "team-a", Name: "actor-1"}, tmpl) + stored, err := w.ensureSuspendedFinalized(ctx, resources.ActorRef{Atespace: "team-a", Name: "actor-1"}) if err != nil { t.Fatalf("ensureSuspendedFinalized: %v", err) } @@ -313,19 +334,20 @@ func TestEnsureSuspendedFinalized_NoAssignment(t *testing.T) { if stored.GetStatus().GetState() != ateapipb.ActorState_ACTOR_STATE_SUSPENDED { t.Errorf("state = %v, want SUSPENDED", stored.GetStatus().GetState()) } - if got := stored.GetStatus().GetExternalSnapshot().GetSnapshotUri(); got != snapshotURI { + gotSnap, gotSt := findLatestSnapshotStorage(stored.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + if got := gotSt.GetObject().GetSnapshotUri(); got != snapshotURI { t.Errorf("SnapshotUri = %q, want %q", got, snapshotURI) } // The snapshot carries the template it was captured under, so a later // repoint can tell that its guest state no longer matches. - if got := stored.GetStatus().GetExternalSnapshot().GetActorTemplateUid(); got != "tmpl-uid-1" { - t.Errorf("ExternalSnapshot.ActorTemplateUid = %q, want %q", got, "tmpl-uid-1") + if got := gotSnap.GetActorTemplateUid(); got != "tmpl-uid-1" { + t.Errorf("Snapshot.ActorTemplateUid = %q, want %q", got, "tmpl-uid-1") } - if got := stored.GetStatus().GetInProgressSnapshotUri(); got != "" { - t.Errorf("InProgressSnapshotUri = %q, want cleared", got) + if _, got := findLatestSnapshotStorage(stored.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS); got != nil { + t.Errorf("in-progress durable snapshot = %v, want cleared", got) } - if stored.GetStatus().GetLocalSnapshot() != nil { - t.Errorf("LocalSnapshot = %v, want cleared", stored.GetStatus().GetLocalSnapshot()) + if _, local := findLatestSnapshotStorage(stored.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED); local != nil { + t.Errorf("LocalSnapshot = %v, want cleared", local) } if got := stored.GetStatus().GetAssignedNode(); got != "" { t.Errorf("AssignedNode = %q, want cleared", got) @@ -375,25 +397,31 @@ func TestEnsureSuspendedFinalized_ReleasesReplacedSnapshot(t *testing.T) { }) previous := mustActorSnapshotURI(t, template, actor, "old") + prevType := ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR if tt.tagOwnedSnapshot { previous = mustTagSnapshotURI(t, template, "team-a", "v1-snapshot") + prevType = ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG } fresh := mustActorSnapshotURI(t, template, actor, snapshotName) objects.PutSnapshot(t, previous, "manifest.json") objects.PutSnapshot(t, fresh, "manifest.json") mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapipb.ActorStatus) { - s.InProgressSnapshotUri = fresh.String() - s.ExternalSnapshot = &ateapipb.ExternalSnapshot{SnapshotUri: previous.String()} + s.LastAssignedGeneration = 2 + s.Snapshots = []*ateapipb.Snapshot{ + newDurableSnapshot(1, prevType, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", previous.String(), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + newDurableSnapshot(2, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", fresh.String(), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS), + } }) - stored, err := w.ensureSuspendedFinalized(ctx, actorRef, template) + stored, err := w.ensureSuspendedFinalized(ctx, actorRef) if err != nil { t.Fatalf("ensureSuspendedFinalized: %v", err) } // Whichever way the previous snapshot went, the actor now owns the // one it just wrote and is no longer borrowing. - if got := stored.GetStatus().GetExternalSnapshot().GetSnapshotUri(); got != fresh.String() { + _, gotSt := findLatestSnapshotStorage(stored.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + if got := gotSt.GetObject().GetSnapshotUri(); got != fresh.String() { t.Errorf("external snapshot = %q, want the one this suspend wrote, %q", got, fresh) } if released := len(objects.Snapshot(t, previous)) == 0; released != tt.wantReleased { @@ -409,6 +437,13 @@ func TestEnsureSuspendedFinalized_ReleasesReplacedSnapshot(t *testing.T) { // errObjectStore stands in for object storage being unreachable. var errObjectStore = errors.New("object storage is unavailable") +// isObjectStoreErr reports whether err came from errObjectStore. The object +// store sits behind the snapshot plugin's gRPC boundary, which carries the +// message but not the error value, so errors.Is cannot match it. +func isObjectStoreErr(err error) bool { + return err != nil && strings.Contains(err.Error(), errObjectStore.Error()) +} + // TestEnsureSuspendedFinalized_CommitsDespiteObjectStoreFailure verifies that // failing to collect the snapshot a suspend replaced does not fail the // suspend. The worker is already released by then, so aborting would leave the @@ -434,23 +469,27 @@ func TestEnsureSuspendedFinalized_CommitsDespiteObjectStoreFailure(t *testing.T) objects.PutSnapshot(t, fresh, "manifest.json") mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapipb.ActorStatus) { - s.InProgressSnapshotUri = fresh.String() - s.ExternalSnapshot = &ateapipb.ExternalSnapshot{SnapshotUri: previous.String()} + s.LastAssignedGeneration = 2 + s.Snapshots = []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", previous.String(), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + newDurableSnapshot(2, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", fresh.String(), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS), + } }) objects.OnDelete = func(string, string) error { return errObjectStore } - stored, err := w.ensureSuspendedFinalized(ctx, actorRef, template) + stored, err := w.ensureSuspendedFinalized(ctx, actorRef) if err != nil { t.Fatalf("ensureSuspendedFinalized: %v", err) } if got := stored.GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_SUSPENDED { t.Errorf("state = %v, want SUSPENDED", got) } - if got := stored.GetStatus().GetExternalSnapshot().GetSnapshotUri(); got != fresh.String() { + _, gotSt := findLatestSnapshotStorage(stored.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + if got := gotSt.GetObject().GetSnapshotUri(); got != fresh.String() { t.Errorf("actor snapshot uri = %q, want the one this suspend wrote, %q", got, fresh) } - if got := stored.GetStatus().GetInProgressSnapshotUri(); got != "" { - t.Errorf("InProgressSnapshotUri = %q, want cleared", got) + if _, got := findLatestSnapshotStorage(stored.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS); got != nil { + t.Errorf("in-progress durable snapshot = %v, want cleared", got) } if len(objects.Snapshot(t, previous)) == 0 { t.Error("replaced external snapshot was collected despite every delete failing") @@ -482,12 +521,15 @@ func TestEnsureSuspendedFinalized_KeepsReplacedSnapshotOnConflict(t *testing.T) objects.PutSnapshot(t, previous, "manifest.json") objects.PutSnapshot(t, fresh, "manifest.json") mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapipb.ActorStatus) { - s.InProgressSnapshotUri = fresh.String() - s.ExternalSnapshot = &ateapipb.ExternalSnapshot{SnapshotUri: previous.String()} + s.LastAssignedGeneration = 2 + s.Snapshots = []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", previous.String(), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + newDurableSnapshot(2, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", fresh.String(), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS), + } }) - w := &ActorWorkflow{store: &conflictingUpdateStore{Interface: persistence}, objectStore: objects} - if _, err := w.ensureSuspendedFinalized(ctx, actorRef, template); apierror.Code(err) != codes.Aborted { + w := &ActorWorkflow{store: &conflictingUpdateStore{Interface: persistence}, snapshotPlugin: objectstoreplugintest.ControlClient(objects)} + if _, err := w.ensureSuspendedFinalized(ctx, actorRef); apierror.Code(err) != codes.Aborted { t.Fatalf("ensureSuspendedFinalized = %v, want code Aborted", err) } if len(objects.Snapshot(t, previous)) == 0 { @@ -545,7 +587,10 @@ func TestEnsureSuspendedFinalized_ReleasesOnlyOwnWorker(t *testing.T) { WorkerPod: "pod-1", WorkerPodUid: testWorkerUID("pod-1"), }, - InProgressSnapshotUri: someActorSnapshotURI(t, testStorageLocation, "team-a", "snapshot-1"), + LastAssignedGeneration: 1, + Snapshots: []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", someActorSnapshotURI(t, testStorageLocation, "team-a", "snapshot-1"), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS), + }, }, } created := storetest.MustCreateActor(t, ctx, persistence, actor) @@ -571,8 +616,7 @@ func TestEnsureSuspendedFinalized_ReleasesOnlyOwnWorker(t *testing.T) { }) w := &ActorWorkflow{store: persistence} - tmpl := &ateapipb.ActorTemplate{SnapshotConfig: &ateapipb.SnapshotConfig{StorageLocation: "gs://bucket/root"}} - if _, err := w.ensureSuspendedFinalized(ctx, resources.ActorRef{Atespace: "team-a", Name: "shared"}, tmpl); err != nil { + if _, err := w.ensureSuspendedFinalized(ctx, resources.ActorRef{Atespace: "team-a", Name: "shared"}); err != nil { t.Fatalf("ensureSuspendedFinalized: %v", err) } @@ -584,32 +628,32 @@ func TestEnsureSuspendedFinalized_ReleasesOnlyOwnWorker(t *testing.T) { } } -// TestCommitSnapshotScope verifies golden actors always commit Full — new -// actors resume the golden snapshot Full, so the template's onCommit must not +// TestPreferredFidelity verifies golden actors always commit Full — new +// actors resume the golden snapshot Full, so the template's preferredFidelity must not // thin it down to a data-only capture. -func TestCommitSnapshotScope(t *testing.T) { - tmpl := func(onCommit ateapipb.SnapshotContentScope) *ateapipb.ActorTemplate { +func TestPreferredFidelity(t *testing.T) { + tmpl := func(fidelity ateapipb.SnapshotFidelity) *ateapipb.ActorTemplate { return &ateapipb.ActorTemplate{ - SnapshotConfig: &ateapipb.SnapshotConfig{OnCommit: onCommit}, + SnapshotConfig: &ateapipb.SnapshotConfig{PreferredFidelity: fidelity}, } } - fullScope := ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL - dataScope := ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA + fullScope := ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY + dataScope := ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES tests := []struct { name string atespace string - onCommit ateapipb.SnapshotContentScope - want ateapipb.SnapshotContentScope + fidelity ateapipb.SnapshotFidelity + want ateapipb.SnapshotFidelity }{ - {"golden actor ignores Data onCommit", resources.GoldenActorAtespace, dataScope, fullScope}, - {"golden actor keeps Full onCommit", resources.GoldenActorAtespace, fullScope, fullScope}, - {"regular actor uses Data onCommit", "team-a", dataScope, dataScope}, - {"regular actor uses Full onCommit", "team-a", fullScope, fullScope}, + {"golden actor ignores Data fidelity", resources.GoldenActorAtespace, dataScope, fullScope}, + {"golden actor keeps Full fidelity", resources.GoldenActorAtespace, fullScope, fullScope}, + {"regular actor uses Data fidelity", "team-a", dataScope, dataScope}, + {"regular actor uses Full fidelity", "team-a", fullScope, fullScope}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - if got := commitSnapshotScope(tc.atespace, tmpl(tc.onCommit)); got != tc.want { - t.Errorf("commitSnapshotScope(%q, onCommit=%s) = %s, want %s", tc.atespace, tc.onCommit, got, tc.want) + if got := preferredFidelity(tc.atespace, tmpl(tc.fidelity)); got != tc.want { + t.Errorf("preferredFidelity(%q, fidelity=%s) = %s, want %s", tc.atespace, tc.fidelity, got, tc.want) } }) } @@ -621,16 +665,18 @@ func TestCommitSnapshotScope(t *testing.T) { // no worker assignment, means the suspend uploads a local snapshot. func TestIsPausedOriginSuspend(t *testing.T) { assignment := &ateapipb.WorkerAssignment{WorkerNamespace: "ns", WorkerPool: "pool", WorkerPod: "pod-1"} - localInfo := &ateapipb.LocalSnapshot{SnapshotName: "snap"} + localSnaps := []*ateapipb.Snapshot{ + newLocalSnapshot(1, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "snap", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + } tests := []struct { name string actor *ateapipb.Actor want bool }{ - {"paused actor", &ateapipb.Actor{Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_PAUSED, AssignedNode: "node1", LocalSnapshot: localInfo}}, true}, - {"suspending retry of a paused-origin suspend", &ateapipb.Actor{Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_SUSPENDING, AssignedNode: "node1", LocalSnapshot: localInfo}}, true}, - {"running actor with stale local snapshot info", &ateapipb.Actor{Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_RUNNING, LocalSnapshot: localInfo, WorkerAssignment: assignment}}, false}, - {"suspending retry of a running-origin suspend with stale local snapshot info", &ateapipb.Actor{Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_SUSPENDING, LocalSnapshot: localInfo, WorkerAssignment: assignment}}, false}, + {"paused actor", &ateapipb.Actor{Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_PAUSED, AssignedNode: "node1", Snapshots: localSnaps}}, true}, + {"suspending retry of a paused-origin suspend", &ateapipb.Actor{Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_SUSPENDING, AssignedNode: "node1", Snapshots: localSnaps}}, true}, + {"running actor with stale local snapshot info", &ateapipb.Actor{Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_RUNNING, Snapshots: localSnaps, WorkerAssignment: assignment}}, false}, + {"suspending retry of a running-origin suspend with stale local snapshot info", &ateapipb.Actor{Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_SUSPENDING, Snapshots: localSnaps, WorkerAssignment: assignment}}, false}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { @@ -654,8 +700,11 @@ func TestEnsurePausedSnapshotUploaded_Preconditions(t *testing.T) { created := storetest.MustCreateActor(t, ctx, persistence, &ateapipb.Actor{ Metadata: &ateapipb.ResourceMetadata{Atespace: "team-a", Name: "actor-1"}, Status: &ateapipb.ActorStatus{ - State: ateapipb.ActorState_ACTOR_STATE_SUSPENDING, - LocalSnapshot: &ateapipb.LocalSnapshot{SnapshotName: "snap"}, + State: ateapipb.ActorState_ACTOR_STATE_SUSPENDING, + LastAssignedGeneration: 1, + Snapshots: []*ateapipb.Snapshot{ + newLocalSnapshot(1, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "snap", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + }, }, }) @@ -677,13 +726,20 @@ func TestEnsurePausedSnapshotUploaded_Preconditions(t *testing.T) { persistence := newTestPersistence(t) w := &ActorWorkflow{store: persistence, dialer: newDanglingDialer()} + snap := newLocalSnapshot(1, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "snap", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + snap.Storage = append(snap.Storage, &ateapipb.SnapshotStorage{ + Durability: ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, + Status: ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS, + Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + Object: &ateapipb.ObjectSnapshot{SnapshotUri: someActorSnapshotURI(t, testStorageLocation, "team-a", "snap-dest")}, + }) created := storetest.MustCreateActor(t, ctx, persistence, &ateapipb.Actor{ Metadata: &ateapipb.ResourceMetadata{Atespace: "team-a", Name: "actor-1"}, Status: &ateapipb.ActorStatus{ - State: ateapipb.ActorState_ACTOR_STATE_SUSPENDING, - AssignedNode: "node1", - InProgressSnapshotUri: someActorSnapshotURI(t, testStorageLocation, "team-a", "snap-dest"), - LocalSnapshot: &ateapipb.LocalSnapshot{SnapshotName: "snap"}, + State: ateapipb.ActorState_ACTOR_STATE_SUSPENDING, + AssignedNode: "node1", + LastAssignedGeneration: 1, + Snapshots: []*ateapipb.Snapshot{snap}, }, }) diff --git a/cmd/ateapi/internal/controlapi/workflow_tag.go b/cmd/ateapi/internal/controlapi/workflow_tag.go index 8b6fa949f4..634f781ce9 100644 --- a/cmd/ateapi/internal/controlapi/workflow_tag.go +++ b/cmd/ateapi/internal/controlapi/workflow_tag.go @@ -21,7 +21,6 @@ import ( "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/objectstore" "github.com/agent-substrate/substrate/internal/resources" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" ) @@ -68,16 +67,12 @@ func (w *ActorWorkflow) TagActorSnapshot(ctx context.Context, tag *ateapipb.Tag) if err != nil { return nil, err } - snapshot := actor.GetStatus().GetExternalSnapshot() + snapshot, _ := findLatestSnapshotStorage(actor.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) - reserved, err := w.ensureTagReserved(leaseCtx, tagRef, actor, actorTemplate, tag) + reserved, dst, err := w.ensureTagReserved(leaseCtx, tagRef, actor, actorTemplate, tag) if err != nil { return nil, err } - dst, err := resources.NewTagSnapshotURI(reserved.GetStatus().GetStorageLocation(), tagRef.Atespace, reserved.GetMetadata().GetUid()) - if err != nil { - return nil, fmt.Errorf("while building the snapshot URI for tag %s: %w", tagRef, err) - } if err := w.ensureTagSnapshotCopied(leaseCtx, reserved, snapshot, dst); err != nil { return nil, err } @@ -155,16 +150,16 @@ func (w *ActorWorkflow) ensureTagSnapshotReleased(ctx context.Context, tag *atea ctx, done := stepSpan(ctx, "ReleaseTagSnapshot") defer func() { err = done(err) }() - if w.objectStore == nil { + if w.snapshotPlugin == nil { markSkipped(ctx, "no object store configured") return nil } tagRef := resources.TagRefFromTag(tag) - uri, err := resources.NewTagSnapshotURI(tag.GetStatus().GetStorageLocation(), tagRef.Atespace, tag.GetMetadata().GetUid()) + uri, err := resources.ParseSnapshotURI(tagDurableSnapshotURI(tag, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_UNSPECIFIED)) if err != nil { return fmt.Errorf("while resolving the external snapshot of tag %s: %w", tagRef, err) } - if err := objectstore.DeletePrefix(ctx, w.objectStore, uri.Prefix()); err != nil { + if err := w.cleanupSnapshot(ctx, uri.Prefix()); err != nil { return fmt.Errorf("while releasing the external snapshot %q of tag %s: %w", uri, tagRef, err) } return nil @@ -206,8 +201,8 @@ func (w *ActorWorkflow) loadActorForTag(ctx context.Context, actorRef resources. if got := actor.GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_SUSPENDED { return nil, nil, apierror.FailedPrecondition("Actor %s must be %s to be tagged (got: %v)", actorRef, ateapipb.ActorState_ACTOR_STATE_SUSPENDED, got) } - snapshotURI := actor.GetStatus().GetExternalSnapshot().GetSnapshotUri() - if snapshotURI == "" { + snap, st := findLatestSnapshotStorage(actor.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + if st == nil { return nil, nil, apierror.FailedPrecondition("Actor %s holds no external snapshot to tag", actorRef) } // Every way an Actor comes to hold an external snapshot records the @@ -215,7 +210,7 @@ func (w *ActorWorkflow) loadActorForTag(ctx context.Context, actorRef resources. // ensureSuspendedFinalized, a create from a tag through the tag's own UID. // A snapshot without one is a broken row, and tagging it would mint a tag // that names no template. - if actor.GetStatus().GetExternalSnapshot().GetActorTemplateUid() == "" { + if snap.GetActorTemplateUid() == "" { return nil, nil, apierror.Internal("Actor %s holds an external snapshot but records no template it was built under", actorRef) } actorTemplate, err := resolveActorTemplate(ctx, w.store, actor) @@ -225,21 +220,26 @@ func (w *ActorWorkflow) loadActorForTag(ctx context.Context, actorRef resources. return actor, actorTemplate, nil } -// ensureTagReserved takes the tag's name and records the storage location. +// ensureTagReserved takes the tag's name and records the in-progress snapshot. // // A name already taken is AlreadyExists, whether the tag holding it is finished // or was left pending by a create that died. Resuming a pending row would mean // deciding whether the objects under it still belong to the snapshot being // tagged, and the row may not even be this actor's; deleting the tag collects // them and frees the name, so a retry is a delete followed by a create. -func (w *ActorWorkflow) ensureTagReserved(ctx context.Context, tagRef resources.TagRef, actor *ateapipb.Actor, actorTemplate *ateapipb.ActorTemplate, tag *ateapipb.Tag) (_ *ateapipb.Tag, err error) { +func (w *ActorWorkflow) ensureTagReserved(ctx context.Context, tagRef resources.TagRef, actor *ateapipb.Actor, actorTemplate *ateapipb.ActorTemplate, tag *ateapipb.Tag) (_ *ateapipb.Tag, _ resources.SnapshotURI, err error) { ctx, done := stepSpan(ctx, "ReserveTag") defer func() { err = done(err) }() location := actorTemplate.GetSnapshotConfig().GetStorageLocation() if err := resources.ValidateSnapshotLocation(location); err != nil { - return nil, fmt.Errorf("invalid storage location for tag %s: %w", tagRef, err) + return nil, resources.SnapshotURI{}, fmt.Errorf("invalid storage location for tag %s: %w", tagRef, err) } + dst, err := resources.NewTagSnapshotURI(location, tagRef.Atespace, resources.NewSnapshotName()) + if err != nil { + return nil, resources.SnapshotURI{}, fmt.Errorf("while building the snapshot URI for tag %s: %w", tagRef, err) + } + snap, st := findLatestSnapshotStorage(actor.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) tagToCreate := &ateapipb.Tag{ Metadata: &ateapipb.ResourceMetadata{Atespace: tagRef.Atespace, Name: tagRef.Name}, Scope: tag.GetScope(), @@ -250,57 +250,71 @@ func (w *ActorWorkflow) ensureTagReserved(ctx context.Context, tagRef resources. // and a tag that claimed the new template would hand clones the old template's // memory under the new one's identity, past the data-only downgrade a resume of // the actor itself would take. - ActorTemplateUid: actor.GetStatus().GetExternalSnapshot().GetActorTemplateUid(), - StorageLocation: location, + ActorTemplateUid: snap.GetActorTemplateUid(), + Snapshot: newDurableSnapshot( + 1, + ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG, + st.GetFidelity(), + "", + dst.String(), + ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS, + ), }, } stored, err := w.store.CreateTag(ctx, tagToCreate) switch { case err == nil: - return stored, nil + return stored, dst, nil case errors.Is(err, store.ErrFailedPrecondition): - return nil, apierror.FailedPrecondition("Atespace %s not found", tagRef.Atespace) + return nil, resources.SnapshotURI{}, apierror.FailedPrecondition("Atespace %s not found", tagRef.Atespace) case errors.Is(err, store.ErrAlreadyExists): - return nil, apierror.AlreadyExists("Tag %s already exists; delete it and create it again to retry", tagRef) + return nil, resources.SnapshotURI{}, apierror.AlreadyExists("Tag %s already exists; delete it and create it again to retry", tagRef) } - return nil, fmt.Errorf("while reserving tag %s: %w", tagRef, err) + return nil, resources.SnapshotURI{}, fmt.Errorf("while reserving tag %s: %w", tagRef, err) } // ensureTagSnapshotCopied copies the actor's external snapshot to the tag's own -// prefix, derived from the reserved row's freshly minted UID. The prefix is -// empty by construction, so the copy never blends with another attempt's objects. -func (w *ActorWorkflow) ensureTagSnapshotCopied(ctx context.Context, tag *ateapipb.Tag, snapshot *ateapipb.ExternalSnapshot, dst resources.SnapshotURI) (err error) { +// prefix. The prefix is empty by construction, so the copy never blends with +// another attempt's objects. +func (w *ActorWorkflow) ensureTagSnapshotCopied(ctx context.Context, tag *ateapipb.Tag, snapshot *ateapipb.Snapshot, dst resources.SnapshotURI) (err error) { ctx, done := stepSpan(ctx, "CopyTagSnapshot") defer func() { err = done(err) }() - if w.objectStore == nil { + if w.snapshotPlugin == nil { markSkipped(ctx, "no object store configured") return nil } tagRef := resources.TagRefFromTag(tag) - src, err := resources.ParseSnapshotURI(snapshot.GetSnapshotUri()) + srcURI := findSnapshotStorage(snapshot, ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE).GetObject().GetSnapshotUri() + src, err := resources.ParseSnapshotURI(srcURI) if err != nil { - return fmt.Errorf("while parsing the external snapshot %q of the source actor: %w", snapshot.GetSnapshotUri(), err) + return fmt.Errorf("while parsing the external snapshot %q of the source actor: %w", srcURI, err) } - if err := objectstore.CopyPrefix(ctx, w.objectStore, src.Prefix(), dst.Prefix()); err != nil { + if err := w.copySnapshot(ctx, src.Prefix(), dst.Prefix()); err != nil { return fmt.Errorf("while copying the external snapshot for tag %s: %w", tagRef, err) } return nil } -// ensureTagFinalized publishes the copy by setting status.snapshot. Until this -// lands the tag is pending and unusable; deleting it collects any partial copy. -func (w *ActorWorkflow) ensureTagFinalized(ctx context.Context, tag *ateapipb.Tag, snapshot *ateapipb.ExternalSnapshot, dst resources.SnapshotURI) (_ *ateapipb.Tag, err error) { +// ensureTagFinalized publishes the copy by marking status.snapshot COMPLETED. +// Until this lands the tag is pending and unusable; deleting it collects any +// partial copy. +func (w *ActorWorkflow) ensureTagFinalized(ctx context.Context, tag *ateapipb.Tag, snapshot *ateapipb.Snapshot, dst resources.SnapshotURI) (_ *ateapipb.Tag, err error) { ctx, done := stepSpan(ctx, "FinalizeTag") defer func() { err = done(err) }() tagRef := resources.TagRefFromTag(tag) // The copy is byte-identical to the source, so it carries the same content. - finalSnapshot := &ateapipb.ExternalSnapshot{ - SnapshotUri: dst.String(), - ContentScope: snapshot.GetContentScope(), - } + srcSt := findSnapshotStorage(snapshot, ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE) + finalSnapshot := newDurableSnapshot( + 1, + ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG, + srcSt.GetFidelity(), + "", + dst.String(), + ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED, + ) stored, err := w.store.UpdateTag(ctx, tagRef, store.PreconditionFrom(tag), func(toUpdate *ateapipb.Tag) error { toUpdate.Status.Snapshot = finalSnapshot return nil diff --git a/cmd/ateapi/internal/controlapi/workflow_tag_test.go b/cmd/ateapi/internal/controlapi/workflow_tag_test.go index a3a15e0fa3..1e7ee697b3 100644 --- a/cmd/ateapi/internal/controlapi/workflow_tag_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_tag_test.go @@ -44,10 +44,9 @@ func seedTagSource(t *testing.T, ctx context.Context, persistence store.Interfac uri := mustActorSnapshotURI(t, template, actor, name+"-snapshot") objects.PutSnapshot(t, uri, objectNames...) actor = mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapipb.ActorStatus) { - s.ExternalSnapshot = &ateapipb.ExternalSnapshot{ - SnapshotUri: uri.String(), - ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, - ActorTemplateUid: template.GetMetadata().GetUid(), + s.LastAssignedGeneration = 1 + s.Snapshots = []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, template.GetMetadata().GetUid(), uri.String(), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), } }) return actor, uri @@ -90,7 +89,7 @@ func TestTagActorSnapshot(t *testing.T) { t.Fatalf("TagActorSnapshot: %v", err) } - tagSnapshot := tag.GetStatus().GetSnapshot().GetSnapshotUri() + tagSnapshot := tagDurableSnapshotURI(tag, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) if tagSnapshot == "" || tagSnapshot == actorSnapshot.String() { t.Fatalf("tag snapshot uri = %q, want a copy of its own rather than the actor's %q", tagSnapshot, actorSnapshot) } @@ -100,8 +99,8 @@ func TestTagActorSnapshot(t *testing.T) { if got, want := tag.GetStatus().GetActorTemplateUid(), template.GetMetadata().GetUid(); got != want { t.Errorf("actor template uid = %q, want %q", got, want) } - if got, want := tag.GetStatus().GetSnapshot().GetContentScope(), ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL; got != want { - t.Errorf("content scope = %v, want the source's %v", got, want) + if got, want := findSnapshotStorage(tag.GetStatus().GetSnapshot(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE).GetFidelity(), ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY; got != want { + t.Errorf("fidelity = %v, want the source's %v", got, want) } // Both prefixes hold the same objects: the tag copied rather than moved. @@ -198,9 +197,13 @@ func TestTagActorSnapshot_Preconditions(t *testing.T) { uri := mustActorSnapshotURI(t, template, actor, "actor-1-snapshot") objects.PutSnapshot(t, uri, "manifest.json") mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapipb.ActorStatus) { - s.ExternalSnapshot = &ateapipb.ExternalSnapshot{SnapshotUri: uri.String()} + var tmplUID string if tt.builtOnTemplate { - s.ExternalSnapshot.ActorTemplateUid = template.GetMetadata().GetUid() + tmplUID = template.GetMetadata().GetUid() + } + s.LastAssignedGeneration = 1 + s.Snapshots = []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, tmplUID, uri.String(), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), } }) } @@ -239,7 +242,7 @@ func TestTagActorSnapshot_RecreateAfterCopyFailure(t *testing.T) { } return nil } - if _, err := w.TagActorSnapshot(ctx, tagToCreate(actorRef, "v1")); !errors.Is(err, errObjectStore) { + if _, err := w.TagActorSnapshot(ctx, tagToCreate(actorRef, "v1")); !isObjectStoreErr(err) { t.Fatalf("TagActorSnapshot = %v, want an error wrapping %v", err, errObjectStore) } objects.OnCopy = nil @@ -248,7 +251,7 @@ func TestTagActorSnapshot_RecreateAfterCopyFailure(t *testing.T) { if err != nil { t.Fatalf("GetTag after the external store failure: %v", err) } - if got := pending.GetStatus().GetSnapshot().GetSnapshotUri(); got != "" { + if got := tagDurableSnapshotURI(pending, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED); got != "" { t.Errorf("snapshot uri after the failure = %q, want unset: the copy never finished", got) } strandedURI := mustReservedTagSnapshotURI(t, pending) @@ -287,7 +290,7 @@ func TestTagActorSnapshot_RecreateAfterCopyFailure(t *testing.T) { if err != nil { t.Fatalf("TagActorSnapshot after the delete: %v", err) } - recreated := tag.GetStatus().GetSnapshot().GetSnapshotUri() + recreated := tagDurableSnapshotURI(tag, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) if recreated == "" || recreated == stranded { t.Fatalf("snapshot uri after the delete = %q, want a fresh prefix rather than the stranded %q", recreated, stranded) } @@ -350,7 +353,7 @@ func TestTagActorSnapshot_RacesDelete(t *testing.T) { // Nothing was collected out from under the copy, and the row in the store it // is still there. - if got := tag.GetStatus().GetSnapshot().GetSnapshotUri(); got != pendingURI { + if got := tagDurableSnapshotURI(tag, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED); got != pendingURI { t.Errorf("snapshot uri = %q, want the prefix the create reserved, %q", got, pendingURI) } if diff := cmp.Diff([]string{"manifest.json", "memory.zst"}, objects.Snapshot(t, mustParseSnapshotURI(t, pendingURI))); diff != "" { @@ -382,7 +385,7 @@ func TestTagActorSnapshot_NameTakenByAnotherActor(t *testing.T) { } return nil } - if _, err := w.TagActorSnapshot(ctx, tagToCreate(resources.ActorRefFromActor(first), "v1")); !errors.Is(err, errObjectStore) { + if _, err := w.TagActorSnapshot(ctx, tagToCreate(resources.ActorRefFromActor(first), "v1")); !isObjectStoreErr(err) { t.Fatalf("TagActorSnapshot(actor-1) = %v, want an error wrapping %v", err, errObjectStore) } pending, err := persistence.GetTag(ctx, tagRef) @@ -433,7 +436,7 @@ func TestDeleteTag_ReleasesExternalSnapshot(t *testing.T) { // A delete that cannot reach object storage must not drop the row: it is // the only handle left on the snapshot. objects.OnDelete = func(string, string) error { return errObjectStore } - if _, err := w.DeleteTag(ctx, tagRef, store.DeletePreconditions{}); !errors.Is(err, errObjectStore) { + if _, err := w.DeleteTag(ctx, tagRef, store.DeletePreconditions{}); !isObjectStoreErr(err) { t.Fatalf("DeleteTag = %v, want an error wrapping %v", err, errObjectStore) } if _, err := persistence.GetTag(ctx, tagRef); err != nil { @@ -480,7 +483,7 @@ func TestDeleteTag_ReleasesPendingSnapshot(t *testing.T) { t.Fatalf("DeleteActor: %v", err) } objects.OnDelete = func(string, string) error { return errObjectStore } - if _, err := w.DeleteTag(ctx, tagRef, store.DeletePreconditions{}); !errors.Is(err, errObjectStore) { + if _, err := w.DeleteTag(ctx, tagRef, store.DeletePreconditions{}); !isObjectStoreErr(err) { t.Fatalf("DeleteTag = %v, want an error wrapping %v", err, errObjectStore) } if _, err := persistence.GetTag(ctx, tagRef); err != nil { @@ -516,9 +519,9 @@ func TestDeleteTag_NotFound(t *testing.T) { func mustReservedTagSnapshotURI(t *testing.T, tag *ateapipb.Tag) resources.SnapshotURI { t.Helper() - uri, err := resources.NewTagSnapshotURI(tag.GetStatus().GetStorageLocation(), tag.GetMetadata().GetAtespace(), tag.GetMetadata().GetUid()) + uri, err := resources.ParseSnapshotURI(tagDurableSnapshotURI(tag, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_UNSPECIFIED)) if err != nil { - t.Fatalf("NewTagSnapshotURI: %v", err) + t.Fatalf("ParseSnapshotURI: %v", err) } return uri } diff --git a/cmd/ateapi/internal/controlapi/workflow_test.go b/cmd/ateapi/internal/controlapi/workflow_test.go index cb3b48ea96..c89f43f9aa 100644 --- a/cmd/ateapi/internal/controlapi/workflow_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_test.go @@ -59,8 +59,8 @@ func TestActorStateChangeRecords(t *testing.T) { { name: "pause marks the actor pausing", seedState: ateapipb.ActorState_ACTOR_STATE_RUNNING, - transition: func(t *testing.T, w *ActorWorkflow, ref resources.ActorRef, actor *ateapipb.Actor, _ *ateapipb.ActorTemplate) { - if _, err := w.ensureMarkedPausing(context.Background(), ref, actor); err != nil { + transition: func(t *testing.T, w *ActorWorkflow, ref resources.ActorRef, actor *ateapipb.Actor, tmpl *ateapipb.ActorTemplate) { + if _, err := w.ensureMarkedPausing(context.Background(), ref, actor, tmpl); err != nil { t.Fatalf("ensureMarkedPausing: %v", err) } }, @@ -258,7 +258,10 @@ func TestActorStateChangeRecordSkippedOnConflict(t *testing.T) { // Bump the stored version so the workflow's precondition is stale. if _, err := persistence.UpdateActor(ctx, actorRef, store.PreconditionFrom(stale), func(toUpdate *ateapipb.Actor) error { - toUpdate.Status.InProgressSnapshotUri = someActorSnapshotURI(t, testStorageLocation, "team-a", "someone-else") + toUpdate.Status.LastAssignedGeneration = 1 + toUpdate.Status.Snapshots = []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", someActorSnapshotURI(t, testStorageLocation, "team-a", "someone-else"), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS), + } return nil }); err != nil { t.Fatalf("bump version: %v", err) diff --git a/cmd/ateapi/internal/controlapi/workflow_testutil_test.go b/cmd/ateapi/internal/controlapi/workflow_testutil_test.go index 20095ecf90..8c79960c15 100644 --- a/cmd/ateapi/internal/controlapi/workflow_testutil_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_testutil_test.go @@ -25,6 +25,7 @@ import ( "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest" "github.com/agent-substrate/substrate/internal/apierror" "github.com/agent-substrate/substrate/internal/objectstore/objectstoretest" + "github.com/agent-substrate/substrate/internal/objectstoreplugin/objectstoreplugintest" "github.com/agent-substrate/substrate/internal/resources" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/google/uuid" @@ -41,8 +42,8 @@ func testWorkerUID(podName string) string { // with one minimal ActorTemplate stored in tmplAtespace. Dependencies the // unit tests never reach (worker cache, atelet dialer, k8s clients) are nil, // so a step that unexpectedly executes against them fails the test loudly. -// External snapshots go to an in-memory object store, reachable from a test as -// w.objectStore.(*objectstoretest.Fake). +// External snapshots go to an in-memory object store through the control +// snapshot plugin. func newTestActorWorkflow(t *testing.T, st store.Interface, tmplAtespace, tmplName string) *ActorWorkflow { t.Helper() storetest.MustCreateAtespace(t, context.Background(), st, tmplAtespace) @@ -58,7 +59,7 @@ func newTestActorWorkflow(t *testing.T, st store.Interface, tmplAtespace, tmplNa }); err != nil && !errors.Is(err, store.ErrAlreadyExists) { t.Fatalf("create test ActorTemplate: %v", err) } - return NewActorWorkflow(st, nil, nil, nil, nil, nil, "", nil, time.Minute, objectstoretest.New()) + return NewActorWorkflow(st, nil, nil, nil, nil, nil, "", nil, time.Minute, objectstoreplugintest.ControlClient(objectstoretest.New())) } // newFinalizeWorkflow builds an ActorWorkflow over persistence with an @@ -66,7 +67,7 @@ func newTestActorWorkflow(t *testing.T, st store.Interface, tmplAtespace, tmplNa // directly rather than going through newTestActorWorkflow. func newFinalizeWorkflow(persistence store.Interface) (*ActorWorkflow, *objectstoretest.Fake) { objects := objectstoretest.New() - return &ActorWorkflow{store: persistence, workflowDeadline: time.Minute, objectStore: objects}, objects + return &ActorWorkflow{store: persistence, workflowDeadline: time.Minute, snapshotPlugin: objectstoreplugintest.ControlClient(objects)}, objects } // mustActorSnapshotURI builds the URI of a snapshot the actor took under diff --git a/cmd/ateapi/internal/controlapi/workflow_worker_delete.go b/cmd/ateapi/internal/controlapi/workflow_worker_delete.go index 5d8847376e..742f7179ce 100644 --- a/cmd/ateapi/internal/controlapi/workflow_worker_delete.go +++ b/cmd/ateapi/internal/controlapi/workflow_worker_delete.go @@ -227,10 +227,6 @@ func (w *WorkerWorkflow) crashBoundActor(ctx context.Context, worker *ateapipb.W toUpdate.Status.Crash = newActorCrash(opName, crashMsg) } toUpdate.Status.WorkerAssignment = nil - // Local in-progress checkpoint dies with the sandbox that was writing - // it. The external in-progress checkpoint is kept so delete or revert - // can delete it. - toUpdate.Status.InProgressLocalSnapshotName = "" return nil }) switch { diff --git a/cmd/ateapi/internal/controlapi/workflow_worker_delete_test.go b/cmd/ateapi/internal/controlapi/workflow_worker_delete_test.go index bbfe22b461..15ebdc91ba 100644 --- a/cmd/ateapi/internal/controlapi/workflow_worker_delete_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_worker_delete_test.go @@ -109,9 +109,12 @@ func TestDeleteWorkerWorkflow_ReleasesBoundActor(t *testing.T) { actor := seedAPIActor(t, ctx, persistence, ateapipb.ActorState_ACTOR_STATE_RUNNING, func(a *ateapipb.Actor) { // Both in-progress checkpoints are set so the assertion covers the // shared crash path, which cannot know which workflow was in flight. - a.Status.InProgressSnapshotUri = someActorSnapshotURI(t, testStorageLocation, apiActorRef.Atespace, "partial-snapshot") - a.Status.InProgressLocalSnapshotName = "partial-local-snapshot" - a.Status.ExternalSnapshot = &ateapipb.ExternalSnapshot{SnapshotUri: someActorSnapshotURI(t, testStorageLocation, apiActorRef.Atespace, "last")} + a.Status.LastAssignedGeneration = 3 + a.Status.Snapshots = []*ateapipb.Snapshot{ + newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", someActorSnapshotURI(t, testStorageLocation, apiActorRef.Atespace, "last"), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED), + newDurableSnapshot(2, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", someActorSnapshotURI(t, testStorageLocation, apiActorRef.Atespace, "partial-snapshot"), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS), + newLocalSnapshot(3, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "partial-local-snapshot", ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS), + } }) assignAPIWorker(t, ctx, persistence, apiWorkerName, actor.GetMetadata().GetUid()) @@ -132,19 +135,21 @@ func TestDeleteWorkerWorkflow_ReleasesBoundActor(t *testing.T) { if got.GetStatus().GetWorkerAssignment() != nil { t.Errorf("actor worker assignment = %v, want it cleared", got.GetStatus().GetWorkerAssignment()) } - // The local checkpoint lived on the node that went away, so it dies with - // the worker. - if got.GetStatus().GetInProgressLocalSnapshotName() != "" { - t.Errorf("in-progress local checkpoint not cleared: %v", got.GetStatus()) + // In-progress checkpoints are kept so delete or revert can clean them up + // while assigned_node or the object prefix is still recorded. + if _, gotSt := findLatestSnapshotStorage(got.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS); gotSt.GetLocal().GetSnapshotName() != "partial-local-snapshot" { + t.Errorf("in-progress local checkpoint = %q, want %q preserved", gotSt.GetLocal().GetSnapshotName(), "partial-local-snapshot") } // The durable one is kept: it names the prefix whatever atelet already // uploaded lives under, which delete or revert needs to collect it. - if want := someActorSnapshotURI(t, testStorageLocation, apiActorRef.Atespace, "partial-snapshot"); got.GetStatus().GetInProgressSnapshotUri() != want { + _, gotInProgressSt := findLatestSnapshotStorage(got.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS) + if want := someActorSnapshotURI(t, testStorageLocation, apiActorRef.Atespace, "partial-snapshot"); gotInProgressSt.GetObject().GetSnapshotUri() != want { t.Errorf("in-progress external checkpoint not preserved: %v", got.GetStatus()) } // The last completed snapshot is what makes the actor resumable, so it stays. - if want := someActorSnapshotURI(t, testStorageLocation, apiActorRef.Atespace, "last"); got.GetStatus().GetExternalSnapshot().GetSnapshotUri() != want { - t.Errorf("external snapshot = %q, want it preserved as %q", got.GetStatus().GetExternalSnapshot().GetSnapshotUri(), want) + _, gotCompletedSt := findLatestSnapshotStorage(got.GetStatus(), ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED) + if want := someActorSnapshotURI(t, testStorageLocation, apiActorRef.Atespace, "last"); gotCompletedSt.GetObject().GetSnapshotUri() != want { + t.Errorf("external snapshot = %q, want it preserved as %q", gotCompletedSt.GetObject().GetSnapshotUri(), want) } } diff --git a/cmd/ateapi/internal/controlapi/workload_spec.go b/cmd/ateapi/internal/controlapi/workload_spec.go index a8c934195a..2c666994bf 100644 --- a/cmd/ateapi/internal/controlapi/workload_spec.go +++ b/cmd/ateapi/internal/controlapi/workload_spec.go @@ -175,8 +175,8 @@ func appendExternalVolumes(workloadSpec *ateletpb.WorkloadSpec, template *ateapi var storageVolID string var volType string var volCtx map[string]string - for _, dbVol := range actor.GetStatus().GetActorVolumes() { - if dbVol.GetVolumeName() == vol.GetName() { + for _, dbVol := range actor.GetStatus().GetExternalVolumes() { + if dbVol.GetName() == vol.GetName() { storageVolID = dbVol.GetStorageVolumeId() volType = dbVol.GetVolumeType() volCtx = dbVol.GetVolumeContext() diff --git a/cmd/ateapi/internal/controlapi/workload_spec_test.go b/cmd/ateapi/internal/controlapi/workload_spec_test.go index ec9fa9a0b6..b39a90bbfd 100644 --- a/cmd/ateapi/internal/controlapi/workload_spec_test.go +++ b/cmd/ateapi/internal/controlapi/workload_spec_test.go @@ -421,9 +421,9 @@ func TestAppendExternalVolumes(t *testing.T) { Name: "actor-123", }, Status: &ateapipb.ActorStatus{ - ActorVolumes: []*ateapipb.ExternalVolume{ + ExternalVolumes: []*ateapipb.ExternalVolume{ { - VolumeName: "vol-1", + Name: "vol-1", StorageVolumeId: "vol-gce-pd-123", VolumeType: "pd-standard", VolumeContext: map[string]string{"foo": "bar"}, @@ -480,7 +480,7 @@ func TestAppendExternalVolumes(t *testing.T) { Atespace: "space-abc", Name: "actor-123", }, - Status: &ateapipb.ActorStatus{ActorVolumes: []*ateapipb.ExternalVolume{}}, + Status: &ateapipb.ActorStatus{ExternalVolumes: []*ateapipb.ExternalVolume{}}, } if err := appendExternalVolumes(&ateletpb.WorkloadSpec{}, template, missingActor, nil); err == nil { t.Errorf("appendExternalVolumes expected error for missing volume, got nil") diff --git a/cmd/ateapi/internal/defaults/defaults.go b/cmd/ateapi/internal/defaults/defaults.go index 3b4cfde90e..76c8bb2c7b 100644 --- a/cmd/ateapi/internal/defaults/defaults.go +++ b/cmd/ateapi/internal/defaults/defaults.go @@ -61,8 +61,8 @@ func applySnapshotConfigDefaults(sc *ateapipb.SnapshotConfig) { if sc == nil { return } - if sc.OnCommit == ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_UNSPECIFIED { - sc.OnCommit = ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL + if sc.PreferredFidelity == ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED { + sc.PreferredFidelity = ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY } } diff --git a/cmd/ateapi/internal/defaults/defaults_test.go b/cmd/ateapi/internal/defaults/defaults_test.go index 33ce8d1f9c..ca8082713b 100644 --- a/cmd/ateapi/internal/defaults/defaults_test.go +++ b/cmd/ateapi/internal/defaults/defaults_test.go @@ -25,8 +25,8 @@ import ( func TestApply(t *testing.T) { const ( - scopeFull = ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL - scopeData = ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA + scopeFull = ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY + scopeData = ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES ) tests := []struct { name string @@ -44,15 +44,15 @@ func TestApply(t *testing.T) { name: "empty snapshot_config gets every default", in: &ateapipb.ActorTemplate{SnapshotConfig: &ateapipb.SnapshotConfig{}}, want: &ateapipb.ActorTemplate{SnapshotConfig: &ateapipb.SnapshotConfig{ - OnCommit: scopeFull, + PreferredFidelity: scopeFull, }}, }, { name: "set scopes are kept", in: &ateapipb.ActorTemplate{SnapshotConfig: &ateapipb.SnapshotConfig{ - OnCommit: scopeData, + PreferredFidelity: scopeData, }}, want: &ateapipb.ActorTemplate{SnapshotConfig: &ateapipb.SnapshotConfig{ - OnCommit: scopeData, + PreferredFidelity: scopeData, }}, }, { name: "container without wakeup probe stays without one", diff --git a/cmd/ateapi/internal/store/atepg/access_policy.go b/cmd/ateapi/internal/store/atepg/access_policy.go index 475498ddb3..e09880f33d 100644 --- a/cmd/ateapi/internal/store/atepg/access_policy.go +++ b/cmd/ateapi/internal/store/atepg/access_policy.go @@ -27,7 +27,6 @@ import ( func (p *Persistence) CreateGlobalAccessPolicy(ctx context.Context, policy *ateapipb.AccessPolicy) (*ateapipb.AccessPolicy, error) { dbPolicy := proto.Clone(policy).(*ateapipb.AccessPolicy) - dbPolicy.Metadata = &ateapipb.ResourceMetadata{Name: "default"} setCreateMetadata(dbPolicy.Metadata) protoBytes, err := proto.Marshal(dbPolicy) if err != nil { @@ -111,7 +110,6 @@ func (p *Persistence) UpdateGlobalAccessPolicy(ctx context.Context, precondition func (p *Persistence) CreateAtespaceAccessPolicy(ctx context.Context, name string, policy *ateapipb.AccessPolicy) (*ateapipb.AccessPolicy, error) { dbPolicy := proto.Clone(policy).(*ateapipb.AccessPolicy) - dbPolicy.Metadata = &ateapipb.ResourceMetadata{Name: "default"} setCreateMetadata(dbPolicy.Metadata) protoBytes, err := proto.Marshal(dbPolicy) if err != nil { diff --git a/cmd/ateapi/internal/store/atepg/access_policy_test.go b/cmd/ateapi/internal/store/atepg/access_policy_test.go index 5f646e54f0..48abda0138 100644 --- a/cmd/ateapi/internal/store/atepg/access_policy_test.go +++ b/cmd/ateapi/internal/store/atepg/access_policy_test.go @@ -36,6 +36,7 @@ func TestGlobalAccessPolicy_Lifecycle(t *testing.T) { } policy := &ateapipb.AccessPolicy{ + Metadata: &ateapipb.ResourceMetadata{Name: "default"}, Bindings: []*ateapipb.Binding{ {Role: authz.RoleViewer, Members: []string{"user:bob"}}, {Role: authz.RoleOwner, Members: []string{"user:alice"}}, @@ -87,6 +88,7 @@ func TestAtespaceAccessPolicy_LifecycleAndCascade(t *testing.T) { ctx := t.Context() policy := &ateapipb.AccessPolicy{ + Metadata: &ateapipb.ResourceMetadata{Name: "default"}, Bindings: []*ateapipb.Binding{ {Role: authz.RoleEditor, Members: []string{"user:bob"}}, }, diff --git a/cmd/ateapi/internal/store/atepg/actor_template_test.go b/cmd/ateapi/internal/store/atepg/actor_template_test.go index 0254d38bd7..0e04e5b6c3 100644 --- a/cmd/ateapi/internal/store/atepg/actor_template_test.go +++ b/cmd/ateapi/internal/store/atepg/actor_template_test.go @@ -97,7 +97,7 @@ func TestGetActorTemplate_BackfillsDefaults(t *testing.T) { t.Fatalf("GetActorTemplate failed: %v", err) } want := &ateapipb.SnapshotConfig{ - OnCommit: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, + PreferredFidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, } if diff := cmp.Diff(want, got.GetSnapshotConfig(), protocmp.Transform()); diff != "" { t.Errorf("GetActorTemplate did not backfill snapshot_config defaults (-want +got):\n%s", diff) diff --git a/cmd/ateapi/internal/store/atepg/tag.go b/cmd/ateapi/internal/store/atepg/tag.go index c902a2bdd5..543e1fa15a 100644 --- a/cmd/ateapi/internal/store/atepg/tag.go +++ b/cmd/ateapi/internal/store/atepg/tag.go @@ -185,6 +185,22 @@ func (p *Persistence) CreateTag(ctx context.Context, tag *ateapipb.Tag) (*ateapi return dbTag, nil } +func isAllowedTagSnapshotTransition(stored, mutated *ateapipb.Snapshot) bool { + if proto.Equal(stored, mutated) || stored == nil { + return true + } + if mutated == nil || len(stored.GetStorage()) != 1 || len(mutated.GetStorage()) != 1 { + return false + } + if stored.GetStorage()[0].GetStatus() != ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS || + mutated.GetStorage()[0].GetStatus() != ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED { + return false + } + expected := proto.CloneOf(stored) + expected.Storage[0].Status = ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED + return proto.Equal(expected, mutated) +} + func validateUpdateTagMutation(storedTag, mutatedTag *ateapipb.Tag) error { if stored, mutated := storedTag.GetMetadata().GetAtespace(), mutatedTag.GetMetadata().GetAtespace(); stored != mutated { return fmt.Errorf("metadata.atespace is immutable: mutation changed it from %q to %q", stored, mutated) @@ -192,12 +208,9 @@ func validateUpdateTagMutation(storedTag, mutatedTag *ateapipb.Tag) error { if stored, mutated := storedTag.GetMetadata().GetName(), mutatedTag.GetMetadata().GetName(); stored != mutated { return fmt.Errorf("metadata.name is immutable: mutation changed it from %q to %q", stored, mutated) } - if stored, mutated := storedTag.GetStatus().GetSnapshot(), mutatedTag.GetStatus().GetSnapshot(); stored != nil && !proto.Equal(stored, mutated) { + if stored, mutated := storedTag.GetStatus().GetSnapshot(), mutatedTag.GetStatus().GetSnapshot(); !isAllowedTagSnapshotTransition(stored, mutated) { return fmt.Errorf("status.snapshot is immutable once set: mutation changed it from %s to %s", stored, mutated) } - if stored, mutated := storedTag.GetStatus().GetStorageLocation(), mutatedTag.GetStatus().GetStorageLocation(); stored != mutated { - return fmt.Errorf("status.storage_location is immutable: mutation changed it from %q to %q", stored, mutated) - } if stored, mutated := storedTag.GetStatus().GetActorTemplateUid(), mutatedTag.GetStatus().GetActorTemplateUid(); stored != mutated { return fmt.Errorf("status.actor_template_uid is immutable: mutation changed it from %q to %q", stored, mutated) } diff --git a/cmd/ateapi/internal/store/atepg/tag_test.go b/cmd/ateapi/internal/store/atepg/tag_test.go index 50ae7b4826..3d51f7a6eb 100644 --- a/cmd/ateapi/internal/store/atepg/tag_test.go +++ b/cmd/ateapi/internal/store/atepg/tag_test.go @@ -35,7 +35,16 @@ func createTestTag(t *testing.T, s *Persistence, tagAtespace, tagName string) *a Metadata: &ateapipb.ResourceMetadata{Atespace: tagAtespace, Name: tagName}, Scope: ateapipb.TagScope_TAG_SCOPE_ATESPACE, Status: &ateapipb.TagStatus{ - Snapshot: &ateapipb.ExternalSnapshot{SnapshotUri: "gs://bucket/atespaces/" + tagAtespace + "/tags/" + tagName}, + Snapshot: &ateapipb.Snapshot{ + Generation: 1, + Owner: ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG, + Storage: []*ateapipb.SnapshotStorage{{ + Durability: ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, + Status: ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED, + Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + Object: &ateapipb.ObjectSnapshot{SnapshotUri: "gs://bucket/atespaces/" + tagAtespace + "/tags/" + tagName}, + }}, + }, }, }) if err != nil { @@ -84,9 +93,7 @@ func TestCreateTag_TagForeignKeyErrors(t *testing.T) { // A tag in an atespace that does not exist trips the tag's atespace FK. _, err := s.CreateTag(ctx, &ateapipb.Tag{ Metadata: &ateapipb.ResourceMetadata{Atespace: "gone", Name: "latest"}, - Status: &ateapipb.TagStatus{ - StorageLocation: "gs://bucket", - }, + Status: &ateapipb.TagStatus{}, }) if !errors.Is(err, store.ErrFailedPrecondition) { t.Errorf("missing tag atespace error = %v, want ErrFailedPrecondition", err) diff --git a/cmd/ateapi/internal/store/storecontract/contract.go b/cmd/ateapi/internal/store/storecontract/contract.go index 5a82a3b411..3589e6849c 100644 --- a/cmd/ateapi/internal/store/storecontract/contract.go +++ b/cmd/ateapi/internal/store/storecontract/contract.go @@ -865,16 +865,36 @@ func runActorContractTests(t *testing.T, setup func(t *testing.T) store.Interfac Metadata: &ateapipb.ResourceMetadata{Name: "id1", Atespace: testAtespace}, ActorTemplate: &ateapipb.ObjectRef{Atespace: "ns1", Name: "tmpl1"}, Status: &ateapipb.ActorStatus{ - State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, - ExternalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: testActorSnapshotURI("gs://bucket", testAtespace, "snapshot-1")}, + State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, + LastAssignedGeneration: 1, + Snapshots: []*ateapipb.Snapshot{{ + Generation: 1, + Owner: ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, + Storage: []*ateapipb.SnapshotStorage{{ + Durability: ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, + Status: ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED, + Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + Object: &ateapipb.ObjectSnapshot{SnapshotUri: testActorSnapshotURI("gs://bucket", testAtespace, "snapshot-1")}, + }}, + }}, }, } actor2 := &ateapipb.Actor{ Metadata: &ateapipb.ResourceMetadata{Name: "id2", Atespace: testAtespace}, ActorTemplate: &ateapipb.ObjectRef{Atespace: "ns1", Name: "tmpl1"}, Status: &ateapipb.ActorStatus{ - State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, - ExternalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: testActorSnapshotURI("gs://bucket", testAtespace, "snapshot-2")}, + State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, + LastAssignedGeneration: 1, + Snapshots: []*ateapipb.Snapshot{{ + Generation: 1, + Owner: ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, + Storage: []*ateapipb.SnapshotStorage{{ + Durability: ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, + Status: ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED, + Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + Object: &ateapipb.ObjectSnapshot{SnapshotUri: testActorSnapshotURI("gs://bucket", testAtespace, "snapshot-2")}, + }}, + }}, }, } if _, err := s.CreateActor(ctx, actor1); err != nil { @@ -1137,25 +1157,32 @@ func runActorTemplateContractTests(t *testing.T, setup func(t *testing.T) store. }) } -// newTestSuspendedActor builds an actor that already holds an external +// newTestSuspendedActor builds an actor that already holds a durable // snapshot, the state CreateTag tags from. func newTestSuspendedActor(atespace, name string) *ateapipb.Actor { return &ateapipb.Actor{ Metadata: &ateapipb.ResourceMetadata{Atespace: atespace, Name: name}, ActorTemplate: &ateapipb.ObjectRef{Atespace: "ns1", Name: "tmpl1"}, Status: &ateapipb.ActorStatus{ - State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, - ExternalSnapshot: &ateapipb.ExternalSnapshot{ - SnapshotUri: testActorSnapshotURI("gs://private", atespace, name), - ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, - }, + State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, + LastAssignedGeneration: 1, + Snapshots: []*ateapipb.Snapshot{{ + Generation: 1, + Owner: ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, + Storage: []*ateapipb.SnapshotStorage{{ + Durability: ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, + Status: ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED, + Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + Object: &ateapipb.ObjectSnapshot{SnapshotUri: testActorSnapshotURI("gs://private", atespace, name)}, + }}, + }}, }, } } // newTestInProgressTag builds the row CreateTag reserves for a -// tag of actor: ATESPACE-scoped, with no snapshot yet and the destination of -// the copy still to come derived from status.storage_location and its UID. +// tag of actor: ATESPACE-scoped, with an IN_PROGRESS snapshot naming where the +// copy is going. func newTestInProgressTag(name string, actor *ateapipb.Actor) *ateapipb.Tag { atespace := actor.GetMetadata().GetAtespace() return &ateapipb.Tag{ @@ -1163,7 +1190,18 @@ func newTestInProgressTag(name string, actor *ateapipb.Actor) *ateapipb.Tag { Scope: ateapipb.TagScope_TAG_SCOPE_ATESPACE, Status: &ateapipb.TagStatus{ ActorTemplateUid: "template-uid", - StorageLocation: "gs://private", + Snapshot: &ateapipb.Snapshot{ + Generation: 1, + Owner: ateapipb.SnapshotOwner_SNAPSHOT_OWNER_TAG, + Storage: []*ateapipb.SnapshotStorage{{ + Durability: ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, + Status: ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS, + Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + Object: &ateapipb.ObjectSnapshot{ + SnapshotUri: testTagSnapshotURI("gs://private", atespace, name), + }, + }}, + }, }, } } @@ -1171,7 +1209,7 @@ func newTestInProgressTag(name string, actor *ateapipb.Actor) *ateapipb.Tag { func runTagContractTests(t *testing.T, setup func(t *testing.T) store.Interface) { t.Helper() - // seedSuspendedActor seeds an atespace and an actor holding an external + // seedSuspendedActor seeds an atespace and an actor holding a durable // snapshot, the precondition every tag test starts from. seedSuspendedActor := func(t *testing.T, s store.Interface, atespace, name string) *ateapipb.Actor { t.Helper() @@ -1204,10 +1242,10 @@ func runTagContractTests(t *testing.T, setup func(t *testing.T) store.Interface) if err != nil { t.Fatalf("GetTag failed: %v", err) } - // A reserved tag names where its copy is going and nothing else: it is - // not usable until the copy lands and finalize names it. - if tag.GetStatus().GetSnapshot() != nil { - t.Errorf("reserved tag snapshot = %v, want unset", tag.GetStatus().GetSnapshot()) + // A reserved tag has an IN_PROGRESS snapshot storage entry and is not + // usable until the copy lands and finalize marks it COMPLETED. + if got := tag.GetStatus().GetSnapshot().GetStorage()[0].GetStatus(); got != ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS { + t.Errorf("reserved tag snapshot storage status = %v, want IN_PROGRESS", got) } if diff := cmp.Diff(inProgressTag, tag, protocmp.Transform(), ignoreUID, ignoreVersion, ignoreTimestamps); diff != "" { t.Errorf("stored tag mismatch (-want +got):\n%s", diff) @@ -1223,12 +1261,16 @@ func runTagContractTests(t *testing.T, setup func(t *testing.T) store.Interface) if err != nil { t.Fatalf("finalizing tag failed: %v", err) } - wantURI := testTagSnapshotURI(tag.GetStatus().GetStorageLocation(), "team-a", tag.GetMetadata().GetUid()) - if got := ready.GetStatus().GetSnapshot().GetSnapshotUri(); got != wantURI { - t.Errorf("finalized tag snapshot uri = %q, want %q", got, wantURI) + wantURI := testTagSnapshotURI("gs://private", "team-a", "production") + var gotURI string + if len(ready.GetStatus().GetSnapshot().GetStorage()) > 0 { + gotURI = ready.GetStatus().GetSnapshot().GetStorage()[0].GetObject().GetSnapshotUri() + } + if gotURI != wantURI { + t.Errorf("finalized tag snapshot uri = %q, want %q", gotURI, wantURI) } - if got := ready.GetStatus().GetStorageLocation(); got != tag.GetStatus().GetStorageLocation() { - t.Errorf("finalized tag storage location = %q, want unchanged", got) + if got := ready.GetStatus().GetSnapshot().GetStorage()[0].GetStatus(); got != ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED { + t.Errorf("finalized tag snapshot storage status = %v, want COMPLETED", got) } updated, err := s.UpdateTag(ctx, resources.TagRef{Atespace: "team-a", Name: "production"}, store.PreconditionFrom(ready), func(toUpdate *ateapipb.Tag) error { @@ -1332,13 +1374,13 @@ func runTagContractTests(t *testing.T, setup func(t *testing.T) store.Interface) { name: "snapshot uri", mutate: func(toUpdate *ateapipb.Tag) { - toUpdate.Status.Snapshot.SnapshotUri = "gs://private/elsewhere" + toUpdate.Status.Snapshot.Storage[0].Object.SnapshotUri = "gs://private/elsewhere" }, }, { name: "snapshot content scope", mutate: func(toUpdate *ateapipb.Tag) { - toUpdate.Status.Snapshot.ContentScope = ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA + toUpdate.Status.Snapshot.Storage[0].Fidelity = ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES }, }, { @@ -1347,16 +1389,6 @@ func runTagContractTests(t *testing.T, setup func(t *testing.T) store.Interface) toUpdate.Status.Snapshot = nil }, }, - { - name: "storage location", - mutate: func(toUpdate *ateapipb.Tag) { - toUpdate.Status.StorageLocation = "gs://elsewhere" - }, - }, - { - name: "clearing the storage location", - mutate: func(toUpdate *ateapipb.Tag) { toUpdate.Status.StorageLocation = "" }, - }, { name: "actor template uid", mutate: func(toUpdate *ateapipb.Tag) { toUpdate.Status.ActorTemplateUid = "other-template-uid" }, @@ -1494,12 +1526,9 @@ func runTagContractTests(t *testing.T, setup func(t *testing.T) store.Interface) } // finalizeTag mutates a reserved tag the way the tag workflow's second -// transaction does: it names the copy that landed under the tag UID. +// transaction does: it marks the in-progress snapshot storage COMPLETED. func finalizeTag(toUpdate *ateapipb.Tag) error { - toUpdate.Status.Snapshot = &ateapipb.ExternalSnapshot{ - SnapshotUri: testTagSnapshotURI(toUpdate.GetStatus().GetStorageLocation(), toUpdate.GetMetadata().GetAtespace(), toUpdate.GetMetadata().GetUid()), - ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, - } + toUpdate.Status.Snapshot.Storage[0].Status = ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED return nil } diff --git a/cmd/ateapi/internal/workerservice/register.go b/cmd/ateapi/internal/workerservice/register.go index ad6b6aee1a..2ae394e889 100644 --- a/cmd/ateapi/internal/workerservice/register.go +++ b/cmd/ateapi/internal/workerservice/register.go @@ -15,10 +15,12 @@ package workerservice import ( + "cmp" "context" "errors" "fmt" "log/slog" + "slices" "github.com/agent-substrate/substrate/cmd/ateapi/internal/apivalidation" "github.com/agent-substrate/substrate/cmd/ateapi/internal/ateletauth" @@ -29,9 +31,9 @@ import ( "google.golang.org/protobuf/proto" ) -// RegisterWorker records a Worker's reported capacity and the hardware identity of -// its node in one write. As with MintCert, the caller must be an atelet -// running on the Worker's node. +// RegisterWorker records a Worker's reported capacity and sandbox runtimes in +// one write. As with MintCert, the caller must be an atelet running on the +// Worker's node. func (s *Server) RegisterWorker(ctx context.Context, req *ateapipb.RegisterWorkerRequest) (*ateapipb.RegisterWorkerResponse, error) { // TODO(identity): This check should be handled by OpenFGA. caller, err := ateletauth.Authenticate(ctx, s.ateletSPIFFEID) @@ -42,8 +44,13 @@ func (s *Server) RegisterWorker(ctx context.Context, req *ateapipb.RegisterWorke return nil, resources.ToAPIError(errs) } reported := req.GetCapacity() - reportedHardware := req.GetHardware() + defaultRuntime := req.GetDefaultRuntime() + restorable := req.GetRestorableRuntimes() name := req.GetWorker().GetName() + sortAttributes(defaultRuntime) + for _, rt := range restorable { + sortAttributes(rt) + } // Use authoritative state to authorize the write. worker, err := s.store.GetWorker(ctx, name) @@ -62,16 +69,24 @@ func (s *Server) RegisterWorker(ctx context.Context, req *ateapipb.RegisterWorke slog.String("caller_pod", caller.PodName)) return nil, apierror.NotFound("Worker %s not found", name) } + if err := validateRuntimeClasses(worker.GetSandboxClass(), req); err != nil { + return nil, err + } - if proto.Equal(worker.GetStatus().GetCapacity(), reported) && proto.Equal(worker.GetStatus().GetHardware(), reportedHardware) { + status := worker.GetStatus() + if proto.Equal(status.GetCapacity(), reported) && + proto.Equal(status.GetDefaultRuntime(), defaultRuntime) && + sameRuntimes(status.GetRestorableRuntimes(), restorable) { return &ateapipb.RegisterWorkerResponse{Worker: worker}, nil } updated, err := s.store.UpdateWorker(ctx, name, store.PreconditionFrom(worker), func(toUpdate *ateapipb.Worker) error { // Replaces rather than merges: a Worker reports everything it has, so a - // dimension this report leaves out is one it no longer supplies. + // dimension or runtime this report leaves out is one it no longer + // supplies. toUpdate.Status.Capacity = reported - toUpdate.Status.Hardware = reportedHardware + toUpdate.Status.DefaultRuntime = defaultRuntime + toUpdate.Status.RestorableRuntimes = restorable return nil }) switch { @@ -83,10 +98,48 @@ func (s *Server) RegisterWorker(ctx context.Context, req *ateapipb.RegisterWorke default: return nil, fmt.Errorf("while recording capacity for worker %s: %w", name, err) } - slog.InfoContext(ctx, "Worker registered its capacity and hardware", + slog.InfoContext(ctx, "Worker registered its capacity and sandbox runtimes", slog.String("worker", name), slog.String("was", worker.GetStatus().GetCapacity().String()), slog.String("now", updated.GetStatus().GetCapacity().String()), - slog.String("hardware", updated.GetStatus().GetHardware().String())) + slog.String("default_runtime", updated.GetStatus().GetDefaultRuntime().String()), + slog.Int("restorable_runtimes", len(updated.GetStatus().GetRestorableRuntimes()))) return &ateapipb.RegisterWorkerResponse{Worker: updated}, nil } + +// validateRuntimeClasses rejects a runtime of a class other than the Worker's +// own. The class is fixed when the pool creates the Worker and names the +// sandbox its ateom runs, so a runtime of another class contradicts the +// recorded Worker state. A Worker recorded without a class has nothing for a +// report to contradict. +func validateRuntimeClasses(class string, req *ateapipb.RegisterWorkerRequest) error { + if class == "" { + return nil + } + if got := req.GetDefaultRuntime().GetSandboxClass(); got != class { + return apierror.FailedPrecondition("default_runtime.sandbox_class %q does not match Worker's sandbox_class %q", got, class) + } + for i, rt := range req.GetRestorableRuntimes() { + if got := rt.GetSandboxClass(); got != class { + return apierror.FailedPrecondition("restorable_runtimes[%d].sandbox_class %q does not match Worker's sandbox_class %q", i, got, class) + } + } + return nil +} + +// sameRuntimes reports whether two runtime lists are identical entry for +// entry, in order. A report is compared as sent, so a Worker that reorders its +// runtimes between reports is recorded again. +func sameRuntimes(a, b []*ateapipb.SandboxRuntime) bool { + return slices.EqualFunc(a, b, func(x, y *ateapipb.SandboxRuntime) bool { return proto.Equal(x, y) }) +} + +// sortAttributes orders a runtime's compatibility attributes by key, in place. +// Attribute order carries no meaning, but proto equality is order-sensitive, so +// a report is canonicalized before it is compared with what is stored: the same +// attributes in another order are the same runtime and write nothing. +func sortAttributes(rt *ateapipb.SandboxRuntime) { + slices.SortFunc(rt.GetVersion().GetAttributes(), func(a, b *ateapipb.AttributeEntry) int { + return cmp.Compare(a.GetKey(), b.GetKey()) + }) +} diff --git a/cmd/ateapi/internal/workerservice/register_test.go b/cmd/ateapi/internal/workerservice/register_test.go index 2761cbce4f..52e0cc15be 100644 --- a/cmd/ateapi/internal/workerservice/register_test.go +++ b/cmd/ateapi/internal/workerservice/register_test.go @@ -16,9 +16,11 @@ package workerservice import ( "context" + "fmt" "testing" "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" "google.golang.org/protobuf/testing/protocmp" "github.com/agent-substrate/substrate/cmd/ateapi/internal/ateletauth/ateletauthtest" @@ -29,13 +31,28 @@ import ( "google.golang.org/grpc/codes" ) -var testHardware = &ateapipb.HardwareIdentity{Attributes: map[string]string{"architecture": "amd64"}} +// testRuntime is a SandboxRuntime of the given class whose compatibility +// identity is a single architecture attribute, which is enough to tell two +// runtimes apart. +func testRuntime(class, arch string) *ateapipb.SandboxRuntime { + return &ateapipb.SandboxRuntime{ + SandboxClass: class, + Version: &ateapipb.VersionedSandboxCompat{ + SchemaVersion: "v1", + Attributes: []*ateapipb.AttributeEntry{{Key: "architecture", Value: arch}}, + }, + } +} + +// testDefaultRuntime is what seedReportedWorker records as already reported, +// of the class it gives the Worker. +var testDefaultRuntime = testRuntime("gvisor", "amd64") func setRequest(actors int32) *ateapipb.RegisterWorkerRequest { return &ateapipb.RegisterWorkerRequest{ - Worker: &ateapipb.ObjectRef{Name: testWorkerName}, - Capacity: &ateapipb.WorkerResources{Actors: actors}, - Hardware: testHardware, + Worker: &ateapipb.ObjectRef{Name: testWorkerName}, + Capacity: &ateapipb.WorkerResources{Actors: actors}, + DefaultRuntime: testDefaultRuntime, } } @@ -62,35 +79,138 @@ func TestRegisterWorker(t *testing.T) { } } -func TestRegisterWorker_RecordsHardware(t *testing.T) { +// A report's runtimes are checked against the Worker's class and recorded only +// when they differ from what is stored. Capacity is held at what was seeded, +// so a write here is one the runtimes alone caused. +func TestRegisterWorker_Runtimes(t *testing.T) { st, cleanup := storetest.SetupTestStore(t) defer cleanup() s := New(st, &fakeSuspender{}, testAteletSPIFFEID, nil) - seedReportedWorker(t, st, testNode, &ateapipb.WorkerResources{Actors: 1}) authed := ateletauthtest.ContextWith(ateletauthtest.CertOn(t, testNode)) - wantHW := &ateapipb.HardwareIdentity{Attributes: map[string]string{"architecture": "arm64"}} - req := &ateapipb.RegisterWorkerRequest{ - Worker: &ateapipb.ObjectRef{Name: testWorkerName}, - Capacity: &ateapipb.WorkerResources{Actors: 4094}, - Hardware: wantHW, - } - got, err := s.RegisterWorker(authed, req) - if err != nil { - t.Fatalf("RegisterWorker() failed: %v", err) - } - if diff := cmp.Diff(wantHW, got.GetWorker().GetStatus().GetHardware(), protocmp.Transform()); diff != "" { - t.Errorf("hardware mismatch (-want +got):\n%s", diff) + gvisorAMD64, gvisorARM64 := testRuntime("gvisor", "amd64"), testRuntime("gvisor", "arm64") + gvisorV8, gvisorV9 := testRuntime("gvisor", "arm64-v8"), testRuntime("gvisor", "arm64-v9") + microvm := testRuntime("microvm", "amd64") + // Every Worker here has already reported gvisorAMD64 with gvisorV8 and + // gvisorV9 restorable, in that order, at a capacity of one actor. + seeded := func() *ateapipb.WorkerStatus { + return &ateapipb.WorkerStatus{ + State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, + Capacity: &ateapipb.WorkerResources{Actors: 1}, + DefaultRuntime: gvisorAMD64, + RestorableRuntimes: []*ateapipb.SandboxRuntime{gvisorV8, gvisorV9}, + } } - // Repeating the identical capacity and hardware must not bump version. - v1 := got.GetWorker().GetMetadata().GetVersion() - again, err := s.RegisterWorker(authed, req) - if err != nil { - t.Fatalf("RegisterWorker() repeat failed: %v", err) - } - if gotV := again.GetWorker().GetMetadata().GetVersion(); gotV != v1 { - t.Errorf("version = %d after identical capacity+hardware report, want %d unchanged", gotV, v1) + tests := []struct { + name string + // workerClass is the class the pool created the Worker with; "" is + // a Worker recorded without one. + workerClass string + def *ateapipb.SandboxRuntime + restorable []*ateapipb.SandboxRuntime + wantCode codes.Code + // wantWrite is whether the report changes the record. + wantWrite bool + }{{ + name: "identical report writes nothing", + workerClass: "gvisor", + def: gvisorAMD64, + restorable: []*ateapipb.SandboxRuntime{gvisorV8, gvisorV9}, + }, { + name: "changed default runtime writes", + workerClass: "gvisor", + def: gvisorARM64, + restorable: []*ateapipb.SandboxRuntime{gvisorV8, gvisorV9}, + wantWrite: true, + }, { + name: "changed restorable runtime writes", + workerClass: "gvisor", + def: gvisorAMD64, + restorable: []*ateapipb.SandboxRuntime{gvisorV8, gvisorARM64}, + wantWrite: true, + }, { + // A report is compared as sent: the list is atomic, so a new order + // is a new report. + name: "reordered restorable runtimes write", + workerClass: "gvisor", + def: gvisorAMD64, + restorable: []*ateapipb.SandboxRuntime{gvisorV9, gvisorV8}, + wantWrite: true, + }, { + // Runtimes are replaced like capacity: a report that lists no + // restorable runtimes is a Worker that can restore from none but its + // default, as when a version is disabled, and the record must not + // keep advertising them. + name: "dropped restorable runtimes write", + workerClass: "gvisor", + def: gvisorAMD64, + wantWrite: true, + }, { + // The class is fixed by the pool that created the Worker. A runtime + // of another class is one it cannot run, so the report is refused + // rather than recorded for the scheduler to act on. + name: "default runtime of another class is refused", + workerClass: "gvisor", + def: microvm, + restorable: []*ateapipb.SandboxRuntime{gvisorV8, gvisorV9}, + wantCode: codes.FailedPrecondition, + }, { + name: "restorable runtime of another class is refused", + workerClass: "gvisor", + def: gvisorAMD64, + restorable: []*ateapipb.SandboxRuntime{gvisorV8, microvm}, + wantCode: codes.FailedPrecondition, + }, { + // A Worker recorded without a class has nothing for a report to + // contradict. + name: "unclassed Worker takes a default runtime of any class", + def: microvm, + restorable: []*ateapipb.SandboxRuntime{gvisorV8, gvisorV9}, + wantWrite: true, + }, { + name: "unclassed Worker takes restorable runtimes of mixed classes", + def: gvisorAMD64, + restorable: []*ateapipb.SandboxRuntime{gvisorV8, microvm}, + wantWrite: true, + }} + for i, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + name := fmt.Sprintf("1a7e4c83-6d20-4f95-b3c8-%012d", i) + before := seedWorker(t, st, name, testNode, tt.workerClass, seeded()) + + got, err := s.RegisterWorker(authed, &ateapipb.RegisterWorkerRequest{ + Worker: &ateapipb.ObjectRef{Name: name}, + Capacity: &ateapipb.WorkerResources{Actors: 1}, + DefaultRuntime: tt.def, + RestorableRuntimes: tt.restorable, + }) + if code := apierror.Code(err); code != tt.wantCode { + t.Fatalf("code = %v (err %v), want %v", code, err, tt.wantCode) + } + after, err := st.GetWorker(context.Background(), name) + if err != nil { + t.Fatalf("GetWorker: %v", err) + } + if wrote := after.GetMetadata().GetVersion() != before.GetMetadata().GetVersion(); wrote != tt.wantWrite { + t.Fatalf("version %d -> %d, want a write: %v", before.GetMetadata().GetVersion(), after.GetMetadata().GetVersion(), tt.wantWrite) + } + if !tt.wantWrite { + if diff := cmp.Diff(before.GetStatus(), after.GetStatus(), protocmp.Transform()); diff != "" { + t.Errorf("status changed without a write (-want +got):\n%s", diff) + } + return + } + if diff := cmp.Diff(tt.def, after.GetStatus().GetDefaultRuntime(), protocmp.Transform()); diff != "" { + t.Errorf("default_runtime mismatch (-want +got):\n%s", diff) + } + if diff := cmp.Diff(tt.restorable, after.GetStatus().GetRestorableRuntimes(), protocmp.Transform(), cmpopts.EquateEmpty()); diff != "" { + t.Errorf("restorable_runtimes mismatch (-want +got):\n%s", diff) + } + if diff := cmp.Diff(after, got.GetWorker(), protocmp.Transform()); diff != "" { + t.Errorf("response is not the stored Worker (-want +got):\n%s", diff) + } + }) } } @@ -141,6 +261,52 @@ func TestRegisterWorker_UnchangedDoesNotWrite(t *testing.T) { } } +// Attribute order is the producer's business, not the record's: the same +// attributes in another order are the same runtime, so they are stored sorted +// by key and a reordered repeat writes nothing. +func TestRegisterWorker_SortsAttributes(t *testing.T) { + st, cleanup := storetest.SetupTestStore(t) + defer cleanup() + s := New(st, &fakeSuspender{}, testAteletSPIFFEID, nil) + seedReportedWorker(t, st, testNode, &ateapipb.WorkerResources{Actors: 1}) + authed := ateletauthtest.ContextWith(ateletauthtest.CertOn(t, testNode)) + + runtimeWith := func(keys ...string) *ateapipb.SandboxRuntime { + rt := testRuntime("gvisor", "amd64") + rt.Version.Attributes = nil + for _, k := range keys { + rt.Version.Attributes = append(rt.Version.Attributes, &ateapipb.AttributeEntry{Key: k, Value: "v"}) + } + return rt + } + register := func(def, restorable *ateapipb.SandboxRuntime) *ateapipb.Worker { + t.Helper() + got, err := s.RegisterWorker(authed, &ateapipb.RegisterWorkerRequest{ + Worker: &ateapipb.ObjectRef{Name: testWorkerName}, + Capacity: &ateapipb.WorkerResources{Actors: 1}, + DefaultRuntime: def, + RestorableRuntimes: []*ateapipb.SandboxRuntime{restorable}, + }) + if err != nil { + t.Fatalf("RegisterWorker() failed: %v", err) + } + return got.GetWorker() + } + + first := register(runtimeWith("cpu_features", "architecture"), runtimeWith("gvisor_asset_hash", "architecture")) + if diff := cmp.Diff(runtimeWith("architecture", "cpu_features"), first.GetStatus().GetDefaultRuntime(), protocmp.Transform()); diff != "" { + t.Errorf("default_runtime not stored sorted by key (-want +got):\n%s", diff) + } + if diff := cmp.Diff([]*ateapipb.SandboxRuntime{runtimeWith("architecture", "gvisor_asset_hash")}, first.GetStatus().GetRestorableRuntimes(), protocmp.Transform()); diff != "" { + t.Errorf("restorable_runtimes not stored sorted by key (-want +got):\n%s", diff) + } + + again := register(runtimeWith("architecture", "cpu_features"), runtimeWith("architecture", "gvisor_asset_hash")) + if got, want := again.GetMetadata().GetVersion(), first.GetMetadata().GetVersion(); got != want { + t.Errorf("version = %d after a reordered repeat, want %d unchanged", got, want) + } +} + func TestRegisterWorker_Errors(t *testing.T) { st, cleanup := storetest.SetupTestStore(t) defer cleanup() @@ -156,21 +322,21 @@ func TestRegisterWorker_Errors(t *testing.T) { }{ {"unauthenticated", ateletauthtest.ContextWith(nil), setRequest(2), codes.Unauthenticated}, {"no worker ref", authed, &ateapipb.RegisterWorkerRequest{ - Capacity: &ateapipb.WorkerResources{Actors: 2}, - Hardware: testHardware, + Capacity: &ateapipb.WorkerResources{Actors: 2}, + DefaultRuntime: testDefaultRuntime, }, codes.InvalidArgument}, {"no capacity", authed, &ateapipb.RegisterWorkerRequest{ - Worker: &ateapipb.ObjectRef{Name: testWorkerName}, - Hardware: testHardware, + Worker: &ateapipb.ObjectRef{Name: testWorkerName}, + DefaultRuntime: testDefaultRuntime, }, codes.InvalidArgument}, - {"no hardware", authed, &ateapipb.RegisterWorkerRequest{ + {"no default runtime", authed, &ateapipb.RegisterWorkerRequest{ Worker: &ateapipb.ObjectRef{Name: testWorkerName}, Capacity: &ateapipb.WorkerResources{Actors: 2}, }, codes.InvalidArgument}, {"absent worker", authed, &ateapipb.RegisterWorkerRequest{ - Worker: &ateapipb.ObjectRef{Name: "3b9f1e77-2c4d-4a80-91be-6d5c8f0a7e21"}, - Capacity: &ateapipb.WorkerResources{Actors: 2}, - Hardware: testHardware, + Worker: &ateapipb.ObjectRef{Name: "3b9f1e77-2c4d-4a80-91be-6d5c8f0a7e21"}, + Capacity: &ateapipb.WorkerResources{Actors: 2}, + DefaultRuntime: testDefaultRuntime, }, codes.NotFound}, } for _, tc := range tests { @@ -206,9 +372,9 @@ func TestRegisterWorker_RejectsNonsense(t *testing.T) { } { t.Run(tc.name, func(t *testing.T) { _, err := s.RegisterWorker(authed, &ateapipb.RegisterWorkerRequest{ - Worker: &ateapipb.ObjectRef{Name: testWorkerName}, - Capacity: tc.capacity, - Hardware: testHardware, + Worker: &ateapipb.ObjectRef{Name: testWorkerName}, + Capacity: tc.capacity, + DefaultRuntime: testDefaultRuntime, }) if got := apierror.Code(err); got != codes.InvalidArgument { t.Fatalf("code = %v (err %v), want %v", got, err, codes.InvalidArgument) diff --git a/cmd/ateapi/internal/workerservice/service_test.go b/cmd/ateapi/internal/workerservice/service_test.go index 1ad20d5e2c..8e26beebdc 100644 --- a/cmd/ateapi/internal/workerservice/service_test.go +++ b/cmd/ateapi/internal/workerservice/service_test.go @@ -49,25 +49,35 @@ func (f *fakeSuspender) SuspendActor(_ context.Context, req *ateapipb.SuspendAct } // seedReportedWorker registers a Worker on nodeName that has already reported -// capacity and hardware: identity from the pod, the way the syncer writes it, -// plus the result of an earlier registration. A Worker that has never -// registered carries none, so what a fresh registration replaces is what this -// seeds. +// capacity and its default runtime: identity from the pod, the way the syncer +// writes it, plus the result of an earlier registration. A Worker that has +// never registered carries none, so what a fresh registration replaces is what +// this seeds. func seedReportedWorker(t *testing.T, st store.Interface, nodeName string, capacity *ateapipb.WorkerResources) *ateapipb.Worker { + t.Helper() + return seedWorker(t, st, testWorkerName, nodeName, testDefaultRuntime.GetSandboxClass(), &ateapipb.WorkerStatus{ + State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, Capacity: capacity, DefaultRuntime: testDefaultRuntime, + }) +} + +// seedWorker registers a Worker named name, of sandbox class class, on +// nodeName, with status already recorded. name doubles as the pod UID, so it +// must be UUID-shaped. class "" is a Worker recorded without one. +func seedWorker(t *testing.T, st store.Interface, name, nodeName, class string, status *ateapipb.WorkerStatus) *ateapipb.Worker { t.Helper() created, err := st.CreateWorker(context.Background(), &ateapipb.Worker{ - Metadata: &ateapipb.ResourceMetadata{Name: testWorkerName}, + Metadata: &ateapipb.ResourceMetadata{Name: name}, WorkerNamespace: "ate-system", WorkerPool: "pool-1", WorkerPod: "worker-pod-1", - WorkerPodUid: testWorkerName, + WorkerPodUid: name, NodeName: nodeName, Ips: []string{"10.1.2.3"}, - SandboxClass: "gvisor", - Status: &ateapipb.WorkerStatus{State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, Capacity: capacity, Hardware: testHardware}, + SandboxClass: class, + Status: status, }) if err != nil { - t.Fatalf("seeding worker: %v", err) + t.Fatalf("seeding worker %s: %v", name, err) } return created } diff --git a/cmd/ateapi/main.go b/cmd/ateapi/main.go index 3c40b4c6b2..bf16048c0e 100644 --- a/cmd/ateapi/main.go +++ b/cmd/ateapi/main.go @@ -27,7 +27,6 @@ import ( "syscall" "time" - "cloud.google.com/go/storage" "github.com/agent-substrate/substrate/cmd/ateapi/internal/apiauthn" "github.com/agent-substrate/substrate/cmd/ateapi/internal/authz" "github.com/agent-substrate/substrate/cmd/ateapi/internal/controlapi" @@ -40,7 +39,7 @@ import ( "github.com/agent-substrate/substrate/internal/installdefaults" "github.com/agent-substrate/substrate/internal/localca" "github.com/agent-substrate/substrate/internal/localjwtauthority" - "github.com/agent-substrate/substrate/internal/objectstore" + "github.com/agent-substrate/substrate/internal/objectstoreplugin" "github.com/agent-substrate/substrate/internal/oidcdiscovery" "github.com/agent-substrate/substrate/internal/serverboot" "github.com/agent-substrate/substrate/internal/version" @@ -48,8 +47,7 @@ import ( "github.com/agent-substrate/substrate/pkg/client/clientset/versioned" "github.com/agent-substrate/substrate/pkg/client/informers/externalversions" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" - "github.com/aws/aws-sdk-go-v2/config" - "github.com/aws/aws-sdk-go-v2/service/s3" + objectstorev1 "github.com/agent-substrate/substrate/pkg/proto/objectstorepb/v1" "github.com/jackc/pgx/v5/pgconn" "github.com/spf13/pflag" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" @@ -87,7 +85,7 @@ var ( authzBootstrapOwners = pflag.StringSlice("authz-bootstrap-owners", nil, "Principal IDs that are always global owners while listed, independent of the stored global AccessPolicy. Removing an ID revokes its access on restart. At least one is required when --experimental-enable-authz is set.") actorIDJWTPoolFile = pflag.String("actor-id-jwt-pool", "", "The file that contains the serialized JWT authority pool for signing actor JWTs") - actorJWTIssuer = pflag.String("actor-jwt-issuer", "", "Issuer URL placed in the iss claim of actor JWTs. Relying parties fetch /.well-known/openid-configuration to verify them. Must be https with no query or fragment. Empty means https://"+installdefaults.IDPServiceName+"..svc.") + actorJWTIssuer = pflag.String("actor-jwt-issuer", "", "Issuer URL placed in the iss claim of actor JWTs. Relying parties fetch /.well-known/openid-configuration to verify them. Must be https with no query or fragment. Required.") defaultEgressGatewayAddress = pflag.String("default-egress-gateway-address", "", "Default address (host:port) of the egress PEP that each actor's atunnel dials. Sent on every atelet Run and Restore, so it takes effect at the actor's next activation. Empty leaves actors with no TCP egress.") actorIDCAPoolFile = pflag.String("actor-id-ca-pool", "", "The file that contains the CA pool for signing actor JWTs") @@ -99,6 +97,8 @@ var ( drainDelay = pflag.Duration("drain-delay", 13*time.Second, "How long to keep accepting new work after SIGTERM, before starting the gRPC drain.") drainTimeout = pflag.Duration("drain-timeout", 15*time.Second, "Deadline for the graceful gRPC drain on shutdown. In-flight RPCs still running past it are forcefully cancelled.") + snapshotPluginSocket = pflag.String("snapshot-plugin-socket", "/run/snapshot-plugin/control.sock", "Unix socket of the control snapshot plugin that external snapshots are deleted and copied through.") + templateResyncInterval = pflag.Duration("template-resync-interval", 20*time.Second, fmt.Sprintf("Interval between actor template resyncs. Must be at least %s.", minResyncInterval)) actorWorkflowDeadline = pflag.Duration("actor-workflow-deadline", 5*time.Minute, "Maximum wall-clock duration of a single Resume/Suspend workflow; raise it for slow image registries.") @@ -120,15 +120,16 @@ func main() { if err := loadFlagsFromEnv(); err != nil { serverboot.Fatal(ctx, "Invalid PostgreSQL configuration", err) } + if err := rejectStorageEnv(); err != nil { + serverboot.Fatal(ctx, "Storage settings moved to the snapshot-plugin sidecar", err) + } slog.InfoContext(ctx, "ateapi starting", slog.String("version", version.Version)) if *templateResyncInterval < minResyncInterval { serverboot.Fatal(ctx, "Invalid --template-resync-interval", fmt.Errorf("must be at least %s", minResyncInterval)) } - resolvedActorJWTIssuer, err := resolveActorJWTIssuer(*actorJWTIssuer, installdefaults.NamespaceFromPodEnv()) - if err != nil { + if err := validateActorJWTIssuer(*actorJWTIssuer); err != nil { serverboot.Fatal(ctx, "Invalid --actor-jwt-issuer", err) } - slog.InfoContext(ctx, "Resolved actor JWT issuer", slog.String("actor-jwt-issuer", resolvedActorJWTIssuer)) // Kept separate from ctx so that in-progress work (clients, informers) is // not cancelled the moment SIGTERM arrives. The drainOnShutdown @@ -260,9 +261,18 @@ func main() { serverboot.Fatal(ctx, "Failed to create metric instruments", err) } - objectStore, err := newObjectStore(ctx) + snapshotPluginConn, err := objectstoreplugin.Dial(*snapshotPluginSocket, objectstoreplugin.ReadyWait) if err != nil { - serverboot.Fatal(ctx, "Failed to set up the object storage backend", err) + serverboot.Fatal(ctx, "Failed to set up the snapshot plugin client", err) + } + defer snapshotPluginConn.Close() + // A plugin that never serves would fail every call. Fail at startup + // instead, giving the sidecar time to come up. + readyCtx, cancelReady := context.WithTimeout(ctx, time.Minute) + err = objectstoreplugin.WaitReady(readyCtx, snapshotPluginConn) + cancelReady() + if err != nil { + serverboot.Fatal(ctx, "Snapshot plugin is not serving", err) } volPlugins := make(map[string]volume.VolumePluginControlPlane) @@ -293,8 +303,8 @@ func main() { *defaultEgressGatewayAddress, *actorWorkflowDeadline, volPlugins, - objectStore, - resolvedActorJWTIssuer, + objectstorev1.NewControlProviderClient(snapshotPluginConn), + *actorJWTIssuer, actorIDJWTAuthorityPool, actorIDCAPool, ) @@ -399,6 +409,7 @@ func loadFlagsFromEnv() error { {postgresReadWriteRole, "ATE_API_POSTGRES_READ_WRITE_ROLE"}, {postgresOwnerRole, "ATE_API_POSTGRES_OWNER_ROLE"}, {postgresSchema, "ATE_API_POSTGRES_SCHEMA"}, + {actorJWTIssuer, "ATE_API_ACTOR_JWT_ISSUER"}, } for _, o := range overrides { if *o.flag == "@env" { @@ -420,6 +431,19 @@ func loadFlagsFromEnv() error { return nil } +// rejectStorageEnv fails if ATE_STORAGE_BACKEND is set on ate-api-server, +// which no longer reads it; objectstorage.UsesS3 reads it in the sidecar. An +// install that still patches it onto this container would otherwise come up +// healthy with a sidecar on the default backend, and fail at the first +// snapshot cleanup or copy. Only this variable is a signal: EKS pod identity +// sets AWS_* in every container. +func rejectStorageEnv() error { + if v, ok := os.LookupEnv("ATE_STORAGE_BACKEND"); ok { + return fmt.Errorf("ATE_STORAGE_BACKEND=%q is set on ate-api-server, which no longer reads it; set the storage backend on its snapshot-plugin sidecar", v) + } + return nil +} + func logFlagValues(ctx context.Context) { slog.InfoContext(ctx, "Final flag values", slog.String("grpc-listen-addr", *listenAddr), @@ -445,35 +469,6 @@ func logFlagValues(ctx context.Context) { ) } -// newObjectStore builds the client ate-api manages external snapshots with. -// The backend is selected the same way atelet selects the one it reads and -// writes snapshots through, so both ends of a snapshot's life agree on where -// it lives. -func newObjectStore(ctx context.Context) (objectstore.Store, error) { - switch backend := os.Getenv("ATE_STORAGE_BACKEND"); backend { - case "s3": - slog.InfoContext(ctx, "Using S3 storage backend") - // Depends on the standard AWS environment variables, which have to be - // set on the ate-api pod. - cfg, err := config.LoadDefaultConfig(ctx) - if err != nil { - return nil, fmt.Errorf("loading S3 config: %w", err) - } - return objectstore.NewS3(s3.NewFromConfig(cfg, func(o *s3.Options) { - if os.Getenv("AWS_S3_USE_PATH_STYLE") == "true" { - o.UsePathStyle = true - } - })), nil - // GCS is currently the default, TODO: we assume workload identity / ADC - default: - client, err := storage.NewClient(ctx) - if err != nil { - return nil, fmt.Errorf("creating GCS client: %w", err) - } - return objectstore.NewGCS(client), nil - } -} - // postgresConnectionAttr describes the connection string for the startup log // without echoing it. The bundled and Cloud SQL IAM setups use passwordless // strings, but an external database DSN can carry a password, and the raw @@ -643,15 +638,9 @@ func buildJWTProviders(ctx context.Context, cfg *apiauthn.AuthenticationConfig) return serverCfg, nil } -// resolveActorJWTIssuer applies the install default to an empty -// --actor-jwt-issuer and validates the result. -func resolveActorJWTIssuer(flagValue, namespace string) (string, error) { - issuer := flagValue +func validateActorJWTIssuer(issuer string) error { if issuer == "" { - issuer = installdefaults.ActorJWTIssuer(namespace) - } - if err := oidcdiscovery.ValidateIssuer(issuer); err != nil { - return "", err + return errors.New("required") } - return issuer, nil + return oidcdiscovery.ValidateIssuer(issuer) } diff --git a/cmd/ateapi/main_test.go b/cmd/ateapi/main_test.go index 89d269dde8..6abf56c61d 100644 --- a/cmd/ateapi/main_test.go +++ b/cmd/ateapi/main_test.go @@ -104,35 +104,34 @@ func TestLoadFlagsFromEnvPoolMaxConns(t *testing.T) { } } -func TestResolveActorJWTIssuer(t *testing.T) { - tests := []struct { - name string - flagValue string - namespace string - want string - wantErr bool +func TestValidateActorJWTIssuer(t *testing.T) { + for _, tt := range []struct { + issuer string + wantErr bool }{ - {name: "unset uses the namespace's idp Service", namespace: "ate-system", want: "https://idp.ate-system.svc"}, - {name: "unset in a relocated install", namespace: "team-a", want: "https://idp.team-a.svc"}, - {name: "set is used as given", flagValue: "https://idp.example.com/prod/", namespace: "ate-system", want: "https://idp.example.com/prod/"}, - {name: "set but invalid", flagValue: "http://idp.example.com", namespace: "ate-system", wantErr: true}, + {issuer: "", wantErr: true}, + {issuer: "http://idp.example.com", wantErr: true}, + {issuer: "https://idp.ate-system.svc"}, + {issuer: "https://idp.example.com/prod/"}, + } { + err := validateActorJWTIssuer(tt.issuer) + if gotErr := err != nil; gotErr != tt.wantErr { + t.Errorf("validateActorJWTIssuer(%q) = %v, want error %t", tt.issuer, err, tt.wantErr) + } } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := resolveActorJWTIssuer(tt.flagValue, tt.namespace) - if tt.wantErr { - if err == nil { - t.Fatalf("resolveActorJWTIssuer(%q, %q) = %q, want error", tt.flagValue, tt.namespace, got) - } - return - } - if err != nil { - t.Fatalf("resolveActorJWTIssuer(%q, %q) returned error: %v", tt.flagValue, tt.namespace, err) - } - if got != tt.want { - t.Errorf("resolveActorJWTIssuer(%q, %q) = %q, want %q", tt.flagValue, tt.namespace, got, tt.want) - } - }) +} + +func TestLoadFlagsFromEnvActorJWTIssuer(t *testing.T) { + old := *actorJWTIssuer + t.Cleanup(func() { *actorJWTIssuer = old }) + *actorJWTIssuer = "@env" + t.Setenv("ATE_API_ACTOR_JWT_ISSUER", "https://idp.example.com") + + if err := loadFlagsFromEnv(); err != nil { + t.Fatal(err) + } + if *actorJWTIssuer != "https://idp.example.com" { + t.Errorf("--actor-jwt-issuer = %q, want %q", *actorJWTIssuer, "https://idp.example.com") } } @@ -268,7 +267,12 @@ func TestBuildServerTLSConfigReloadsCACertsWithoutRestart(t *testing.T) { t.Fatalf("GetConfigForClient() first call error = %v", err) } - writeCA(t, path, "ca-two") + // Replace the bundle like a projected-volume rotation, independent of mtime. + rotatedPath := path + ".rotated" + writeCA(t, rotatedPath, "ca-two") + if err := os.Rename(rotatedPath, path); err != nil { + t.Fatal(err) + } after, err := cfg.GetConfigForClient(nil) if err != nil { @@ -304,3 +308,34 @@ func writeCA(t *testing.T, path, cn string) { t.Fatalf("WriteFile(%s) error = %v", path, err) } } + +func TestRejectStorageEnv(t *testing.T) { + for _, tc := range []struct { + name string + value *string // nil leaves the variable unset + want bool // an error naming the sidecar + }{ + {name: "unset", value: nil, want: false}, + {name: "s3", value: ptr("s3"), want: true}, + // The old manifests set the default explicitly, so a stale patch + // can carry it. + {name: "gcs", value: ptr("gcs"), want: true}, + {name: "empty", value: ptr(""), want: true}, + } { + t.Run(tc.name, func(t *testing.T) { + // t.Setenv restores the variable after the subtest, which also + // undoes the Unsetenv. + t.Setenv("ATE_STORAGE_BACKEND", "") + os.Unsetenv("ATE_STORAGE_BACKEND") + if tc.value != nil { + os.Setenv("ATE_STORAGE_BACKEND", *tc.value) + } + err := rejectStorageEnv() + if got := err != nil && strings.Contains(err.Error(), "snapshot-plugin sidecar"); got != tc.want { + t.Errorf("rejectStorageEnv() = %v, want error naming the sidecar: %t", err, tc.want) + } + }) + } +} + +func ptr(s string) *string { return &s } diff --git a/cmd/atecontroller/internal/controllers/workerpool_apply.go b/cmd/atecontroller/internal/controllers/workerpool_apply.go index a11d6f2613..afa2a3edd5 100644 --- a/cmd/atecontroller/internal/controllers/workerpool_apply.go +++ b/cmd/atecontroller/internal/controllers/workerpool_apply.go @@ -164,7 +164,7 @@ func buildDeploymentApplyConfig(wp *atev1alpha1.WorkerPool, otel ateomOTelSettin WithHTTPGet(corev1ac.HTTPGetAction(). WithPath("/readyz"). WithPort(intstr.FromString("readyz")))). - WithSecurityContext(ateomSecurityContext(wp.Spec.SandboxClass)). + WithSecurityContext(ateomSecurityContext(wp.Spec.DefaultSandboxClass())). WithEnv(ateomContainerEnv(otel)...). WithVolumeMounts( corev1ac.VolumeMount(). @@ -234,8 +234,8 @@ func buildDeploymentApplyConfig(wp *atev1alpha1.WorkerPool, otel ateomOTelSettin ) applyWorkerPoolPodTemplate(podSpecAC, containerAC, wp.Spec.Template) - applySandboxClassToleration(podSpecAC, wp.Spec.SandboxClass) - maybeApplyMicroVMPodShape(podSpecAC, containerAC, wp.Spec.SandboxClass) + applySandboxClassToleration(podSpecAC, wp.Spec.DefaultSandboxClass()) + maybeApplyMicroVMPodShape(podSpecAC, containerAC, wp.Spec.DefaultSandboxClass()) podSpecAC.WithContainers(containerAC) podSpecAC.WithTerminationGracePeriodSeconds(workerTerminationGracePeriodSeconds) diff --git a/cmd/atecontroller/internal/controllers/workerpool_apply_test.go b/cmd/atecontroller/internal/controllers/workerpool_apply_test.go index 49edc1265b..d5bd9283ec 100644 --- a/cmd/atecontroller/internal/controllers/workerpool_apply_test.go +++ b/cmd/atecontroller/internal/controllers/workerpool_apply_test.go @@ -299,7 +299,7 @@ func TestSandboxClassToleration(t *testing.T) { Effect: corev1.TaintEffectNoSchedule, }}, }) - wp.Spec.SandboxClass = tt.class + wp.Spec.SandboxClasses = []atev1alpha1.WorkerPoolSandboxClass{{Name: tt.class}} ps := buildDeploymentApplyConfig(wp, ateomOTelSettings{}, installdefaults.SystemNamespace, installdefaults.AteletServiceAccount, installdefaults.RouterServiceAccount).Spec.Template.Spec var classValues []string @@ -347,7 +347,7 @@ func TestMicroVMPodShape(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { wp := testWorkerPoolApplyConfig(nil) - wp.Spec.SandboxClass = tt.class + wp.Spec.SandboxClasses = []atev1alpha1.WorkerPoolSandboxClass{{Name: tt.class}} ps := buildDeploymentApplyConfig(wp, ateomOTelSettings{}, installdefaults.SystemNamespace, installdefaults.AteletServiceAccount, installdefaults.RouterServiceAccount).Spec.Template.Spec // /dev/kvm must come from the device plugin, never a hostPath: a @@ -434,7 +434,7 @@ func TestMicroVMDeviceRequestsPreserveTemplateResources(t *testing.T) { Limits: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("2Gi")}, }, }) - wp.Spec.SandboxClass = atev1alpha1.SandboxClassMicroVM + wp.Spec.SandboxClasses = []atev1alpha1.WorkerPoolSandboxClass{{Name: atev1alpha1.SandboxClassMicroVM}} c := buildDeploymentApplyConfig(wp, ateomOTelSettings{}, installdefaults.SystemNamespace, installdefaults.AteletServiceAccount, installdefaults.RouterServiceAccount).Spec.Template.Spec.Containers[0] if got, ok := deviceLimit(c, string(corev1.ResourceMemory)); !ok || got != "2Gi" { diff --git a/cmd/atecontroller/internal/controllers/workerpool_controller_test.go b/cmd/atecontroller/internal/controllers/workerpool_controller_test.go index 7915d1501b..784f3de2f1 100644 --- a/cmd/atecontroller/internal/controllers/workerpool_controller_test.go +++ b/cmd/atecontroller/internal/controllers/workerpool_controller_test.go @@ -590,8 +590,9 @@ func makeWorkerPool(name, ns string, replicas int32, image string) *atev1alpha1. return &atev1alpha1.WorkerPool{ ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns}, Spec: atev1alpha1.WorkerPoolSpec{ - Replicas: replicas, - WorkerImage: image, + Replicas: replicas, + WorkerImage: image, + SandboxClasses: []atev1alpha1.WorkerPoolSandboxClass{{Name: atev1alpha1.SandboxClassGvisor}}, }, } } diff --git a/cmd/atecontroller/internal/workersync/syncer.go b/cmd/atecontroller/internal/workersync/syncer.go index 40f820a7af..62cf8c144d 100644 --- a/cmd/atecontroller/internal/workersync/syncer.go +++ b/cmd/atecontroller/internal/workersync/syncer.go @@ -357,7 +357,7 @@ func (s *WorkerPoolSyncer) createOrUpdateWorker(ctx context.Context, key workerK Ips: podIPs(pod), WorkerPodUid: string(pod.UID), NodeName: pod.Spec.NodeName, - SandboxClass: string(pool.Spec.SandboxClass), + SandboxClass: string(pool.Spec.DefaultSandboxClass()), Labels: pool.GetLabels(), Epoch: podEpoch(pod), // Capacity is the Worker's to report, not the syncer's to infer @@ -395,7 +395,7 @@ func (s *WorkerPoolSyncer) createOrUpdateWorker(ctx context.Context, key workerK w.Labels = pool.GetLabels() changed = true } - if w.GetSandboxClass() != string(pool.Spec.SandboxClass) { + if w.GetSandboxClass() != string(pool.Spec.DefaultSandboxClass()) { // Expected mid-rollout: sandboxClass drives the worker pod's shape, so // editing it on the pool replaces every pod rather than reclassifying // any. This pod predates that edit and is on its way out; its successor @@ -403,7 +403,7 @@ func (s *WorkerPoolSyncer) createOrUpdateWorker(ctx context.Context, key workerK // back would be rejected, and would misreport this pod's shape to the // scheduler if it were not. slog.DebugContext(ctx, "Syncer: registered worker sandbox class predates its pool", - append(key.logAttrs(), slog.String("registered", w.GetSandboxClass()), slog.String("pool", string(pool.Spec.SandboxClass)))...) + append(key.logAttrs(), slog.String("registered", w.GetSandboxClass()), slog.String("pool", string(pool.Spec.DefaultSandboxClass())))...) } if ips := podIPs(pod); !slices.Equal(w.GetIps(), ips) { // TODO: I don't think this is possible, but handling this case so we can diff --git a/cmd/atecontroller/internal/workersync/syncer_test.go b/cmd/atecontroller/internal/workersync/syncer_test.go index 6e37c99238..642e2e7eeb 100644 --- a/cmd/atecontroller/internal/workersync/syncer_test.go +++ b/cmd/atecontroller/internal/workersync/syncer_test.go @@ -84,7 +84,7 @@ func workerPod(ns, name, poolName, uid, ip string) *corev1.Pod { func workerPool(ns, name, sandboxClass string, labels map[string]string) *atev1alpha1.WorkerPool { return &atev1alpha1.WorkerPool{ ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns, Labels: labels}, - Spec: atev1alpha1.WorkerPoolSpec{SandboxClass: atev1alpha1.SandboxClass(sandboxClass)}, + Spec: atev1alpha1.WorkerPoolSpec{SandboxClasses: []atev1alpha1.WorkerPoolSandboxClass{{Name: atev1alpha1.SandboxClass(sandboxClass)}}}, } } diff --git a/cmd/atelet/ateomsupport.go b/cmd/atelet/ateomsupport.go index 4481257b91..c6e778a91a 100644 --- a/cmd/atelet/ateomsupport.go +++ b/cmd/atelet/ateomsupport.go @@ -102,16 +102,16 @@ func verifyClientOnSameNode(node *substratex509.PodIdentity) func(tls.Connection } // RegisterWorker registers the calling worker with the control plane: what the -// worker says its capacity and hardware identity are, in one -// WorkerService.RegisterWorker call so capacity and hardware land atomically. +// worker says its capacity and sandbox runtimes are, in one +// WorkerService.RegisterWorker call so capacity and runtimes land atomically. // // It returns the control plane's error unwrapped so the caller retries: a // worker reports once, so an accepted call is the only thing that puts -// capacity and hardware on the Worker, and a Worker record the syncer has not +// capacity and runtimes on the Worker, and a Worker record the syncer has not // created yet is the ordinary reason for a first attempt to fail. func (s *ateomSupportServer) RegisterWorker(ctx context.Context, req *ateletpb.RegisterWorkerRequest) (*ateletpb.RegisterWorkerResponse, error) { // Identity comes only from the mTLS certificate, never from the request: - // a worker can report its own capacity and hardware and no one else's. + // a worker can report its own capacity and runtimes and no one else's. workerIdentity, err := authenticatedWorkerIdentity(ctx) if err != nil { return nil, err @@ -124,14 +124,18 @@ func (s *ateomSupportServer) RegisterWorker(ctx context.Context, req *ateletpb.R } if _, err := s.workers.RegisterWorker(ctx, &ateapipb.RegisterWorkerRequest{ // Workers are global-scoped and named by their pod UID. - Worker: &ateapipb.ObjectRef{Name: workerIdentity.PodUID}, - Capacity: toWorkerResources(req.GetCapacity()), - Hardware: toHardwareIdentity(req.GetHardware()), + Worker: &ateapipb.ObjectRef{Name: workerIdentity.PodUID}, + Capacity: toWorkerResources(req.GetCapacity()), + DefaultRuntime: toAteAPISandboxRuntime(req.GetDefaultRuntime()), + RestorableRuntimes: toAteAPISandboxRuntimes(req.GetRestorableRuntimes()), }); err != nil { return nil, err } - slog.InfoContext(ctx, "Registered worker capacity and hardware", - slog.String("pod_uid", workerIdentity.PodUID), slog.Any("capacity", req.GetCapacity()), slog.Any("hardware", req.GetHardware())) + slog.InfoContext(ctx, "Registered worker capacity and sandbox runtimes", + slog.String("pod_uid", workerIdentity.PodUID), + slog.Any("capacity", req.GetCapacity()), + slog.Any("default_runtime", req.GetDefaultRuntime()), + slog.Any("restorable_runtimes", req.GetRestorableRuntimes())) return &ateletpb.RegisterWorkerResponse{}, nil } @@ -151,22 +155,33 @@ func toWorkerResources(in *ateletpb.WorkerResources) *ateapipb.WorkerResources { return out } -// toHardwareIdentity converts atelet's HardwareIdentity to the control plane's, -// which it mirrors field for field. -func toHardwareIdentity(in *ateletpb.HardwareIdentity) *ateapipb.HardwareIdentity { +// toAteAPISandboxRuntime converts atelet's SandboxRuntime to the control +// plane's, which it mirrors field for field. +func toAteAPISandboxRuntime(in *ateletpb.SandboxRuntime) *ateapipb.SandboxRuntime { if in == nil { return nil } - out := &ateapipb.HardwareIdentity{} - if attrs := in.GetAttributes(); attrs != nil { - out.Attributes = make(map[string]string, len(attrs)) - for k, v := range attrs { - out.Attributes[k] = v + out := &ateapipb.SandboxRuntime{SandboxClass: in.GetSandboxClass()} + if v := in.GetVersion(); v != nil { + out.Version = &ateapipb.VersionedSandboxCompat{SchemaVersion: v.GetSchemaVersion()} + for _, a := range v.GetAttributes() { + out.Version.Attributes = append(out.Version.Attributes, &ateapipb.AttributeEntry{Key: a.GetKey(), Value: a.GetValue()}) } } return out } +func toAteAPISandboxRuntimes(in []*ateletpb.SandboxRuntime) []*ateapipb.SandboxRuntime { + if in == nil { + return nil + } + out := make([]*ateapipb.SandboxRuntime, 0, len(in)) + for _, r := range in { + out = append(out, toAteAPISandboxRuntime(r)) + } + return out +} + // RequestActorSuspend forwards a worker's request to suspend an actor it hosts // to the control plane, which owns their lifecycle. The worker observes; the // control plane decides. diff --git a/cmd/atelet/ateomsupport_test.go b/cmd/atelet/ateomsupport_test.go index 8c704209a6..d6f096be19 100644 --- a/cmd/atelet/ateomsupport_test.go +++ b/cmd/atelet/ateomsupport_test.go @@ -114,13 +114,41 @@ func (s *fakeWorkerService) MintAteomActorCertificate(_ context.Context, in *ate } func TestRegisterWorker(t *testing.T) { - reqHW := &ateletpb.HardwareIdentity{Attributes: map[string]string{"architecture": "amd64"}} - wantHW := &ateapipb.HardwareIdentity{Attributes: map[string]string{"architecture": "amd64"}} - forwarded := func(capacity *ateapipb.WorkerResources) []*ateapipb.RegisterWorkerRequest { + reqRuntime := func(hash string) *ateletpb.SandboxRuntime { + return &ateletpb.SandboxRuntime{ + SandboxClass: "gvisor", + Version: &ateletpb.VersionedSandboxCompat{ + SchemaVersion: "v1", + Attributes: []*ateletpb.AttributeEntry{ + {Key: "architecture", Value: "amd64"}, + {Key: "gvisor_asset_hash", Value: hash}, + }, + }, + } + } + wantRuntime := func(hash string) *ateapipb.SandboxRuntime { + return &ateapipb.SandboxRuntime{ + SandboxClass: "gvisor", + Version: &ateapipb.VersionedSandboxCompat{ + SchemaVersion: "v1", + Attributes: []*ateapipb.AttributeEntry{ + {Key: "architecture", Value: "amd64"}, + {Key: "gvisor_asset_hash", Value: hash}, + }, + }, + } + } + reqDefault, wantDefault := reqRuntime("9988"), wantRuntime("9988") + forwarded := func(capacity *ateapipb.WorkerResources, restorable ...*ateapipb.SandboxRuntime) []*ateapipb.RegisterWorkerRequest { // The Worker is named after the worker pod UID, taken from the - // certificate rather than the request; capacity and hardware come from - // what the worker reported. - return []*ateapipb.RegisterWorkerRequest{{Worker: &ateapipb.ObjectRef{Name: "pod-a"}, Capacity: capacity, Hardware: wantHW}} + // certificate rather than the request; capacity and runtimes come + // from what the worker reported. + return []*ateapipb.RegisterWorkerRequest{{ + Worker: &ateapipb.ObjectRef{Name: "pod-a"}, + Capacity: capacity, + DefaultRuntime: wantDefault, + RestorableRuntimes: restorable, + }} } tests := []struct { @@ -138,16 +166,24 @@ func TestRegisterWorker(t *testing.T) { Capacity: &ateletpb.WorkerResources{Actors: 4, Resources: &ateletpb.Resources{ Limits: []*ateletpb.Limits{{Name: "cpu", Quantity: "2"}, {Name: "memory", Quantity: "4Gi"}}, }}, - Hardware: reqHW, + DefaultRuntime: reqDefault, }, wantForwarded: forwarded(&ateapipb.WorkerResources{Actors: 4, Resources: resources.CPUMemory(2000, 4294967296)}), + }, { + name: "forwards restorable runtimes", + req: &ateletpb.RegisterWorkerRequest{ + Capacity: &ateletpb.WorkerResources{Actors: 1}, + DefaultRuntime: reqDefault, + RestorableRuntimes: []*ateletpb.SandboxRuntime{reqRuntime("d547")}, + }, + wantForwarded: forwarded(&ateapipb.WorkerResources{Actors: 1}, wantRuntime("d547")), }, { name: "omits undetermined compute", - req: &ateletpb.RegisterWorkerRequest{Capacity: &ateletpb.WorkerResources{Actors: 1}, Hardware: reqHW}, + req: &ateletpb.RegisterWorkerRequest{Capacity: &ateletpb.WorkerResources{Actors: 1}, DefaultRuntime: reqDefault}, wantForwarded: forwarded(&ateapipb.WorkerResources{Actors: 1}), }, { name: "forwards empty resources", - req: &ateletpb.RegisterWorkerRequest{Capacity: &ateletpb.WorkerResources{Resources: &ateletpb.Resources{}}, Hardware: reqHW}, + req: &ateletpb.RegisterWorkerRequest{Capacity: &ateletpb.WorkerResources{Resources: &ateletpb.Resources{}}, DefaultRuntime: reqDefault}, wantForwarded: forwarded(&ateapipb.WorkerResources{Resources: &ateapipb.Resources{}}), }, { name: "rejects invalid capacity without forwarding", @@ -155,22 +191,22 @@ func TestRegisterWorker(t *testing.T) { Capacity: &ateletpb.WorkerResources{Resources: &ateletpb.Resources{ Limits: []*ateletpb.Limits{{Name: "gpu", Quantity: "1"}}, }}, - Hardware: reqHW, + DefaultRuntime: reqDefault, }, wantCode: codes.InvalidArgument, }, { name: "rejects missing capacity without forwarding", - req: &ateletpb.RegisterWorkerRequest{Hardware: reqHW}, + req: &ateletpb.RegisterWorkerRequest{DefaultRuntime: reqDefault}, wantCode: codes.InvalidArgument, }, { - name: "rejects missing hardware without forwarding", + name: "rejects missing default runtime without forwarding", req: &ateletpb.RegisterWorkerRequest{Capacity: &ateletpb.WorkerResources{Actors: 1}}, wantCode: codes.InvalidArgument, }, { // A worker may report only what its certificate proves it is. name: "requires a certificate", unauthenticated: true, - req: &ateletpb.RegisterWorkerRequest{Capacity: &ateletpb.WorkerResources{Actors: 1}, Hardware: reqHW}, + req: &ateletpb.RegisterWorkerRequest{Capacity: &ateletpb.WorkerResources{Actors: 1}, DefaultRuntime: reqDefault}, wantCode: codes.Unauthenticated, }, { // The Worker record may not exist yet. The error must reach the @@ -178,7 +214,7 @@ func TestRegisterWorker(t *testing.T) { // leaves the Worker with no capacity forever. name: "surfaces the control plane's rejection", serviceErr: errors.New("no such worker"), - req: &ateletpb.RegisterWorkerRequest{Capacity: &ateletpb.WorkerResources{Actors: 1}, Hardware: reqHW}, + req: &ateletpb.RegisterWorkerRequest{Capacity: &ateletpb.WorkerResources{Actors: 1}, DefaultRuntime: reqDefault}, wantCode: codes.Unknown, }} for _, tt := range tests { diff --git a/cmd/atelet/internal/apivalidation/validation_test.go b/cmd/atelet/internal/apivalidation/validation_test.go index 7e38efb0bd..938a5b3e28 100644 --- a/cmd/atelet/internal/apivalidation/validation_test.go +++ b/cmd/atelet/internal/apivalidation/validation_test.go @@ -16,6 +16,7 @@ package apivalidation import ( "context" + "fmt" "strings" "testing" @@ -28,6 +29,28 @@ func assertValidateErr(t *testing.T, got field.ErrorList, want field.ErrorList) field.ErrorMatcher{}.ByType().ByField().ByOrigin().Test(t, want, got) } +// runtimeWithAttributes returns a valid gVisor runtime carrying n attributes +// with distinct keys. +func runtimeWithAttributes(n int) *ateletpb.SandboxRuntime { + attrs := make([]*ateletpb.AttributeEntry, n) + for i := range attrs { + attrs[i] = &ateletpb.AttributeEntry{Key: fmt.Sprintf("k%d", i), Value: "v"} + } + return &ateletpb.SandboxRuntime{ + SandboxClass: "gvisor", + Version: &ateletpb.VersionedSandboxCompat{SchemaVersion: "v1", Attributes: attrs}, + } +} + +// sandboxRuntimes returns n runtimes built by mk. +func sandboxRuntimes(n int, mk func() *ateletpb.SandboxRuntime) []*ateletpb.SandboxRuntime { + rts := make([]*ateletpb.SandboxRuntime, n) + for i := range rts { + rts[i] = mk() + } + return rts +} + func TestValidateRequestActorSuspendRequest(t *testing.T) { valid := func(mutate ...func(*ateletpb.RequestActorSuspendRequest)) *ateletpb.RequestActorSuspendRequest { r := &ateletpb.RequestActorSuspendRequest{ @@ -149,14 +172,26 @@ func TestValidateMintActorCertificateRequest(t *testing.T) { } func TestValidateRegisterWorkerRequest(t *testing.T) { - testHW := &ateletpb.HardwareIdentity{Attributes: map[string]string{"architecture": "amd64"}} + testRuntime := func() *ateletpb.SandboxRuntime { + return &ateletpb.SandboxRuntime{ + SandboxClass: "gvisor", + Version: &ateletpb.VersionedSandboxCompat{ + SchemaVersion: "v1", + Attributes: []*ateletpb.AttributeEntry{ + {Key: "architecture", Value: "amd64"}, + }, + }, + } + } withLimits := func(limits ...*ateletpb.Limits) *ateletpb.RegisterWorkerRequest { return &ateletpb.RegisterWorkerRequest{ - Capacity: &ateletpb.WorkerResources{Resources: &ateletpb.Resources{Limits: limits}}, - Hardware: testHW, + Capacity: &ateletpb.WorkerResources{Resources: &ateletpb.Resources{Limits: limits}}, + DefaultRuntime: testRuntime(), } } limitsPath := field.NewPath("capacity", "resources", "limits") + defaultRuntimePath := field.NewPath("default_runtime") + restorablePath := field.NewPath("restorable_runtimes") tests := []struct { name string @@ -164,43 +199,206 @@ func TestValidateRegisterWorkerRequest(t *testing.T) { want field.ErrorList }{{ name: "valid", - obj: &ateletpb.RegisterWorkerRequest{Capacity: &ateletpb.WorkerResources{}, Hardware: testHW}, + obj: &ateletpb.RegisterWorkerRequest{Capacity: &ateletpb.WorkerResources{}, DefaultRuntime: testRuntime()}, + }, { + name: "valid with restorable runtimes", + obj: &ateletpb.RegisterWorkerRequest{ + Capacity: &ateletpb.WorkerResources{}, + DefaultRuntime: testRuntime(), + RestorableRuntimes: []*ateletpb.SandboxRuntime{testRuntime()}, + }, }, { name: "missing capacity", - obj: &ateletpb.RegisterWorkerRequest{Hardware: testHW}, + obj: &ateletpb.RegisterWorkerRequest{DefaultRuntime: testRuntime()}, want: field.ErrorList{field.Required(field.NewPath("capacity"), "")}, }, { - name: "missing hardware", + name: "missing default_runtime", obj: &ateletpb.RegisterWorkerRequest{Capacity: &ateletpb.WorkerResources{}}, - want: field.ErrorList{field.Required(field.NewPath("hardware"), "")}, + want: field.ErrorList{field.Required(defaultRuntimePath, "")}, + }, { + name: "missing default_runtime.sandbox_class", + obj: &ateletpb.RegisterWorkerRequest{ + Capacity: &ateletpb.WorkerResources{}, + DefaultRuntime: &ateletpb.SandboxRuntime{ + Version: testRuntime().Version, + }, + }, + want: field.ErrorList{field.Required(defaultRuntimePath.Child("sandbox_class"), "")}, + }, { + name: "default_runtime.sandbox_class too long", + obj: &ateletpb.RegisterWorkerRequest{ + Capacity: &ateletpb.WorkerResources{}, + DefaultRuntime: &ateletpb.SandboxRuntime{ + SandboxClass: strings.Repeat("c", 64), + Version: testRuntime().Version, + }, + }, + want: field.ErrorList{field.TooLong(defaultRuntimePath.Child("sandbox_class"), "", 63).WithOrigin("maxLength")}, + }, { + name: "missing default_runtime.version", + obj: &ateletpb.RegisterWorkerRequest{ + Capacity: &ateletpb.WorkerResources{}, + DefaultRuntime: &ateletpb.SandboxRuntime{SandboxClass: "gvisor"}, + }, + want: field.ErrorList{field.Required(defaultRuntimePath.Child("version"), "")}, + }, { + name: "missing default_runtime.version.schema_version", + obj: &ateletpb.RegisterWorkerRequest{ + Capacity: &ateletpb.WorkerResources{}, + DefaultRuntime: &ateletpb.SandboxRuntime{ + SandboxClass: "gvisor", + Version: &ateletpb.VersionedSandboxCompat{ + Attributes: []*ateletpb.AttributeEntry{{Key: "architecture", Value: "amd64"}}, + }, + }, + }, + want: field.ErrorList{field.Required(defaultRuntimePath.Child("version", "schema_version"), "")}, + }, { + name: "default_runtime.version.schema_version too long", + obj: &ateletpb.RegisterWorkerRequest{ + Capacity: &ateletpb.WorkerResources{}, + DefaultRuntime: &ateletpb.SandboxRuntime{ + SandboxClass: "gvisor", + Version: &ateletpb.VersionedSandboxCompat{ + SchemaVersion: strings.Repeat("v", 65), + Attributes: []*ateletpb.AttributeEntry{{Key: "architecture", Value: "amd64"}}, + }, + }, + }, + want: field.ErrorList{field.TooLong(defaultRuntimePath.Child("version", "schema_version"), "", 64).WithOrigin("maxLength")}, + }, { + name: "missing default_runtime.version.attributes", + obj: &ateletpb.RegisterWorkerRequest{ + Capacity: &ateletpb.WorkerResources{}, + DefaultRuntime: &ateletpb.SandboxRuntime{ + SandboxClass: "gvisor", + Version: &ateletpb.VersionedSandboxCompat{SchemaVersion: "v1"}, + }, + }, + want: field.ErrorList{field.Required(defaultRuntimePath.Child("version", "attributes"), "")}, + }, { + name: "duplicate attribute key", + obj: &ateletpb.RegisterWorkerRequest{ + Capacity: &ateletpb.WorkerResources{}, + DefaultRuntime: &ateletpb.SandboxRuntime{ + SandboxClass: "gvisor", + Version: &ateletpb.VersionedSandboxCompat{ + SchemaVersion: "v1", + Attributes: []*ateletpb.AttributeEntry{ + {Key: "architecture", Value: "amd64"}, + {Key: "architecture", Value: "arm64"}, + }, + }, + }, + }, + want: field.ErrorList{field.Duplicate(defaultRuntimePath.Child("version", "attributes").Index(1), nil)}, + }, { + name: "missing attribute key", + obj: &ateletpb.RegisterWorkerRequest{ + Capacity: &ateletpb.WorkerResources{}, + DefaultRuntime: &ateletpb.SandboxRuntime{ + SandboxClass: "gvisor", + Version: &ateletpb.VersionedSandboxCompat{ + SchemaVersion: "v1", + Attributes: []*ateletpb.AttributeEntry{{Value: "v"}}, + }, + }, + }, + want: field.ErrorList{field.Required(defaultRuntimePath.Child("version", "attributes").Index(0).Child("key"), "")}, }, { - name: "valid empty hardware attributes", - obj: &ateletpb.RegisterWorkerRequest{Capacity: &ateletpb.WorkerResources{}, Hardware: &ateletpb.HardwareIdentity{}}, + name: "attribute key too long", + obj: &ateletpb.RegisterWorkerRequest{ + Capacity: &ateletpb.WorkerResources{}, + DefaultRuntime: &ateletpb.SandboxRuntime{ + SandboxClass: "gvisor", + Version: &ateletpb.VersionedSandboxCompat{ + SchemaVersion: "v1", + Attributes: []*ateletpb.AttributeEntry{{Key: strings.Repeat("k", 129), Value: "v"}}, + }, + }, + }, + want: field.ErrorList{field.TooLong(defaultRuntimePath.Child("version", "attributes").Index(0).Child("key"), "", 128).WithOrigin("maxLength")}, }, { - name: "hardware attribute key too long", + name: "missing attribute value", obj: &ateletpb.RegisterWorkerRequest{ Capacity: &ateletpb.WorkerResources{}, - Hardware: &ateletpb.HardwareIdentity{Attributes: map[string]string{strings.Repeat("k", 129): "v"}}, + DefaultRuntime: &ateletpb.SandboxRuntime{ + SandboxClass: "gvisor", + Version: &ateletpb.VersionedSandboxCompat{ + SchemaVersion: "v1", + Attributes: []*ateletpb.AttributeEntry{{Key: "k"}}, + }, + }, }, - want: field.ErrorList{field.TooLong(field.NewPath("hardware", "attributes"), "", 128).WithOrigin("maxLength")}, + want: field.ErrorList{field.Required(defaultRuntimePath.Child("version", "attributes").Index(0).Child("value"), "")}, }, { - name: "hardware attribute value too long", + name: "attribute value too long", obj: &ateletpb.RegisterWorkerRequest{ Capacity: &ateletpb.WorkerResources{}, - Hardware: &ateletpb.HardwareIdentity{Attributes: map[string]string{"k": strings.Repeat("v", 257)}}, + DefaultRuntime: &ateletpb.SandboxRuntime{ + SandboxClass: "gvisor", + Version: &ateletpb.VersionedSandboxCompat{ + SchemaVersion: "v1", + Attributes: []*ateletpb.AttributeEntry{{Key: "k", Value: strings.Repeat("v", 257)}}, + }, + }, + }, + want: field.ErrorList{field.TooLong(defaultRuntimePath.Child("version", "attributes").Index(0).Child("value"), "", 256).WithOrigin("maxLength")}, + }, { + name: "attributes at the bound", + obj: &ateletpb.RegisterWorkerRequest{ + Capacity: &ateletpb.WorkerResources{}, + DefaultRuntime: runtimeWithAttributes(32), + }, + }, { + name: "too many attributes", + obj: &ateletpb.RegisterWorkerRequest{ + Capacity: &ateletpb.WorkerResources{}, + DefaultRuntime: runtimeWithAttributes(33), + }, + want: field.ErrorList{field.TooMany(defaultRuntimePath.Child("version", "attributes"), 33, 32).WithOrigin("maxItems")}, + }, { + name: "nil restorable_runtimes entry", + obj: &ateletpb.RegisterWorkerRequest{ + Capacity: &ateletpb.WorkerResources{}, + DefaultRuntime: testRuntime(), + RestorableRuntimes: []*ateletpb.SandboxRuntime{nil}, + }, + want: field.ErrorList{field.Required(restorablePath.Index(0), "")}, + }, { + name: "invalid restorable_runtimes entry", + obj: &ateletpb.RegisterWorkerRequest{ + Capacity: &ateletpb.WorkerResources{}, + DefaultRuntime: testRuntime(), + RestorableRuntimes: []*ateletpb.SandboxRuntime{{SandboxClass: "gvisor"}}, + }, + want: field.ErrorList{field.Required(restorablePath.Index(0).Child("version"), "")}, + }, { + name: "restorable_runtimes at the bound", + obj: &ateletpb.RegisterWorkerRequest{ + Capacity: &ateletpb.WorkerResources{}, + DefaultRuntime: testRuntime(), + RestorableRuntimes: sandboxRuntimes(32, testRuntime), + }, + }, { + name: "too many restorable_runtimes", + obj: &ateletpb.RegisterWorkerRequest{ + Capacity: &ateletpb.WorkerResources{}, + DefaultRuntime: testRuntime(), + RestorableRuntimes: sandboxRuntimes(33, testRuntime), }, - want: field.ErrorList{field.TooLong(field.NewPath("hardware", "attributes").Key("k"), "", 256).WithOrigin("maxLength")}, + want: field.ErrorList{field.TooMany(restorablePath, 33, 32).WithOrigin("maxItems")}, }, { name: "full capacity", obj: &ateletpb.RegisterWorkerRequest{ Capacity: &ateletpb.WorkerResources{Actors: 4, Resources: &ateletpb.Resources{ Limits: []*ateletpb.Limits{{Name: "cpu", Quantity: "4"}, {Name: "memory", Quantity: "8Gi"}}, }}, - Hardware: testHW, + DefaultRuntime: testRuntime(), }, }, { name: "negative actors", - obj: &ateletpb.RegisterWorkerRequest{Capacity: &ateletpb.WorkerResources{Actors: -1}, Hardware: testHW}, + obj: &ateletpb.RegisterWorkerRequest{Capacity: &ateletpb.WorkerResources{Actors: -1}, DefaultRuntime: testRuntime()}, want: field.ErrorList{field.Invalid(field.NewPath("capacity", "actors"), nil, "").WithOrigin("minimum")}, }, { name: "unsupported resource name", diff --git a/cmd/atelet/internal/apivalidation/zz_generated.validation.go b/cmd/atelet/internal/apivalidation/zz_generated.validation.go index 6f59c7bb20..c77dd28826 100644 --- a/cmd/atelet/internal/apivalidation/zz_generated.validation.go +++ b/cmd/atelet/internal/apivalidation/zz_generated.validation.go @@ -29,54 +29,74 @@ import ( field "k8s.io/apimachinery/pkg/util/validation/field" ) -// Validate_HardwareIdentity validates an instance of HardwareIdentity according +// Validate_AttributeEntry validates an instance of AttributeEntry according // to declarative validation rules in the API schema. -func Validate_HardwareIdentity( +func Validate_AttributeEntry( ctx context.Context, op operation.Operation, fldPath *field.Path, - obj, oldObj *ateletpb.HardwareIdentity) (errs field.ErrorList) { + obj, oldObj *ateletpb.AttributeEntry) (errs field.ErrorList) { - { // field ateletpb.HardwareIdentity.Attributes + { // field ateletpb.AttributeEntry.Key fn := func( fldPath *field.Path, - obj, oldObj map[string]string, + obj, oldObj *string, oldValueCorrelated bool) (errs field.ErrorList) { // don't revalidate unchanged data if oldValueCorrelated && op.Type == operation.Update { - if ateDeepEqual(obj, oldObj) { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { return nil } } // call field-attached validations earlyReturn := false - if e := validate.MaxProperties(ctx, op, fldPath, obj, oldObj, 32).MarkShortCircuit(); len(e) != 0 { + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { errs = append(errs, e...) earlyReturn = true } - if e := validate.OptionalMap(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { - earlyReturn = true - } if earlyReturn { return // do not proceed } - if e := validate.EachMapKey(ctx, op, fldPath, obj, oldObj, - func(ctx context.Context, op operation.Operation, fldPath *field.Path, obj, oldObj *string) field.ErrorList { - return validate.MaxLength(ctx, op, fldPath, obj, oldObj, 128) - }); len(e) != 0 { + if e := validate.MaxLength(ctx, op, fldPath, obj, oldObj, 128); len(e) != 0 { errs = append(errs, e...) } - if e := validate.EachMapVal(ctx, op, fldPath, obj, oldObj, validate.DirectEqual, - func(ctx context.Context, op operation.Operation, fldPath *field.Path, obj, oldObj *string) field.ErrorList { - return validate.MaxLength(ctx, op, fldPath, obj, oldObj, 256) - }); len(e) != 0 { + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateletpb.AttributeEntry) *string { + return &oldObj.Key + }) + errs = append(errs, fn(fldPath.Child("key"), &obj.Key, oldVal, oldObj != nil)...) + } + + { // field ateletpb.AttributeEntry.Value + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.MaxLength(ctx, op, fldPath, obj, oldObj, 256); len(e) != 0 { errs = append(errs, e...) } return } oldVal := safe.Field(oldObj, - func(oldObj *ateletpb.HardwareIdentity) map[string]string { - return oldObj.Attributes + func(oldObj *ateletpb.AttributeEntry) *string { + return &oldObj.Value }) - errs = append(errs, fn(fldPath.Child("attributes"), obj.Attributes, oldVal, oldObj != nil)...) + errs = append(errs, fn(fldPath.Child("value"), &obj.Value, oldVal, oldObj != nil)...) } return errs @@ -334,10 +354,10 @@ func Validate_RegisterWorkerRequest( errs = append(errs, fn(fldPath.Child("capacity"), obj.Capacity, oldVal, oldObj != nil)...) } - { // field ateletpb.RegisterWorkerRequest.Hardware + { // field ateletpb.RegisterWorkerRequest.DefaultRuntime fn := func( fldPath *field.Path, - obj, oldObj *ateletpb.HardwareIdentity, + obj, oldObj *ateletpb.SandboxRuntime, oldValueCorrelated bool) (errs field.ErrorList) { // don't revalidate unchanged data if oldValueCorrelated && op.Type == operation.Update { @@ -355,14 +375,54 @@ func Validate_RegisterWorkerRequest( return // do not proceed } // call the type's validation function - errs = append(errs, Validate_HardwareIdentity(ctx, op, fldPath, obj, oldObj)...) + errs = append(errs, Validate_SandboxRuntime(ctx, op, fldPath, obj, oldObj)...) + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateletpb.RegisterWorkerRequest) *ateletpb.SandboxRuntime { + return oldObj.DefaultRuntime + }) + errs = append(errs, fn(fldPath.Child("default_runtime"), obj.DefaultRuntime, oldVal, oldObj != nil)...) + } + + { // field ateletpb.RegisterWorkerRequest.RestorableRuntimes + fn := func( + fldPath *field.Path, + obj, oldObj []*ateletpb.SandboxRuntime, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.PtrSliceNoNils[ateletpb.SandboxRuntime](ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.MaxItems(ctx, op, fldPath, obj, oldObj, 32).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.OptionalSlice(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // iterate the list and call the type's validation function + if e := validate.EachPtrSliceVal(ctx, op, fldPath, obj, oldObj, nil, nil, Validate_SandboxRuntime); len(e) != 0 { + errs = append(errs, e...) + } return } oldVal := safe.Field(oldObj, - func(oldObj *ateletpb.RegisterWorkerRequest) *ateletpb.HardwareIdentity { - return oldObj.Hardware + func(oldObj *ateletpb.RegisterWorkerRequest) []*ateletpb.SandboxRuntime { + return oldObj.RestorableRuntimes }) - errs = append(errs, fn(fldPath.Child("hardware"), obj.Hardware, oldVal, oldObj != nil)...) + errs = append(errs, fn(fldPath.Child("restorable_runtimes"), obj.RestorableRuntimes, oldVal, oldObj != nil)...) } return errs @@ -528,6 +588,166 @@ func Validate_Resources( return errs } +// Validate_SandboxRuntime validates an instance of SandboxRuntime according +// to declarative validation rules in the API schema. +func Validate_SandboxRuntime( + ctx context.Context, op operation.Operation, fldPath *field.Path, + obj, oldObj *ateletpb.SandboxRuntime) (errs field.ErrorList) { + + { // field ateletpb.SandboxRuntime.SandboxClass + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.MaxLength(ctx, op, fldPath, obj, oldObj, 63); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateletpb.SandboxRuntime) *string { + return &oldObj.SandboxClass + }) + errs = append(errs, fn(fldPath.Child("sandbox_class"), &obj.SandboxClass, oldVal, oldObj != nil)...) + } + + { // field ateletpb.SandboxRuntime.Version + fn := func( + fldPath *field.Path, + obj, oldObj *ateletpb.VersionedSandboxCompat, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // call the type's validation function + errs = append(errs, Validate_VersionedSandboxCompat(ctx, op, fldPath, obj, oldObj)...) + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateletpb.SandboxRuntime) *ateletpb.VersionedSandboxCompat { + return oldObj.Version + }) + errs = append(errs, fn(fldPath.Child("version"), obj.Version, oldVal, oldObj != nil)...) + } + + return errs +} + +// Validate_VersionedSandboxCompat validates an instance of VersionedSandboxCompat according +// to declarative validation rules in the API schema. +func Validate_VersionedSandboxCompat( + ctx context.Context, op operation.Operation, fldPath *field.Path, + obj, oldObj *ateletpb.VersionedSandboxCompat) (errs field.ErrorList) { + + { // field ateletpb.VersionedSandboxCompat.SchemaVersion + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.MaxLength(ctx, op, fldPath, obj, oldObj, 64); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateletpb.VersionedSandboxCompat) *string { + return &oldObj.SchemaVersion + }) + errs = append(errs, fn(fldPath.Child("schema_version"), &obj.SchemaVersion, oldVal, oldObj != nil)...) + } + + { // field ateletpb.VersionedSandboxCompat.Attributes + fn := func( + fldPath *field.Path, + obj, oldObj []*ateletpb.AttributeEntry, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.PtrSliceNoNils[ateletpb.AttributeEntry](ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.MaxItems(ctx, op, fldPath, obj, oldObj, 32).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.RequiredSlice(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // lists with map semantics require unique keys + if e := validate.PtrSliceUnique(ctx, op, fldPath, obj, oldObj, + func(a *ateletpb.AttributeEntry, b *ateletpb.AttributeEntry) bool { return a.Key == b.Key }); len(e) != 0 { + errs = append(errs, e...) + } + // iterate the list and call the type's validation function + if e := validate.EachPtrSliceVal(ctx, op, fldPath, obj, oldObj, + func(a *ateletpb.AttributeEntry, b *ateletpb.AttributeEntry) bool { return a.Key == b.Key }, ateDeepEqual, Validate_AttributeEntry); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateletpb.VersionedSandboxCompat) []*ateletpb.AttributeEntry { + return oldObj.Attributes + }) + errs = append(errs, fn(fldPath.Child("attributes"), obj.Attributes, oldVal, oldObj != nil)...) + } + + return errs +} + // Validate_WorkerResources validates an instance of WorkerResources according // to declarative validation rules in the API schema. func Validate_WorkerResources( diff --git a/cmd/atelet/internal/ateletpath/ateletpath.go b/cmd/atelet/internal/ateletpath/ateletpath.go index 9c2a9ebef8..aa80821d7d 100644 --- a/cmd/atelet/internal/ateletpath/ateletpath.go +++ b/cmd/atelet/internal/ateletpath/ateletpath.go @@ -29,6 +29,11 @@ var ( // directories are visible at the same path in atelet (which writes them) // and in every ateom pod (which mounts them as overlay lowerdirs). ImageCacheDir = filepath.Join(nodepath.BasePath, "image-cache") + + // SnapshotScratchDir holds short-lived directories for snapshot manifests + // moved through the snapshot plugin. It lives under BasePath, which the + // node plugin shares with atelet. + SnapshotScratchDir = filepath.Join(nodepath.BasePath, "snapshot-scratch") ) func RunSCBinaryPath(sha256 string) string { @@ -124,8 +129,8 @@ func DurableDirVolumeMountPoint(actorUID, volumeName string) string { // DurableDirVolumeMountsDir (see ateom-microvm's tarDurableVolumes), so // system-info roots are excluded by living in this separate directory. // - gVisor captures by declaration: durable mounts are registered with -// the sandbox (mount-hint annotations for FULL checkpoints, the -// enumerated durable mount paths for DATA fscheckpoints); system-info +// the sandbox (mount-hint annotations for MEMORY checkpoints, the +// enumerated durable mount paths for VOLUMES fscheckpoints); system-info // mounts are plain undeclared binds, never captured regardless of host // layout. // diff --git a/cmd/atelet/lifecycle_test.go b/cmd/atelet/lifecycle_test.go index b32db7e605..80c332c0fe 100644 --- a/cmd/atelet/lifecycle_test.go +++ b/cmd/atelet/lifecycle_test.go @@ -149,7 +149,7 @@ func TestLocalSnapshotGC(t *testing.T) { atespace = "ate-demo" actorName = "counter" actorUID = "actor-uid-1" - ateomUID = "ateom-uid-1" + workerPodUID = "worker-pod-uid-1" snapshotName = "pause-snap-1" ) @@ -191,7 +191,7 @@ func TestLocalSnapshotGC(t *testing.T) { ActorUid: actorUID, ActorTemplateAtespace: "default", ActorTemplateName: "counter", - TargetAteomUid: ateomUID, + WorkerPodUid: workerPodUID, SandboxAssets: sandboxAssets, Spec: spec, }); err != nil { @@ -205,9 +205,9 @@ func TestLocalSnapshotGC(t *testing.T) { ActorUid: actorUID, ActorTemplateAtespace: "default", ActorTemplateName: "counter", - TargetAteomUid: ateomUID, + WorkerPodUid: workerPodUID, Spec: spec, - Scope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL, + Fidelity: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, Type: ateletpb.CheckpointType_CHECKPOINT_TYPE_LOCAL, Config: &ateletpb.CheckpointRequest_LocalConfig{ LocalConfig: &ateletpb.LocalCheckpointConfiguration{SnapshotName: snapshotName}, @@ -227,10 +227,10 @@ func TestLocalSnapshotGC(t *testing.T) { ActorUid: actorUID, ActorTemplateAtespace: "default", ActorTemplateName: "counter", - TargetAteomUid: ateomUID, + WorkerPodUid: workerPodUID, SandboxAssets: sandboxAssets, Spec: spec, - Scope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL, + Fidelity: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, Type: ateletpb.CheckpointType_CHECKPOINT_TYPE_LOCAL, Config: &ateletpb.RestoreRequest_LocalConfig{ LocalConfig: &ateletpb.LocalCheckpointConfiguration{SnapshotName: snapshotName}, @@ -255,7 +255,7 @@ func TestLocalSnapshotGC(t *testing.T) { ActorUid: actorUID, ActorTemplateAtespace: "default", ActorTemplateName: "counter", - TargetAteomUid: ateomUID, + WorkerPodUid: workerPodUID, Spec: spec, }); err != nil { t.Fatalf("Terminate: %v", err) @@ -305,7 +305,7 @@ func TestRestoreUsesRequestSandboxAssets(t *testing.T) { atespace = "ate-demo" actorName = "counter" actorUID = "actor-uid-1" - ateomUID = "ateom-uid-1" + workerPodUID = "worker-pod-uid-1" snapshotName = "pause-snap-1" ) @@ -351,7 +351,7 @@ func TestRestoreUsesRequestSandboxAssets(t *testing.T) { ActorUid: actorUID, ActorTemplateAtespace: "default", ActorTemplateName: "counter", - TargetAteomUid: ateomUID, + WorkerPodUid: workerPodUID, SandboxAssets: assetsWithPause(checkpointPause), Spec: spec, }); err != nil { @@ -364,9 +364,9 @@ func TestRestoreUsesRequestSandboxAssets(t *testing.T) { ActorUid: actorUID, ActorTemplateAtespace: "default", ActorTemplateName: "counter", - TargetAteomUid: ateomUID, + WorkerPodUid: workerPodUID, Spec: spec, - Scope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL, + Fidelity: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, Type: ateletpb.CheckpointType_CHECKPOINT_TYPE_LOCAL, Config: &ateletpb.CheckpointRequest_LocalConfig{ LocalConfig: &ateletpb.LocalCheckpointConfiguration{SnapshotName: snapshotName}, @@ -393,10 +393,10 @@ func TestRestoreUsesRequestSandboxAssets(t *testing.T) { ActorUid: actorUID, ActorTemplateAtespace: "default", ActorTemplateName: "counter", - TargetAteomUid: ateomUID, + WorkerPodUid: workerPodUID, SandboxAssets: assetsWithPause(restorePause), Spec: spec, - Scope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL, + Fidelity: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, Type: ateletpb.CheckpointType_CHECKPOINT_TYPE_LOCAL, Config: &ateletpb.RestoreRequest_LocalConfig{ LocalConfig: &ateletpb.LocalCheckpointConfiguration{SnapshotName: snapshotName}, @@ -490,14 +490,14 @@ func TestActivationFailureBeforeRegistration(t *testing.T) { } if tc.restore { _, err = s.Restore(ctx, &ateletpb.RestoreRequest{ - Atespace: "team-a", ActorName: "actor-1", ActorUid: actorUID, TargetAteomUid: "ateom-uid-1", + Atespace: "team-a", ActorName: "actor-1", ActorUid: actorUID, WorkerPodUid: "worker-pod-uid-1", SandboxAssets: assets, Spec: spec, - Scope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL, Type: ateletpb.CheckpointType_CHECKPOINT_TYPE_LOCAL, + Fidelity: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, Type: ateletpb.CheckpointType_CHECKPOINT_TYPE_LOCAL, Config: &ateletpb.RestoreRequest_LocalConfig{LocalConfig: &ateletpb.LocalCheckpointConfiguration{SnapshotName: snapshotName}}, }) } else { _, err = s.Run(ctx, &ateletpb.RunRequest{ - Atespace: "team-a", ActorName: "actor-1", ActorUid: actorUID, TargetAteomUid: "ateom-uid-1", + Atespace: "team-a", ActorName: "actor-1", ActorUid: actorUID, WorkerPodUid: "worker-pod-uid-1", SandboxAssets: assets, Spec: spec, }) } @@ -560,10 +560,10 @@ func TestRunFailureAfterRegistrationRemovesOwnRegistration(t *testing.T) { }}, } _, err := s.Run(t.Context(), &ateletpb.RunRequest{ - Atespace: "team-a", - ActorName: "actor-run", - ActorUid: "actor-uid-run", - TargetAteomUid: "ateom-uid-1", + Atespace: "team-a", + ActorName: "actor-run", + ActorUid: "actor-uid-run", + WorkerPodUid: "worker-pod-uid-1", SandboxAssets: &ateletpb.SandboxAssets{ SandboxClass: "gvisor", PauseImage: "://invalid-image", @@ -612,10 +612,10 @@ func TestRestoreFailureAfterRegistrationRemovesOwnRegistration(t *testing.T) { systemInfoVolumes: refresher, } _, err := s.Restore(t.Context(), &ateletpb.RestoreRequest{ - Atespace: "team-a", - ActorName: "actor-restore-after", - ActorUid: actorUID, - TargetAteomUid: "ateom-uid-1", + Atespace: "team-a", + ActorName: "actor-restore-after", + ActorUid: actorUID, + WorkerPodUid: "worker-pod-uid-1", SandboxAssets: &ateletpb.SandboxAssets{ SandboxClass: "gvisor", PauseImage: "://invalid-image", @@ -623,9 +623,9 @@ func TestRestoreFailureAfterRegistrationRemovesOwnRegistration(t *testing.T) { runscAssetName: {Url: "gs://test-bucket/runsc", Sha256: assetHash}, }}}, }, - Spec: spec, - Scope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL, - Type: ateletpb.CheckpointType_CHECKPOINT_TYPE_LOCAL, + Spec: spec, + Fidelity: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + Type: ateletpb.CheckpointType_CHECKPOINT_TYPE_LOCAL, Config: &ateletpb.RestoreRequest_LocalConfig{ LocalConfig: &ateletpb.LocalCheckpointConfiguration{SnapshotName: snapshotName}, }, @@ -641,11 +641,11 @@ func TestRestoreFailureAfterRegistrationRemovesOwnRegistration(t *testing.T) { } } -// TestTerminateWithoutTargetAteomUID verifies that Terminate with an empty -// TargetAteomUid (used when cleaning up the assigned node for a paused or +// TestTerminateWithoutWorkerPodUID verifies that Terminate with an empty +// WorkerPodUid (used when cleaning up the assigned node for a paused or // crashed actor that no longer has a worker pod) skips dialing ateom while // still pruning local checkpoints and removing actor directories on the node. -func TestTerminateWithoutTargetAteomUID(t *testing.T) { +func TestTerminateWithoutWorkerPodUID(t *testing.T) { useTempNodeDirs(t) ctx := t.Context() @@ -685,14 +685,14 @@ func TestTerminateWithoutTargetAteomUID(t *testing.T) { ActorUid: actorUID, ActorTemplateAtespace: "default", ActorTemplateName: "counter", - TargetAteomUid: "", + WorkerPodUid: "", Spec: spec, }); err != nil { t.Fatalf("Terminate: %v", err) } if got := ateom.actorDirs["TerminateWorkload"]; got != nil { - t.Errorf("TerminateWorkload was called on ateom (%v), want skipped when TargetAteomUid is empty", got) + t.Errorf("TerminateWorkload was called on ateom (%v), want skipped when WorkerPodUid is empty", got) } if _, err := os.Stat(ateletpath.LocalCheckpointsDir(actorUID)); !os.IsNotExist(err) { t.Errorf("local checkpoint dir survived terminate: %v", err) diff --git a/cmd/atelet/main.go b/cmd/atelet/main.go index d15177d449..47414a031a 100644 --- a/cmd/atelet/main.go +++ b/cmd/atelet/main.go @@ -34,7 +34,6 @@ import ( "github.com/agent-substrate/substrate/cmd/atelet/internal/credentialprovider" "github.com/agent-substrate/substrate/cmd/atelet/internal/sparsefile" "github.com/agent-substrate/substrate/cmd/atelet/internal/trustbundle" - "github.com/agent-substrate/substrate/internal/actorlog" "github.com/agent-substrate/substrate/internal/apierror" "github.com/agent-substrate/substrate/internal/ateapiauth" "github.com/agent-substrate/substrate/internal/ateattr" @@ -44,6 +43,7 @@ import ( "github.com/agent-substrate/substrate/internal/credbundle" "github.com/agent-substrate/substrate/internal/imagecache" "github.com/agent-substrate/substrate/internal/nodepath" + "github.com/agent-substrate/substrate/internal/objectstoreplugin" "github.com/agent-substrate/substrate/internal/ocispec" "github.com/agent-substrate/substrate/internal/otlprelay" "github.com/agent-substrate/substrate/internal/proto/ateletpb" @@ -59,8 +59,7 @@ import ( "github.com/agent-substrate/substrate/pkg/client/informers/externalversions" "github.com/agent-substrate/substrate/pkg/objectstorage" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" - "github.com/aws/aws-sdk-go-v2/config" - "github.com/aws/aws-sdk-go-v2/service/s3" + objectstorev1 "github.com/agent-substrate/substrate/pkg/proto/objectstorepb/v1" "github.com/google/go-containerregistry/pkg/authn" "github.com/spf13/pflag" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" @@ -115,6 +114,8 @@ var ( drainDelay = pflag.Duration("drain-delay", 0, "How long to keep accepting new RPCs after SIGTERM before starting the gRPC drain.") drainTimeout = pflag.Duration("drain-timeout", 5*time.Minute, "Deadline for the graceful gRPC drain on shutdown. In-flight RPCs still running past it are forcefully cancelled.") + + snapshotPluginSocket = pflag.String("snapshot-plugin-socket", "/run/snapshot-plugin/node.sock", "Unix socket of the node snapshot plugin that external snapshots are fetched and uploaded through.") ) var _ imagecache.CandidateKeychain = (*credentialprovider.Keychain)(nil) @@ -126,13 +127,7 @@ func main() { return } ctx := context.Background() - // One synchronized writer in front of stdout, shared by the runtime - // logger and the usage-event drain (see startStatsPoller): uncoordinated - // writers stay tear-free only while every record fits a pipe's - // atomic-write size -- an accident of field sizes, not a contract. Same - // pattern as the ateoms' actor-log forwarders. - logSink := actorlog.NewSyncedWriter(os.Stdout) - serverboot.InitLoggerWithWriter(logSink) + serverboot.InitLogger() if err := serverboot.SetLogLevel(*logLevelFlag); err != nil { serverboot.Fatal(ctx, "Invalid --log-level", err) } @@ -239,28 +234,9 @@ func main() { serverboot.Fatal(ctx, "Failed to create anonymous GCS client", err) } - var wrappedGCS objectstorage.ObjectStorage - storageBackend := os.Getenv("ATE_STORAGE_BACKEND") - switch storageBackend { - case "s3": - slog.InfoContext(ctx, "Using S3 storage backend") - // depend on standard AWS environment variables to configure the client - // these will need to be set on the atelet pods - cfg, err := config.LoadDefaultConfig(ctx) - if err != nil { - serverboot.Fatal(ctx, "Failed to load S3 config", err) - } - wrappedGCS = objectstorage.NewS3Client(s3.NewFromConfig(cfg, func(o *s3.Options) { - if usePathStyle := os.Getenv("AWS_S3_USE_PATH_STYLE"); usePathStyle == "true" { - o.UsePathStyle = true - } - })) - // GCS is currently the default, TODO: we assume workload identity / ADC - default: - wrappedGCS, err = objectstorage.NewGCSClient(ctx) - if err != nil { - serverboot.Fatal(ctx, "Failed to create GCS client", err) - } + wrappedGCS, err := objectstorage.NewFromEnv(ctx) + if err != nil { + serverboot.Fatal(ctx, "Failed to set up the object storage backend", err) } volPlugins := make(map[string]volume.VolumePluginWorkerPlane) @@ -279,7 +255,7 @@ func main() { // crash-looping every actor operation on the node. slog.ErrorContext(ctx, "Actor stats sampling disabled: failed to create instruments", slog.Any("err", err)) } else { - startStatsPoller(ctx, interval, statsInst, k8sClient, logSink) + startStatsPoller(ctx, interval, statsInst, k8sClient) } } @@ -308,11 +284,26 @@ func main() { go trustBundles.Informer().Run(stopCh) cache.WaitForCacheSync(stopCh, trustBundles.Informer().HasSynced) + snapshotPluginConn, err := objectstoreplugin.Dial(*snapshotPluginSocket, objectstoreplugin.ReadyWait) + if err != nil { + serverboot.Fatal(ctx, "Failed to set up the snapshot plugin client", err) + } + defer snapshotPluginConn.Close() + // A plugin that never serves would fail every call. Fail at startup + // instead, giving the sidecar time to come up. + readyCtx, cancelReady := context.WithTimeout(ctx, time.Minute) + err = objectstoreplugin.WaitReady(readyCtx, snapshotPluginConn) + cancelReady() + if err != nil { + serverboot.Fatal(ctx, "Snapshot plugin is not serving", err) + } + wmService := NewService( ctx, ateomDialer, wrappedAnonGCS, wrappedGCS, + objectstorev1.NewNodeProviderClient(snapshotPluginConn), imageCache, instruments, volPlugins, @@ -464,10 +455,17 @@ func (g *directCSIDriverConfigGetter) Get(name string) (*atev1alpha1.CSIDriverCo type AteomHerder struct { ateletpb.UnimplementedAteomHerderServer - ateomDialer *AteomDialer - imageCache *imagecache.Store - anonGCSClient objectstorage.ObjectStorage - gcsClient objectstorage.ObjectStorage + ateomDialer *AteomDialer + imageCache *imagecache.Store + anonGCSClient objectstorage.ObjectStorage + // gcsClient downloads sandbox assets. Snapshot files never use it: they move + // through snapshotPlugin. + gcsClient objectstorage.ObjectStorage + // snapshotPlugin moves external snapshot files between the node and storage. + snapshotPlugin objectstorev1.NodeProviderClient + // snapshotScratchDir holds short-lived manifest directories; it must be + // inside the node plugin's root. + snapshotScratchDir string instruments *Instruments mu sync.RWMutex volumePlugins map[string]volume.VolumePluginWorkerPlane @@ -483,6 +481,7 @@ func NewService( ateomDialer *AteomDialer, anonGCSClient objectstorage.ObjectStorage, gcsClient objectstorage.ObjectStorage, + snapshotPlugin objectstorev1.NodeProviderClient, imageCache *imagecache.Store, instruments *Instruments, volumePlugins map[string]volume.VolumePluginWorkerPlane, @@ -494,6 +493,8 @@ func NewService( imageCache: imageCache, anonGCSClient: anonGCSClient, gcsClient: gcsClient, + snapshotPlugin: snapshotPlugin, + snapshotScratchDir: ateletpath.SnapshotScratchDir, instruments: instruments, volumePlugins: volumePlugins, csiDriverConfigGetter: csiDriverConfigGetter, @@ -544,18 +545,19 @@ func (s *AteomHerder) Run(ctx context.Context, req *ateletpb.RunRequest) (resp * if registration, err = s.systemInfoVolumes.Register(actorUID, actorRef, systemInfoVolumesFor(actorUID, req.GetSpec())); err != nil { return nil, err } - if err := s.prepareOCIBundles(ctx, actorUID, actorRef, - req.GetSpec(), sandboxRec.PauseImage, req.GetTargetAteomUid(), - ); err != nil { + containerSpecs, err := s.prepareOCIBundles(ctx, actorUID, actorRef, + req.GetSpec(), sandboxRec.PauseImage, + ) + if err != nil { return nil, err } - client, err := s.dialAteom(ctx, req.GetTargetAteomUid()) + client, err := s.dialAteom(ctx, req.GetWorkerPodUid()) if err != nil { return nil, err } - spec, err := buildAteomWorkloadSpec(req.GetSpec()) + spec, err := buildAteomWorkloadSpec(req.GetSpec(), containerSpecs) if err != nil { return nil, apierror.InvalidArgument("invalid workload spec: %v", err) } @@ -638,7 +640,7 @@ func (s *AteomHerder) Checkpoint(ctx context.Context, req *ateletpb.CheckpointRe templateNamespace: req.GetActorTemplateAtespace(), templateName: req.GetActorTemplateName(), kind: checkpointSnapshotKind(req), - scope: ateattr.SnapshotScopeValue(req.GetScope()), + fidelity: ateattr.SnapshotFidelityValue(req.GetFidelity()), } attribution := resources.ActorAttribution{ Ref: actorRef, @@ -680,7 +682,7 @@ func (s *AteomHerder) Checkpoint(ctx context.Context, req *ateletpb.CheckpointRe checkpointDir := ateletpath.CheckpointStateDir(actorUID) - client, err := s.dialAteom(ctx, req.GetTargetAteomUid()) + client, err := s.dialAteom(ctx, req.GetWorkerPodUid()) if err != nil { return nil, err } @@ -688,7 +690,7 @@ func (s *AteomHerder) Checkpoint(ctx context.Context, req *ateletpb.CheckpointRe // Tell ateom to take the checkpoint and delete containers. ateom reports the // exact files it wrote so we ship precisely that set (gVisor's image files, // cloud-hypervisor's snapshot set, ...) rather than a hardcoded list. - spec, err := buildAteomWorkloadSpec(req.GetSpec()) + spec, err := buildAteomWorkloadSpec(req.GetSpec(), nil) if err != nil { return nil, apierror.InvalidArgument("invalid workload spec: %v", err) } @@ -702,7 +704,7 @@ func (s *AteomHerder) Checkpoint(ctx context.Context, req *ateletpb.CheckpointRe RunscPath: runscPathFor(assetPaths), RuntimeAssetPaths: assetPaths, Spec: spec, - Scope: toAteomSnapshotScope(req.GetScope()), + Fidelity: toAteomFidelity(req.GetFidelity()), ActorUid: actorUID, ActorDirs: ateletpath.ActorDirs(actorUID), }) @@ -724,7 +726,7 @@ func (s *AteomHerder) Checkpoint(ctx context.Context, req *ateletpb.CheckpointRe sandboxRec.ActorUID = req.GetActorUid() sandboxRec.ActorTemplateAtespace = req.GetActorTemplateAtespace() sandboxRec.ActorTemplateName = req.GetActorTemplateName() - sandboxRec.Scope = ateattr.SnapshotScopeValue(req.GetScope()) + sandboxRec.Fidelity = ateattr.SnapshotFidelityValue(req.GetFidelity()) // No earlier pause snapshot can ever be restored again, so remove them // all: the actor's current state was just captured by CheckpointWorkload, @@ -748,6 +750,10 @@ func (s *AteomHerder) Checkpoint(ctx context.Context, req *ateletpb.CheckpointRe tPersist := time.Now() switch req.GetType() { case ateletpb.CheckpointType_CHECKPOINT_TYPE_EXTERNAL: + // CheckpointWorkload has already deleted the workload, so a retried + // Checkpoint cannot redo this upload. It waits for the snapshot plugin + // as long as ctx allows, and a plugin error reaches the control plane as + // Internal, which crashes the actor. // TODO(#362): Because we do not cache the external snapshot files when upload fails, we have to mark the Actor as CRASHED. if err := s.uploadExternalCheckpoint(ctx, req, checkpointDir, sandboxRec); err != nil { dPersist = time.Since(tPersist) @@ -790,13 +796,19 @@ func checkpointSnapshotFiles(resp *ateompb.CheckpointWorkloadResponse, required return files, dataFiles, nil } -func toAteomSnapshotScope(scope ateletpb.SnapshotScope) ateompb.SnapshotScope { - // assumption the request already been validated and scope is in the valid values set - switch scope { - case ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA: - return ateompb.SnapshotScope_SNAPSHOT_SCOPE_DATA +// toAteomFidelity maps the atelet enum onto the identical ateom one, level by +// level; the request was validated already, so anything else is a bug and +// goes out as UNSPECIFIED for ateom to reject. +func toAteomFidelity(fidelity ateletpb.SnapshotFidelity) ateompb.SnapshotFidelity { + switch fidelity { + case ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES: + return ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES + case ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS: + return ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS + case ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY: + return ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY default: - return ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL + return ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED } } @@ -852,7 +864,7 @@ func (s *AteomHerder) moveLocalCheckpoint(ctx context.Context, req *ateletpb.Che // shouldHaveSnapshots returns true if the checkpoint request is expected to produce snapshot files. func shouldHaveSnapshots(req *ateletpb.CheckpointRequest) bool { - if req.GetScope() != ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA { + if req.GetFidelity() != ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES { return true } @@ -864,67 +876,48 @@ func shouldHaveSnapshots(req *ateletpb.CheckpointRequest) bool { return false } +// uploadExternalCheckpoint uploads the checkpoint CheckpointWorkload just +// took. Its plugin calls pass grpc.WaitForReady, so they wait out a plugin +// outage for as long as ctx allows rather than only objectstoreplugin's ready +// wait, and their errors are not reported as retryable: see Checkpoint. func (s *AteomHerder) uploadExternalCheckpoint(ctx context.Context, req *ateletpb.CheckpointRequest, checkpointDir string, rec *sandboxAssetsRecord) error { uri, err := resources.ParseSnapshotURI(req.GetExternalConfig().GetSnapshotUri()) if err != nil { return err } - return s.uploadSnapshot(ctx, uri, checkpointDir, rec, req.GetActorTemplateAtespace(), req.GetActorTemplateName()) + return s.uploadSnapshot(ctx, uri, checkpointDir, rec, req.GetActorTemplateAtespace(), req.GetActorTemplateName(), grpc.WaitForReady(true)) } -// uploadSnapshot uploads rec's snapshot files from srcDir to uri (each -// zstd-compressed, concurrently), then the marshaled manifest. The manifest -// goes last, never in parallel: its presence is the commit marker — readers -// assume every file it lists is already present. A crash mid-upload thus -// leaves only orphaned files, never a manifest pointing at files that never -// landed; retries overwrite the deterministic object names. -func (s *AteomHerder) uploadSnapshot(ctx context.Context, uri resources.SnapshotURI, srcDir string, rec *sandboxAssetsRecord, templateAtespace, templateName string) error { +// uploadSnapshot uploads rec's snapshot files from srcDir to uri through the +// snapshot plugin, then the marshaled manifest. The manifest goes last, in its +// own call: its presence is the commit marker — readers assume every file it +// lists is already present. A crash mid-upload thus leaves only orphaned +// files, never a manifest pointing at files that never landed; retries +// overwrite the deterministic object names. opts are passed to every plugin +// call, and plugin errors are returned as uploadSnapshotFiles returns them. +func (s *AteomHerder) uploadSnapshot(ctx context.Context, uri resources.SnapshotURI, srcDir string, rec *sandboxAssetsRecord, templateAtespace, templateName string, opts ...grpc.CallOption) error { root, err := os.OpenRoot(srcDir) if err != nil { return fmt.Errorf("while opening snapshot directory: %w", err) } defer root.Close() - g, gCtx := errgroup.WithContext(ctx) for _, fileName := range rec.SnapshotFiles { - g.Go(func() error { - local, err := root.Open(fileName) - if err != nil { - return fmt.Errorf("while opening %s in snapshot directory: %w", fileName, err) - } - defer local.Close() - info, err := local.Stat() - if err != nil { - return fmt.Errorf("while inspecting %s in snapshot directory: %w", fileName, err) - } - if !info.Mode().IsRegular() { - return fmt.Errorf("snapshot file %s is not a regular file", fileName) - } - recordSnapshotSize(ctx, fileName, allocatedBytes(info), templateAtespace, templateName) - - objectURI, err := uri.ObjectURI(fileName + ".zstd") - if err != nil { - return fmt.Errorf("while addressing %s in GCS: %w", fileName, err) - } - if err := objectstorage.SendFileToGCSWithZstd(gCtx, s.gcsClient, objectURI, local); err != nil { - return fmt.Errorf("while uploading %s to GCS: %w", fileName, err) - } - return nil - }) + info, err := root.Lstat(fileName) + if err != nil { + return fmt.Errorf("while inspecting %s in snapshot directory: %w", fileName, err) + } + recordSnapshotSize(ctx, fileName, allocatedBytes(info), templateAtespace, templateName) } - if err := g.Wait(); err != nil { - return err + if err := s.uploadSnapshotFiles(ctx, uri.String(), srcDir, rec.SnapshotFiles, opts...); err != nil { + return fmt.Errorf("while uploading snapshot files: %w", err) } manifest, err := json.Marshal(rec) if err != nil { return fmt.Errorf("while marshaling snapshot manifest: %w", err) } - manifestURI, err := uri.ObjectURI(sandboxManifestName) - if err != nil { - return fmt.Errorf("while addressing snapshot manifest in GCS: %w", err) - } - if err := objectstorage.SendBytesToGCS(ctx, s.gcsClient, manifestURI, manifest); err != nil { + if err := s.uploadManifest(ctx, uri.String(), manifest, opts...); err != nil { return fmt.Errorf("while uploading snapshot manifest: %w", err) } return nil @@ -946,8 +939,8 @@ func (s *AteomHerder) UploadPausedCheckpoint(ctx context.Context, req *ateletpb. templateName: req.GetActorTemplateName(), // Always the actor's durable latest: golden actors are never paused // (validation above rejects the golden atespace). - kind: ateattr.SnapshotKindLatest, - scope: ateattr.SnapshotScopeValue(req.GetDesiredScope()), + kind: ateattr.SnapshotKindLatest, + fidelity: ateattr.SnapshotFidelityValue(req.GetDesiredFidelity()), } defer func() { s.instruments.recordCheckpoint(ctx, op, @@ -983,11 +976,6 @@ func (s *AteomHerder) UploadPausedCheckpoint(ctx context.Context, req *ateletpb. // returns the sandbox class recorded in the snapshot manifest (empty when the // manifest was not read). Parameterized by localDir for tests. func (s *AteomHerder) uploadLocalCheckpointDir(ctx context.Context, req *ateletpb.UploadPausedCheckpointRequest, localDir string, uri resources.SnapshotURI) (string, error) { - manifestURI, err := uri.ObjectURI(sandboxManifestName) - if err != nil { - return "", fmt.Errorf("while addressing snapshot manifest in GCS: %w", err) - } - manifest, err := readSnapshotManifest(localDir) if errors.Is(err, os.ErrNotExist) { // The local snapshot is gone. A previous invocation may have uploaded @@ -995,12 +983,12 @@ func (s *AteomHerder) uploadLocalCheckpointDir(ctx context.Context, req *ateletp // means the whole snapshot is committed and this retry already // succeeded. Absent on both sides, the paused actor's state is // unrecoverable. - _, fetchErr := objectstorage.FetchFromGCS(ctx, s.gcsClient, manifestURI) + _, fetchErr := s.fetchManifest(ctx, uri.String()) if fetchErr == nil { slog.InfoContext(ctx, "Local snapshot already uploaded and pruned; nothing to do", slog.String("snapshot_uri", req.GetDestinationSnapshotUri())) return "", nil } - if errors.Is(fetchErr, objectstorage.ErrObjectNotFound) { + if status.Code(fetchErr) == codes.NotFound { return "", fmt.Errorf("local snapshot %q is gone and no uploaded copy exists: %w", req.GetLocalSnapshotName(), fetchErr) } @@ -1015,25 +1003,27 @@ func (s *AteomHerder) uploadLocalCheckpointDir(ctx context.Context, req *ateletp return "", err } - capturedScope := rec.Scope - if capturedScope == "" { - return rec.SandboxClass, apierror.FailedPrecondition("local snapshot %q has no scope recorded in its manifest (written by an older atelet); resume and pause the actor again before suspending it", req.GetLocalSnapshotName()) + capturedFidelity := rec.Fidelity + if capturedFidelity == "" { + return rec.SandboxClass, apierror.FailedPrecondition("local snapshot %q has no fidelity recorded in its manifest; resume and pause the actor again before suspending it", req.GetLocalSnapshotName()) } - desiredScope := ateattr.SnapshotScopeValue(req.GetDesiredScope()) + desiredFidelity := ateattr.SnapshotFidelityValue(req.GetDesiredFidelity()) switch { - case capturedScope == desiredScope: - case capturedScope == ateattr.SnapshotScopeData && desiredScope == ateattr.SnapshotScopeFull: + case capturedFidelity == desiredFidelity: + case capturedFidelity == ateattr.SnapshotFidelityVolumes && desiredFidelity == ateattr.SnapshotFidelityMemory: // The control plane rejects this before marking SUSPENDING; reaching // it here means the template changed mid-flight or store state drifted. - return rec.SandboxClass, apierror.FailedPrecondition("pause snapshot captured %s; cannot upload it as %s (memory was never captured)", capturedScope, desiredScope) - default: // captured FULL, DATA wanted - if err := narrowFullCaptureToData(rec); err != nil { + return rec.SandboxClass, apierror.FailedPrecondition("pause snapshot captured %s; cannot upload it as %s (memory was never captured)", capturedFidelity, desiredFidelity) + default: // captured MEMORY, VOLUMES wanted + if err := narrowMemoryCaptureToVolumes(rec); err != nil { return rec.SandboxClass, err } } - return rec.SandboxClass, s.uploadSnapshot(ctx, uri, localDir, rec, req.GetActorTemplateAtespace(), req.GetActorTemplateName()) + // The local snapshot stays until the upload succeeds, so a retry can + // upload it again. + return rec.SandboxClass, objectstoreplugin.CallError(s.uploadSnapshot(ctx, uri, localDir, rec, req.GetActorTemplateAtespace(), req.GetActorTemplateName())) } func readSnapshotManifest(dir string) ([]byte, error) { @@ -1045,16 +1035,16 @@ func readSnapshotManifest(dir string) ([]byte, error) { return root.ReadFile(sandboxManifestName) } -// narrowFullCaptureToData rewrites rec so a FULL capture uploads as a DATA -// snapshot holding only the data-scope files ateom reported at checkpoint. -func narrowFullCaptureToData(rec *sandboxAssetsRecord) error { +// narrowMemoryCaptureToVolumes rewrites rec so a MEMORY capture uploads as a +// VOLUMES snapshot holding only the volume files ateom reported at checkpoint. +func narrowMemoryCaptureToVolumes(rec *sandboxAssetsRecord) error { if len(rec.DataSnapshotFiles) == 0 { // Either no durable-dir volumes were attached at pause, or the // manifest predates the list. Neither is retryable. - return apierror.FailedPrecondition("full capture lists no data-scope files; the actor has no durable data to upload as %s", ateattr.SnapshotScopeData) + return apierror.FailedPrecondition("memory capture lists no volume files; the actor has no durable data to upload as %s", ateattr.SnapshotFidelityVolumes) } rec.SnapshotFiles = rec.DataSnapshotFiles - rec.Scope = ateattr.SnapshotScopeData + rec.Fidelity = ateattr.SnapshotFidelityVolumes return nil } @@ -1085,7 +1075,7 @@ func (s *AteomHerder) Restore(ctx context.Context, req *ateletpb.RestoreRequest) op := snapshotOp{ templateNamespace: req.GetActorTemplateAtespace(), templateName: req.GetActorTemplateName(), - scope: ateattr.SnapshotScopeValue(req.GetScope()), + fidelity: ateattr.SnapshotFidelityValue(req.GetFidelity()), sandboxClass: req.GetSandboxAssets().GetSandboxClass(), } attribution := resources.ActorAttribution{ @@ -1144,15 +1134,7 @@ func (s *AteomHerder) Restore(ctx context.Context, req *ateletpb.RestoreRequest) var sandboxRec *sandboxAssetsRecord switch req.GetType() { case ateletpb.CheckpointType_CHECKPOINT_TYPE_EXTERNAL: - uri, err := resources.ParseSnapshotURI(req.GetExternalConfig().GetSnapshotUri()) - if err != nil { - return nil, err - } - manifestURI, err := uri.ObjectURI(sandboxManifestName) - if err != nil { - return nil, err - } - manifest, err := objectstorage.FetchFromGCS(ctx, s.gcsClient, manifestURI) + manifest, err := s.fetchManifest(ctx, req.GetExternalConfig().GetSnapshotUri()) if err != nil { return nil, fmt.Errorf("while fetching snapshot manifest: %w", err) } @@ -1193,6 +1175,7 @@ func (s *AteomHerder) Restore(ctx context.Context, req *ateletpb.RestoreRequest) // fetch + image unpack hides whichever leg is shorter, and on a cold node // (uncached assets + image, ~2.5s unpack) that overlap is large. var assetPaths map[string]string + var containerSpecs []*ateompb.ContainerSpec // One per leg: a single field written from both goroutines would race. var downloadErr, prepErr error var prepFailedPhase string @@ -1232,7 +1215,7 @@ func (s *AteomHerder) Restore(ctx context.Context, req *ateletpb.RestoreRequest) return err } t := time.Now() - err = s.prepareOCIBundles(gctx, actorUID, actorRef, req.GetSpec(), runtimeRec.PauseImage, req.GetTargetAteomUid()) + containerSpecs, err = s.prepareOCIBundles(gctx, actorUID, actorRef, req.GetSpec(), runtimeRec.PauseImage) dBundles = time.Since(t) if err != nil { prepFailedPhase = ateattr.SnapshotPhaseOCIUnpack @@ -1250,14 +1233,14 @@ func (s *AteomHerder) Restore(ctx context.Context, req *ateletpb.RestoreRequest) return nil, err } - client, err := s.dialAteom(ctx, req.GetTargetAteomUid()) + client, err := s.dialAteom(ctx, req.GetWorkerPodUid()) if err != nil { return nil, err } // Tell ateom to do runsc create + runsc restore for pause container and // all application containers. - spec, err := buildAteomWorkloadSpec(req.GetSpec()) + spec, err := buildAteomWorkloadSpec(req.GetSpec(), containerSpecs) if err != nil { return nil, apierror.InvalidArgument("invalid workload spec: %v", err) } @@ -1276,7 +1259,7 @@ func (s *AteomHerder) Restore(ctx context.Context, req *ateletpb.RestoreRequest) RunscPath: runscPathFor(assetPaths), RuntimeAssetPaths: assetPaths, Spec: spec, - Scope: toAteomSnapshotScope(req.GetScope()), + Fidelity: toAteomFidelity(req.GetFidelity()), ActorUid: req.GetActorUid(), ActorDirs: actorDirs, PreserveRestoreDir: directLocal, @@ -1311,7 +1294,7 @@ func (s *AteomHerder) Terminate(ctx context.Context, req *ateletpb.TerminateRequ actorRef := resources.ActorRef{Atespace: req.GetAtespace(), Name: req.GetActorName()} actorUID := req.GetActorUid() - if req.GetTargetAteomUid() != "" { + if req.GetWorkerPodUid() != "" { var assetPaths map[string]string sandboxRec, err := readSandboxRecord(actorUID) if err != nil { @@ -1323,12 +1306,12 @@ func (s *AteomHerder) Terminate(ctx context.Context, req *ateletpb.TerminateRequ } assetPaths = paths - client, err := s.dialAteom(ctx, req.GetTargetAteomUid()) + client, err := s.dialAteom(ctx, req.GetWorkerPodUid()) if err != nil { return nil, fmt.Errorf("failed to dial ateom for terminate (actor: %s, actorUID: %s): %w", actorRef, actorUID, err) } - spec, err := buildAteomWorkloadSpec(req.GetSpec()) + spec, err := buildAteomWorkloadSpec(req.GetSpec(), nil) if err != nil { return nil, apierror.InvalidArgument("invalid workload spec: %v", err) } @@ -1476,40 +1459,9 @@ func copyRootFile(root *os.Root, src, dst string) (int64, error) { } func (s *AteomHerder) downloadExternalCheckpoint(ctx context.Context, snapshotURI string, dstDir string, files []string) error { - uri, err := resources.ParseSnapshotURI(snapshotURI) - if err != nil { - return err - } - root, err := os.OpenRoot(dstDir) - if err != nil { - return fmt.Errorf("while opening restore directory: %w", err) - } - defer root.Close() - - g, gCtx := errgroup.WithContext(ctx) - for _, fileName := range files { - fileName := fileName - g.Go(func() error { - objectURI, err := uri.ObjectURI(fileName + ".zstd") - if err != nil { - return fmt.Errorf("while addressing %s in GCS: %w", fileName, err) - } - local, err := root.OpenFile(fileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600) - if err != nil { - return fmt.Errorf("while opening %s in restore directory: %w", fileName, err) - } - fetchErr := objectstorage.FetchFileFromGCSWithZstd(gCtx, s.gcsClient, objectURI, local) - closeErr := local.Close() - if err := errors.Join(fetchErr, closeErr); err != nil { - return fmt.Errorf("while downloading %s from GCS: %w", fileName, err) - } - return nil - }) + if err := s.fetchSnapshotFiles(ctx, snapshotURI, dstDir, files); err != nil { + return fmt.Errorf("while downloading snapshot files: %w", err) } - if err := g.Wait(); err != nil { - return err - } - return nil } @@ -1518,22 +1470,22 @@ func (s *AteomHerder) downloadExternalCheckpoint(ctx context.Context, snapshotUR // comes from the sandbox record, not the workload spec: it is sandbox // configuration, and on a restore it must be the image the snapshot was taken // with. It is empty for sandboxes without a pause container, which get no -// pause bundle. +// pause bundle. It returns each application container's spec, indexed like +// spec.GetContainers(). func (s *AteomHerder) prepareOCIBundles( ctx context.Context, actorUID string, actorRef resources.ActorRef, spec *ateletpb.WorkloadSpec, pauseImage string, - targetAteomUid string, -) error { +) ([]*ateompb.ContainerSpec, error) { // Prepare host folders for volume types that need them. for _, vol := range spec.GetVolumes() { switch vol.GetSource().(type) { case *ateletpb.Volume_DurableDir: volPath := ateletpath.DurableDirVolumeMountPoint(actorUID, vol.GetName()) if err := os.MkdirAll(volPath, 0o700); err != nil { - return fmt.Errorf("while creating %q: %w", volPath, err) + return nil, fmt.Errorf("while creating %q: %w", volPath, err) } } } @@ -1542,21 +1494,8 @@ func (s *AteomHerder) prepareOCIBundles( if pauseImage != "" { g.Go(func() error { - if err := prepareOCIDirectory( - gCtx, - s.imageCache, - actorUID, - ocispec.PauseContainer, - pauseImage, - []string{"/pause"}, - nil, - nil, - nodepath.ActorNetNSPath(actorUID), - nil, // pause is sandbox infra; it mounts no volumes. - nil, - nil, // pause only reaps; it needs no capabilities. - nil, // pause carries no user-declared limits. - ); err != nil { + // pause is sandbox infra; it mounts no volumes. + if _, err := prepareOCIDirectory(gCtx, s.imageCache, actorUID, ocispec.PauseContainer, pauseImage, nil, nil); err != nil { return wrapFileSystemErr("while creating pause OCI bundle", err) } return nil @@ -1564,50 +1503,51 @@ func (s *AteomHerder) prepareOCIBundles( } // Application containers. - for _, ctr := range spec.GetContainers() { - ctr := ctr + containerSpecs := make([]*ateompb.ContainerSpec, len(spec.GetContainers())) + for i, ctr := range spec.GetContainers() { var envs []string for _, env := range ctr.GetEnv() { envs = append(envs, fmt.Sprintf("%s=%s", env.GetName(), env.GetValue())) } g.Go(func() error { - if err := prepareOCIDirectory( - gCtx, - s.imageCache, - actorUID, - ctr.GetName(), - ctr.GetImage(), - ctr.GetCommand(), - ctr.GetArgs(), - envs, - nodepath.ActorNetNSPath(actorUID), - spec.GetVolumes(), - ctr.GetVolumeMounts(), - resolveCapabilities(ctr.GetSecurityContext().GetCapabilities()), - ctr.GetResources(), - ); err != nil { + imageConfig, err := prepareOCIDirectory(gCtx, s.imageCache, actorUID, ctr.GetName(), ctr.GetImage(), spec.GetVolumes(), ctr.GetVolumeMounts()) + if err != nil { return wrapFileSystemErr(fmt.Sprintf("while creating %q OCI bundle", ctr.GetName()), err) } + args, err := resolveProcessArgs(imageConfig, ctr.GetCommand(), ctr.GetArgs()) + if err != nil { + return fmt.Errorf("while resolving process args for container %q: %w", ctr.GetName(), err) + } + containerSpecs[i] = &ateompb.ContainerSpec{ + Args: args, + Env: resolveActorEnv(imageConfig, envs), + Capabilities: resolveCapabilities(ctr.GetSecurityContext().GetCapabilities()), + Resources: toAteomResourceLimits(ctr.GetResources()), + } return nil }) } - - return g.Wait() + if err := g.Wait(); err != nil { + return nil, err + } + return containerSpecs, nil } // dialAteom opens (or reuses) the gRPC connection to the target ateom // pod and returns an ateom client. -func (s *AteomHerder) dialAteom(ctx context.Context, targetAteomUid string) (ateompb.AteomClient, error) { - conn, err := s.ateomDialer.DialAteomPod(ctx, targetAteomUid) +func (s *AteomHerder) dialAteom(ctx context.Context, workerPodUID string) (ateompb.AteomClient, error) { + conn, err := s.ateomDialer.DialAteomPod(ctx, workerPodUID) if err != nil { - return nil, fmt.Errorf("while getting ateom conn for %s: %w", targetAteomUid, err) + return nil, fmt.Errorf("while getting ateom conn for %s: %w", workerPodUID, err) } return ateompb.NewAteomClient(conn), nil } // buildAteomWorkloadSpec projects the atelet-facing workload spec onto -// the ateom-facing one. -func buildAteomWorkloadSpec(spec *ateletpb.WorkloadSpec) (*ateompb.WorkloadSpec, error) { +// the ateom-facing one. containerSpecs holds each container's spec from +// prepareOCIBundles, indexed like spec.GetContainers(); it is nil for RPCs that +// start no process. +func buildAteomWorkloadSpec(spec *ateletpb.WorkloadSpec, containerSpecs []*ateompb.ContainerSpec) (*ateompb.WorkloadSpec, error) { volumes := make(map[string]*ateletpb.Volume) for _, vol := range spec.GetVolumes() { name := vol.GetName() @@ -1618,7 +1558,7 @@ func buildAteomWorkloadSpec(spec *ateletpb.WorkloadSpec) (*ateompb.WorkloadSpec, } out := &ateompb.WorkloadSpec{} - for _, ctr := range spec.GetContainers() { + for i, ctr := range spec.GetContainers() { var ddMounts []*ateompb.DurableDirVolumeMount var csiMounts []*ateompb.VolumeMount var siMounts []*ateompb.SystemInfoVolumeMount @@ -1655,18 +1595,29 @@ func buildAteomWorkloadSpec(spec *ateletpb.WorkloadSpec) (*ateompb.WorkloadSpec, return nil, fmt.Errorf("container %q mounts volume %q with unsupported source %T", ctr.GetName(), volName, vol.GetSource()) } } - out.Containers = append(out.Containers, &ateompb.Container{ + container := &ateompb.Container{ Name: ctr.GetName(), DurableDirVolumeMounts: ddMounts, CsiVolumeMounts: csiMounts, SystemInfoVolumeMounts: siMounts, ImageVolumeMounts: imgMounts, WakeupProbe: toAteomWakeupProbe(ctr.GetWakeupProbe()), - }) + } + if containerSpecs != nil { + container.ContainerSpec = containerSpecs[i] + } + out.Containers = append(out.Containers, container) } return out, nil } +func toAteomResourceLimits(limits *ateletpb.ResourceLimits) *ateompb.ResourceLimits { + if limits == nil { + return nil + } + return &ateompb.ResourceLimits{MemoryBytes: limits.GetMemoryBytes(), CpuMillis: limits.GetCpuMillis()} +} + func toAteomEgressGateway(gateway *ateletpb.EgressGateway) *ateompb.EgressGateway { if gateway == nil { return nil @@ -1760,7 +1711,7 @@ func validateRunRequest(req *ateletpb.RunRequest) error { return errs.ToAggregate() } // TODO: Migrate all validations below to the validation framework. - if err := resources.ValidateAteomUID(req.GetTargetAteomUid()); err != nil { + if err := resources.ValidateWorkerPodUID(req.GetWorkerPodUid()); err != nil { return err } names := make([]string, 0, len(req.GetSpec().GetContainers())) @@ -1779,7 +1730,7 @@ func validateCheckpointRequest(req *ateletpb.CheckpointRequest) error { return errs.ToAggregate() } // TODO: Migrate all validations below to the validation framework. - if err := resources.ValidateAteomUID(req.GetTargetAteomUid()); err != nil { + if err := resources.ValidateWorkerPodUID(req.GetWorkerPodUid()); err != nil { return err } names := make([]string, 0, len(req.GetSpec().GetContainers())) @@ -1790,7 +1741,7 @@ func validateCheckpointRequest(req *ateletpb.CheckpointRequest) error { return err } - if err := validateSnapshotScope(req.GetScope()); err != nil { + if err := validateFidelity(req.GetFidelity()); err != nil { return err } @@ -1818,7 +1769,7 @@ func validateRestoreRequest(req *ateletpb.RestoreRequest) error { return errs.ToAggregate() } // TODO: Migrate all validations below to the validation framework. - if err := resources.ValidateAteomUID(req.GetTargetAteomUid()); err != nil { + if err := resources.ValidateWorkerPodUID(req.GetWorkerPodUid()); err != nil { return err } names := make([]string, 0, len(req.GetSpec().GetContainers())) @@ -1829,7 +1780,7 @@ func validateRestoreRequest(req *ateletpb.RestoreRequest) error { return err } - if err := validateSnapshotScope(req.GetScope()); err != nil { + if err := validateFidelity(req.GetFidelity()); err != nil { return err } @@ -1860,8 +1811,8 @@ func validateTerminateRequest(req *ateletpb.TerminateRequest) error { if len(errs) > 0 { return errs.ToAggregate() } - if req.GetTargetAteomUid() != "" { - if err := resources.ValidateAteomUID(req.GetTargetAteomUid()); err != nil { + if req.GetWorkerPodUid() != "" { + if err := resources.ValidateWorkerPodUID(req.GetWorkerPodUid()); err != nil { return err } } @@ -1872,15 +1823,20 @@ func validateTerminateRequest(req *ateletpb.TerminateRequest) error { return resources.ValidateContainerNames(names) } -func validateSnapshotScope(scope ateletpb.SnapshotScope) error { - switch scope { - case ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL, - ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA: +// validateFidelity rejects a fidelity no runtime can serve. ROOTFS is in the +// enum but not captured by any sandbox runtime yet, so it is refused here as +// well as at template admission. +func validateFidelity(fidelity ateletpb.SnapshotFidelity) error { + switch fidelity { + case ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES: return nil - case ateletpb.SnapshotScope_SNAPSHOT_SCOPE_UNSPECIFIED: - return fmt.Errorf("snapshot scope must be non-zero") + case ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED: + return fmt.Errorf("snapshot fidelity must be non-zero") + case ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS: + return fmt.Errorf("ROOTFS fidelity is not supported yet") default: - return fmt.Errorf("invalid snapshot scope: %v", scope) + return fmt.Errorf("invalid snapshot fidelity: %v", fidelity) } } @@ -1898,12 +1854,14 @@ func validateUploadPausedCheckpointRequest(req *ateletpb.UploadPausedCheckpointR if _, err := resources.ParseSnapshotURI(req.GetDestinationSnapshotUri()); err != nil { errs = append(errs, field.Invalid(field.NewPath("destination_snapshot_uri"), req.GetDestinationSnapshotUri(), err.Error())) } - // Uploads only ever produce FULL or DATA snapshots. - switch req.GetDesiredScope() { - case ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL, ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA: + // Uploads only ever produce MEMORY or VOLUMES snapshots. + switch req.GetDesiredFidelity() { + case ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES: + case ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS: + errs = append(errs, field.Invalid(field.NewPath("desired_fidelity"), req.GetDesiredFidelity().String(), "ROOTFS fidelity is not supported yet")) default: - errs = append(errs, field.NotSupported(field.NewPath("desired_scope"), req.GetDesiredScope(), - []string{ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL.String(), ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA.String()})) + errs = append(errs, field.NotSupported(field.NewPath("desired_fidelity"), req.GetDesiredFidelity(), + []string{ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY.String(), ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES.String()})) } return errs.ToAggregate() } diff --git a/cmd/atelet/main_test.go b/cmd/atelet/main_test.go index 3507e67604..921267e907 100644 --- a/cmd/atelet/main_test.go +++ b/cmd/atelet/main_test.go @@ -37,6 +37,7 @@ import ( "github.com/agent-substrate/substrate/internal/apierror" "github.com/agent-substrate/substrate/internal/ateattr" "github.com/agent-substrate/substrate/internal/atelet" + "github.com/agent-substrate/substrate/internal/imagecache" "github.com/agent-substrate/substrate/internal/nodepath" "github.com/agent-substrate/substrate/internal/ocispec" "github.com/agent-substrate/substrate/internal/proto/ateletpb" @@ -173,38 +174,38 @@ func TestSnapshotManifestActorMetadata(t *testing.T) { ActorUID: "actor-uid", ActorTemplateAtespace: "templates", ActorTemplateName: "agent", - Scope: ateattr.SnapshotScopeFull, + Fidelity: ateattr.SnapshotFidelityMemory, } got, err := json.Marshal(rec) if err != nil { t.Fatal(err) } - for _, want := range []string{`"atespace":"team-a"`, `"actorName":"actor-1"`, `"actorUid":"actor-uid"`, `"actorTemplateAtespace":"templates"`, `"actorTemplateName":"agent"`, `"scope":"full"`} { + for _, want := range []string{`"atespace":"team-a"`, `"actorName":"actor-1"`, `"actorUid":"actor-uid"`, `"actorTemplateAtespace":"templates"`, `"actorTemplateName":"agent"`, `"fidelity":"memory"`} { if !bytes.Contains(got, []byte(want)) { t.Errorf("manifest %s missing %s", got, want) } } } -// TestSnapshotManifestScopeAbsent pins backward compatibility: manifests -// written before the scope field existed must still parse, reporting an empty -// scope, and a scope-less record must not serialize a scope key at all. -func TestSnapshotManifestScopeAbsent(t *testing.T) { +// TestSnapshotManifestFidelityAbsent pins that the on-node record written at +// Run/Restore, which has no fidelity, parses back with an empty fidelity and +// does not serialize a fidelity key at all. +func TestSnapshotManifestFidelityAbsent(t *testing.T) { legacy := []byte(`{"sandboxClass":"gvisor","pauseImage":"` + testPauseImage + `","snapshotFiles":["checkpoint.img"]}`) rec, err := unmarshalSandboxRecord(legacy) if err != nil { t.Fatalf("unmarshalSandboxRecord(legacy manifest): %v", err) } - if rec.Scope != "" { - t.Errorf("legacy manifest scope = %q, want empty", rec.Scope) + if rec.Fidelity != "" { + t.Errorf("manifest fidelity = %q, want empty", rec.Fidelity) } got, err := json.Marshal(sandboxAssetsRecord{SandboxClass: "gvisor"}) if err != nil { t.Fatal(err) } - if bytes.Contains(got, []byte(`"scope"`)) { - t.Errorf("scope-less record serialized a scope key: %s", got) + if bytes.Contains(got, []byte(`"fidelity"`)) { + t.Errorf("fidelity-less record serialized a fidelity key: %s", got) } } @@ -258,10 +259,10 @@ func TestPrepareOCIBundlesPause(t *testing.T) { useTempNodeDirs(t) const actorUID = "actor-uid-1" s := &AteomHerder{imageCache: newImageVolumeStore(t)} - if err := s.prepareOCIBundles(t.Context(), actorUID, resources.ActorRef{}, spec, tc.pauseImage, "ateom-uid-1"); err != nil { + if _, err := s.prepareOCIBundles(t.Context(), actorUID, resources.ActorRef{}, spec, tc.pauseImage); err != nil { t.Fatalf("prepareOCIBundles: %v", err) } - if _, err := os.Stat(filepath.Join(ateletpath.OCIBundlePath(actorUID, "app"), "config.json")); err != nil { + if _, err := os.Stat(filepath.Join(ateletpath.OCIBundlePath(actorUID, "app"), imagecache.OverlaySpecFileName)); err != nil { t.Errorf("app bundle: %v", err) } _, err := os.Stat(ateletpath.OCIBundlePath(actorUID, ocispec.PauseContainer)) @@ -272,6 +273,43 @@ func TestPrepareOCIBundlesPause(t *testing.T) { } } +// TestPrepareOCIBundlesFillsContainerSpec pins that each container's spec +// carries its resolved args, env, capabilities and resource limits. +func TestPrepareOCIBundlesFillsContainerSpec(t *testing.T) { + useTempNodeDirs(t) + host := imageVolumeTestRegistry(t) + image := host + "/actor:v1" + pushTestImage(t, image, singleFileLayer(t, "bin/app", "app")) + spec := &ateletpb.WorkloadSpec{ + Containers: []*ateletpb.Container{{ + Name: "app", + Image: image, + Command: []string{"/bin/app"}, + Args: []string{"serve"}, + Env: []*ateletpb.EnvEntry{{Name: "GREETING", Value: "hello"}}, + SecurityContext: &ateletpb.SecurityContext{ + Capabilities: &ateletpb.Capabilities{Add: []string{"NET_ADMIN"}, Drop: []string{"KILL"}}, + }, + Resources: &ateletpb.ResourceLimits{MemoryBytes: 1 << 30, CpuMillis: 500}, + }}, + } + + s := &AteomHerder{imageCache: newImageVolumeStore(t)} + got, err := s.prepareOCIBundles(t.Context(), "actor-uid-1", resources.ActorRef{}, spec, "") + if err != nil { + t.Fatalf("prepareOCIBundles: %v", err) + } + want := []*ateompb.ContainerSpec{{ + Args: []string{"/bin/app", "serve"}, + Env: []string{"GREETING=hello", "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"}, + Capabilities: []string{"CAP_AUDIT_WRITE", "CAP_NET_ADMIN", "CAP_NET_BIND_SERVICE"}, + Resources: &ateompb.ResourceLimits{MemoryBytes: 1 << 30, CpuMillis: 500}, + }} + if diff := cmp.Diff(want, got, protocmp.Transform()); diff != "" { + t.Errorf("prepareOCIBundles container specs mismatch (-want +got):\n%s", diff) + } +} + func TestSnapshotManifestRejectsNonLocalFile(t *testing.T) { manifest, err := json.Marshal(sandboxAssetsRecord{ SandboxClass: "gvisor", @@ -351,7 +389,7 @@ func TestUploadSnapshotRejectsSymlinkOutsideRoot(t *testing.T) { t.Fatal(err) } store := &recordingObjectStorage{} - err = (&AteomHerder{gcsClient: store}).uploadSnapshot(context.Background(), uri, checkpointDir, + err = newPluginHerder(t, store).uploadSnapshot(context.Background(), uri, checkpointDir, &sandboxAssetsRecord{SnapshotFiles: []string{"checkpoint.img"}}, "test", "test") if err == nil { t.Fatal("uploadSnapshot() followed a symlink outside the checkpoint directory") @@ -383,7 +421,7 @@ func TestDownloadExternalCheckpointRejectsSymlinkOutsideRoot(t *testing.T) { if err := objectstorage.SendLocalFileToGCSWithZstd(context.Background(), store, testSnapshotURI+"/checkpoint.img.zstd", payload); err != nil { t.Fatal(err) } - err := (&AteomHerder{gcsClient: store}).downloadExternalCheckpoint( + err := newPluginHerder(t, store).downloadExternalCheckpoint( context.Background(), testSnapshotURI, restoreDir, []string{"checkpoint.img"}) if err == nil { t.Fatal("downloadExternalCheckpoint() followed a symlink outside the restore directory") @@ -508,7 +546,7 @@ func validRunRequest() *ateletpb.RunRequest { ActorName: "counter-1", ActorTemplateAtespace: "ate-demo", ActorTemplateName: "counter", - TargetAteomUid: "422938ba-8860-4983-a25d-d6bcb0a69d4e", + WorkerPodUid: "422938ba-8860-4983-a25d-d6bcb0a69d4e", ActorUid: "123e4567-e89b-12d3-a456-426614174000", Spec: &ateletpb.WorkloadSpec{Containers: []*ateletpb.Container{{Name: "worker"}}}, } @@ -520,7 +558,7 @@ func validCheckpointRequest() *ateletpb.CheckpointRequest { ActorName: "counter-1", ActorTemplateAtespace: "ate-demo", ActorTemplateName: "counter", - TargetAteomUid: "422938ba-8860-4983-a25d-d6bcb0a69d4e", + WorkerPodUid: "422938ba-8860-4983-a25d-d6bcb0a69d4e", ActorUid: "123e4567-e89b-12d3-a456-426614174000", Spec: &ateletpb.WorkloadSpec{Containers: []*ateletpb.Container{{Name: "worker"}}}, Type: ateletpb.CheckpointType_CHECKPOINT_TYPE_EXTERNAL, @@ -529,7 +567,7 @@ func validCheckpointRequest() *ateletpb.CheckpointRequest { SnapshotUri: testSnapshotURI, }, }, - Scope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL, + Fidelity: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, } } @@ -539,7 +577,7 @@ func validRestoreRequest() *ateletpb.RestoreRequest { ActorName: "counter-1", ActorTemplateAtespace: "ate-demo", ActorTemplateName: "counter", - TargetAteomUid: "422938ba-8860-4983-a25d-d6bcb0a69d4e", + WorkerPodUid: "422938ba-8860-4983-a25d-d6bcb0a69d4e", ActorUid: "123e4567-e89b-12d3-a456-426614174000", Spec: &ateletpb.WorkloadSpec{Containers: []*ateletpb.Container{{Name: "worker"}}}, Type: ateletpb.CheckpointType_CHECKPOINT_TYPE_EXTERNAL, @@ -548,7 +586,7 @@ func validRestoreRequest() *ateletpb.RestoreRequest { SnapshotUri: testSnapshotURI, }, }, - Scope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL, + Fidelity: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, SandboxAssets: &ateletpb.SandboxAssets{ SandboxClass: "gvisor", PauseImage: testPauseImage, @@ -563,7 +601,7 @@ func TestValidateRunRequest(t *testing.T) { wantErr bool }{ {"valid", func(*ateletpb.RunRequest) {}, false}, - {"invalid ateom uid", func(r *ateletpb.RunRequest) { r.TargetAteomUid = "../escape" }, true}, + {"invalid worker pod uid", func(r *ateletpb.RunRequest) { r.WorkerPodUid = "../escape" }, true}, {"invalid atespace", func(r *ateletpb.RunRequest) { r.Atespace = "../escape" }, true}, {"invalid actor name", func(r *ateletpb.RunRequest) { r.ActorName = "../escape" }, true}, {"invalid actor uid", func(r *ateletpb.RunRequest) { r.ActorUid = "../escape" }, true}, @@ -603,7 +641,7 @@ func TestValidateCheckpointRequest(t *testing.T) { {"valid", makeReq(), false}, {"empty snapshot uri", makeReq(func(r *ateletpb.CheckpointRequest) { r.GetExternalConfig().SnapshotUri = "" }), true}, {"bucketless snapshot uri", makeReq(func(r *ateletpb.CheckpointRequest) { r.GetExternalConfig().SnapshotUri = "relative/path" }), true}, - {"invalid ateom uid", makeReq(func(r *ateletpb.CheckpointRequest) { r.TargetAteomUid = "../escape" }), true}, + {"invalid worker pod uid", makeReq(func(r *ateletpb.CheckpointRequest) { r.WorkerPodUid = "../escape" }), true}, {"invalid atespace", makeReq(func(r *ateletpb.CheckpointRequest) { r.Atespace = "../escape" }), true}, {"invalid actor name", makeReq(func(r *ateletpb.CheckpointRequest) { r.ActorName = "../escape" }), true}, {"invalid actor uid", makeReq(func(r *ateletpb.CheckpointRequest) { r.ActorUid = "../escape" }), true}, @@ -631,8 +669,13 @@ func TestValidateCheckpointRequest(t *testing.T) { r.Config = &ateletpb.CheckpointRequest_LocalConfig{LocalConfig: &ateletpb.LocalCheckpointConfiguration{SnapshotName: ".."}} }), true}, {"unspecified snapshot type", makeReq(func(r *ateletpb.CheckpointRequest) { r.Type = ateletpb.CheckpointType_CHECKPOINT_TYPE_UNSPECIFIED }), true}, - {"unspecified snapshot scope", makeReq(func(r *ateletpb.CheckpointRequest) { r.Scope = ateletpb.SnapshotScope_SNAPSHOT_SCOPE_UNSPECIFIED }), true}, - {"invalid snapshot scope", makeReq(func(r *ateletpb.CheckpointRequest) { r.Scope = ateletpb.SnapshotScope(23) }), true}, + {"unspecified snapshot fidelity", makeReq(func(r *ateletpb.CheckpointRequest) { + r.Fidelity = ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED + }), true}, + {"invalid snapshot fidelity", makeReq(func(r *ateletpb.CheckpointRequest) { r.Fidelity = ateletpb.SnapshotFidelity(23) }), true}, + {"rootfs fidelity not supported yet", makeReq(func(r *ateletpb.CheckpointRequest) { + r.Fidelity = ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS + }), true}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { @@ -661,7 +704,7 @@ func TestValidateRestoreRequest(t *testing.T) { {"missing sandbox assets", makeReq(func(r *ateletpb.RestoreRequest) { r.SandboxAssets = nil }), true}, {"empty snapshot uri", makeReq(func(r *ateletpb.RestoreRequest) { r.GetExternalConfig().SnapshotUri = "" }), true}, {"bucketless snapshot uri", makeReq(func(r *ateletpb.RestoreRequest) { r.GetExternalConfig().SnapshotUri = "relative/path" }), true}, - {"invalid ateom uid", makeReq(func(r *ateletpb.RestoreRequest) { r.TargetAteomUid = "../escape" }), true}, + {"invalid worker pod uid", makeReq(func(r *ateletpb.RestoreRequest) { r.WorkerPodUid = "../escape" }), true}, {"invalid atespace", makeReq(func(r *ateletpb.RestoreRequest) { r.Atespace = "../escape" }), true}, {"invalid actor name", makeReq(func(r *ateletpb.RestoreRequest) { r.ActorName = "../escape" }), true}, {"invalid actor uid", makeReq(func(r *ateletpb.RestoreRequest) { r.ActorUid = "../escape" }), true}, @@ -689,8 +732,11 @@ func TestValidateRestoreRequest(t *testing.T) { r.Config = &ateletpb.RestoreRequest_LocalConfig{LocalConfig: &ateletpb.LocalCheckpointConfiguration{SnapshotName: ".."}} }), true}, {"unspecified snapshot type", makeReq(func(r *ateletpb.RestoreRequest) { r.Type = ateletpb.CheckpointType_CHECKPOINT_TYPE_UNSPECIFIED }), true}, - {"unspecified snapshot scope", makeReq(func(r *ateletpb.RestoreRequest) { r.Scope = ateletpb.SnapshotScope_SNAPSHOT_SCOPE_UNSPECIFIED }), true}, - {"invalid snapshot scope", makeReq(func(r *ateletpb.RestoreRequest) { r.Scope = ateletpb.SnapshotScope(23) }), true}, + {"unspecified snapshot fidelity", makeReq(func(r *ateletpb.RestoreRequest) { r.Fidelity = ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED }), true}, + {"invalid snapshot fidelity", makeReq(func(r *ateletpb.RestoreRequest) { r.Fidelity = ateletpb.SnapshotFidelity(23) }), true}, + {"rootfs fidelity not supported yet", makeReq(func(r *ateletpb.RestoreRequest) { + r.Fidelity = ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS + }), true}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { @@ -702,17 +748,20 @@ func TestValidateRestoreRequest(t *testing.T) { } // Every valid atelet scope must map to its ateom counterpart. -func TestToAteomSnapshotScope(t *testing.T) { +func TestToAteomFidelity(t *testing.T) { tests := []struct { - in ateletpb.SnapshotScope - want ateompb.SnapshotScope + in ateletpb.SnapshotFidelity + want ateompb.SnapshotFidelity }{ - {ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL, ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL}, - {ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA, ateompb.SnapshotScope_SNAPSHOT_SCOPE_DATA}, + {ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY}, + {ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES}, + {ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS, ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS}, + {ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED, ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED}, + {ateletpb.SnapshotFidelity(99), ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED}, } for _, tc := range tests { - if got := toAteomSnapshotScope(tc.in); got != tc.want { - t.Errorf("toAteomSnapshotScope(%v) = %v, want %v", tc.in, got, tc.want) + if got := toAteomFidelity(tc.in); got != tc.want { + t.Errorf("toAteomFidelity(%v) = %v, want %v", tc.in, got, tc.want) } } } @@ -861,14 +910,14 @@ func TestFetchAssetStreaming(t *testing.T) { func TestRPCBoundariesReject(t *testing.T) { s := &AteomHerder{} ctx := context.Background() - badUID := "../escape" // valid actor ref, invalid ateom UID + badUID := "../escape" // valid actor ref, invalid worker pod UID const okAtespace, okID, okActorUID = "ate-demo", "counter-1", "123e4567-e89b-12d3-a456-426614174000" okSpec := &ateletpb.WorkloadSpec{Containers: []*ateletpb.Container{{Name: "worker"}}} wantInvalidArgument := func(t *testing.T, rpc string, err error) { t.Helper() if err == nil { - t.Errorf("%s accepted an invalid target ateom UID", rpc) + t.Errorf("%s accepted an invalid worker pod UID", rpc) return } if code := apierror.Code(err); code != codes.InvalidArgument { @@ -879,21 +928,21 @@ func TestRPCBoundariesReject(t *testing.T) { t.Run("Run", func(t *testing.T) { _, err := s.Run(ctx, &ateletpb.RunRequest{ Atespace: okAtespace, ActorName: okID, - ActorUid: okActorUID, TargetAteomUid: badUID, Spec: okSpec, + ActorUid: okActorUID, WorkerPodUid: badUID, Spec: okSpec, }) wantInvalidArgument(t, "Run", err) }) t.Run("Checkpoint", func(t *testing.T) { _, err := s.Checkpoint(ctx, &ateletpb.CheckpointRequest{ Atespace: okAtespace, ActorName: okID, - ActorUid: okActorUID, TargetAteomUid: badUID, Spec: okSpec, + ActorUid: okActorUID, WorkerPodUid: badUID, Spec: okSpec, }) wantInvalidArgument(t, "Checkpoint", err) }) t.Run("Restore", func(t *testing.T) { _, err := s.Restore(ctx, &ateletpb.RestoreRequest{ Atespace: okAtespace, ActorName: okID, - ActorUid: okActorUID, TargetAteomUid: badUID, Spec: okSpec, + ActorUid: okActorUID, WorkerPodUid: badUID, Spec: okSpec, }) wantInvalidArgument(t, "Restore", err) }) @@ -901,7 +950,7 @@ func TestRPCBoundariesReject(t *testing.T) { _, err := s.Terminate(ctx, &ateletpb.TerminateRequest{ Atespace: okAtespace, ActorName: okID, ActorUid: okActorUID, ActorTemplateAtespace: "default", ActorTemplateName: "template", - TargetAteomUid: badUID, Spec: okSpec, + WorkerPodUid: badUID, Spec: okSpec, }) wantInvalidArgument(t, "Terminate", err) }) @@ -935,7 +984,30 @@ func TestBuildAteomWorkloadSpecForwardsWakeupProbe(t *testing.T) { {Name: "without-probe"}, }, } - got, err := buildAteomWorkloadSpec(in) + got, err := buildAteomWorkloadSpec(in, nil) + if err != nil { + t.Fatalf("buildAteomWorkloadSpec failed: %v", err) + } + if diff := cmp.Diff(want, got, protocmp.Transform()); diff != "" { + t.Errorf("buildAteomWorkloadSpec mismatch (-want +got):\n%s", diff) + } +} + +func TestBuildAteomWorkloadSpecForwardsContainerSpecs(t *testing.T) { + in := &ateletpb.WorkloadSpec{ + Containers: []*ateletpb.Container{{Name: "main"}, {Name: "sidecar"}}, + } + containerSpecs := []*ateompb.ContainerSpec{ + {Args: []string{"/app", "serve"}}, + {Args: []string{"/sidecar"}}, + } + want := &ateompb.WorkloadSpec{ + Containers: []*ateompb.Container{ + {Name: "main", ContainerSpec: &ateompb.ContainerSpec{Args: []string{"/app", "serve"}}}, + {Name: "sidecar", ContainerSpec: &ateompb.ContainerSpec{Args: []string{"/sidecar"}}}, + }, + } + got, err := buildAteomWorkloadSpec(in, containerSpecs) if err != nil { t.Fatalf("buildAteomWorkloadSpec failed: %v", err) } @@ -997,7 +1069,7 @@ func TestBuildAteomWorkloadSpecForwardsDurableDirMounts(t *testing.T) { {Name: "no-volumes"}, }, } - got, err := buildAteomWorkloadSpec(in) + got, err := buildAteomWorkloadSpec(in, nil) if err != nil { t.Fatalf("buildAteomWorkloadSpec failed: %v", err) } @@ -1068,7 +1140,7 @@ func TestBuildAteomWorkloadSpecValidation(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - _, err := buildAteomWorkloadSpec(tc.in) + _, err := buildAteomWorkloadSpec(tc.in, nil) if err == nil { t.Fatal("expected error, got nil") } @@ -1348,7 +1420,7 @@ func TestBuildAteomWorkloadSpec_ImageVolumeMounts(t *testing.T) { }}, } - got, err := buildAteomWorkloadSpec(spec) + got, err := buildAteomWorkloadSpec(spec, nil) if err != nil { t.Fatalf("buildAteomWorkloadSpec: %v", err) } @@ -1444,7 +1516,7 @@ func validUploadPausedCheckpointRequest() *ateletpb.UploadPausedCheckpointReques ActorTemplateName: "counter", LocalSnapshotName: "pause-snap-1", DestinationSnapshotUri: pausedSnapshotURI, - DesiredScope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL, + DesiredFidelity: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, } } @@ -1460,7 +1532,7 @@ func TestUploadLocalCheckpointDir(t *testing.T) { PauseImage: testPauseImage, SnapshotFiles: []string{"config.json", "memory-ranges", "data.tar"}, DataSnapshotFiles: []string{"data.tar"}, - Scope: ateattr.SnapshotScopeFull, + Fidelity: ateattr.SnapshotFidelityMemory, } } @@ -1479,7 +1551,7 @@ func TestUploadLocalCheckpointDir(t *testing.T) { t.Run("matching scope uploads all files", func(t *testing.T) { store := &recordingObjectStorage{} - s := &AteomHerder{gcsClient: store} + s := newPluginHerder(t, store) dir := filepath.Join(t.TempDir(), "pause-snap-1") writeLocalSnapshot(t, dir, fullRec("microvm"), map[string]string{ "config.json": "cfg", "memory-ranges": "mem", "data.tar": "data", @@ -1497,21 +1569,21 @@ func TestUploadLocalCheckpointDir(t *testing.T) { if got := store.keys(); !slices.Equal(got, want) { t.Errorf("uploaded objects = %v, want %v", got, want) } - if rec := remoteManifest(t, store); rec.Scope != ateattr.SnapshotScopeFull { - t.Errorf("uploaded manifest scope = %q, want %q", rec.Scope, ateattr.SnapshotScopeFull) + if rec := remoteManifest(t, store); rec.Fidelity != ateattr.SnapshotFidelityMemory { + t.Errorf("uploaded manifest fidelity = %q, want %q", rec.Fidelity, ateattr.SnapshotFidelityMemory) } }) t.Run("full capture uploads the reported data files alone as data", func(t *testing.T) { store := &recordingObjectStorage{} - s := &AteomHerder{gcsClient: store} + s := newPluginHerder(t, store) dir := filepath.Join(t.TempDir(), "pause-snap-1") writeLocalSnapshot(t, dir, fullRec("microvm"), map[string]string{ "config.json": "cfg", "memory-ranges": "mem", "data.tar": "data", }) req := validUploadPausedCheckpointRequest() - req.DesiredScope = ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA + req.DesiredFidelity = ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES if _, err := s.uploadLocalCheckpointDir(ctx, req, dir, uri); err != nil { t.Fatalf("uploadLocalCheckpointDir: %v", err) } @@ -1523,8 +1595,8 @@ func TestUploadLocalCheckpointDir(t *testing.T) { t.Errorf("uploaded objects = %v, want %v", got, want) } rec := remoteManifest(t, store) - if rec.Scope != ateattr.SnapshotScopeData { - t.Errorf("uploaded manifest scope = %q, want %q", rec.Scope, ateattr.SnapshotScopeData) + if rec.Fidelity != ateattr.SnapshotFidelityVolumes { + t.Errorf("uploaded manifest fidelity = %q, want %q", rec.Fidelity, ateattr.SnapshotFidelityVolumes) } if want := []string{"data.tar"}; !slices.Equal(rec.SnapshotFiles, want) { t.Errorf("uploaded manifest files = %v, want %v", rec.SnapshotFiles, want) @@ -1533,7 +1605,7 @@ func TestUploadLocalCheckpointDir(t *testing.T) { t.Run("full capture listing no data files is rejected", func(t *testing.T) { store := &recordingObjectStorage{} - s := &AteomHerder{gcsClient: store} + s := newPluginHerder(t, store) dir := filepath.Join(t.TempDir(), "pause-snap-1") rec := fullRec("microvm") rec.DataSnapshotFiles = nil @@ -1542,7 +1614,7 @@ func TestUploadLocalCheckpointDir(t *testing.T) { }) req := validUploadPausedCheckpointRequest() - req.DesiredScope = ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA + req.DesiredFidelity = ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES _, err := s.uploadLocalCheckpointDir(ctx, req, dir, uri) if got := apierror.Code(err); got != codes.FailedPrecondition { t.Fatalf("status.Code = %v (err %v), want FailedPrecondition", got, err) @@ -1553,13 +1625,13 @@ func TestUploadLocalCheckpointDir(t *testing.T) { }) t.Run("data capture cannot become full", func(t *testing.T) { - s := &AteomHerder{gcsClient: &recordingObjectStorage{}} + s := newPluginHerder(t, &recordingObjectStorage{}) dir := filepath.Join(t.TempDir(), "pause-snap-1") writeLocalSnapshot(t, dir, sandboxAssetsRecord{ SandboxClass: "microvm", PauseImage: testPauseImage, SnapshotFiles: []string{"data.tar"}, - Scope: ateattr.SnapshotScopeData, + Fidelity: ateattr.SnapshotFidelityVolumes, }, map[string]string{"data.tar": "data"}) _, err := s.uploadLocalCheckpointDir(ctx, validUploadPausedCheckpointRequest(), dir, uri) @@ -1570,7 +1642,7 @@ func TestUploadLocalCheckpointDir(t *testing.T) { t.Run("manifest without scope is rejected", func(t *testing.T) { store := &recordingObjectStorage{} - s := &AteomHerder{gcsClient: store} + s := newPluginHerder(t, store) dir := filepath.Join(t.TempDir(), "pause-snap-1") writeLocalSnapshot(t, dir, sandboxAssetsRecord{ SandboxClass: "microvm", @@ -1579,7 +1651,7 @@ func TestUploadLocalCheckpointDir(t *testing.T) { }, map[string]string{"data.tar": "data"}) req := validUploadPausedCheckpointRequest() - req.DesiredScope = ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA + req.DesiredFidelity = ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES _, err := s.uploadLocalCheckpointDir(ctx, req, dir, uri) if got := apierror.Code(err); got != codes.FailedPrecondition { t.Fatalf("status.Code = %v (err %v), want FailedPrecondition for a scope-less manifest", got, err) @@ -1593,7 +1665,7 @@ func TestUploadLocalCheckpointDir(t *testing.T) { store := &recordingObjectStorage{objects: map[string][]byte{ pausedSnapshotPath + "/manifest.json": []byte(`{"sandboxClass":"microvm"}`), }} - s := &AteomHerder{gcsClient: store} + s := newPluginHerder(t, store) if _, err := s.uploadLocalCheckpointDir(ctx, validUploadPausedCheckpointRequest(), filepath.Join(t.TempDir(), "never-created"), uri); err != nil { t.Fatalf("uploadLocalCheckpointDir: %v", err) @@ -1601,7 +1673,7 @@ func TestUploadLocalCheckpointDir(t *testing.T) { }) t.Run("gone locally and remotely crashes the actor", func(t *testing.T) { - s := &AteomHerder{gcsClient: &recordingObjectStorage{}} + s := newPluginHerder(t, &recordingObjectStorage{}) _, err := s.uploadLocalCheckpointDir(ctx, validUploadPausedCheckpointRequest(), filepath.Join(t.TempDir(), "never-created"), uri) if err == nil { @@ -1613,7 +1685,7 @@ func TestUploadLocalCheckpointDir(t *testing.T) { }) t.Run("upload failure returns the error", func(t *testing.T) { - s := &AteomHerder{gcsClient: &recordingObjectStorage{putErr: errors.New("boom")}} + s := newPluginHerder(t, &recordingObjectStorage{putErr: errors.New("boom")}) dir := filepath.Join(t.TempDir(), "pause-snap-1") writeLocalSnapshot(t, dir, fullRec("microvm"), map[string]string{ "config.json": "cfg", "memory-ranges": "mem", "data.tar": "data", @@ -1633,8 +1705,8 @@ func TestValidateUploadPausedCheckpointRequest(t *testing.T) { wantErr bool }{ {"valid", func(*ateletpb.UploadPausedCheckpointRequest) {}, false}, - {"valid data scope", func(r *ateletpb.UploadPausedCheckpointRequest) { - r.DesiredScope = ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA + {"valid volumes fidelity", func(r *ateletpb.UploadPausedCheckpointRequest) { + r.DesiredFidelity = ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES }, false}, {"invalid atespace", func(r *ateletpb.UploadPausedCheckpointRequest) { r.Atespace = "../escape" }, true}, {"golden atespace rejected", func(r *ateletpb.UploadPausedCheckpointRequest) { r.Atespace = resources.GoldenActorAtespace }, true}, @@ -1646,8 +1718,11 @@ func TestValidateUploadPausedCheckpointRequest(t *testing.T) { }, false}, {"invalid snapshot name", func(r *ateletpb.UploadPausedCheckpointRequest) { r.LocalSnapshotName = "../escape" }, true}, {"invalid snapshot uri", func(r *ateletpb.UploadPausedCheckpointRequest) { r.DestinationSnapshotUri = "not-a-uri" }, true}, - {"unspecified scope", func(r *ateletpb.UploadPausedCheckpointRequest) { - r.DesiredScope = ateletpb.SnapshotScope_SNAPSHOT_SCOPE_UNSPECIFIED + {"unspecified fidelity", func(r *ateletpb.UploadPausedCheckpointRequest) { + r.DesiredFidelity = ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED + }, true}, + {"rootfs fidelity not supported yet", func(r *ateletpb.UploadPausedCheckpointRequest) { + r.DesiredFidelity = ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS }, true}, } for _, tc := range tests { @@ -1670,14 +1745,14 @@ func TestShouldHaveSnapshots(t *testing.T) { { name: "full scope always expects snapshots", req: &ateletpb.CheckpointRequest{ - Scope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL, + Fidelity: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, }, want: true, }, { name: "data scope with durable volumes expects snapshots", req: &ateletpb.CheckpointRequest{ - Scope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA, + Fidelity: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, Spec: &ateletpb.WorkloadSpec{ Volumes: []*ateletpb.Volume{ {Name: "durable", Source: &ateletpb.Volume_DurableDir{DurableDir: &ateletpb.DurableDirVolume{}}}, @@ -1689,7 +1764,7 @@ func TestShouldHaveSnapshots(t *testing.T) { { name: "data scope with only CSI volumes does not expect snapshots", req: &ateletpb.CheckpointRequest{ - Scope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA, + Fidelity: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, Spec: &ateletpb.WorkloadSpec{ Volumes: []*ateletpb.Volume{ {Name: "csi", Source: &ateletpb.Volume_External{External: &ateletpb.ExternalVolumeSource{}}}, @@ -1701,7 +1776,7 @@ func TestShouldHaveSnapshots(t *testing.T) { { name: "data scope with both durable and CSI volumes expects snapshots", req: &ateletpb.CheckpointRequest{ - Scope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA, + Fidelity: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, Spec: &ateletpb.WorkloadSpec{ Volumes: []*ateletpb.Volume{ {Name: "durable", Source: &ateletpb.Volume_DurableDir{DurableDir: &ateletpb.DurableDirVolume{}}}, @@ -1714,8 +1789,8 @@ func TestShouldHaveSnapshots(t *testing.T) { { name: "data scope with no volumes does not expect snapshots", req: &ateletpb.CheckpointRequest{ - Scope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA, - Spec: &ateletpb.WorkloadSpec{}, + Fidelity: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, + Spec: &ateletpb.WorkloadSpec{}, }, want: false, }, diff --git a/cmd/atelet/metrics.go b/cmd/atelet/metrics.go index b3dc99ef54..b7ae516955 100644 --- a/cmd/atelet/metrics.go +++ b/cmd/atelet/metrics.go @@ -77,7 +77,7 @@ type snapshotOp struct { templateNamespace string templateName string kind string - scope string + fidelity string sandboxClass string } @@ -89,7 +89,7 @@ func (o snapshotOp) attrs() []attribute.KeyValue { attrs = append(attrs, ateattr.TemplateAtespaceKey.String(o.templateNamespace), ateattr.TemplateNameKey.String(o.templateName), - ateattr.SnapshotScopeKey.String(o.scope), + ateattr.SnapshotFidelityKey.String(o.fidelity), ) if o.kind != "" { attrs = append(attrs, ateattr.SnapshotKindKey.String(o.kind)) diff --git a/cmd/atelet/metrics_test.go b/cmd/atelet/metrics_test.go index 83d98c33d3..5c58a7ed3b 100644 --- a/cmd/atelet/metrics_test.go +++ b/cmd/atelet/metrics_test.go @@ -102,7 +102,7 @@ func TestRestoreDurationShape(t *testing.T) { templateNamespace: testTemplateNamespace, templateName: testTemplateName, kind: ateattr.SnapshotKindLatest, - scope: ateattr.SnapshotScopeData, + fidelity: ateattr.SnapshotFidelityVolumes, sandboxClass: "gvisor", } inst.recordRestore(context.Background(), op, @@ -129,7 +129,7 @@ func TestRestoreDurationShape(t *testing.T) { {ateattr.TemplateAtespaceKey, testTemplateNamespace}, {ateattr.TemplateNameKey, testTemplateName}, {ateattr.SnapshotKindKey, ateattr.SnapshotKindLatest}, - {ateattr.SnapshotScopeKey, ateattr.SnapshotScopeData}, + {ateattr.SnapshotFidelityKey, ateattr.SnapshotFidelityVolumes}, {ateattr.SandboxClassKey, "gvisor"}, } { if v := attrString(t, got, tc.key); v != tc.want { @@ -151,7 +151,7 @@ func TestCheckpointDurationShape(t *testing.T) { templateNamespace: testTemplateNamespace, templateName: testTemplateName, kind: ateattr.SnapshotKindLocal, - scope: ateattr.SnapshotScopeFull, + fidelity: ateattr.SnapshotFidelityMemory, sandboxClass: "microvm", }, phase{ateattr.SnapshotPhasePersist, time.Second}) @@ -174,7 +174,7 @@ func TestRecordPhasesSkipsZeroPhases(t *testing.T) { inst, reader := newTestInstruments(t) inst.recordRestore(context.Background(), - snapshotOp{scope: ateattr.SnapshotScopeFull}, + snapshotOp{fidelity: ateattr.SnapshotFidelityMemory}, phase{ateattr.SnapshotPhaseManifestFetch, 50 * time.Millisecond}, phase{ateattr.SnapshotPhaseDownload, 2 * time.Second}, phase{ateattr.SnapshotPhaseAteomRestore, 0}, @@ -196,7 +196,7 @@ func TestSnapshotOpAttrsOmitsUnknownDimensions(t *testing.T) { attrs := snapshotOp{ templateNamespace: testTemplateNamespace, templateName: testTemplateName, - scope: ateattr.SnapshotScopeFull, + fidelity: ateattr.SnapshotFidelityMemory, }.attrs() for _, kv := range attrs { if kv.Key == ateattr.SnapshotKindKey || kv.Key == ateattr.SandboxClassKey { diff --git a/cmd/atelet/oci.go b/cmd/atelet/oci.go index f82c279d7c..21bbb0040f 100644 --- a/cmd/atelet/oci.go +++ b/cmd/atelet/oci.go @@ -24,7 +24,6 @@ import ( "github.com/agent-substrate/substrate/cmd/atelet/internal/ateletpath" "github.com/agent-substrate/substrate/internal/imagecache" - "github.com/agent-substrate/substrate/internal/ocispec" "github.com/agent-substrate/substrate/internal/proto/ateletpb" v1 "github.com/google/go-containerregistry/pkg/v1" "go.opentelemetry.io/otel" @@ -75,7 +74,7 @@ func resolveCapabilities(caps *ateletpb.Capabilities) []string { return out } -func prepareOCIDirectory(ctx context.Context, imageCache *imagecache.Store, actorUID, containerName, ref string, command, args []string, env []string, netns string, volumes []*ateletpb.Volume, volumeMounts []*ateletpb.VolumeMount, capabilities []string, resources *ateletpb.ResourceLimits) error { +func prepareOCIDirectory(ctx context.Context, imageCache *imagecache.Store, actorUID, containerName, ref string, volumes []*ateletpb.Volume, volumeMounts []*ateletpb.VolumeMount) (*v1.Config, error) { tracer := otel.Tracer("prepareOCIDirectory") ctx, span := tracer.Start(ctx, "prepareOCIDirectory") @@ -87,7 +86,7 @@ func prepareOCIDirectory(ctx context.Context, imageCache *imagecache.Store, acto // Clear any previous bundle contents (belt and suspenders: resetActorDirs // already wiped the bundle dir on the Run/Restore path). if err := imagecache.RemoveAllWritable(bundlePath); err != nil { - return fmt.Errorf("while clearing bundle %q: %w", bundlePath, err) + return nil, fmt.Errorf("while clearing bundle %q: %w", bundlePath, err) } // The bundle's rootfs is composed by ateom as an overlay mount just before @@ -98,7 +97,7 @@ func prepareOCIDirectory(ctx context.Context, imageCache *imagecache.Store, acto // it deliberately runs with no capabilities, so it cannot mount. for _, d := range []string{"rootfs", "upper", "work"} { if err := os.MkdirAll(path.Join(bundlePath, d), 0o700); err != nil { - return fmt.Errorf("in os.MkdirAll for container bundle dir: %w", err) + return nil, fmt.Errorf("in os.MkdirAll for container bundle dir: %w", err) } } @@ -120,16 +119,8 @@ func prepareOCIDirectory(ctx context.Context, imageCache *imagecache.Store, acto return err }) if err := g.Wait(); err != nil { - return err - } - - // Argv and env need only the image config; resolve them before writing - // any spec so an invalid container config fails fast. - resolvedArgs, err := resolveProcessArgs(&img.Config, command, args) - if err != nil { - return fmt.Errorf("while resolving process args for container %q: %w", containerName, err) + return nil, err } - resolvedEnv := resolveActorEnv(&img.Config, env) // Every bind target must exist in the rootfs for the mount to attach; // ateom creates them through the mounted overlay (they land in the @@ -144,27 +135,10 @@ func prepareOCIDirectory(ctx context.Context, imageCache *imagecache.Store, acto ExtraDirs: extraDirs, ImageVolumes: imageVolumes, }); err != nil { - return fmt.Errorf("while writing overlay spec: %w", err) - } - - // Write the runtime-neutral OCI spec to config.json. - if err := ocispec.Save(bundlePath, ocispec.Build(ocispec.Options{ - Args: resolvedArgs, - Env: resolvedEnv, - NetNSPath: netns, - Volumes: volumes, - VolumeMounts: volumeMounts, - Capabilities: capabilities, - Resources: resources, - DurableDirVolumeMountsDir: ateletpath.DurableDirVolumeMountsDir(actorUID), - VolumesDir: ateletpath.VolumesDir(actorUID), - SystemInfoVolumeRootsDir: ateletpath.SystemInfoVolumeRootsDir(actorUID), - BundlePath: bundlePath, - })); err != nil { - return fmt.Errorf("while writing OCI spec: %w", err) + return nil, fmt.Errorf("while writing overlay spec: %w", err) } - return nil + return &img.Config, nil } // resolveImageVolumes pulls the image behind every image-typed volume this diff --git a/cmd/atelet/sandbox_assets.go b/cmd/atelet/sandbox_assets.go index 6b291371f9..0e59dd5852 100644 --- a/cmd/atelet/sandbox_assets.go +++ b/cmd/atelet/sandbox_assets.go @@ -101,14 +101,14 @@ type sandboxAssetsRecord struct { // on-node record written at Run/Restore; populated at Checkpoint. SnapshotFiles []string `json:"snapshotFiles,omitempty"` // DataSnapshotFiles is the subset of SnapshotFiles that restores the actor - // at DATA scope on its own, as reported by CheckpointWorkloadResponse. + // at VOLUMES fidelity on its own, as reported by CheckpointWorkloadResponse. // Empty when the capture holds no durable data. DataSnapshotFiles []string `json:"dataSnapshotFiles,omitempty"` - // Scope is the snapshot scope the checkpoint captured, as the shared - // ateattr label ("full" or "data"), so a snapshot's content is knowable - // from the manifest alone. Empty in the on-node record written at - // Run/Restore and in snapshot manifests written before this field existed. - Scope string `json:"scope,omitempty"` + // Fidelity is the snapshot fidelity the checkpoint captured, as the shared + // ateattr label ("memory" or "volumes"), so a snapshot's content is + // knowable from the manifest alone. Empty in the on-node record written at + // Run/Restore. + Fidelity string `json:"fidelity,omitempty"` } // recordFromRequest projects a request's per-architecture SandboxAssets onto the @@ -516,17 +516,13 @@ func unmarshalSandboxRecord(data []byte) (*sandboxAssetsRecord, error) { // Actual file access must still use os.Root so symlinks cannot escape that // directory. func validateSnapshotFiles(files []string) error { - seen := make(map[string]bool, len(files)) + if err := resources.ValidateSnapshotFileNames(files); err != nil { + return err + } for i, name := range files { - switch { - case name != filepath.Base(name) || !filepath.IsLocal(name) || name == ".": - return fmt.Errorf("snapshotFiles[%d] %q is not a file name in the checkpoint directory", i, name) - case name == sandboxManifestName: + if name == sandboxManifestName { return fmt.Errorf("snapshotFiles[%d] %q is reserved for the snapshot manifest", i, name) - case seen[name]: - return fmt.Errorf("snapshotFiles[%d] %q is duplicated", i, name) } - seen[name] = true } return nil } diff --git a/cmd/atelet/sandbox_prewarm.go b/cmd/atelet/sandbox_prewarm.go index deaa413057..1395643cea 100644 --- a/cmd/atelet/sandbox_prewarm.go +++ b/cmd/atelet/sandbox_prewarm.go @@ -257,18 +257,27 @@ func (p *sandboxPrewarmer) prewarm(ctx context.Context, cfg *v1alpha1.SandboxCon defer cancel() t := time.Now() + // TODO: prewarm every Enabled version, not just the default. + ver, ok := cfg.Spec.DefaultVersionConfig() + if !ok { + // The CRD rejects this shape, so this should not be reachable. + slog.WarnContext(ctx, "Skipping sandbox asset prewarm: config has no enabled default version", + slog.String("config", cfg.Name), slog.String("defaultVersion", cfg.Spec.DefaultVersion)) + return nil + } + var imageErr error var wg sync.WaitGroup // schedule prewarm pause image if provided - if cfg.Spec.PauseImage != "" { + if ver.PauseImage != "" { wg.Go(func() { - if _, err := p.images.EnsureImage(ctx, cfg.Spec.PauseImage); err != nil { - imageErr = fmt.Errorf("while prewarming pause image %q: %w", cfg.Spec.PauseImage, err) + if _, err := p.images.EnsureImage(ctx, ver.PauseImage); err != nil { + imageErr = fmt.Errorf("while prewarming pause image %q: %w", ver.PauseImage, err) } }) } // schedule prewarm sandbox assets if provided - rec, assetErr := recordFromSandboxConfig(cfg) + rec, assetErr := recordFromSandboxConfig(cfg, ver) switch { case errors.Is(assetErr, errNoAssetsForArch): // Permanent until the config changes, and a change re-enqueues: @@ -292,8 +301,9 @@ func (p *sandboxPrewarmer) prewarm(ctx context.Context, cfg *v1alpha1.SandboxCon } slog.InfoContext(ctx, "Sandbox assets prewarmed", slog.String("config", cfg.Name), + slog.String("version", ver.Name), slog.Int("assets", assets), - slog.String("pauseImage", cfg.Spec.PauseImage), + slog.String("pauseImage", ver.PauseImage), slog.Duration("duration", time.Since(t))) return nil } @@ -303,17 +313,18 @@ func (p *sandboxPrewarmer) prewarm(ctx context.Context, cfg *v1alpha1.SandboxCon // config changes, so prewarm treats it as nothing-to-do rather than retrying. var errNoAssetsForArch = errors.New("no sandbox assets for this architecture") -// recordFromSandboxConfig projects a SandboxConfig's per-architecture assets -// onto the local node's architecture, mirroring recordFromRequest. -func recordFromSandboxConfig(cfg *v1alpha1.SandboxConfig) (*sandboxAssetsRecord, error) { +// recordFromSandboxConfig projects the per-architecture assets of one version +// of a SandboxConfig onto the local node's architecture, mirroring +// recordFromRequest. +func recordFromSandboxConfig(cfg *v1alpha1.SandboxConfig, ver *v1alpha1.SandboxVersionConfig) (*sandboxAssetsRecord, error) { arch := runtime.GOARCH - files := cfg.Spec.Assets[arch] + files := ver.Assets[arch] if len(files) == 0 { - return nil, fmt.Errorf("sandbox config %q, architecture %q: %w", cfg.Name, arch, errNoAssetsForArch) + return nil, fmt.Errorf("sandbox config %q, version %q, architecture %q: %w", cfg.Name, ver.Name, arch, errNoAssetsForArch) } rec := &sandboxAssetsRecord{ SandboxClass: string(cfg.Spec.SandboxClass), - PauseImage: cfg.Spec.PauseImage, + PauseImage: ver.PauseImage, Assets: make(map[string]assetEntry, len(files)), } for name, f := range files { diff --git a/cmd/atelet/sandbox_prewarm_test.go b/cmd/atelet/sandbox_prewarm_test.go index 283d4c4cd0..7ff74a939e 100644 --- a/cmd/atelet/sandbox_prewarm_test.go +++ b/cmd/atelet/sandbox_prewarm_test.go @@ -30,6 +30,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/cache" + "k8s.io/utils/ptr" "github.com/agent-substrate/substrate/cmd/atelet/internal/ateletpath" "github.com/agent-substrate/substrate/internal/imagecache" @@ -72,13 +73,17 @@ func gvisorConfig(name, url, sha string) *v1alpha1.SandboxConfig { return &v1alpha1.SandboxConfig{ ObjectMeta: metav1.ObjectMeta{Name: name}, Spec: v1alpha1.SandboxConfigSpec{ - SandboxClass: v1alpha1.SandboxClassGvisor, - PauseImage: "registry.k8s.io/pause@sha256:abc", - Assets: map[string]map[string]v1alpha1.AssetFile{ - runtime.GOARCH: { - runscAssetName: {URL: url, SHA256: sha}, + SandboxClass: v1alpha1.SandboxClassGvisor, + DefaultVersion: "v1", + Versions: []v1alpha1.SandboxVersionConfig{{ + Name: "v1", + PauseImage: "registry.k8s.io/pause@sha256:abc", + Assets: map[string]map[string]v1alpha1.AssetFile{ + runtime.GOARCH: { + runscAssetName: {URL: url, SHA256: sha}, + }, }, - }, + }}, }, } } @@ -87,15 +92,15 @@ func TestRecordFromSandboxConfig(t *testing.T) { sha := fmt.Sprintf("%x", sha256.Sum256([]byte("runsc"))) cfg := gvisorConfig("gvisor-default", "gs://bucket/runsc", sha) - rec, err := recordFromSandboxConfig(cfg) + rec, err := recordFromSandboxConfig(cfg, &cfg.Spec.Versions[0]) if err != nil { t.Fatalf("recordFromSandboxConfig: %v", err) } if rec.SandboxClass != string(v1alpha1.SandboxClassGvisor) { t.Errorf("SandboxClass = %q, want %q", rec.SandboxClass, v1alpha1.SandboxClassGvisor) } - if rec.PauseImage != cfg.Spec.PauseImage { - t.Errorf("PauseImage = %q, want %q", rec.PauseImage, cfg.Spec.PauseImage) + if rec.PauseImage != cfg.Spec.Versions[0].PauseImage { + t.Errorf("PauseImage = %q, want %q", rec.PauseImage, cfg.Spec.Versions[0].PauseImage) } want := assetEntry{URL: "gs://bucket/runsc", SHA256: sha} if got := rec.Assets[runscAssetName]; got != want { @@ -105,10 +110,10 @@ func TestRecordFromSandboxConfig(t *testing.T) { // A config with no assets for this node's architecture cannot be // projected, and the error carries the sentinel that keeps prewarm from // retrying a condition only a config change can clear. - cfg.Spec.Assets = map[string]map[string]v1alpha1.AssetFile{ + cfg.Spec.Versions[0].Assets = map[string]map[string]v1alpha1.AssetFile{ "other-arch": {runscAssetName: {URL: "gs://bucket/runsc", SHA256: sha}}, } - if _, err := recordFromSandboxConfig(cfg); !errors.Is(err, errNoAssetsForArch) { + if _, err := recordFromSandboxConfig(cfg, &cfg.Spec.Versions[0]); !errors.Is(err, errNoAssetsForArch) { t.Errorf("recordFromSandboxConfig with no assets for the local architecture = %v, want errNoAssetsForArch", err) } } @@ -170,7 +175,7 @@ func TestPrewarmProcessRetries(t *testing.T) { ctx := context.Background() cfg := gvisorConfig("gvisor-default", "gs://bucket/runsc", fmt.Sprintf("%x", sha256.Sum256([]byte("runsc")))) // No pause image, so a failing prewarm exercises only the asset path. - cfg.Spec.PauseImage = "" + cfg.Spec.Versions[0].PauseImage = "" indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{}) if err := indexer.Add(cfg); err != nil { @@ -268,7 +273,7 @@ func TestPrewarmPauseImage(t *testing.T) { ctx := context.Background() content := []byte("runsc binary bytes") cfg := gvisorConfig("gvisor-default", "gs://bucket/runsc", fmt.Sprintf("%x", sha256.Sum256(content))) - cfg.Spec.PauseImage = pauseRef + cfg.Spec.Versions[0].PauseImage = pauseRef p := &sandboxPrewarmer{ assets: &AteomHerder{anonGCSClient: fakeObjectStorage{data: content}}, @@ -282,7 +287,7 @@ func TestPrewarmPauseImage(t *testing.T) { // failing object storage is actually consulted — and must not keep the // pause image from being pulled. failCfg := gvisorConfig("gvisor-default", "gs://bucket/runsc", fmt.Sprintf("%x", sha256.Sum256([]byte("other runsc")))) - failCfg.Spec.PauseImage = pauseRef + failCfg.Spec.Versions[0].PauseImage = pauseRef p = &sandboxPrewarmer{ assets: &AteomHerder{anonGCSClient: fakeObjectStorage{err: errors.New("bucket unavailable")}}, images: failStore, @@ -295,10 +300,10 @@ func TestPrewarmPauseImage(t *testing.T) { // pause image pulled, and prewarm succeeds: the missing assets are // permanent until the config changes, not a retryable failure. archCfg := gvisorConfig("gvisor-default", "gs://bucket/runsc", fmt.Sprintf("%x", sha256.Sum256(content))) - archCfg.Spec.Assets = map[string]map[string]v1alpha1.AssetFile{ - "other-arch": {runscAssetName: archCfg.Spec.Assets[runtime.GOARCH][runscAssetName]}, + archCfg.Spec.Versions[0].Assets = map[string]map[string]v1alpha1.AssetFile{ + "other-arch": {runscAssetName: archCfg.Spec.Versions[0].Assets[runtime.GOARCH][runscAssetName]}, } - archCfg.Spec.PauseImage = pauseRef + archCfg.Spec.Versions[0].PauseImage = pauseRef p = &sandboxPrewarmer{images: archStore} if err := p.prewarm(ctx, archCfg); err != nil { t.Errorf("prewarm with no assets for the local architecture: %v", err) @@ -335,7 +340,7 @@ func TestPrewarmTimeout(t *testing.T) { t.Cleanup(func() { nodepath.StaticFilesDir, prewarmTimeout = origDir, origTimeout }) cfg := gvisorConfig("gvisor-default", "gs://bucket/runsc", fmt.Sprintf("%x", sha256.Sum256([]byte("hung runsc")))) - cfg.Spec.PauseImage = "" + cfg.Spec.Versions[0].PauseImage = "" p := &sandboxPrewarmer{assets: &AteomHerder{anonGCSClient: hangingObjectStorage{}}} done := make(chan error, 1) @@ -350,6 +355,49 @@ func TestPrewarmTimeout(t *testing.T) { } } +// TestPrewarmDefaultVersionOnly verifies prewarm fetches only the version +// defaultVersion names, and that a config without its default version, or +// whose default version is Disabled, is nothing to do rather than a retryable +// failure. +func TestPrewarmDefaultVersionOnly(t *testing.T) { + origDir := nodepath.StaticFilesDir + nodepath.StaticFilesDir = t.TempDir() + t.Cleanup(func() { nodepath.StaticFilesDir = origDir }) + + ctx := context.Background() + content := []byte("runsc binary bytes") + cfg := gvisorConfig("gvisor-default", "gs://bucket/runsc", fmt.Sprintf("%x", sha256.Sum256(content))) + cfg.Spec.Versions[0].PauseImage = "" + // A non-default version whose asset the fake bucket cannot serve: fetching + // it would fail the hash check. + otherSHA := fmt.Sprintf("%x", sha256.Sum256([]byte("other runsc"))) + cfg.Spec.Versions = append([]v1alpha1.SandboxVersionConfig{{ + Name: "v0", + Assets: map[string]map[string]v1alpha1.AssetFile{runtime.GOARCH: {runscAssetName: {URL: "gs://bucket/other-runsc", SHA256: otherSHA}}}, + }}, cfg.Spec.Versions...) + + p := &sandboxPrewarmer{assets: &AteomHerder{anonGCSClient: fakeObjectStorage{data: content}}} + if err := p.prewarm(ctx, cfg); err != nil { + t.Fatalf("prewarm: %v", err) + } + if _, err := os.Stat(ateletpath.RunSCBinaryPath(otherSHA)); !errors.Is(err, os.ErrNotExist) { + t.Errorf("non-default version asset was prewarmed (stat err = %v)", err) + } + + // Any fetch would fail, so a nil error means nothing was attempted. + cfg.Spec.DefaultVersion = "missing" + p = &sandboxPrewarmer{assets: &AteomHerder{anonGCSClient: fakeObjectStorage{err: errors.New("bucket unavailable")}}} + if err := p.prewarm(ctx, cfg); err != nil { + t.Errorf("prewarm of a config without its default version: %v", err) + } + + cfg.Spec.DefaultVersion = "v0" + cfg.Spec.Versions[0].State = ptr.To(v1alpha1.SandboxVersionStateDisabled) + if err := p.prewarm(ctx, cfg); err != nil { + t.Errorf("prewarm of a config whose default version is Disabled: %v", err) + } +} + // TestSandboxAssetPrewarmDownloads runs the whole path: a SandboxConfig in a // fake clientset flows through the informer into the prewarm worker, which // lands the asset in the static-files cache without any Run/Restore request. @@ -362,7 +410,7 @@ func TestSandboxAssetPrewarmDownloads(t *testing.T) { content := []byte("runsc binary bytes") sha := fmt.Sprintf("%x", sha256.Sum256(content)) cfg := gvisorConfig("gvisor-default", "gs://bucket/runsc", sha) - cfg.Spec.PauseImage = "" + cfg.Spec.Versions[0].PauseImage = "" ctx := t.Context() client := fake.NewSimpleClientset(cfg) diff --git a/cmd/atelet/snapshot_transfer.go b/cmd/atelet/snapshot_transfer.go new file mode 100644 index 0000000000..485490d1b5 --- /dev/null +++ b/cmd/atelet/snapshot_transfer.go @@ -0,0 +1,100 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "fmt" + "os" + "path/filepath" + + "github.com/agent-substrate/substrate/internal/objectstoreplugin" + objectstorev1 "github.com/agent-substrate/substrate/pkg/proto/objectstorepb/v1" + "google.golang.org/grpc" +) + +// External snapshots move between the node and storage only through the +// snapshot plugin. These helpers adapt the plugin's file-oriented API to the +// manifest bytes atelet works with. + +// fetchSnapshotFiles downloads the named snapshot files into dstDir. +func (s *AteomHerder) fetchSnapshotFiles(ctx context.Context, snapshotURI, dstDir string, files []string) error { + if len(files) == 0 { + return nil + } + _, err := s.snapshotPlugin.FetchSnapshot(ctx, &objectstorev1.FetchSnapshotRequest{ + SnapshotUri: snapshotURI, + WritePath: dstDir, + Files: files, + }) + return objectstoreplugin.CallError(err) +} + +// uploadSnapshotFiles uploads the named files in srcDir to the snapshot, +// passing opts to the plugin call. It returns the plugin's error as is: an +// upload that can be retried reports it through objectstoreplugin.CallError, +// one that cannot does not. +func (s *AteomHerder) uploadSnapshotFiles(ctx context.Context, snapshotURI, srcDir string, files []string, opts ...grpc.CallOption) error { + if len(files) == 0 { + return nil + } + _, err := s.snapshotPlugin.UploadSnapshot(ctx, &objectstorev1.UploadSnapshotRequest{ + SnapshotUri: snapshotURI, + LocalPath: srcDir, + Files: files, + }, opts...) + return err +} + +// fetchManifest returns a snapshot's manifest. The returned error is the +// plugin's gRPC status, so a missing manifest reports codes.NotFound. +func (s *AteomHerder) fetchManifest(ctx context.Context, snapshotURI string) ([]byte, error) { + dir, err := s.snapshotScratch() + if err != nil { + return nil, err + } + defer func() { _ = os.RemoveAll(dir) }() + if err := s.fetchSnapshotFiles(ctx, snapshotURI, dir, []string{sandboxManifestName}); err != nil { + return nil, err + } + return os.ReadFile(filepath.Join(dir, sandboxManifestName)) +} + +// uploadManifest uploads manifest as the snapshot's manifest, as +// uploadSnapshotFiles does. +func (s *AteomHerder) uploadManifest(ctx context.Context, snapshotURI string, manifest []byte, opts ...grpc.CallOption) error { + dir, err := s.snapshotScratch() + if err != nil { + return err + } + defer func() { _ = os.RemoveAll(dir) }() + if err := os.WriteFile(filepath.Join(dir, sandboxManifestName), manifest, 0o600); err != nil { + return fmt.Errorf("while staging snapshot manifest: %w", err) + } + return s.uploadSnapshotFiles(ctx, snapshotURI, dir, []string{sandboxManifestName}, opts...) +} + +// snapshotScratch creates a fresh directory for one manifest transfer. The +// caller removes it. +func (s *AteomHerder) snapshotScratch() (string, error) { + if err := os.MkdirAll(s.snapshotScratchDir, 0o700); err != nil { + return "", fmt.Errorf("while creating snapshot scratch dir: %w", err) + } + dir, err := os.MkdirTemp(s.snapshotScratchDir, "manifest-") + if err != nil { + return "", fmt.Errorf("while creating snapshot scratch dir: %w", err) + } + return dir, nil +} diff --git a/cmd/atelet/snapshot_transfer_test.go b/cmd/atelet/snapshot_transfer_test.go new file mode 100644 index 0000000000..82d7c3ca94 --- /dev/null +++ b/cmd/atelet/snapshot_transfer_test.go @@ -0,0 +1,368 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "path/filepath" + "slices" + "strings" + "sync" + "testing" + "time" + + "github.com/agent-substrate/substrate/internal/apierror" + "github.com/agent-substrate/substrate/internal/ateattr" + "github.com/agent-substrate/substrate/internal/ateinterceptors" + "github.com/agent-substrate/substrate/internal/objectstoreplugin" + "github.com/agent-substrate/substrate/internal/objectstoreplugin/objectstoreplugintest" + "github.com/agent-substrate/substrate/internal/resources" + "github.com/agent-substrate/substrate/pkg/objectstorage" + objectstorev1 "github.com/agent-substrate/substrate/pkg/proto/objectstorepb/v1" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/health" + healthpb "google.golang.org/grpc/health/grpc_health_v1" + "google.golang.org/grpc/status" +) + +// newPluginHerder returns an AteomHerder whose snapshots go through the +// object-store plugin backed by store. +func newPluginHerder(t *testing.T, store objectstorage.ObjectStorage) *AteomHerder { + t.Helper() + // Test directories come from t.TempDir, so admit any absolute path. + plugin, err := objectstoreplugin.NewNodePlugin(store, "/") + if err != nil { + t.Fatal(err) + } + return &AteomHerder{ + snapshotPlugin: objectstoreplugintest.NodeClient(plugin), + snapshotScratchDir: t.TempDir(), + } +} + +// orderedObjectStorage records the order objects are stored in. +type orderedObjectStorage struct { + mu sync.Mutex + puts []string + data map[string][]byte +} + +func (o *orderedObjectStorage) PutObject(_ context.Context, bucket, object string, r io.Reader) error { + b, err := io.ReadAll(r) + if err != nil { + return err + } + o.mu.Lock() + defer o.mu.Unlock() + o.puts = append(o.puts, object) + if o.data == nil { + o.data = map[string][]byte{} + } + o.data[bucket+"/"+object] = b + return nil +} + +func (o *orderedObjectStorage) GetObject(_ context.Context, bucket, object string) (io.ReadCloser, error) { + o.mu.Lock() + defer o.mu.Unlock() + b, ok := o.data[bucket+"/"+object] + if !ok { + return nil, fmt.Errorf("%w: %s/%s", objectstorage.ErrObjectNotFound, bucket, object) + } + return io.NopCloser(bytes.NewReader(b)), nil +} + +func TestUploadSnapshotManifestLast(t *testing.T) { + store := &orderedObjectStorage{} + s := newPluginHerder(t, store) + uri, err := resources.ParseSnapshotURI(pausedSnapshotURI) + if err != nil { + t.Fatal(err) + } + dir := t.TempDir() + rec := sandboxAssetsRecord{SandboxClass: "gvisor", PauseImage: testPauseImage, SnapshotFiles: []string{"a", "b", "c"}} + for _, f := range rec.SnapshotFiles { + if err := os.WriteFile(dir+"/"+f, []byte(f), 0o600); err != nil { + t.Fatal(err) + } + } + + if err := s.uploadSnapshot(context.Background(), uri, dir, &rec, "ate-demo", "counter"); err != nil { + t.Fatalf("uploadSnapshot: %v", err) + } + if len(store.puts) != 4 { + t.Fatalf("puts = %v, want 3 files then the manifest", store.puts) + } + for i, p := range store.puts { + isManifest := strings.HasSuffix(p, "/"+sandboxManifestName) + if isManifest != (i == len(store.puts)-1) { + t.Fatalf("puts = %v, want the manifest last and only last", store.puts) + } + } + + // The manifest round-trips, and scratch space is cleaned up. + got, err := s.fetchManifest(context.Background(), uri.String()) + if err != nil { + t.Fatalf("fetchManifest: %v", err) + } + back, err := unmarshalSandboxRecord(got) + if err != nil { + t.Fatal(err) + } + if !slices.Equal(back.SnapshotFiles, rec.SnapshotFiles) { + t.Errorf("manifest files = %v, want %v", back.SnapshotFiles, rec.SnapshotFiles) + } + if entries, _ := os.ReadDir(s.snapshotScratchDir); len(entries) != 0 { + t.Errorf("scratch dir not cleaned up: %v", entries) + } +} + +func TestSnapshotTransferSkipsEmptyFileLists(t *testing.T) { + store := &orderedObjectStorage{} + s := newPluginHerder(t, store) + if err := s.downloadExternalCheckpoint(context.Background(), pausedSnapshotURI, t.TempDir(), nil); err != nil { + t.Errorf("downloadExternalCheckpoint(no files) = %v, want nil", err) + } + if err := s.uploadSnapshotFiles(context.Background(), pausedSnapshotURI, t.TempDir(), nil); err != nil { + t.Errorf("uploadSnapshotFiles(no files) = %v, want nil", err) + } + if len(store.puts) != 0 { + t.Errorf("objects stored = %v, want none", store.puts) + } +} + +// failingNodePlugin fails every call with err. +type failingNodePlugin struct { + err error +} + +func (p failingNodePlugin) FetchSnapshot(context.Context, *objectstorev1.FetchSnapshotRequest, ...grpc.CallOption) (*objectstorev1.FetchSnapshotResponse, error) { + return nil, p.err +} + +func (p failingNodePlugin) UploadSnapshot(context.Context, *objectstorev1.UploadSnapshotRequest, ...grpc.CallOption) (*objectstorev1.UploadSnapshotResponse, error) { + return nil, p.err +} + +// A snapshot plugin that cannot be reached reaches ate-api-server as +// Unavailable, which it retries, rather than Internal, which crashes the +// actor. The upload after CheckpointWorkload is the exception: a retried +// Checkpoint cannot redo it, so it stays Internal. Other plugin errors reach +// ate-api-server as Internal. +func TestSnapshotPluginErrorCodeThroughAtelet(t *testing.T) { + uri, err := resources.ParseSnapshotURI(pausedSnapshotURI) + if err != nil { + t.Fatal(err) + } + rec := sandboxAssetsRecord{ + SandboxClass: "gvisor", + PauseImage: testPauseImage, + SnapshotFiles: []string{"a"}, + Fidelity: ateattr.SnapshotFidelityMemory, + } + transfers := []struct { + name string + retryable bool + run func(ctx context.Context, s *AteomHerder) error + }{ + {"download", true, func(ctx context.Context, s *AteomHerder) error { + return s.downloadExternalCheckpoint(ctx, pausedSnapshotURI, t.TempDir(), []string{"a"}) + }}, + {"paused upload", true, func(ctx context.Context, s *AteomHerder) error { + dir := t.TempDir() + writeLocalSnapshot(t, dir, rec, map[string]string{"a": "a"}) + _, err := s.uploadLocalCheckpointDir(ctx, validUploadPausedCheckpointRequest(), dir, uri) + return err + }}, + {"checkpoint upload", false, func(ctx context.Context, s *AteomHerder) error { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "a"), []byte("a"), 0o600); err != nil { + return err + } + return s.uploadExternalCheckpoint(ctx, validCheckpointRequest(), dir, &rec) + }}, + } + for _, pluginCode := range []codes.Code{codes.Unavailable, codes.Internal, codes.NotFound} { + for _, tr := range transfers { + t.Run(pluginCode.String()+"/"+tr.name, func(t *testing.T) { + want := codes.Internal + if pluginCode == codes.Unavailable && tr.retryable { + want = codes.Unavailable + } + s := &AteomHerder{ + snapshotPlugin: failingNodePlugin{err: status.Error(pluginCode, "plugin failed")}, + snapshotScratchDir: t.TempDir(), + } + // atelet's server interceptor decides the code ate-api-server sees. + _, err := ateinterceptors.InternalServerUnaryInterceptor(context.Background(), nil, &grpc.UnaryServerInfo{FullMethod: "/test"}, + func(ctx context.Context, _ any) (any, error) { return nil, tr.run(ctx, s) }) + if got := status.Code(err); got != want { + t.Errorf("code = %s (%v), want %s", got, err, want) + } + }) + } + } +} + +// pluginSidecar serves a NodePlugin on a Unix socket and can be stopped and +// started again on the same path, as kubelet restarts a sidecar container. +type pluginSidecar struct { + t *testing.T + path string + plugin *objectstoreplugin.NodePlugin + + mu sync.Mutex + srv *grpc.Server +} + +func newPluginSidecar(t *testing.T, store objectstorage.ObjectStorage) *pluginSidecar { + t.Helper() + plugin, err := objectstoreplugin.NewNodePlugin(store, "/") + if err != nil { + t.Fatal(err) + } + // Unix socket paths are length-limited; t.TempDir can exceed that on macOS. + dir, err := os.MkdirTemp("", "snapplug") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { os.RemoveAll(dir) }) + s := &pluginSidecar{t: t, path: filepath.Join(dir, "plugin.sock"), plugin: plugin} + t.Cleanup(s.stop) + return s +} + +// start may run on any goroutine, so it reports failures with Error. +func (s *pluginSidecar) start() { + lis, err := objectstoreplugin.Listen(s.path) + if err != nil { + s.t.Errorf("Listen: %v", err) + return + } + srv := grpc.NewServer() + objectstorev1.RegisterNodeProviderServer(srv, s.plugin) + healthpb.RegisterHealthServer(srv, health.NewServer()) + go srv.Serve(lis) + s.mu.Lock() + s.srv = srv + s.mu.Unlock() +} + +// stop ends the server and every connection to it at once, as a crashed +// sidecar does. +func (s *pluginSidecar) stop() { + s.mu.Lock() + srv := s.srv + s.srv = nil + s.mu.Unlock() + if srv != nil { + srv.Stop() + } +} + +// The upload after CheckpointWorkload waits out a plugin outage longer than +// the ready wait, since a retried Checkpoint cannot redo it. The paused +// upload, which a retry can redo, still fails within the ready wait. +func TestCheckpointUploadWaitsOutPluginOutage(t *testing.T) { + const wait = 200 * time.Millisecond + store := &orderedObjectStorage{} + sidecar := newPluginSidecar(t, store) + sidecar.start() + conn, err := objectstoreplugin.Dial(sidecar.path, wait) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + if err := objectstoreplugin.WaitReady(ctx, conn); err != nil { + t.Fatalf("WaitReady = %v", err) + } + s := &AteomHerder{ + snapshotPlugin: objectstorev1.NewNodeProviderClient(conn), + snapshotScratchDir: t.TempDir(), + } + rec := sandboxAssetsRecord{ + SandboxClass: "gvisor", + PauseImage: testPauseImage, + SnapshotFiles: []string{"a"}, + Fidelity: ateattr.SnapshotFidelityMemory, + } + pausedDir := t.TempDir() + writeLocalSnapshot(t, pausedDir, rec, map[string]string{"a": "a"}) + checkpointDir := t.TempDir() + if err := os.WriteFile(filepath.Join(checkpointDir, "a"), []byte("a"), 0o600); err != nil { + t.Fatal(err) + } + uri, err := resources.ParseSnapshotURI(pausedSnapshotURI) + if err != nil { + t.Fatal(err) + } + + // Lose the plugin, and wait until the connection has seen it go so the + // uploads start on a lost connection rather than racing the loss. + sidecar.stop() + for conn.GetState() == connectivity.Ready { + if !conn.WaitForStateChange(ctx, connectivity.Ready) { + t.Fatal("connection still ready after the plugin stopped") + } + } + + if _, err := s.uploadLocalCheckpointDir(ctx, validUploadPausedCheckpointRequest(), pausedDir, uri); apierror.Code(err) != codes.Unavailable { + t.Fatalf("paused upload during the outage = %v, want %s", err, codes.Unavailable) + } + + const outage = 10 * wait + restarted := make(chan struct{}) + go func() { + defer close(restarted) + time.Sleep(outage) + sidecar.start() + }() + defer func() { <-restarted }() + + start := time.Now() + if err := s.uploadExternalCheckpoint(ctx, validCheckpointRequest(), checkpointDir, &rec); err != nil { + t.Fatalf("checkpoint upload across a plugin outage = %v, want success", err) + } + if elapsed := time.Since(start); elapsed < outage/2 { + t.Errorf("checkpoint upload succeeded after %s, want it to have waited out the %s outage", elapsed, outage) + } + store.mu.Lock() + defer store.mu.Unlock() + object := strings.TrimPrefix(testSnapshotPath, "bucket/") + if want := []string{object + "/a.zstd", object + "/" + sandboxManifestName}; !slices.Equal(store.puts, want) { + t.Errorf("objects stored = %v, want %v", store.puts, want) + } +} + +// fetchManifest keeps the plugin's NotFound readable, which the paused +// snapshot upload uses to tell a missing snapshot from a failed probe. +func TestFetchManifestKeepsNotFound(t *testing.T) { + s := &AteomHerder{ + snapshotPlugin: failingNodePlugin{err: status.Error(codes.NotFound, "no manifest")}, + snapshotScratchDir: t.TempDir(), + } + _, err := s.fetchManifest(context.Background(), pausedSnapshotURI) + if status.Code(err) != codes.NotFound { + t.Errorf("fetchManifest = %v, want %s", err, codes.NotFound) + } +} diff --git a/cmd/atelet/snapshotlog_test.go b/cmd/atelet/snapshotlog_test.go index a2b69424b4..0cf5276358 100644 --- a/cmd/atelet/snapshotlog_test.go +++ b/cmd/atelet/snapshotlog_test.go @@ -106,7 +106,7 @@ func TestSnapshotLogAttrsSeconds(t *testing.T) { templateNamespace: testTemplateNamespace, templateName: testTemplateName, kind: ateattr.SnapshotKindLatest, - scope: ateattr.SnapshotScopeFull, + fidelity: ateattr.SnapshotFidelityMemory, sandboxClass: "gvisor", } attrs := snapshotLogAttrs(testAttribution(), op, restoreDurationMetric, nil, @@ -125,7 +125,7 @@ func TestSnapshotLogAttrs(t *testing.T) { templateNamespace: testTemplateNamespace, templateName: testTemplateName, kind: ateattr.SnapshotKindLatest, - scope: ateattr.SnapshotScopeFull, + fidelity: ateattr.SnapshotFidelityMemory, sandboxClass: "gvisor", } @@ -157,7 +157,7 @@ func TestSnapshotLogAttrs(t *testing.T) { "ate.template.atespace": testTemplateNamespace, "ate.template.name": testTemplateName, "ate.snapshot.kind": ateattr.SnapshotKindLatest, - "ate.snapshot.scope": ateattr.SnapshotScopeFull, + "ate.snapshot.fidelity": ateattr.SnapshotFidelityMemory, "ate.sandbox.class": "gvisor", }, wantNumbers: map[string]float64{ @@ -229,7 +229,7 @@ func TestSnapshotLogAttrs(t *testing.T) { op: snapshotOp{ templateNamespace: testTemplateNamespace, templateName: testTemplateName, - scope: ateattr.SnapshotScopeFull, + fidelity: ateattr.SnapshotFidelityMemory, }, phases: []phase{{ateattr.SnapshotPhaseTotal, 700 * time.Millisecond}}, wantStrings: map[string]string{ @@ -245,7 +245,7 @@ func TestSnapshotLogAttrs(t *testing.T) { op: snapshotOp{ templateNamespace: testTemplateNamespace, templateName: testTemplateName, - scope: ateattr.SnapshotScopeFull, + fidelity: ateattr.SnapshotFidelityMemory, sandboxClass: "../../etc/passwd", }, phases: []phase{{ateattr.SnapshotPhaseTotal, time.Second}}, diff --git a/cmd/atelet/statsevents.go b/cmd/atelet/statsevents.go deleted file mode 100644 index fd2672ec1e..0000000000 --- a/cmd/atelet/statsevents.go +++ /dev/null @@ -1,184 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "context" - "io" - "log/slog" - "sync" - "sync/atomic" - - "cloud.google.com/go/compute/metadata" - - "github.com/agent-substrate/substrate/internal/actorlog" - "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/contextlogging" - "github.com/agent-substrate/substrate/internal/proto/ateompb" - "github.com/agent-substrate/substrate/internal/resources" -) - -// The events channel is the per-actor half of the usage telemetry split: the -// metrics (statspoller.go) aggregate to the bounded template-level label set, -// and everything with actor or atespace identity travels here instead, as -// structured log events -- the same stream and label vocabulary as actorlog's -// lifecycle events, never a TSDB series. - -// usageSampleMsg is the message every usage event carries; consumers filter on -// it plus the "kind" field. -const usageSampleMsg = "Actor usage sample" - -// eventKindPeriodic marks the events riding the poller's sweep -- today the -// only kind, carried on the wire so future kinds (lifecycle brackets, say) -// can join without reshaping the record. -const eventKindPeriodic = "periodic" - -// defaultLabelsKey resolves the actor-identity label group's spelling -- -// actorlog's, so usage events and lifecycle events promote into Cloud -// Logging labels the same way. metadata.OnGCE probes the metadata server, so -// startStatsPoller warms this off the boot path on a throwaway goroutine; -// sync.OnceValue makes an emit that arrives first wait for the in-flight -// probe. That wait is first-tick-only and bounded: milliseconds on GCE, the -// transport's 2s dial timeout off GCE, a 5s cap on the one pathological -// branch (SMBIOS says GCE, probes disagree) -- and no request path ever -// touches it. -var defaultLabelsKey = sync.OnceValue(func() string { - return actorlog.LabelsKey(metadata.OnGCE()) -}) - -// statsEventEmitter writes per-actor usage events to the process log stream. -// A nil emitter is a valid no-op, so call sites need no guard. -type statsEventEmitter struct { - log *slog.Logger - labelsKey func() string -} - -// newStatsEventEmitter builds an emitter over its own fixed-level handler on -// w rather than the serverboot logger: these records are a data feed, not -// leveled diagnostics, and quieting the node with --log-level=warn must not -// silently sever them -- the subsystem's one off-switch is -// --actor-stats-poll-interval=0. The records still carry level INFO on the -// wire, so nothing downstream changes. In production w is an asyncWriter -// over the process's synchronized stdout writer (see startStatsPoller). -func newStatsEventEmitter(w io.Writer, labelsKey func() string) *statsEventEmitter { - return &statsEventEmitter{ - log: slog.New(contextlogging.NewHandler(slog.NewJSONHandler(w, nil))), - labelsKey: labelsKey, - } -} - -// usageEventQueueDepth is the asyncWriter's buffer, in records: about one -// tick of a packed node. A healthy drain outruns the RPC-paced producers by -// orders of magnitude, so the queue only fills once the pipe is dead -- -// where any finite depth drops, and the choice is cosmetic. -const usageEventQueueDepth = 256 - -// asyncWriter decouples event emission from the log consumer's health. A -// write to a full stdout pipe blocks forever, and emit runs inside the -// sweep's errgroup, so one stalled log consumer (wedged rotation, disk-full -// fallout) would otherwise park a probe, wedge the sweep, and silently -// freeze the metrics channel that shares it. Writes land in a bounded queue -// drained by one goroutine; a full queue drops the record and counts it. -// Dropping is safe: the samples are point-in-time readings the next healthy -// tick repairs, and when stdout is dead the events are lost either way -- -// the choice is whether the metrics die with them. -type asyncWriter struct { - w io.Writer - ch chan []byte - - // report carries the drop warning over the same fixed-level pipeline as - // the records: the loss signal is part of the feed's integrity, so it - // must be exactly as unkillable by --log-level as the feed itself. - report *slog.Logger - - dropped atomic.Int64 -} - -func newAsyncWriter(ctx context.Context, w io.Writer, depth int) *asyncWriter { - aw := &asyncWriter{ - w: w, - ch: make(chan []byte, depth), - report: slog.New(contextlogging.NewHandler(slog.NewJSONHandler(w, nil))), - } - go aw.run(ctx) - return aw -} - -func (aw *asyncWriter) run(ctx context.Context) { - for { - select { - case <-ctx.Done(): - return - case b := <-aw.ch: - // Best-effort: a failed write has nobody to report to. - _, _ = aw.w.Write(b) - // Report drops only after a successful write, when the stream - // can carry the warning -- written directly, so the drain does - // not queue behind itself. - if n := aw.dropped.Swap(0); n > 0 { - aw.report.WarnContext(ctx, "Usage events dropped while the log stream stalled", slog.Int64("count", n)) - } - } - } -} - -// Write queues one record without ever blocking. It reports full success -// even on a drop: the emitter has no recovery to offer, and the drop is -// already counted for the writer goroutine to report. -func (aw *asyncWriter) Write(p []byte) (int, error) { - // The slog handler reuses its buffer after Write returns, so the queue - // must own a copy. - b := append([]byte(nil), p...) - select { - case aw.ch <- b: - default: - aw.dropped.Add(1) - } - return len(p), nil -} - -// emit writes one usage event. The identity comes solely from the sample's -// echo, per the stats RPCs' attribution contract. pool is the caller's -// pod-to-pool resolution -- the same enrichment the metric labels carry, so a -// pool-level metric spike can pivot to the actors behind it. A zero-valued -// pool omits the label pair rather than emitting empty strings, following the -// metric channel's rule. -func (e *statsEventEmitter) emit(ctx context.Context, kind string, s *ateompb.WorkloadStatsSample, pool workerPoolRef) { - if e == nil || s == nil { - return - } - a := resources.ActorAttribution{ - Ref: resources.ActorRef{Atespace: s.GetAtespace(), Name: s.GetActorName()}, - UID: s.GetActorUid(), - TemplateAtespace: s.GetActorTemplateAtespace(), - TemplateName: s.GetActorTemplateName(), - } - labels := ateattr.ActorLogLabels(a, "") - if pool != (workerPoolRef{}) { - labels[string(ateattr.WorkerPoolNamespaceKey)] = pool.namespace - labels[string(ateattr.WorkerPoolNameKey)] = pool.name - } - e.log.LogAttrs(ctx, slog.LevelInfo, usageSampleMsg, - slog.Any(e.labelsKey(), labels), - slog.String("kind", kind), - slog.String("sandbox_class", sandboxClassLabel(s.GetSandboxClass())), - slog.String("source", statsSourceLabel(s.GetSource())), - slog.Uint64("memory_current_bytes", s.GetMemoryCurrentBytes()), - slog.Uint64("memory_peak_bytes", s.GetMemoryPeakBytes()), - slog.Uint64("memory_working_set_bytes", s.GetMemoryWorkingSetBytes()), - slog.Uint64("cpu_usage_usec", s.GetCpuUsageUsec()), - slog.Int64("observed_at_unix_nano", s.GetObservedAtUnixNano()), - ) -} diff --git a/cmd/atelet/statsevents_test.go b/cmd/atelet/statsevents_test.go deleted file mode 100644 index cffcfd362c..0000000000 --- a/cmd/atelet/statsevents_test.go +++ /dev/null @@ -1,307 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "bytes" - "context" - "encoding/json" - "sync" - "testing" - "time" - - "github.com/agent-substrate/substrate/internal/actorlog" - "github.com/agent-substrate/substrate/internal/proto/ateompb" - "github.com/agent-substrate/substrate/internal/serverboot" -) - -// eventSample is the fully-populated sample the event tests emit; distinct -// values per field so a crossed wire is visible in the JSON. -func eventSample() *ateompb.WorkloadStatsSample { - return &ateompb.WorkloadStatsSample{ - Atespace: "space-a", - ActorName: "actor-a", - ActorUid: "uid-a", - ActorTemplateAtespace: "ns-a", - ActorTemplateName: "template-a", - SandboxClass: ateompb.SandboxClass_SANDBOX_CLASS_MICROVM, - Source: ateompb.StatsSource_STATS_SOURCE_GUEST_AGENT, - MemoryCurrentBytes: 1000, - MemoryPeakBytes: 2000, - MemoryWorkingSetBytes: 700, - CpuUsageUsec: 1234, - ObservedAtUnixNano: 42, - } -} - -// syncBuffer is a mutex-guarded buffer, so tests stay valid if an emitter -// call site ever moves onto a goroutine. -type syncBuffer struct { - mu sync.Mutex - buf bytes.Buffer -} - -func (b *syncBuffer) Write(p []byte) (int, error) { - b.mu.Lock() - defer b.mu.Unlock() - return b.buf.Write(p) -} - -func (b *syncBuffer) Bytes() []byte { - b.mu.Lock() - defer b.mu.Unlock() - return append([]byte(nil), b.buf.Bytes()...) -} - -func (b *syncBuffer) Len() int { - b.mu.Lock() - defer b.mu.Unlock() - return b.buf.Len() -} - -func (b *syncBuffer) String() string { return string(b.Bytes()) } - -// newBufferEmitter returns an emitter writing JSON records into buf. The -// labels-key closure stands in for defaultLabelsKey, whose metadata-server -// probe has no place in a unit test. -func newBufferEmitter(buf *syncBuffer, isOnGCE bool) *statsEventEmitter { - return newStatsEventEmitter(buf, func() string { return actorlog.LabelsKey(isOnGCE) }) -} - -func TestStatsEventEmitterEmit(t *testing.T) { - var buf syncBuffer - e := newBufferEmitter(&buf, false) - - e.emit(context.Background(), eventKindPeriodic, eventSample(), workerPoolRef{}) - - var rec map[string]any - if err := json.Unmarshal(buf.Bytes(), &rec); err != nil { - t.Fatalf("event is not one JSON record: %v (%q)", err, buf.String()) - } - if got := rec["msg"]; got != usageSampleMsg { - t.Errorf("msg = %v, want %q", got, usageSampleMsg) - } - if got := rec["kind"]; got != "periodic" { - t.Errorf("kind = %v, want periodic", got) - } - // The identity travels as the actorlog-style label group, so usage events - // join lifecycle events and container logs under the same keys. - labels, ok := rec["labels"].(map[string]any) - if !ok { - t.Fatalf("labels group missing or wrong shape: %v", rec["labels"]) - } - for key, want := range map[string]string{ - "ate.atespace": "space-a", - "ate.actor.name": "actor-a", - "ate.actor.uid": "uid-a", - "ate.template.atespace": "ns-a", - "ate.template.name": "template-a", - } { - if got := labels[key]; got != want { - t.Errorf("labels[%q] = %v, want %q", key, got, want) - } - } - for key, want := range map[string]float64{ - "memory_current_bytes": 1000, - "memory_peak_bytes": 2000, - "memory_working_set_bytes": 700, - "cpu_usage_usec": 1234, - "observed_at_unix_nano": 42, - } { - if got := rec[key]; got != want { - t.Errorf("%s = %v, want %v", key, got, want) - } - } - if got := rec["sandbox_class"]; got != "microvm" { - t.Errorf("sandbox_class = %v, want microvm", got) - } - if got := rec["source"]; got != "guest-agent" { - t.Errorf("source = %v, want guest-agent", got) - } - // An unresolved pool omits the pair rather than emitting empty strings. - for _, key := range []string{"ate.workerpool.namespace", "ate.workerpool.name"} { - if _, ok := labels[key]; ok { - t.Errorf("zero pool ref emitted label %q: %v", key, labels[key]) - } - } -} - -// TestStatsEventEmitterPoolLabels pins the pool enrichment on events: a -// resolved pod's events carry the same ate.workerpool label pair as the -// metric channel, inside the promoted label group. -func TestStatsEventEmitterPoolLabels(t *testing.T) { - var buf syncBuffer - e := newBufferEmitter(&buf, false) - - e.emit(context.Background(), eventKindPeriodic, eventSample(), workerPoolRef{namespace: "pool-ns", name: "pool-a"}) - - var rec map[string]any - if err := json.Unmarshal(buf.Bytes(), &rec); err != nil { - t.Fatalf("unmarshal: %v", err) - } - labels, ok := rec["labels"].(map[string]any) - if !ok { - t.Fatalf("labels group missing or wrong shape: %v", rec["labels"]) - } - if got := labels["ate.workerpool.namespace"]; got != "pool-ns" { - t.Errorf("labels[ate.workerpool.namespace] = %v, want pool-ns", got) - } - if got := labels["ate.workerpool.name"]; got != "pool-a" { - t.Errorf("labels[ate.workerpool.name] = %v, want pool-a", got) - } -} - -// TestStatsEventEmitterGCELabelsKey pins the label-group spelling split: on -// GCE the group must sit under the key Cloud Logging promotes. -func TestStatsEventEmitterGCELabelsKey(t *testing.T) { - var buf syncBuffer - e := newBufferEmitter(&buf, true) - - e.emit(context.Background(), eventKindPeriodic, eventSample(), workerPoolRef{}) - - var rec map[string]any - if err := json.Unmarshal(buf.Bytes(), &rec); err != nil { - t.Fatalf("unmarshal: %v", err) - } - if _, ok := rec["logging.googleapis.com/labels"]; !ok { - t.Errorf("GCE emitter did not use the promoted labels key; record keys: %v", buf.String()) - } -} - -// TestStatsEventEmitterIgnoresLogLevel pins the emitter's independence from -// the serverboot verbosity knob: usage events are a data feed, and quieting -// the node with --log-level=warn (or error) must not silently sever them. -func TestStatsEventEmitterIgnoresLogLevel(t *testing.T) { - if err := serverboot.SetLogLevel("error"); err != nil { - t.Fatal(err) - } - t.Cleanup(func() { - if err := serverboot.SetLogLevel("info"); err != nil { - t.Fatal(err) - } - }) - - var buf syncBuffer - newBufferEmitter(&buf, false).emit(context.Background(), eventKindPeriodic, eventSample(), workerPoolRef{}) - - if buf.Len() == 0 { - t.Error("raising the serverboot log level suppressed a usage event") - } -} - -// TestStatsEventEmitterNil: a nil emitter and a nil sample are both valid -// no-ops, so call sites stay unconditional. -func TestStatsEventEmitterNil(t *testing.T) { - var e *statsEventEmitter - e.emit(context.Background(), eventKindPeriodic, eventSample(), workerPoolRef{}) // must not panic - - var buf syncBuffer - newBufferEmitter(&buf, false).emit(context.Background(), eventKindPeriodic, nil, workerPoolRef{}) - if buf.Len() != 0 { - t.Errorf("nil sample emitted a record: %q", buf.String()) - } -} - -// stallableWriter blocks every Write until released -- the stalled log -// consumer the asyncWriter exists for. -type stallableWriter struct { - gate chan struct{} - wrote syncBuffer -} - -func (w *stallableWriter) Write(p []byte) (int, error) { - <-w.gate - return w.wrote.Write(p) -} - -// TestAsyncWriterNeverBlocks pins the contract: with the underlying -// writer wedged, every Write returns immediately -- overflow drops and -// counts instead of blocking -- and releasing the wedge drains what the -// queue held. -func TestAsyncWriterNeverBlocks(t *testing.T) { - under := &stallableWriter{gate: make(chan struct{})} - ctx, cancel := context.WithCancel(context.Background()) - t.Cleanup(cancel) - const depth = 4 - aw := newAsyncWriter(ctx, under, depth) - - // The writer goroutine dequeues one record and wedges on it; the queue - // holds depth more. Everything past that must drop, not block. The - // in-flight handoff is asynchronous, so tolerate one record of slack. - const writes = depth + 8 - done := make(chan struct{}) - go func() { - defer close(done) - for i := 0; i < writes; i++ { - if _, err := aw.Write([]byte("x")); err != nil { - t.Errorf("Write returned %v, want nil", err) - } - } - }() - select { - case <-done: - case <-time.After(5 * time.Second): - t.Fatal("Write blocked on a wedged log stream") - } - dropped := int(aw.dropped.Load()) - if dropped < writes-depth-1 || dropped > writes-depth { - t.Errorf("dropped = %d, want %d or %d", dropped, writes-depth-1, writes-depth) - } - - // Conservation: everything not dropped -- the wedged in-flight record - // plus the queue -- drains once the stream recovers, and the drop report - // arrives over the same sink, unkillable by any verbosity knob. - close(under.gate) - wantWritten := writes - dropped - waitFor(t, func() bool { return bytes.Count(under.wrote.Bytes(), []byte("x")) >= wantWritten }) - if got := bytes.Count(under.wrote.Bytes(), []byte("x")); got != wantWritten { - t.Errorf("drained %d records, want %d", got, wantWritten) - } - waitFor(t, func() bool { return bytes.Contains(under.wrote.Bytes(), []byte("Usage events dropped")) }) -} - -// TestAsyncWriterCopies: the slog handler reuses its buffer after -// Write returns, so the queue must hold copies, not aliases. -func TestAsyncWriterCopies(t *testing.T) { - under := &stallableWriter{gate: make(chan struct{})} - ctx, cancel := context.WithCancel(context.Background()) - t.Cleanup(cancel) - aw := newAsyncWriter(ctx, under, 4) - - p := []byte("original") - if _, err := aw.Write(p); err != nil { - t.Fatal(err) - } - copy(p, "clobber!") - - close(under.gate) - waitFor(t, func() bool { return under.wrote.Len() > 0 }) - if got := under.wrote.String(); got != "original" { - t.Errorf("record = %q, want the copy taken at Write time", got) - } -} - -// waitFor polls until cond returns true or the deadline passes. -func waitFor(t *testing.T, cond func() bool) { - t.Helper() - deadline := time.Now().Add(5 * time.Second) - for time.Now().Before(deadline) { - if cond() { - return - } - time.Sleep(5 * time.Millisecond) - } - t.Fatal("condition not reached before deadline") -} diff --git a/cmd/atelet/statspoller.go b/cmd/atelet/statspoller.go index b60c1b86a7..cdd9aade76 100644 --- a/cmd/atelet/statspoller.go +++ b/cmd/atelet/statspoller.go @@ -37,6 +37,7 @@ import ( "k8s.io/client-go/kubernetes" "github.com/agent-substrate/substrate/internal/ateattr" + "github.com/agent-substrate/substrate/internal/ateomstats" "github.com/agent-substrate/substrate/internal/nodepath" "github.com/agent-substrate/substrate/internal/proto/ateompb" ) @@ -48,15 +49,16 @@ import ( const workerPoolLabel = "ate.dev/worker-pool" // minActorStatsPollInterval is the floor a configured poll interval is clamped -// to: one actor's worst-case micro-VM read, maxActorContainers (25) containers -// at statsCallTimeout (2s) each. Sweeps never overlap, so it bounds -// guest-agent load, not correctness. +// to. The ateoms serve GetActiveWorkloadStats from their own sampling timer, so +// a poll faster than their sample interval only reads the same sample again. +// The floor also bounds the worker pod list each sweep sends the apiserver +// (resolveWorkerPools), so do not lower it to cut lag. const minActorStatsPollInterval = 50 * time.Second -// statsRPCTimeout bounds one ateom's GetActiveWorkloadStats call. The micro-VM -// ateom sets its statsSweepBudget (45s) below this timeout and reports guests -// it did not reach as pending, so the call does not grow with actor count. -const statsRPCTimeout = 55 * time.Second +// statsRPCTimeout bounds one ateom's GetActiveWorkloadStats call. The ateom +// answers from memory in under a millisecond, whatever its actor count or +// guest speed, so the timeout is only for a stuck socket. +const statsRPCTimeout = 5 * time.Second // statsSweepConcurrency bounds how many ateoms one sweep probes at once, one // call per ateom whatever it hosts. It caps how many stuck sockets can hold a @@ -91,9 +93,10 @@ type activeStatsClient interface { // It holds no worker-to-actor mapping and never asks the control plane: every // ateom registers itself on disk by creating its socket directory at boot (the // same sockets the lifecycle RPCs dial), so one readdir plus one probe per -// socket is complete discovery, and an atelet restart loses nothing because -// nothing was held. Attribution comes solely from the identity echoed inside -// each sample, per the RPC's contract. +// socket is complete discovery, and discovery survives an atelet restart. +// The CPU used between the old atelet's last sweep and the new one's first is +// not charged. Attribution comes solely from the identity echoed inside each +// sample, per the RPC's contract. type statsPoller struct { // interval between the end of one sweep and the start of the next. Sweeps // never overlap: a slow sweep delays the next tick rather than stacking a @@ -122,37 +125,52 @@ type statsPoller struct { inst *statsInstruments - // eventEmitter receives one usage event per sample per sweep -- the - // per-actor channel the aggregates deliberately erase identity from. - // Nil disables emission; emit is nil-safe. - eventEmitter *statsEventEmitter - // cachedPools carries pool resolutions across sweeps, so one failed pod // list cannot re-home a tick's samples -- and the CPU counter's // increments, which can never be re-attributed -- onto a pool-less label // set. Safe because a pod's pool is immutable for the pod's lifetime: an // entry can be stale, never wrong. Only the sweep loop touches it; // resolveWorkerPools prunes it to the pods whose ateom directories still - // exist, the same bound lastCPU keeps. + // exist. cachedPools map[string]workerPoolRef - // lastCPU is the last measured cpu_usage_usec per actor uid, the baseline + // lastCPU is the last measured cpu_usage_usec per activation, the baseline // for the next delta; read-only during a sweep and replaced whole at the - // end. Pending actors keep their entry, unseen actors are dropped. - lastCPU map[string]uint64 + // end. An activation keeps its entry while its worker reports it, pending + // included, and while its worker does not answer; it is dropped once that + // worker answers without it or the worker's directory is gone. A worker + // that dies without removing its directory keeps its entries until atelet + // restarts. + lastCPU map[cpuKey]cpuBaseline + + // startedAt is when this atelet started, in unix nanoseconds. An activation + // that began later was never charged, so its first sample counts in full. + startedAt int64 +} + +// cpuKey is one activation of one actor, the span a CPU counter counts within. +type cpuKey struct { + actorUID string + epoch int64 +} + +// cpuBaseline is an activation's last CPU reading and the worker that sent it. +type cpuBaseline struct { + usec uint64 + podUID string } // templateAggregate is one tick's sums for one templateKey group: the // bounded label set #174 permits on a TSDB series. Actor and atespace -// identity deliberately never reach a metric label; per-actor detail is the -// events channel's job, not this one's. +// identity deliberately never reach a metric label; per-actor detail is on +// the ateoms' usage records. // // The memory fields are point-in-time sums the gauges observe. cpuDeltaUsec is -// different: cpu_usage_usec is a cumulative per-epoch counter per actor, so -// summing the raw values across a churning actor set would be meaningless to -// rate() -- instead the poller tracks each actor's last seen value and this -// carries the sweep's INCREASE, which tick adds onto a monotonic counter. -// Counter semantics survive actors joining, leaving, and resetting epochs by +// different: cpu_usage_usec is a cumulative counter per activation, so summing +// the raw values across a churning actor set would be meaningless to rate() -- +// instead the poller tracks each activation's last seen value and this carries +// the sweep's INCREASE, which tick adds onto a monotonic counter. Counter +// semantics survive actors joining, leaving, and starting new activations by // construction. type templateAggregate struct { sampledActors int64 @@ -233,8 +251,8 @@ func (p *statsPoller) tick(ctx context.Context) { // their directory but not yet listened, and workers torn down mid-sweep. The // no-sample answers are equally routine: an empty samples list is an idle // worker, and a pending entry (source UNSPECIFIED) is a workload the ateom -// could not measure (boot, restore, teardown, or an unreached guest), which -// adds nothing but keeps its CPU baseline. +// has not measured (boot, restore, or an unreached guest), which adds nothing +// but keeps its CPU baseline. func (p *statsPoller) collect(ctx context.Context) map[templateKey]*templateAggregate { entries, err := os.ReadDir(p.ateomsDir) if err != nil { @@ -245,9 +263,11 @@ func (p *statsPoller) collect(ctx context.Context) map[templateKey]*templateAggr } podUIDs := make([]string, 0, len(entries)) + present := make(map[string]bool, len(entries)) for _, e := range entries { if e.IsDir() { podUIDs = append(podUIDs, e.Name()) + present[e.Name()] = true } } pools := p.resolveWorkerPools(ctx, podUIDs) @@ -255,8 +275,10 @@ func (p *statsPoller) collect(ctx context.Context) map[templateKey]*templateAggr var ( mu sync.Mutex aggs = make(map[templateKey]*templateAggregate) - seenCPU = make(map[string]uint64) - g errgroup.Group + seenCPU = make(map[cpuKey]cpuBaseline) + // answered is the workers whose probe succeeded this sweep. + answered = make(map[string]bool) + g errgroup.Group ) g.SetLimit(statsSweepConcurrency) for _, podUID := range podUIDs { @@ -282,29 +304,29 @@ func (p *statsPoller) collect(ctx context.Context) map[templateKey]*templateAggr return nil } + mu.Lock() + answered[podUID] = true + mu.Unlock() + // One entry per workload the ateom is hosting; empty when it is // available. for _, sample := range resp.GetSamples() { + cpuAt := cpuKey{actorUID: sample.GetActorUid(), epoch: sample.GetEpochUnixNano()} if sample.GetSource() == ateompb.StatsSource_STATS_SOURCE_UNSPECIFIED { // Pending: hosted but not measured, so it adds nothing - // and its CPU baseline carries forward. A measured value - // from another worker this sweep (restore in flight) wins. - if last, ok := p.lastCPU[sample.GetActorUid()]; ok { + // and its CPU baseline carries forward. + if last, ok := p.lastCPU[cpuAt]; ok { mu.Lock() - if _, measured := seenCPU[sample.GetActorUid()]; !measured { - seenCPU[sample.GetActorUid()] = last - } + seenCPU[cpuAt] = last mu.Unlock() } continue } - p.eventEmitter.emit(ctx, eventKindPeriodic, sample, pools[podUID]) - key := templateKey{ templateNamespace: sample.GetActorTemplateAtespace(), templateName: sample.GetActorTemplateName(), - sandboxClass: sandboxClassLabel(sample.GetSandboxClass()), - source: statsSourceLabel(sample.GetSource()), + sandboxClass: ateomstats.SandboxClassLabel(sample.GetSandboxClass()), + source: ateomstats.StatsSourceLabel(sample.GetSource()), workerPool: pools[podUID], } mu.Lock() @@ -317,22 +339,21 @@ func (p *statsPoller) collect(ctx context.Context) map[templateKey]*templateAggr agg.memoryCurrentBytes = addSat(agg.memoryCurrentBytes, sample.GetMemoryCurrentBytes()) agg.memoryWorkingSetBytes = addSat(agg.memoryWorkingSetBytes, sample.GetMemoryWorkingSetBytes()) - // The counter increase since the baseline. On a decrease, a - // cgroup counter restarted at zero, so the new value is the - // usage since then. A guest-agent decrease is ambiguous (a - // resume at another guest's value, a restart at zero, or a - // container exit), so it charges nothing. A sample with no - // baseline charges nothing: atelet cannot tell a new actor - // from its own restart. + // The counter increase since the activation's baseline; the + // ateom never lets it decrease within an activation. An + // activation with no baseline that began after this atelet + // started was never charged, so its value counts in full. One + // that began earlier may have been charged by the atelet before + // this one, so it only sets the baseline. Both times come from + // this node's clock: atelet reads only the ateoms on its node. cpu := sample.GetCpuUsageUsec() - seenCPU[sample.GetActorUid()] = cpu - if last, ok := p.lastCPU[sample.GetActorUid()]; ok { - switch { - case last <= cpu: - agg.cpuDeltaUsec = addSat(agg.cpuDeltaUsec, cpu-last) - case sample.GetSource() == ateompb.StatsSource_STATS_SOURCE_CGROUP: - agg.cpuDeltaUsec = addSat(agg.cpuDeltaUsec, cpu) + seenCPU[cpuAt] = cpuBaseline{usec: cpu, podUID: podUID} + if last, ok := p.lastCPU[cpuAt]; ok { + if cpu > last.usec { + agg.cpuDeltaUsec = addSat(agg.cpuDeltaUsec, cpu-last.usec) } + } else if cpuAt.epoch > p.startedAt { + agg.cpuDeltaUsec = addSat(agg.cpuDeltaUsec, cpu) } mu.Unlock() } @@ -342,8 +363,15 @@ func (p *statsPoller) collect(ctx context.Context) map[templateKey]*templateAggr // The tasks only ever return nil: a probe that fails is "not a target this // tick", never a failed sweep. _ = g.Wait() - // Replacing (not merging) the baselines drops actors this sweep did not - // see, so lastCPU cannot grow with actor churn. + // A worker that did not answer keeps its activations' baselines: dropping + // one would charge the activation in full again once the worker answers. + // Everything else this sweep did not see is dropped, so lastCPU does not + // grow with actor churn on workers that answer. + for k, b := range p.lastCPU { + if _, seen := seenCPU[k]; !seen && present[b.podUID] && !answered[b.podUID] { + seenCPU[k] = b + } + } p.lastCPU = seenCPU return aggs } @@ -365,31 +393,6 @@ func addSat(agg int64, v uint64) int64 { return agg + int64(v) } -// sandboxClassLabel maps the wire enum to the ate.sandbox.class label values -// the rest of the system uses. -func sandboxClassLabel(c ateompb.SandboxClass) string { - switch c { - case ateompb.SandboxClass_SANDBOX_CLASS_GVISOR: - return "gvisor" - case ateompb.SandboxClass_SANDBOX_CLASS_MICROVM: - return "microvm" - default: - return ateattr.SandboxClassUnknown - } -} - -// statsSourceLabel maps the wire enum to the ate.stats.source label values. -func statsSourceLabel(s ateompb.StatsSource) string { - switch s { - case ateompb.StatsSource_STATS_SOURCE_CGROUP: - return ateattr.StatsSourceCgroup - case ateompb.StatsSource_STATS_SOURCE_GUEST_AGENT: - return ateattr.StatsSourceGuestAgent - default: - return ateattr.StatsSourceUnspecified - } -} - // resolveWorkerPools returns this sweep's pod-UID-to-pool map: fresh // resolutions win, a failed or partial list falls back to cachedPools, and // the result -- rebuilt restricted to the pods whose ateom directories exist @@ -573,21 +576,13 @@ func (i *statsInstruments) addCPU(ctx context.Context, aggs map[templateKey]*tem } } -// startStatsPoller assembles the sampling subsystem -- the metrics poller and -// the periodic events channel -- and starts the poller. Split from main's -// boot sequence so the subsystem has one obvious entry point. +// startStatsPoller assembles the metrics poller and starts it. Split from +// main's boot sequence so the subsystem has one obvious entry point. // // The poller dials its own short-lived connection per probe (see // dialAteomStats) and takes no AteomDialer: the isolation from the lifecycle // RPCs' connection cache is structural, not just behavioral. -// logSink is the process's synchronized stdout writer, shared with the -// runtime logger so the event drain and slog can never tear each other's -// records. -func startStatsPoller(ctx context.Context, interval time.Duration, inst *statsInstruments, k8sClient kubernetes.Interface, logSink io.Writer) { - // Warm the labels-key resolution so the first emit does not pay the - // metadata probe either; see defaultLabelsKey. - go defaultLabelsKey() - +func startStatsPoller(ctx context.Context, interval time.Duration, inst *statsInstruments, k8sClient kubernetes.Interface) { poller := &statsPoller{ interval: interval, ateomsDir: nodepath.AteomsDir(), @@ -598,8 +593,8 @@ func startStatsPoller(ctx context.Context, interval time.Duration, inst *statsIn } return ateompb.NewAteomClient(conn), closer, nil }, - inst: inst, - eventEmitter: newStatsEventEmitter(newAsyncWriter(ctx, logSink, usageEventQueueDepth), defaultLabelsKey), + inst: inst, + startedAt: time.Now().UnixNano(), } // NODE_NAME comes from the Downward API; without it the samples still // flow, just grouped without pool labels. diff --git a/cmd/atelet/statspoller_test.go b/cmd/atelet/statspoller_test.go index a90cc3665f..f6f6f9636f 100644 --- a/cmd/atelet/statspoller_test.go +++ b/cmd/atelet/statspoller_test.go @@ -15,15 +15,12 @@ package main import ( - "bytes" "context" - "encoding/json" "errors" "io" "math" "os" "path/filepath" - "sort" "sync" "testing" "time" @@ -53,19 +50,9 @@ type fakeStatsAteom struct { // sawDeadline records whether the probe's context carried one, pinning the // per-call timeout. sawDeadline bool - // gate, when set, is received from before answering, so a test can order - // this probe after another worker's response has been folded. - gate <-chan struct{} } func (f *fakeStatsAteom) GetActiveWorkloadStats(ctx context.Context, req *ateompb.GetActiveWorkloadStatsRequest, opts ...grpc.CallOption) (*ateompb.GetActiveWorkloadStatsResponse, error) { - if f.gate != nil { - select { - case <-f.gate: - case <-ctx.Done(): - return nil, ctx.Err() // a misconfigured gate fails the probe, not the run - } - } f.mu.Lock() f.calls++ _, f.sawDeadline = ctx.Deadline() @@ -112,6 +99,22 @@ func pendingSample(actorUID, templateNS, templateName string) *ateompb.WorkloadS } } +// testStartedAt is when the fixture's atelet started. Samples default to epoch +// zero, an activation that began before it; later epochs began after. +const testStartedAt = 1000 + +// atEpoch sets the activation a sample belongs to. +func atEpoch(s *ateompb.WorkloadStatsSample, epoch int64) *ateompb.WorkloadStatsSample { + s.EpochUnixNano = epoch + return s +} + +// baseline is the CPU baseline of one activation, and whether there is one. +func baseline(p *statsPoller, actorUID string, epoch int64) (uint64, bool) { + b, ok := p.lastCPU[cpuKey{actorUID: actorUID, epoch: epoch}] + return b.usec, ok +} + // availableResponse is an idle ateom's answer: the empty list. func availableResponse() *ateompb.GetActiveWorkloadStatsResponse { return &ateompb.GetActiveWorkloadStatsResponse{} @@ -131,21 +134,15 @@ func pendingResponse(actorUID string) *ateompb.GetActiveWorkloadStatsResponse { } // closeRecorder counts Close calls, standing in for a probe's connection. -// collect closes it after folding the response, so onClose is the point at -// which that worker's entries are in the aggregates. type closeRecorder struct { - mu sync.Mutex - closes int - onClose func() + mu sync.Mutex + closes int } func (c *closeRecorder) Close() error { c.mu.Lock() c.closes++ c.mu.Unlock() - if c.onClose != nil { - c.onClose() - } return nil } @@ -166,6 +163,7 @@ func newPollerFixture(t *testing.T, fakes map[string]*fakeStatsAteom) (*statsPol } return &statsPoller{ ateomsDir: dir, + startedAt: testStartedAt, dial: func(_ context.Context, podUID string) (activeStatsClient, io.Closer, error) { f, ok := fakes[podUID] if !ok || f == nil { @@ -353,19 +351,18 @@ func cpuResponse(actorUID string, cpuUsec uint64) *ateompb.GetActiveWorkloadStat } // TestStatsPollerCPUDeltas pins the increase computation across sweeps: the -// first sight of an actor establishes a baseline and charges nothing (atelet -// cannot tell a new actor from its own restart, and re-charging an epoch the -// previous atelet counted would spike the counter), a later sweep charges -// only the increase, a cgroup decrease is an epoch reset whose new value is -// the usage since the reset, and an actor that disappears stops contributing and -// is dropped from the baselines. +// first sight of an activation that began before atelet started establishes a +// baseline and charges nothing (the previous atelet may have counted it), a +// later sweep charges only the increase, a new activation that began after +// atelet started counts in full, and an actor that disappears stops +// contributing and is dropped from the baselines. func TestStatsPollerCPUDeltas(t *testing.T) { key := templateKey{templateNamespace: "ns-a", templateName: "tmpl-a", sandboxClass: "gvisor", source: "cgroup"} fake := &fakeStatsAteom{resp: cpuResponse("uid-a", 1000)} p, _ := newPollerFixture(t, map[string]*fakeStatsAteom{"uid-1": fake}) if got := p.collect(context.Background())[key].cpuDeltaUsec; got != 0 { - t.Errorf("first sweep delta = %d, want 0 (baseline only on first sight)", got) + t.Errorf("first sweep delta = %d, want 0 (an activation from before atelet started only sets its baseline)", got) } fake.resp = cpuResponse("uid-a", 1600) @@ -373,11 +370,13 @@ func TestStatsPollerCPUDeltas(t *testing.T) { t.Errorf("second sweep delta = %d, want 600 (the increase)", got) } - // Epoch reset: the counter went backwards, so the new value is the usage - // since the reset. - fake.resp = cpuResponse("uid-a", 250) + // A new activation that began after atelet started: its value is all + // usage atelet has not charged. + fake.resp = &ateompb.GetActiveWorkloadStatsResponse{Samples: []*ateompb.WorkloadStatsSample{ + atEpoch(cpuResponse("uid-a", 250).Samples[0], testStartedAt+1), + }} if got := p.collect(context.Background())[key].cpuDeltaUsec; got != 250 { - t.Errorf("post-reset sweep delta = %d, want 250", got) + t.Errorf("new activation sweep delta = %d, want 250", got) } // The actor leaves: nothing to contribute, and its baseline must be @@ -419,68 +418,6 @@ func TestStatsPollerWorkerPoolLabels(t *testing.T) { } } -// TestStatsPollerPeriodicEvents pins the events channel: one event per -// executing sample per sweep, none for idle or mid-boot ateoms, identity -// taken from the echo, pool labels from the sweep's own resolution. -func TestStatsPollerPeriodicEvents(t *testing.T) { - fakes := map[string]*fakeStatsAteom{ - "uid-1": {resp: executingResponse("ns-a", "tmpl-a", ateompb.SandboxClass_SANDBOX_CLASS_GVISOR, ateompb.StatsSource_STATS_SOURCE_CGROUP, 1000, 700)}, - "uid-2": {resp: availableResponse()}, - } - p, _ := newPollerFixture(t, fakes) - var buf syncBuffer - p.eventEmitter = newBufferEmitter(&buf, false) - p.fetchWorkerPools = func(context.Context) map[string]workerPoolRef { - return map[string]workerPoolRef{"uid-1": {namespace: "pool-ns", name: "pool-a"}} - } - - p.collect(context.Background()) - - lines := bytes.Count(bytes.TrimSpace(buf.Bytes()), []byte("\n")) + 1 - if buf.Len() == 0 { - t.Fatal("no periodic event emitted for the executing ateom") - } - if lines != 1 { - t.Fatalf("emitted %d events, want 1 (idle ateoms emit nothing): %q", lines, buf.String()) - } - var rec map[string]any - if err := json.Unmarshal(bytes.TrimSpace(buf.Bytes()), &rec); err != nil { - t.Fatalf("unmarshal: %v", err) - } - if got := rec["kind"]; got != "periodic" { - t.Errorf("kind = %v, want periodic", got) - } - labels, _ := rec["labels"].(map[string]any) - if got := labels["ate.workerpool.name"]; got != "pool-a" { - t.Errorf("labels[ate.workerpool.name] = %v, want pool-a", got) - } -} - -func TestAddSat(t *testing.T) { - tests := []struct { - name string - agg int64 - v uint64 - want int64 - }{ - {name: "normal add", agg: 100, v: 50, want: 150}, - {name: "zero add", agg: 100, v: 0, want: 100}, - // A wire value above MaxInt64 -- a corrupt or hostile guest reading -- - // must pin at the ceiling, not wrap the aggregate negative. - {name: "value above MaxInt64 saturates", agg: 0, v: math.MaxUint64, want: math.MaxInt64}, - // The addition itself can also overflow once inputs are clamped. - {name: "sum overflow saturates", agg: math.MaxInt64 - 10, v: 100, want: math.MaxInt64}, - {name: "exactly at ceiling", agg: math.MaxInt64 - 5, v: 5, want: math.MaxInt64}, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - if got := addSat(tc.agg, tc.v); got != tc.want { - t.Errorf("addSat(%d, %d) = %d, want %d", tc.agg, tc.v, tc.want, got) - } - }) - } -} - // TestStatsPollerCollectSaturatesCorruptSamples pins the end-to-end behavior: // one guest reporting absurd counters must not flip a template's aggregates // negative -- a negative gauge misreads as "no memory", and a negative CPU @@ -731,7 +668,7 @@ func TestStatsPollerMultiActorCPUBaselines(t *testing.T) { if got := p.collect(context.Background())[key].cpuDeltaUsec; got != 0 { t.Fatalf("first sweep delta = %d, want 0 (baselines only)", got) } - if _, ok := p.lastCPU["actor-3"]; ok { + if _, ok := baseline(p, "actor-3", 0); ok { t.Error("pending actor was given a CPU baseline") } @@ -745,11 +682,11 @@ func TestStatsPollerMultiActorCPUBaselines(t *testing.T) { t.Errorf("second sweep delta = %d, want 850 (600 + 250)", got) } - // actor-2 resets its epoch; actor-1 keeps advancing. The reset charges - // the new value, not a negative delta. + // actor-2 starts a new activation; actor-1 keeps advancing. The new + // activation charges its value, not a negative delta. fake.resp = &ateompb.GetActiveWorkloadStatsResponse{Samples: []*ateompb.WorkloadStatsSample{ measuredSample("actor-1", "ns-a", "tmpl-a", 1, 1, 1700), - measuredSample("actor-2", "ns-a", "tmpl-a", 1, 1, 40), + atEpoch(measuredSample("actor-2", "ns-a", "tmpl-a", 1, 1, 40), testStartedAt+1), }} if got := p.collect(context.Background())[key].cpuDeltaUsec; got != 140 { t.Errorf("reset sweep delta = %d, want 140 (100 + 40)", got) @@ -759,38 +696,6 @@ func TestStatsPollerMultiActorCPUBaselines(t *testing.T) { } } -// TestStatsPollerMultiActorEvents pins one usage event per measured entry -// and none for a pending one. -func TestStatsPollerMultiActorEvents(t *testing.T) { - fakes := map[string]*fakeStatsAteom{ - "uid-w1": {resp: &ateompb.GetActiveWorkloadStatsResponse{Samples: []*ateompb.WorkloadStatsSample{ - measuredSample("actor-1", "ns-a", "tmpl-a", 1000, 700, 10), - measuredSample("actor-2", "ns-a", "tmpl-a", 500, 300, 20), - pendingSample("actor-3", "ns-a", "tmpl-a"), - }}}, - } - p, _ := newPollerFixture(t, fakes) - var buf syncBuffer - p.eventEmitter = newBufferEmitter(&buf, false) - - p.collect(context.Background()) - - var uids []string - for _, line := range bytes.Split(bytes.TrimSpace(buf.Bytes()), []byte("\n")) { - var rec map[string]any - if err := json.Unmarshal(line, &rec); err != nil { - t.Fatalf("event line is not JSON: %v: %s", err, line) - } - labels, _ := rec["labels"].(map[string]any) - uid, _ := labels["ate.actor.uid"].(string) - uids = append(uids, uid) - } - sort.Strings(uids) - if want := []string{"actor-1", "actor-2"}; !cmp.Equal(want, uids) { - t.Errorf("event actor uids = %v, want %v (one per measured entry, none for pending)", uids, want) - } -} - // TestStatsPollerPendingKeepsCPUBaseline pins that a sweep in which an actor // reports pending carries its baseline forward: the interval is charged when // the actor is measured again, not lost. @@ -809,7 +714,7 @@ func TestStatsPollerPendingKeepsCPUBaseline(t *testing.T) { if got := p.collect(context.Background()); len(got) != 0 { t.Fatalf("pending sweep produced aggregates %v, want none", got) } - if got, ok := p.lastCPU["actor-1"]; !ok || got != 1000 { + if got, ok := baseline(p, "actor-1", 0); !ok || got != 1000 { t.Fatalf("baseline after pending sweep = %d (present=%v), want 1000 kept", got, ok) } @@ -821,130 +726,177 @@ func TestStatsPollerPendingKeepsCPUBaseline(t *testing.T) { } } -// TestStatsPollerCPUDecreaseBySource pins the decrease rule, with and without -// a pending sweep between the samples: a cgroup decrease charges the new value, -// since that counter restarts at zero, and a guest-agent decrease charges -// nothing, since that counter can resume at another guest's value. -func TestStatsPollerCPUDecreaseBySource(t *testing.T) { - guestAgent := func(s *ateompb.WorkloadStatsSample) *ateompb.WorkloadStatsSample { - s.SandboxClass = ateompb.SandboxClass_SANDBOX_CLASS_MICROVM - s.Source = ateompb.StatsSource_STATS_SOURCE_GUEST_AGENT - return s +// TestStatsPollerCPUEpochs pins the charge rule for activations: an increase +// within one charges the increase, a decrease charges nothing, and a new one +// charges its first value in full only if it began after atelet started. +func TestStatsPollerCPUEpochs(t *testing.T) { + key := templateKey{templateNamespace: "ns-a", templateName: "tmpl-a", sandboxClass: "gvisor", source: "cgroup"} + sample := func(epoch int64, cpu uint64) *ateompb.WorkloadStatsSample { + return atEpoch(measuredSample("actor-1", "ns-a", "tmpl-a", 1, 1, cpu), epoch) } for _, tc := range []struct { name string - sample func(cpu uint64) *ateompb.WorkloadStatsSample - key templateKey - pending bool - wantDelta int64 + sweeps []*ateompb.WorkloadStatsSample // nil is a pending sweep + wantDelta int64 // on the last sweep }{ - { - name: "cgroup", - sample: func(cpu uint64) *ateompb.WorkloadStatsSample { - return measuredSample("actor-1", "ns-a", "tmpl-a", 1, 1, cpu) - }, - key: templateKey{templateNamespace: "ns-a", templateName: "tmpl-a", sandboxClass: "gvisor", source: "cgroup"}, - wantDelta: 400, - }, - { - name: "cgroup across pending", - sample: func(cpu uint64) *ateompb.WorkloadStatsSample { - return measuredSample("actor-1", "ns-a", "tmpl-a", 1, 1, cpu) - }, - key: templateKey{templateNamespace: "ns-a", templateName: "tmpl-a", sandboxClass: "gvisor", source: "cgroup"}, - pending: true, - wantDelta: 400, - }, - { - name: "guest-agent", - sample: func(cpu uint64) *ateompb.WorkloadStatsSample { - return guestAgent(measuredSample("actor-1", "ns-a", "tmpl-a", 1, 1, cpu)) - }, - key: templateKey{templateNamespace: "ns-a", templateName: "tmpl-a", sandboxClass: "microvm", source: "guest-agent"}, - wantDelta: 0, - }, - { - name: "guest-agent across pending", - sample: func(cpu uint64) *ateompb.WorkloadStatsSample { - return guestAgent(measuredSample("actor-1", "ns-a", "tmpl-a", 1, 1, cpu)) - }, - key: templateKey{templateNamespace: "ns-a", templateName: "tmpl-a", sandboxClass: "microvm", source: "guest-agent"}, - pending: true, - wantDelta: 0, - }, + {"increase within an activation", []*ateompb.WorkloadStatsSample{sample(2000, 1000), sample(2000, 1400)}, 400}, + {"decrease within an activation", []*ateompb.WorkloadStatsSample{sample(2000, 1000), sample(2000, 400)}, 0}, + {"new activation after start", []*ateompb.WorkloadStatsSample{sample(2000, 1000), sample(3000, 400)}, 400}, + {"new activation across a pending sweep", []*ateompb.WorkloadStatsSample{sample(2000, 1000), nil, sample(3000, 400)}, 400}, + {"first sight of an activation after start", []*ateompb.WorkloadStatsSample{sample(2000, 700)}, 700}, + {"first sight of an activation at start", []*ateompb.WorkloadStatsSample{sample(testStartedAt, 700)}, 0}, + {"first sight of an activation before start", []*ateompb.WorkloadStatsSample{sample(testStartedAt-1, 700)}, 0}, } { t.Run(tc.name, func(t *testing.T) { fake := &fakeStatsAteom{} p, _ := newPollerFixture(t, map[string]*fakeStatsAteom{"uid-w1": fake}) - sweep := func(sample *ateompb.WorkloadStatsSample) map[templateKey]*templateAggregate { - fake.resp = &ateompb.GetActiveWorkloadStatsResponse{Samples: []*ateompb.WorkloadStatsSample{sample}} - return p.collect(context.Background()) - } - - sweep(tc.sample(1000)) // baseline 1000 - if tc.pending { - sweep(pendingSample("actor-1", "ns-a", "tmpl-a")) - } - if got := sweep(tc.sample(400))[tc.key].cpuDeltaUsec; got != tc.wantDelta { - t.Errorf("delta for a decrease 1000 -> 400 = %d, want %d", got, tc.wantDelta) + var got int64 + for _, sw := range tc.sweeps { + if sw == nil { + sw = atEpoch(pendingSample("actor-1", "ns-a", "tmpl-a"), 3000) + } + fake.resp = &ateompb.GetActiveWorkloadStatsResponse{Samples: []*ateompb.WorkloadStatsSample{sw}} + got = 0 + if agg := p.collect(context.Background())[key]; agg != nil { + got = agg.cpuDeltaUsec + } } - // Either way 400 is the new baseline. - if got := sweep(tc.sample(500))[tc.key].cpuDeltaUsec; got != 100 { - t.Errorf("delta after the decrease = %d, want 100", got) + if got != tc.wantDelta { + t.Errorf("last sweep delta = %d, want %d", got, tc.wantDelta) } }) } } -// TestStatsPollerRestoreInFlightMeasuredWins pins that when one worker -// measures an actor and another reports it pending in the same sweep, the -// measured value is the baseline in either fold order. -func TestStatsPollerRestoreInFlightMeasuredWins(t *testing.T) { +// TestStatsPollerRestoreChargesEachActivationOnce pins a restore across two +// workers: the source's activation and the destination's are separate +// baselines, so nothing is charged twice and the source's is dropped once it +// leaves. +func TestStatsPollerRestoreChargesEachActivationOnce(t *testing.T) { key := templateKey{templateNamespace: "ns-a", templateName: "tmpl-a", sandboxClass: "gvisor", source: "cgroup"} - for _, order := range []string{"measured first", "pending first", "unordered"} { - t.Run(order, func(t *testing.T) { - src := &fakeStatsAteom{resp: &ateompb.GetActiveWorkloadStatsResponse{Samples: []*ateompb.WorkloadStatsSample{ - measuredSample("actor-1", "ns-a", "tmpl-a", 1, 1, 1000), - }}} - dst := &fakeStatsAteom{resp: availableResponse()} - p, closers := newPollerFixture(t, map[string]*fakeStatsAteom{"uid-src": src, "uid-dst": dst}) - - p.collect(context.Background()) // baseline C0 = 1000 - - // Sweep N: source measured at C1 = 1300; destination pending. - src.resp = &ateompb.GetActiveWorkloadStatsResponse{Samples: []*ateompb.WorkloadStatsSample{ - measuredSample("actor-1", "ns-a", "tmpl-a", 1, 1, 1300), - }} - dst.resp = &ateompb.GetActiveWorkloadStatsResponse{Samples: []*ateompb.WorkloadStatsSample{ - pendingSample("actor-1", "ns-a", "tmpl-a"), - }} - // The gated probe answers only after the other worker's response has - // been folded: collect closes a probe's connection after the fold. - gate := make(chan struct{}) - release := sync.OnceFunc(func() { close(gate) }) - switch order { - case "measured first": - dst.gate = gate - closers["uid-src"].onClose = release - case "pending first": - src.gate = gate - closers["uid-dst"].onClose = release - } - if got := p.collect(context.Background())[key].cpuDeltaUsec; got != 300 { - t.Fatalf("sweep N delta = %d, want 300", got) - } - if got := p.lastCPU["actor-1"]; got != 1300 { - t.Fatalf("baseline after sweep N = %d, want the measured 1300, not the carried 1000", got) - } + const before, after = 2000, 3000 + src := &fakeStatsAteom{resp: &ateompb.GetActiveWorkloadStatsResponse{Samples: []*ateompb.WorkloadStatsSample{ + atEpoch(measuredSample("actor-1", "ns-a", "tmpl-a", 1, 1, 1000), before), + }}} + dst := &fakeStatsAteom{resp: availableResponse()} + p, _ := newPollerFixture(t, map[string]*fakeStatsAteom{"uid-src": src, "uid-dst": dst}) + p.collect(context.Background()) - // Sweep N+1: only the destination hosts it now, measured at C2 = 1500. - closers["uid-src"].onClose, closers["uid-dst"].onClose = nil, nil - src.resp, src.gate = availableResponse(), nil - dst.resp, dst.gate = &ateompb.GetActiveWorkloadStatsResponse{Samples: []*ateompb.WorkloadStatsSample{ - measuredSample("actor-1", "ns-a", "tmpl-a", 1, 1, 1500), - }}, nil - if got := p.collect(context.Background())[key].cpuDeltaUsec; got != 200 { - t.Errorf("sweep N+1 delta = %d, want 200 (C2-C1); 500 would double-charge C1-C0", got) + // The source still measures its activation; the destination hosts the new + // one, not yet measured. + src.resp = &ateompb.GetActiveWorkloadStatsResponse{Samples: []*ateompb.WorkloadStatsSample{ + atEpoch(measuredSample("actor-1", "ns-a", "tmpl-a", 1, 1, 1300), before), + }} + dst.resp = &ateompb.GetActiveWorkloadStatsResponse{Samples: []*ateompb.WorkloadStatsSample{ + atEpoch(pendingSample("actor-1", "ns-a", "tmpl-a"), after), + }} + if got := p.collect(context.Background())[key].cpuDeltaUsec; got != 300 { + t.Fatalf("restore sweep delta = %d, want 300", got) + } + + // Only the destination hosts it now, 200 into its activation. + src.resp = availableResponse() + dst.resp = &ateompb.GetActiveWorkloadStatsResponse{Samples: []*ateompb.WorkloadStatsSample{ + atEpoch(measuredSample("actor-1", "ns-a", "tmpl-a", 1, 1, 200), after), + }} + if got := p.collect(context.Background())[key].cpuDeltaUsec; got != 200 { + t.Errorf("post-restore delta = %d, want 200", got) + } + if _, ok := baseline(p, "actor-1", before); ok { + t.Error("the source activation's baseline survived its worker answering without it") + } +} + +// TestStatsPollerKeepsBaselineOfSilentWorker pins that a worker that does not +// answer keeps its activations' baselines, so the next answer charges the +// increase rather than the activation in full. +func TestStatsPollerKeepsBaselineOfSilentWorker(t *testing.T) { + key := templateKey{templateNamespace: "ns-a", templateName: "tmpl-a", sandboxClass: "gvisor", source: "cgroup"} + fake := &fakeStatsAteom{resp: &ateompb.GetActiveWorkloadStatsResponse{Samples: []*ateompb.WorkloadStatsSample{ + atEpoch(measuredSample("actor-1", "ns-a", "tmpl-a", 1, 1, 1000), 2000), + }}} + p, _ := newPollerFixture(t, map[string]*fakeStatsAteom{"uid-w1": fake}) + p.collect(context.Background()) + + fake.err = errors.New("deadline exceeded") + p.collect(context.Background()) + if got, ok := baseline(p, "actor-1", 2000); !ok || got != 1000 { + t.Fatalf("baseline after a silent sweep = %d (present %v), want 1000 kept", got, ok) + } + + fake.err = nil + fake.resp = &ateompb.GetActiveWorkloadStatsResponse{Samples: []*ateompb.WorkloadStatsSample{ + atEpoch(measuredSample("actor-1", "ns-a", "tmpl-a", 1, 1, 1600), 2000), + }} + if got := p.collect(context.Background())[key].cpuDeltaUsec; got != 600 { + t.Errorf("delta after the silent sweep = %d, want 600, not the full 1600", got) + } +} + +// TestStatsPollerSilentWorkerKeepsBaselinesUntilDirGone pins that a worker +// that stops answering for good, as one killed without cleanup, keeps its +// baselines for as long as its directory stays. +func TestStatsPollerSilentWorkerKeepsBaselinesUntilDirGone(t *testing.T) { + fake := &fakeStatsAteom{resp: &ateompb.GetActiveWorkloadStatsResponse{Samples: []*ateompb.WorkloadStatsSample{ + atEpoch(measuredSample("actor-1", "ns-a", "tmpl-a", 1, 1, 1000), 2000), + }}} + p, _ := newPollerFixture(t, map[string]*fakeStatsAteom{"uid-w1": fake}) + p.collect(context.Background()) + + fake.err = errors.New("connection refused") + for range 3 { + p.collect(context.Background()) + } + if _, ok := baseline(p, "actor-1", 2000); !ok { + t.Fatal("baseline dropped while the silent worker's directory stays") + } + if err := os.Remove(filepath.Join(p.ateomsDir, "uid-w1")); err != nil { + t.Fatal(err) + } + p.collect(context.Background()) + if len(p.lastCPU) != 0 { + t.Errorf("baselines after the directory went = %v, want none", p.lastCPU) + } +} + +// TestStatsPollerDropsBaselineOfGoneWorker pins that a worker whose +// directory is gone takes its baselines with it. +func TestStatsPollerDropsBaselineOfGoneWorker(t *testing.T) { + fake := &fakeStatsAteom{resp: &ateompb.GetActiveWorkloadStatsResponse{Samples: []*ateompb.WorkloadStatsSample{ + atEpoch(measuredSample("actor-1", "ns-a", "tmpl-a", 1, 1, 1000), 2000), + }}} + p, _ := newPollerFixture(t, map[string]*fakeStatsAteom{"uid-w1": fake}) + p.collect(context.Background()) + + if err := os.Remove(filepath.Join(p.ateomsDir, "uid-w1")); err != nil { + t.Fatal(err) + } + p.collect(context.Background()) + if len(p.lastCPU) != 0 { + t.Errorf("baselines after the worker left = %v, want none", p.lastCPU) + } +} + +func TestAddSat(t *testing.T) { + tests := []struct { + name string + agg int64 + v uint64 + want int64 + }{ + {name: "normal add", agg: 100, v: 50, want: 150}, + {name: "zero add", agg: 100, v: 0, want: 100}, + // A wire value above MaxInt64 -- a corrupt or hostile guest reading -- + // must pin at the ceiling, not wrap the aggregate negative. + {name: "value above MaxInt64 saturates", agg: 0, v: math.MaxUint64, want: math.MaxInt64}, + // The addition itself can also overflow once inputs are clamped. + {name: "sum overflow saturates", agg: math.MaxInt64 - 10, v: 100, want: math.MaxInt64}, + {name: "exactly at ceiling", agg: math.MaxInt64 - 5, v: 5, want: math.MaxInt64}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := addSat(tc.agg, tc.v); got != tc.want { + t.Errorf("addSat(%d, %d) = %d, want %d", tc.agg, tc.v, tc.want, got) } }) } diff --git a/cmd/atelet/volumes_test.go b/cmd/atelet/volumes_test.go index 375153e020..e3b13ac9b3 100644 --- a/cmd/atelet/volumes_test.go +++ b/cmd/atelet/volumes_test.go @@ -76,7 +76,7 @@ var _ volume.VolumePluginWorkerPlane = (*fakeWorkerPlugin)(nil) // withTempActorsDir redirects nodepath.ActorsDir at a temp dir for the // duration of the test. Every path derived from ActorsDir moves with it, -// including the ones resetActorDirs and the OCI spec builder compute +// including the ones resetActorDirs and the bundle preparation compute // independently of the volume mount code. func withTempActorsDir(t *testing.T) { t.Helper() diff --git a/cmd/atenet/internal/router/dataplane.go b/cmd/atenet/internal/router/dataplane.go index 1169b93df9..e56f6417a8 100644 --- a/cmd/atenet/internal/router/dataplane.go +++ b/cmd/atenet/internal/router/dataplane.go @@ -19,6 +19,7 @@ import ( "fmt" "log/slog" "net" + "strconv" "time" "golang.org/x/sync/errgroup" @@ -87,10 +88,11 @@ func (s *RouterServer) startEnvoyDataplane(ctx context.Context, g *errgroup.Grou // Envoy receives all routing configuration from the local xDS server. g.Go(func() error { - slog.InfoContext(ctx, "Starting Envoy xDS Server", slog.Int("port", s.cfg.XdsPort)) - lis, err := net.Listen("tcp", fmt.Sprintf(":%d", s.cfg.XdsPort)) + addr := xdsListenAddress(s.cfg.XdsPort) + slog.InfoContext(ctx, "Starting Envoy xDS Server", slog.String("address", addr)) + lis, err := net.Listen("tcp", addr) if err != nil { - return fmt.Errorf("failed to listen on port %d: %w", s.cfg.XdsPort, err) + return fmt.Errorf("failed to listen on %s: %w", addr, err) } defer lis.Close() @@ -98,3 +100,12 @@ func (s *RouterServer) startEnvoyDataplane(ctx context.Context, g *errgroup.Grou }) return nil } + +// xdsListenAddress is where the router serves xDS to the Envoy in its own pod. +// The xDS server is plaintext and unauthenticated, and its ADS and SDS streams +// describe every listener, route, cluster and secret path in the dataplane, so +// it binds loopback only: Envoy dials it at 127.0.0.1 from the same network +// namespace, and nothing else in the cluster has any business reaching it. +func xdsListenAddress(port int) string { + return net.JoinHostPort("127.0.0.1", strconv.Itoa(port)) +} diff --git a/cmd/atenet/internal/router/egress/actorjwt/actorjwt.go b/cmd/atenet/internal/router/egress/actorjwt/actorjwt.go new file mode 100644 index 0000000000..3c65c0783e --- /dev/null +++ b/cmd/atenet/internal/router/egress/actorjwt/actorjwt.go @@ -0,0 +1,110 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package actorjwt mints the actor JWTs that egress policies inject. +package actorjwt + +import ( + "context" + "fmt" + "slices" + "time" + + "golang.org/x/sync/singleflight" + "google.golang.org/protobuf/proto" + "k8s.io/apimachinery/pkg/util/cache" + + "github.com/agent-substrate/substrate/internal/resources" + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" +) + +// Minter mints actor JWTs and reuses each token until a third of its lifetime +// remains. Tokens are cached by actor name, so an actor recreated under the +// same name can get its predecessor's token until then. +type Minter struct { + client ateapipb.ControlClient + // tokens maps a tokenKey to a JWT until the token's refresh point. + tokens *cache.Expiring + // flight collapses concurrent mints of one token into a single call. + flight singleflight.Group +} + +// New returns a Minter that mints through client. +func New(client ateapipb.ControlClient) *Minter { + return &Minter{client: client, tokens: cache.NewExpiring()} +} + +// Token returns a JWT for the actor ref names, bound to src's audiences and +// lifetime. MintActorJWT errors are returned wrapped. +func (m *Minter) Token(ctx context.Context, ref resources.ActorRef, src *ateapipb.ActorJWTSource) (string, error) { + req := &ateapipb.MintActorJWTRequest{ + Actor: ref.ToObjectRef(), + Audiences: slices.Sorted(slices.Values(src.GetAudiences())), + ExpirationSeconds: src.GetExpirationSeconds(), + } + key, err := tokenKey(req) + if err != nil { + return "", err + } + if jwt, ok := m.cached(key); ok { + return jwt, nil + } + + // The mint outlives the caller: the leader of a flight going away must + // not fail the callers that joined it. It has no timeout of its own, so a + // slow mint still lands in the cache for the next request. + ch := m.flight.DoChan(key, func() (any, error) { + return m.mint(context.WithoutCancel(ctx), key, ref, req) + }) + select { + case <-ctx.Done(): + return "", ctx.Err() + case res := <-ch: + if res.Err != nil { + return "", res.Err + } + return res.Val.(string), nil + } +} + +// mint sends req and caches the token under key, unless a flight that just +// finished has already stored one. +func (m *Minter) mint(ctx context.Context, key string, ref resources.ActorRef, req *ateapipb.MintActorJWTRequest) (string, error) { + if jwt, ok := m.cached(key); ok { + return jwt, nil + } + resp, err := m.client.MintActorJWT(ctx, req) + if err != nil { + return "", fmt.Errorf("minting an actor JWT for %s: %w", ref, err) + } + reuseFor := time.Until(resp.GetExpiresAt().AsTime()) - time.Duration(req.GetExpirationSeconds())*time.Second/3 + m.tokens.Set(key, resp.GetActorJwt(), reuseFor) + return resp.GetActorJwt(), nil +} + +func (m *Minter) cached(key string) (string, bool) { + v, ok := m.tokens.Get(key) + s, _ := v.(string) + return s, ok +} + +// tokenKey identifies the token req mints by req's own encoding. Deterministic +// encoding is stable within one binary, which is all an in-process cache needs. +func tokenKey(req *ateapipb.MintActorJWTRequest) (string, error) { + b, err := proto.MarshalOptions{Deterministic: true}.Marshal(req) + if err != nil { + return "", fmt.Errorf("encoding the token key: %w", err) + } + return string(b), nil +} diff --git a/cmd/atenet/internal/router/egress/actorjwt/actorjwt_test.go b/cmd/atenet/internal/router/egress/actorjwt/actorjwt_test.go new file mode 100644 index 0000000000..71a79dfc73 --- /dev/null +++ b/cmd/atenet/internal/router/egress/actorjwt/actorjwt_test.go @@ -0,0 +1,204 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package actorjwt + +import ( + "context" + "errors" + "fmt" + "net" + "sync" + "sync/atomic" + "testing" + "testing/synctest" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/status" + "google.golang.org/grpc/test/bufconn" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/timestamppb" + + "github.com/agent-substrate/substrate/internal/resources" + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" +) + +var testActor = resources.ActorRef{Atespace: "default", Name: "my-actor"} + +// fakeControl's nth MintActorJWT call returns "jwt-", expiring +// expiration_seconds from now. err, when set, is returned instead, and gate, +// when non-nil, blocks each call until it is closed. +type fakeControl struct { + ateapipb.UnimplementedControlServer + calls atomic.Int32 + last atomic.Pointer[ateapipb.MintActorJWTRequest] + err error + gate chan struct{} +} + +func (f *fakeControl) MintActorJWT(ctx context.Context, req *ateapipb.MintActorJWTRequest) (*ateapipb.MintActorJWTResponse, error) { + n := f.calls.Add(1) + f.last.Store(req) + if f.gate != nil { + select { + case <-f.gate: + case <-ctx.Done(): + return nil, status.FromContextError(ctx.Err()).Err() + } + } + if f.err != nil { + return nil, f.err + } + return &ateapipb.MintActorJWTResponse{ + ActorJwt: fmt.Sprintf("jwt-%d", n), + ExpiresAt: timestamppb.New(time.Now().Add(time.Duration(req.GetExpirationSeconds()) * time.Second)), + }, nil +} + +// newMinter returns a Minter whose client reaches ctl over an in-memory gRPC +// connection. +func newMinter(t *testing.T, ctl *fakeControl) *Minter { + t.Helper() + lis := bufconn.Listen(1 << 20) + srv := grpc.NewServer() + ateapipb.RegisterControlServer(srv, ctl) + go func() { _ = srv.Serve(lis) }() + t.Cleanup(srv.Stop) + + conn, err := grpc.NewClient("passthrough:///bufconn", + grpc.WithContextDialer(func(ctx context.Context, _ string) (net.Conn, error) { return lis.DialContext(ctx) }), + grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.NewClient: %v", err) + } + t.Cleanup(func() { _ = conn.Close() }) + return New(ateapipb.NewControlClient(conn)) +} + +func jwtSource(lifetime int64, audiences ...string) *ateapipb.ActorJWTSource { + return &ateapipb.ActorJWTSource{Audiences: audiences, ExpirationSeconds: lifetime} +} + +// wantToken calls m.Token and fails the test unless it returns want. +func wantToken(t *testing.T, m *Minter, ref resources.ActorRef, src *ateapipb.ActorJWTSource, want string) { + t.Helper() + got, err := m.Token(context.Background(), ref, src) + if err != nil { + t.Fatalf("Token: %v", err) + } + if got != want { + t.Fatalf("Token = %q, want %q", got, want) + } +} + +func TestReuseTokenUntilAThirdOfItsLifetimeRemains(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + ctl := &fakeControl{} + m := newMinter(t, ctl) + src := jwtSource(900, "https://b.example", "https://a.example") + + wantToken(t, m, testActor, src, "jwt-1") + want := &ateapipb.MintActorJWTRequest{ + Actor: &ateapipb.ObjectRef{Atespace: "default", Name: "my-actor"}, + Audiences: []string{"https://a.example", "https://b.example"}, + ExpirationSeconds: 900, + } + if got := ctl.last.Load(); !proto.Equal(got, want) { + t.Errorf("MintActorJWT request = %v, want %v", got, want) + } + + time.Sleep(600*time.Second - time.Nanosecond) + wantToken(t, m, testActor, src, "jwt-1") + time.Sleep(time.Nanosecond) + wantToken(t, m, testActor, src, "jwt-2") + }) +} + +func TestKeyOnActorAudiencesAndLifetime(t *testing.T) { + m := newMinter(t, &fakeControl{}) + other := resources.ActorRef{Atespace: "default", Name: "other-actor"} + + wantToken(t, m, testActor, jwtSource(900, "a", "b"), "jwt-1") + wantToken(t, m, testActor, jwtSource(900, "b", "a"), "jwt-1") + wantToken(t, m, testActor, jwtSource(900, "a"), "jwt-2") + wantToken(t, m, testActor, jwtSource(600, "a", "b"), "jwt-3") + wantToken(t, m, other, jwtSource(900, "a", "b"), "jwt-4") + wantToken(t, m, testActor, jwtSource(900, "a b"), "jwt-5") +} + +func TestDoNotCacheErrors(t *testing.T) { + ctl := &fakeControl{err: status.Error(codes.Unavailable, "ateapi is down")} + m := newMinter(t, ctl) + src := jwtSource(900, "a") + + if _, err := m.Token(context.Background(), testActor, src); status.Code(err) != codes.Unavailable { + t.Fatalf("Token = %v, want the Unavailable MintActorJWT error", err) + } + ctl.err = nil + wantToken(t, m, testActor, src, "jwt-2") +} + +func TestCollapseConcurrentMints(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + ctl := &fakeControl{gate: make(chan struct{})} + m := newMinter(t, ctl) + + const callers = 8 + var wg sync.WaitGroup + for range callers { + wg.Go(func() { + if _, err := m.Token(context.Background(), testActor, jwtSource(900, "a")); err != nil { + t.Errorf("Token: %v", err) + } + }) + } + synctest.Wait() + close(ctl.gate) + wg.Wait() + if calls := ctl.calls.Load(); calls != 1 { + t.Errorf("MintActorJWT calls = %d, want 1 for %d concurrent callers", calls, callers) + } + }) +} + +// The leader's cancellation must not fail the mint it started, and the token +// still lands in the cache. +func TestMintOutlivesCanceledCaller(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + ctl := &fakeControl{gate: make(chan struct{})} + m := newMinter(t, ctl) + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { + _, err := m.Token(ctx, testActor, jwtSource(900, "a")) + done <- err + }() + synctest.Wait() + cancel() + if err := <-done; !errors.Is(err, context.Canceled) { + t.Fatalf("canceled caller got %v, want context.Canceled", err) + } + + close(ctl.gate) + synctest.Wait() + wantToken(t, m, testActor, jwtSource(900, "a"), "jwt-1") + if calls := ctl.calls.Load(); calls != 1 { + t.Errorf("MintActorJWT calls = %d, want 1", calls) + } + }) +} diff --git a/cmd/atenet/internal/router/egress/credentials.go b/cmd/atenet/internal/router/egress/credentials.go index 54ae689559..733ba95474 100644 --- a/cmd/atenet/internal/router/egress/credentials.go +++ b/cmd/atenet/internal/router/egress/credentials.go @@ -16,6 +16,7 @@ package egress import ( "context" + "fmt" "log/slog" "strings" @@ -49,6 +50,21 @@ func mapCredentialProviderError(err error) error { } } +// mapActorJWTError converts an actor JWT mint failure into a client-facing +// ext_proc denial. An actor that was deleted (NotFound) denies as 403, a +// transient control-plane failure as a retryable 503, and anything else, a +// gateway or policy misconfiguration that retrying cannot fix, as 500. +func mapActorJWTError(err error) error { + switch status.Code(err) { + case codes.NotFound: + return extproc.WrapReqError(envoy_type.StatusCode_Forbidden, err, deniedBody) + case codes.Unavailable, codes.DeadlineExceeded: + return extproc.WrapReqError(envoy_type.StatusCode_ServiceUnavailable, err, deniedBody) + default: + return extproc.WrapReqError(envoy_type.StatusCode_InternalServerError, err, deniedBody) + } +} + // applyEffects resolves a matched rule's credential injections and returns the // header mutations to add to the request, or an error that denies it. Only a // header the request carries is replaced (headers is keyed by lowercased @@ -56,9 +72,8 @@ func mapCredentialProviderError(err error) error { // fetched. // // Any failure to produce a credential the request needs denies it, including -// having no provider configured. -// -// This gateway cannot mint actor JWTs yet, so a request that needs one is denied. +// having no provider configured for a credential_uri entry. Actor JWTs need no +// provider. func (h *Handler) applyEffects(ctx context.Context, ref resources.ActorRef, dest egresspolicy.Destination, headers map[string]string, effects *ateapipb.HttpRuleEffects) ([]*corev3.HeaderValueOption, error) { var injections []*ateapipb.CredentialHeader for _, inj := range effects.GetReplaceHeaders() { @@ -70,24 +85,6 @@ func (h *Handler) applyEffects(ctx context.Context, ref resources.ActorRef, dest return nil, nil } - // TODO(identity): mint actor JWTs through Control.MintActorJWT. - for _, inj := range injections { - if inj.GetActorJwt() != nil { - slog.ErrorContext(ctx, "egress denied: this gateway cannot inject actor JWTs yet", - slog.Any("actor", ref), slog.String("host", dest.Hostname), slog.String("header", inj.GetHeader())) - return nil, extproc.NewReqError(envoy_type.StatusCode_NotImplemented, deniedBody) - } - } - if h.provider == nil { - slog.ErrorContext(ctx, "egress denied: policy requires credential injection but no credential provider is configured", - slog.Any("actor", ref), slog.String("host", dest.Hostname)) - return nil, extproc.NewReqError(envoy_type.StatusCode_InternalServerError, deniedBody) - } - - // Atunnel connected to us with an ateom-for-actor SPIFFE ID; translate it - // to a pure actor SPIFFE ID for plugins to make decisions on. - actorSpiffeID := resources.ActorSPIFFEID(ref).String() - setHeaders := make([]*corev3.HeaderValueOption, 0, len(injections)) for _, inj := range injections { if err := validateInjectHeader(inj.GetHeader()); err != nil { @@ -96,43 +93,20 @@ func (h *Handler) applyEffects(ctx context.Context, ref resources.ActorRef, dest return nil, extproc.WrapReqError(envoy_type.StatusCode_InternalServerError, err, deniedBody) } - // Confirm the credential URI names the provider this gateway serves - // before dialing: the configured connection fronts one provider, so a URI - // naming another cannot be resolved here and must fail closed rather than - // be sent to the wrong provider. - if h.providerName != "" { - name, err := providerNameFromURI(inj.GetCredentialUri()) + var secret []byte + var err error + if src := inj.GetActorJwt(); src != nil { + secret, err = h.actorJWT(ctx, ref, dest, src) if err != nil { - slog.ErrorContext(ctx, "egress denied: policy names an unparseable credential URI", - slog.Any("actor", ref), slog.String("host", dest.Hostname), slog.String("uri", inj.GetCredentialUri()), slog.Any("err", err)) - return nil, extproc.WrapReqError(envoy_type.StatusCode_InternalServerError, err, deniedBody) + return nil, fmt.Errorf("minting an actor JWT for header %s: %w", inj.GetHeader(), err) } - if name != h.providerName { - slog.ErrorContext(ctx, "egress denied: credential URI names a provider this gateway does not serve", - slog.Any("actor", ref), slog.String("host", dest.Hostname), slog.String("uri", inj.GetCredentialUri()), - slog.String("provider", name), slog.String("serves", h.providerName)) - return nil, extproc.NewReqError(envoy_type.StatusCode_InternalServerError, deniedBody) + } else { + secret, err = h.providerCredential(ctx, ref, dest, inj.GetCredentialUri()) + if err != nil { + return nil, fmt.Errorf("fetching credential %s for header %s: %w", inj.GetCredentialUri(), inj.GetHeader(), err) } } - resp, err := h.provider.FetchSecret(ctx, &credproviderpb.FetchSecretRequest{ - Uri: inj.GetCredentialUri(), - ActorSpiffeId: actorSpiffeID, - }) - if err != nil { - // Fail closed: a credential the policy required but we could not fetch - // must not let the request out without it. - slog.ErrorContext(ctx, "egress denied: credential fetch failed", - slog.Any("actor", ref), slog.String("host", dest.Hostname), slog.String("uri", inj.GetCredentialUri()), slog.Any("err", err)) - return nil, mapCredentialProviderError(err) - } - secret, err := sanitizeSecret(resp.GetOpaqueBytes()) - if err != nil { - slog.ErrorContext(ctx, "egress denied: unusable credential", - slog.Any("actor", ref), slog.String("host", dest.Hostname), slog.String("uri", inj.GetCredentialUri()), slog.Any("err", err)) - return nil, extproc.WrapReqError(envoy_type.StatusCode_ServiceUnavailable, err, deniedBody) - } - // Overwrite any header the actor set itself, so a client cannot pre-seed a // value that survives injection. setHeaders = append(setHeaders, &corev3.HeaderValueOption{ @@ -142,3 +116,65 @@ func (h *Handler) applyEffects(ctx context.Context, ref resources.ActorRef, dest } return setHeaders, nil } + +// actorJWT mints the JWT src asks for. Every error it returns is already a +// client-facing denial. +func (h *Handler) actorJWT(ctx context.Context, ref resources.ActorRef, dest egresspolicy.Destination, src *ateapipb.ActorJWTSource) ([]byte, error) { + jwt, err := h.actorJWTs.Token(ctx, ref, src) + if err != nil { + slog.ErrorContext(ctx, "egress denied: actor JWT mint failed", + slog.Any("actor", ref), slog.String("host", dest.Hostname), slog.Any("audiences", src.GetAudiences()), slog.Any("err", err)) + return nil, mapActorJWTError(err) + } + return []byte(jwt), nil +} + +// providerCredential fetches the secret uri names from the credential +// provider. Every error it returns is already a client-facing denial. +func (h *Handler) providerCredential(ctx context.Context, ref resources.ActorRef, dest egresspolicy.Destination, uri string) ([]byte, error) { + if h.provider == nil { + slog.ErrorContext(ctx, "egress denied: policy requires credential injection but no credential provider is configured", + slog.Any("actor", ref), slog.String("host", dest.Hostname)) + return nil, extproc.NewReqError(envoy_type.StatusCode_InternalServerError, deniedBody) + } + + // Confirm the credential URI names the provider this gateway serves + // before dialing: the configured connection fronts one provider, so a URI + // naming another cannot be resolved here and must fail closed rather than + // be sent to the wrong provider. + if h.providerName != "" { + name, err := providerNameFromURI(uri) + if err != nil { + slog.ErrorContext(ctx, "egress denied: policy names an unparseable credential URI", + slog.Any("actor", ref), slog.String("host", dest.Hostname), slog.String("uri", uri), slog.Any("err", err)) + return nil, extproc.WrapReqError(envoy_type.StatusCode_InternalServerError, err, deniedBody) + } + if name != h.providerName { + slog.ErrorContext(ctx, "egress denied: credential URI names a provider this gateway does not serve", + slog.Any("actor", ref), slog.String("host", dest.Hostname), slog.String("uri", uri), + slog.String("provider", name), slog.String("serves", h.providerName)) + return nil, extproc.NewReqError(envoy_type.StatusCode_InternalServerError, deniedBody) + } + } + + // Atunnel connected to us with an ateom-for-actor SPIFFE ID; translate it + // to a pure actor SPIFFE ID for plugins to make decisions on. + resp, err := h.provider.FetchSecret(ctx, &credproviderpb.FetchSecretRequest{ + Uri: uri, + ActorSpiffeId: resources.ActorSPIFFEID(ref).String(), + }) + if err != nil { + // Fail closed: a credential the policy required but we could not fetch + // must not let the request out without it. + slog.ErrorContext(ctx, "egress denied: credential fetch failed", + slog.Any("actor", ref), slog.String("host", dest.Hostname), slog.String("uri", uri), slog.Any("err", err)) + return nil, mapCredentialProviderError(err) + } + secret, err := sanitizeSecret(resp.GetOpaqueBytes()) + if err != nil { + slog.ErrorContext(ctx, "egress denied: unusable credential", + slog.Any("actor", ref), slog.String("host", dest.Hostname), slog.String("uri", uri), slog.Any("err", err)) + return nil, extproc.WrapReqError(envoy_type.StatusCode_ServiceUnavailable, err, deniedBody) + } + return secret, nil +} diff --git a/cmd/atenet/internal/router/egress/credentials_test.go b/cmd/atenet/internal/router/egress/credentials_test.go index d7d39166d7..49630191c9 100644 --- a/cmd/atenet/internal/router/egress/credentials_test.go +++ b/cmd/atenet/internal/router/egress/credentials_test.go @@ -15,6 +15,7 @@ package egress import ( + "cmp" "context" "errors" "testing" @@ -24,6 +25,7 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" "github.com/agent-substrate/substrate/cmd/atenet/internal/router/extproc" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" @@ -252,47 +254,118 @@ func TestInjectionDenials(t *testing.T) { } } -func TestActorJWTInjectionNotImplemented(t *testing.T) { - effects := &ateapipb.HttpRuleEffects{ReplaceHeaders: []*ateapipb.CredentialHeader{{ - Header: "authorization", +// actorJWTHeader replaces header with an actor JWT bound to api.example.com. +func actorJWTHeader(header string) *ateapipb.CredentialHeader { + return &ateapipb.CredentialHeader{ + Header: header, Prefix: "Bearer ", ActorJwt: &ateapipb.ActorJWTSource{Audiences: []string{"https://api.example.com"}, ExpirationSeconds: 900}, - }}} - httpsPolicy := &ateapipb.EgressPolicy{Rules: []*ateapipb.EgressRule{{Https: &ateapipb.HTTPSRule{Hostnames: []string{"api.example.com"}, Effects: effects}}}} - httpPolicy := &ateapipb.EgressPolicy{Rules: []*ateapipb.EgressRule{{Http: &ateapipb.HTTPRule{Hostnames: []string{"api.example.com"}, Effects: effects}}}} - tests := []struct { - name string - leg string - policy *ateapipb.EgressPolicy - provider *fakeProvider // nil means no provider configured - }{{ - name: "https rule", - leg: extproc.EgressTLSMITMFilterChainName, - policy: httpsPolicy, - provider: &fakeProvider{resp: bearerTokenResponse("s3cr3t")}, - }, { - name: "https rule without a provider", - leg: extproc.EgressTLSMITMFilterChainName, - policy: httpsPolicy, - }, { - name: "http rule", - leg: extproc.EgressCleartextFilterChainName, - policy: httpPolicy, - provider: &fakeProvider{resp: bearerTokenResponse("s3cr3t")}, - }} - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - var h *Handler - if tc.provider == nil { - h = injectionHandlerFor(tc.policy, nil, injectionProviderName) - } else { - h = injectionHandlerFor(tc.policy, tc.provider, injectionProviderName) + } +} + +// replaceHeadersPolicy is a rule for api.example.com on leg's protocol (https +// for the MITM leg, http for the cleartext one) that replaces headers with +// entries. +func replaceHeadersPolicy(leg string, entries ...*ateapipb.CredentialHeader) *ateapipb.EgressPolicy { + hosts := []string{"api.example.com"} + effects := &ateapipb.HttpRuleEffects{ReplaceHeaders: entries} + if leg == extproc.EgressCleartextFilterChainName { + return &ateapipb.EgressPolicy{Rules: []*ateapipb.EgressRule{{Http: &ateapipb.HTTPRule{Hostnames: hosts, Effects: effects}}}} + } + return &ateapipb.EgressPolicy{Rules: []*ateapipb.EgressRule{{Https: &ateapipb.HTTPSRule{Hostnames: hosts, Effects: effects}}}} +} + +var requestLegs = []string{extproc.EgressTLSMITMFilterChainName, extproc.EgressCleartextFilterChainName} + +// An actor JWT is injected with no credential provider configured. The +// rule's credential_uri entry is for a header the request does not carry, so +// the missing provider does not deny it. +func TestActorJWTInjection(t *testing.T) { + for _, leg := range requestLegs { + t.Run(leg, func(t *testing.T) { + client := &egressMockClient{actor: runningActor(), policy: replaceHeadersPolicy(leg, + actorJWTHeader("authorization"), + &ateapipb.CredentialHeader{Header: "x-api-key", CredentialUri: "ate-secret://k8s/default/token"}, + )} + h := New(client, nil, 0, nil, "") + + res, err := h.HandleRequestHeaders(context.Background(), + withAuthorization(innerMetadata(leg, "GET", "api.example.com", nil))) + if err != nil { + t.Fatalf("HandleRequestHeaders: %v", err) } - _, err := h.HandleRequestHeaders(context.Background(), - withAuthorization(innerMetadata(tc.leg, "GET", "api.example.com", nil))) - wantStatus(t, err, envoy_type.StatusCode_NotImplemented) - if tc.provider != nil && tc.provider.got != nil { - t.Errorf("provider was asked for %v, want no fetch", tc.provider.got) + setHeaders := res.Response.GetResponse().GetHeaderMutation().GetSetHeaders() + if len(setHeaders) != 1 { + t.Fatalf("got %d header mutations, want only the actor JWT", len(setHeaders)) + } + h0 := setHeaders[0] + if got := h0.GetHeader().GetKey(); got != "authorization" { + t.Errorf("header key = %q, want authorization", got) + } + if got := string(h0.GetHeader().GetRawValue()); got != "Bearer jwt-1" { + t.Errorf("header value = %q, want %q", got, "Bearer jwt-1") + } + if h0.GetAppendAction() != corev3.HeaderValueOption_OVERWRITE_IF_EXISTS_OR_ADD { + t.Errorf("append action = %v, want OVERWRITE_IF_EXISTS_OR_ADD", h0.GetAppendAction()) + } + want := &ateapipb.MintActorJWTRequest{ + Actor: &ateapipb.ObjectRef{Atespace: testEgressAtespace, Name: testEgressActor}, + Audiences: []string{"https://api.example.com"}, + ExpirationSeconds: 900, + } + if got := client.lastMint.Load(); !proto.Equal(got, want) { + t.Errorf("MintActorJWT request = %v, want %v", got, want) + } + }) + } +} + +func TestActorJWTInjectionSkippedWithoutHeader(t *testing.T) { + client := &egressMockClient{actor: runningActor(), policy: replaceHeadersPolicy(extproc.EgressTLSMITMFilterChainName, actorJWTHeader("authorization"))} + h := New(client, nil, 0, nil, "") + res, err := h.HandleRequestHeaders(context.Background(), + innerMetadata(extproc.EgressTLSMITMFilterChainName, "GET", "api.example.com", nil)) + if err != nil { + t.Fatalf("HandleRequestHeaders: %v", err) + } + if got := res.Response.GetResponse().GetHeaderMutation().GetSetHeaders(); len(got) != 0 { + t.Errorf("got %d injected headers, want 0", len(got)) + } + if calls := client.mintCalls.Load(); calls != 0 { + t.Errorf("MintActorJWT calls = %d, want 0", calls) + } +} + +func TestActorJWTInjectionDenials(t *testing.T) { + tests := []struct { + name string + header string // authorization unless set + mintErr error + want envoy_type.StatusCode + }{ + {name: "actor deleted", mintErr: status.Error(codes.NotFound, "actor not found"), want: envoy_type.StatusCode_Forbidden}, + {name: "ateapi unavailable", mintErr: status.Error(codes.Unavailable, "ateapi is down"), want: envoy_type.StatusCode_ServiceUnavailable}, + {name: "mint timed out", mintErr: status.Error(codes.DeadlineExceeded, "deadline exceeded"), want: envoy_type.StatusCode_ServiceUnavailable}, + {name: "gateway may not mint", mintErr: status.Error(codes.PermissionDenied, "denied"), want: envoy_type.StatusCode_InternalServerError}, + {name: "mint request rejected", mintErr: status.Error(codes.InvalidArgument, "bad lifetime"), want: envoy_type.StatusCode_InternalServerError}, + {name: "system header", header: ":path", want: envoy_type.StatusCode_InternalServerError}, + } + for _, leg := range requestLegs { + t.Run(leg, func(t *testing.T) { + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + header := cmp.Or(tc.header, "authorization") + mock := &egressMockClient{ + actor: runningActor(), + policy: replaceHeadersPolicy(leg, actorJWTHeader(header)), + mintErr: tc.mintErr, + } + h := New(mock, nil, 0, nil, "") + md := innerMetadata(leg, "GET", "api.example.com", nil) + md.Headers[header] = "placeholder" + _, err := h.HandleRequestHeaders(context.Background(), md) + wantStatus(t, err, tc.want) + }) } }) } diff --git a/cmd/atenet/internal/router/egress/egress.go b/cmd/atenet/internal/router/egress/egress.go index 5731b01c48..01b7fcfba3 100644 --- a/cmd/atenet/internal/router/egress/egress.go +++ b/cmd/atenet/internal/router/egress/egress.go @@ -41,6 +41,7 @@ import ( "google.golang.org/grpc/status" "google.golang.org/protobuf/types/known/structpb" + "github.com/agent-substrate/substrate/cmd/atenet/internal/router/egress/actorjwt" "github.com/agent-substrate/substrate/cmd/atenet/internal/router/extproc" "github.com/agent-substrate/substrate/internal/egresspolicy" "github.com/agent-substrate/substrate/internal/resources" @@ -83,9 +84,10 @@ type Handler struct { actorIdentityRoots *x509.CertPool // policies is the per-actor EgressPolicy cache every leg reads through. policies *policyCache - // provider resolves an egress policy's credential injections. Nil means - // credential injection is not configured, and a request that needs one is - // denied. + // actorJWTs mints the actor JWTs that credential injections ask for. + actorJWTs *actorjwt.Minter + // provider resolves an egress policy's credential_uri injections. Nil + // means none are configured, and a request that needs one is denied. provider credproviderpb.CredentialProviderClient // providerName, when set, is the provider this gateway serves (the host of // its ate-secret:// credential URIs); a credential URI naming another @@ -97,16 +99,17 @@ type Handler struct { // trusted_ca; see verifyActorCertificate for why it is checked again here. // policyCacheTTL of 0 fetches the policy on every callout. // -// provider resolves an allowed rule's credential injections on the -// TLS-terminated MITM leg; nil leaves credential injection off, so a request -// matching a rule that requires an injection is denied. providerName, when -// set, is the provider this gateway serves; a credential URI naming another -// provider is refused. +// provider resolves an allowed rule's credential_uri injections on the +// TLS-terminated MITM leg; nil leaves them off, so a request that needs one is +// denied. Actor JWT injections need no provider. providerName, when set, is +// the provider this gateway serves; a credential URI naming another provider +// is refused. func New(apiClient ateapipb.ControlClient, actorIdentityRoots *x509.CertPool, policyCacheTTL time.Duration, provider credproviderpb.CredentialProviderClient, providerName string) *Handler { return &Handler{ apiClient: apiClient, actorIdentityRoots: actorIdentityRoots, policies: newPolicyCache(apiClient, policyCacheTTL), + actorJWTs: actorjwt.New(apiClient), provider: provider, providerName: providerName, } diff --git a/cmd/atenet/internal/router/egress/egress_test.go b/cmd/atenet/internal/router/egress/egress_test.go index 72c8f33cff..3ca10de2eb 100644 --- a/cmd/atenet/internal/router/egress/egress_test.go +++ b/cmd/atenet/internal/router/egress/egress_test.go @@ -40,6 +40,7 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "google.golang.org/protobuf/types/known/structpb" + "google.golang.org/protobuf/types/known/timestamppb" "github.com/agent-substrate/substrate/cmd/atenet/internal/router/extproc" "github.com/agent-substrate/substrate/internal/egresspolicy" @@ -182,6 +183,12 @@ type egressMockClient struct { // policyGate, when non-nil, blocks each GetActorEgressPolicy until it is // closed, so a test can hold several callers on one fetch. policyGate chan struct{} + + // The nth MintActorJWT call returns "jwt-", expiring + // expiration_seconds from now. mintErr, when set, is returned instead. + mintCalls atomic.Int32 + lastMint atomic.Pointer[ateapipb.MintActorJWTRequest] + mintErr error } func (m *egressMockClient) GetActor(context.Context, *ateapipb.GetActorRequest, ...grpc.CallOption) (*ateapipb.Actor, error) { @@ -209,6 +216,18 @@ func (m *egressMockClient) GetActorEgressPolicy(ctx context.Context, _ *ateapipb return m.policy, nil } +func (m *egressMockClient) MintActorJWT(_ context.Context, req *ateapipb.MintActorJWTRequest, _ ...grpc.CallOption) (*ateapipb.MintActorJWTResponse, error) { + n := m.mintCalls.Add(1) + m.lastMint.Store(req) + if m.mintErr != nil { + return nil, m.mintErr + } + return &ateapipb.MintActorJWTResponse{ + ActorJwt: fmt.Sprintf("jwt-%d", n), + ExpiresAt: timestamppb.New(time.Now().Add(time.Duration(req.GetExpirationSeconds()) * time.Second)), + }, nil +} + // allowAllPolicy allows every name and address: cleartext HTTP on any port // and HTTPS on 443. func allowAllPolicy() *ateapipb.EgressPolicy { diff --git a/cmd/atenet/internal/router/egress/request.go b/cmd/atenet/internal/router/egress/request.go index 45e30d22aa..2c34fe2bb1 100644 --- a/cmd/atenet/internal/router/egress/request.go +++ b/cmd/atenet/internal/router/egress/request.go @@ -19,6 +19,7 @@ import ( "errors" "fmt" "log/slog" + "strings" extprocv3 "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" envoy_type "github.com/envoyproxy/go-control-plane/envoy/type/v3" @@ -28,6 +29,8 @@ import ( "github.com/agent-substrate/substrate/internal/resources" ) +const websocketDeniedBody = "WebSocket egress is not supported" + // handleRequest authorizes one request the gateway can read: cleartext HTTP, // or HTTPS the gateway terminated. It runs per request, because the // Host can change between requests on one connection. @@ -38,6 +41,11 @@ import ( // request is dialed by the name it was decided on, on the port the actor // dialed. func (h *Handler) handleRequest(ctx context.Context, md *extproc.RequestMetadata, leg string) (extproc.Result, error) { + if isWebSocketUpgrade(md) { + slog.WarnContext(ctx, "egress denied: WebSocket upgrades are not supported", slog.String("leg", leg), slog.String("host", md.Host)) + return extproc.Result{}, extproc.NewReqError(envoy_type.StatusCode_Forbidden, websocketDeniedBody) + } + ref, err := actorFromFilterState(md) if err != nil { slog.WarnContext(ctx, "egress denied: request carries no actor identity", slog.String("leg", leg), slog.Any("err", err)) @@ -121,6 +129,19 @@ func dialHost(dest egresspolicy.Destination) string { } } +func isWebSocketUpgrade(md *extproc.RequestMetadata) bool { + return md.Method == "GET" && hasHTTPToken(md.Header("connection"), "upgrade") && hasHTTPToken(md.Header("upgrade"), "websocket") +} + +func hasHTTPToken(value, want string) bool { + for _, token := range strings.Split(value, ",") { + if strings.EqualFold(strings.TrimSpace(token), want) { + return true + } + } + return false +} + // connectionDestination is the SNI a decrypted connection presented, on the // port the actor dialed, in the form the https rules are matched against. An // SNI that is not a DNS name, or none at all, matches nothing. diff --git a/cmd/atenet/internal/router/egress/request_test.go b/cmd/atenet/internal/router/egress/request_test.go index 3037d2733a..43eed50ebd 100644 --- a/cmd/atenet/internal/router/egress/request_test.go +++ b/cmd/atenet/internal/router/egress/request_test.go @@ -328,6 +328,45 @@ func TestRequestLegPicksRulesByLeg(t *testing.T) { } } +func TestRequestLegDeniesHTTP1WebSocketUpgrade(t *testing.T) { + h := policyHandler(allowAllPolicy()) + for _, leg := range []string{extproc.EgressCleartextFilterChainName, extproc.EgressTLSMITMFilterChainName} { + t.Run(leg, func(t *testing.T) { + md := innerMetadata(leg, "GET", "api.example.com", nil) + md.Headers["connection"] = "keep-alive, UpGrAdE" + md.Headers["upgrade"] = "WebSocket" + _, err := h.HandleRequestHeaders(context.Background(), md) + wantStatus(t, err, envoy_type.StatusCode_Forbidden) + if err.Error() != websocketDeniedBody { + t.Errorf("denial body = %q, want %q", err, websocketDeniedBody) + } + }) + } +} + +func TestRequestLegAllowsOrdinaryHTTPSRequest(t *testing.T) { + h := policyHandler(allowAllPolicy()) + for _, tc := range []struct { + name string + method string + connection string + upgrade string + }{ + {name: "ordinary GET", method: "GET"}, + {name: "other upgrade", method: "GET", connection: "Upgrade", upgrade: "h2c"}, + {name: "upgrade without connection token", method: "GET", connection: "keep-alive", upgrade: "websocket"}, + {name: "non-GET", method: "POST", connection: "Upgrade", upgrade: "websocket"}, + } { + t.Run(tc.name, func(t *testing.T) { + md := innerMetadata(extproc.EgressTLSMITMFilterChainName, tc.method, "api.example.com", nil) + md.Headers["connection"] = tc.connection + md.Headers["upgrade"] = tc.upgrade + res, err := h.HandleRequestHeaders(context.Background(), md) + wantDial(t, res, err, extproc.EgressDialName) + }) + } +} + // Without the identity the outer chain shares, a request cannot be attributed // to any actor and is refused. func TestRequestLegRequiresIdentity(t *testing.T) { diff --git a/cmd/atenet/internal/router/egresspolicymanifest_test.go b/cmd/atenet/internal/router/egresspolicymanifest_test.go index 84e4c9f511..ba73a0b573 100644 --- a/cmd/atenet/internal/router/egresspolicymanifest_test.go +++ b/cmd/atenet/internal/router/egresspolicymanifest_test.go @@ -192,6 +192,67 @@ func TestEgressManifestsNameEveryExtProcChain(t *testing.T) { } } +func TestEgressManifestsDisableWebSocketUpgrades(t *testing.T) { + want := map[string][]string{ + egressManifests[0]: {extproc.EgressCleartextFilterChainName, extproc.EgressTLSMITMFilterChainName}, + } + for _, path := range egressManifests { + t.Run(path, func(t *testing.T) { + tree := bootstrapTree(t, path) + chains := map[string]node{} + for _, lc := range allChains(tree) { + name := str(lc.chain, "name") + if slices.Contains(requestLegs, name) { + chains[name] = lc.chain + } + } + for _, name := range want[path] { + chain, present := chains[name] + if !present { + t.Errorf("expected request-leg chain %q is absent", name) + continue + } + var disabled bool + for _, upgrade := range list(hcm(chain), "upgrade_configs") { + if str(upgrade, "upgrade_type") == "websocket" { + enabled, ok := upgrade["enabled"].(bool) + disabled = ok && !enabled + break + } + } + if !disabled { + t.Errorf("chain %q must explicitly disable websocket upgrades", name) + } + var routes []node + for _, virtualHost := range list(child(hcm(chain), "route_config"), "virtual_hosts") { + routes = append(routes, list(virtualHost, "routes")...) + } + if len(routes) == 0 { + t.Errorf("chain %q has no routes", name) + } + for i, route := range routes { + action := child(route, "route") + if action == nil { + t.Errorf("chain %q route %d has no route action", name, i) + continue + } + disabled := false + for _, upgrade := range list(action, "upgrade_configs") { + if str(upgrade, "upgrade_type") == "websocket" { + enabled, ok := upgrade["enabled"].(bool) + disabled = ok && !enabled + break + } + } + if !disabled { + t.Errorf("chain %q route %d must explicitly disable websocket upgrades", name, i) + } + } + } + }) + } +} + // extProcOf returns the ext_proc filter config of a chain's HCM and its index // among the http_filters, or nil and -1. func extProcOf(chain node) (node, int, []node) { diff --git a/cmd/atenet/internal/router/extproc/egress_websocket_test.go b/cmd/atenet/internal/router/extproc/egress_websocket_test.go new file mode 100644 index 0000000000..f82116830b --- /dev/null +++ b/cmd/atenet/internal/router/extproc/egress_websocket_test.go @@ -0,0 +1,98 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extproc_test + +import ( + "context" + "io" + "testing" + + corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + extprocv3 "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" + "google.golang.org/protobuf/types/known/structpb" + + "github.com/agent-substrate/substrate/cmd/atenet/internal/router/egress" + "github.com/agent-substrate/substrate/cmd/atenet/internal/router/extproc" +) + +const websocketDenialBody = "WebSocket egress is not supported" + +type websocketProcessStream struct { + extprocv3.ExternalProcessor_ProcessServer + ctx context.Context + requests []*extprocv3.ProcessingRequest + responses []*extprocv3.ProcessingResponse +} + +func (s *websocketProcessStream) Context() context.Context { return s.ctx } + +func (s *websocketProcessStream) Recv() (*extprocv3.ProcessingRequest, error) { + if len(s.requests) == 0 { + return nil, io.EOF + } + req := s.requests[0] + s.requests = s.requests[1:] + return req, nil +} + +func (s *websocketProcessStream) Send(resp *extprocv3.ProcessingResponse) error { + s.responses = append(s.responses, resp) + return nil +} + +func TestProcessReturnsWebSocketDenialAsImmediateResponse(t *testing.T) { + for _, leg := range []string{extproc.EgressCleartextFilterChainName, extproc.EgressTLSMITMFilterChainName} { + t.Run(leg, func(t *testing.T) { + request := &extprocv3.ProcessingRequest{ + Request: &extprocv3.ProcessingRequest_RequestHeaders{ + RequestHeaders: &extprocv3.HttpHeaders{ + Headers: &corev3.HeaderMap{Headers: []*corev3.HeaderValue{ + {Key: ":method", RawValue: []byte("GET")}, + {Key: ":authority", RawValue: []byte("api.example.com")}, + {Key: "connection", RawValue: []byte("keep-alive, Upgrade")}, + {Key: "upgrade", RawValue: []byte("websocket")}, + }}, + }, + }, + Attributes: map[string]*structpb.Struct{ + "envoy.filters.http.ext_proc": {Fields: map[string]*structpb.Value{ + extproc.FilterChainNameAttribute: structpb.NewStringValue(leg), + }}, + }, + } + stream := &websocketProcessStream{ctx: context.Background(), requests: []*extprocv3.ProcessingRequest{request}} + server := extproc.NewServer(0, nil, extproc.Handlers{ + extproc.DirectionEgress: egress.New(nil, nil, 0, nil, ""), + }) + + if err := server.Process(stream); err != nil { + t.Fatalf("Process() error = %v", err) + } + if len(stream.responses) != 1 { + t.Fatalf("responses = %d, want 1", len(stream.responses)) + } + response := stream.responses[0].GetImmediateResponse() + if response == nil { + t.Fatalf("response = %v, want immediate denial", stream.responses[0]) + } + if got := response.GetStatus().GetCode(); got != 403 { + t.Errorf("status = %d, want 403", got) + } + if got := string(response.GetBody()); got != websocketDenialBody { + t.Errorf("body = %q, want %q", got, websocketDenialBody) + } + }) + } +} diff --git a/cmd/atenet/internal/router/extproc/metadata.go b/cmd/atenet/internal/router/extproc/metadata.go index ffee0b035d..f7ff69e168 100644 --- a/cmd/atenet/internal/router/extproc/metadata.go +++ b/cmd/atenet/internal/router/extproc/metadata.go @@ -59,7 +59,11 @@ func NewRequestMetadata(headers []*corev3.HeaderValue, attributes map[string]*st val = string(h.RawValue) } - headersMap[k] = val + if existing, ok := headersMap[k]; ok { + headersMap[k] = existing + "," + val + } else { + headersMap[k] = val + } if k == ":path" { path = val } @@ -82,9 +86,40 @@ func NewRequestMetadata(headers []*corev3.HeaderValue, attributes map[string]*st // Header returns the value of a header by name, case-insensitively, or "" when // it was not sent. func (m *RequestMetadata) Header(name string) string { + if m == nil { + return "" + } return m.Headers[strings.ToLower(name)] } +// Get returns the value associated with the passed key, implementing propagation.TextMapCarrier. +func (m *RequestMetadata) Get(key string) string { + return m.Header(key) +} + +// Set stores the key-value pair, implementing propagation.TextMapCarrier. +func (m *RequestMetadata) Set(key, val string) { + if m == nil { + return + } + if m.Headers == nil { + m.Headers = make(map[string]string) + } + m.Headers[strings.ToLower(key)] = val +} + +// Keys lists the keys stored in this carrier, implementing propagation.TextMapCarrier. +func (m *RequestMetadata) Keys() []string { + if m == nil || m.Headers == nil { + return nil + } + keys := make([]string, 0, len(m.Headers)) + for k := range m.Headers { + keys = append(keys, k) + } + return keys +} + // Attribute returns the named CEL request_attributes value, scanning every // filter's entries so callers don't need to hardcode which one reported it, // or "" if none did. A number (Envoy sends int fields as one) renders as its diff --git a/cmd/atenet/internal/router/extproc/metadata_test.go b/cmd/atenet/internal/router/extproc/metadata_test.go index 3789ec8495..09a4b2548b 100644 --- a/cmd/atenet/internal/router/extproc/metadata_test.go +++ b/cmd/atenet/internal/router/extproc/metadata_test.go @@ -103,6 +103,18 @@ func TestExtractMetadata(t *testing.T) { wantPath: "", wantHost: "", }, + { + name: "repeated headers are joined with commas", + headers: []*corev3.HeaderValue{ + {Key: "X-Repeated", Value: "first"}, + {Key: "x-repeated", Value: "second"}, + }, + wantHeaders: map[string]string{ + "x-repeated": "first,second", + }, + wantPath: "", + wantHost: "", + }, } for _, tc := range tests { @@ -122,6 +134,30 @@ func TestExtractMetadata(t *testing.T) { } } +func TestRequestMetadataHeaderCommaJoinsDuplicates(t *testing.T) { + headers := []*corev3.HeaderValue{ + {Key: "X-Single", Value: "single-val"}, + {Key: "X-Multiple", Value: "val-1"}, + {Key: "X-Multiple", Value: "val-2"}, + } + md := NewRequestMetadata(headers, nil) + + if got := md.Header("X-Single"); got != "single-val" { + t.Errorf("Header(X-Single) = %q, want %q", got, "single-val") + } + if got := md.Header("X-Multiple"); got != "val-1,val-2" { + t.Errorf("Header(X-Multiple) = %q, want %q", got, "val-1,val-2") + } + if got := md.Header("X-Missing"); got != "" { + t.Errorf("Header(X-Missing) = %q, want empty string", got) + } + + var nilMD *RequestMetadata + if got := nilMD.Header("X-Single"); got != "" { + t.Errorf("nil.Header() = %q, want empty string", got) + } +} + func TestRequestMetadataAttribute(t *testing.T) { attrs := map[string]*structpb.Struct{ "envoy.filters.http.ext_proc": { diff --git a/cmd/atenet/internal/router/ingress/ingress.go b/cmd/atenet/internal/router/ingress/ingress.go index 6c6a88e406..6b68011cea 100644 --- a/cmd/atenet/internal/router/ingress/ingress.go +++ b/cmd/atenet/internal/router/ingress/ingress.go @@ -34,7 +34,6 @@ import ( extprocv3 "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" envoy_type "github.com/envoyproxy/go-control-plane/envoy/type/v3" "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/propagation" "google.golang.org/protobuf/types/known/structpb" "github.com/agent-substrate/substrate/cmd/atenet/internal/router/extproc" @@ -83,7 +82,7 @@ func (h *Handler) HandleRequestHeaders(ctx context.Context, md *extproc.RequestM // stream's metadata — the per-request traceparent arrives in the // HTTP headers carried inside the ProcessingRequest payload. Extract // from there so our span links to the gateway's ingress span. - ctx = otel.GetTextMapPropagator().Extract(ctx, propagation.MapCarrier(md.Headers)) + ctx = otel.GetTextMapPropagator().Extract(ctx, md) ctx, span := otel.Tracer(extproc.ServiceName).Start(ctx, "ExtProc.RequestHeaders") defer span.End() @@ -177,8 +176,15 @@ func (h *Handler) HandleRequestHeaders(ctx context.Context, md *extproc.RequestM } func routingValue(md *extproc.RequestMetadata, header, attribute string) string { - if value := md.Header(header); value != "" { + if value := md.Attribute(attribute); value != "" && value != "-" { return value } - return md.Attribute(attribute) + // Requests reaching main_internal via a terminated CONNECT tunnel retain + // the tunnel's authority in ConnectAuthorityFilterStateAttribute. For such + // requests, the actor header must have arrived on the outer CONNECT request; + // inner tunneled headers are untrusted and must never be used for routing. + if md.Attribute(extproc.ConnectAuthorityFilterStateAttribute) != "" { + return "" + } + return md.Header(header) } diff --git a/cmd/atenet/internal/router/ingress/ingress_test.go b/cmd/atenet/internal/router/ingress/ingress_test.go index 97275062b1..35c940dba1 100644 --- a/cmd/atenet/internal/router/ingress/ingress_test.go +++ b/cmd/atenet/internal/router/ingress/ingress_test.go @@ -80,6 +80,121 @@ func TestHandleRequestHeadersAcceptsMixedCaseRoutingHeaders(t *testing.T) { } } +func TestHandleRequestHeadersRejectsDuplicateOrCommaHeaders(t *testing.T) { + tests := []struct { + name string + headers []*corev3.HeaderValue + attrs map[string]*structpb.Struct + }{ + { + name: "duplicate headers with different values", + headers: []*corev3.HeaderValue{ + {Key: atenet.TargetActorHeader, Value: "demo/a"}, + {Key: atenet.TargetActorHeader, Value: "secret/x"}, + }, + }, + { + name: "duplicate headers with identical values", + headers: []*corev3.HeaderValue{ + {Key: atenet.TargetActorHeader, Value: "demo/a"}, + {Key: atenet.TargetActorHeader, Value: "demo/a"}, + }, + }, + { + name: "single header with comma-joined value", + headers: []*corev3.HeaderValue{ + {Key: atenet.TargetActorHeader, Value: "demo/a,secret/x"}, + }, + }, + { + name: "filter state attribute with comma-joined value", + attrs: map[string]*structpb.Struct{ + "envoy.filters.http.ext_proc": { + Fields: map[string]*structpb.Value{ + extproc.TargetActorFilterStateAttribute: structpb.NewStringValue("demo/a,secret/x"), + }, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var resumeCalls int32 + clientMock := &mockClient{ + resumeFn: func(_ context.Context, _ *ateapipb.ResumeActorRequest, _ ...grpc.CallOption) (*ateapipb.ResumeActorResponse, error) { + atomic.AddInt32(&resumeCalls, 1) + return nil, errors.New("should not be called") + }, + } + h := New(clientMock, ParkedRequestConfig{}, nil) + md := extproc.NewRequestMetadata(tt.headers, tt.attrs) + + _, err := h.HandleRequestHeaders(context.Background(), md) + if err == nil { + t.Fatalf("HandleRequestHeaders() succeeded, want 404 error") + } + var reqErr *extproc.ReqError + if !errors.As(err, &reqErr) { + t.Fatalf("error = %v, want *extproc.ReqError", err) + } + if reqErr.StatusCode != int(envoy_type.StatusCode_NotFound) { + t.Errorf("StatusCode = %d, want %d", reqErr.StatusCode, envoy_type.StatusCode_NotFound) + } + if !strings.Contains(reqErr.Error(), "invalid actor reference") { + t.Errorf("error string = %q, want containing %q", reqErr.Error(), "invalid actor reference") + } + if calls := atomic.LoadInt32(&resumeCalls); calls != 0 { + t.Errorf("ResumeActor called %d times, want 0", calls) + } + }) + } +} + +func TestHandleRequestHeadersConnectAttributeOverridesInnerHeader(t *testing.T) { + const testUUID = "123e4567-e89b-12d3-a456-426614174000" + var gotActorRef string + clientMock := &mockClient{ + resumeFn: func(_ context.Context, in *ateapipb.ResumeActorRequest, _ ...grpc.CallOption) (*ateapipb.ResumeActorResponse, error) { + gotActorRef = in.GetActor().GetAtespace() + "/" + in.GetActor().GetName() + return &ateapipb.ResumeActorResponse{Actor: &ateapipb.Actor{ + Status: &ateapipb.ActorStatus{WorkerAssignment: &ateapipb.WorkerAssignment{WorkerPodIps: []string{"10.0.0.52"}}}, + }}, nil + }, + } + h := New(clientMock, ParkedRequestConfig{}, nil) + md := extproc.NewRequestMetadata( + []*corev3.HeaderValue{ + {Key: atenet.TargetActorHeader, Value: "secret/x"}, + }, + map[string]*structpb.Struct{ + "envoy.filters.http.ext_proc": { + Fields: map[string]*structpb.Value{ + extproc.TargetActorFilterStateAttribute: structpb.NewStringValue("team-a/" + testUUID), + }, + }, + }, + ) + + res, err := h.HandleRequestHeaders(context.Background(), md) + if err != nil { + t.Fatalf("HandleRequestHeaders() error = %v", err) + } + + if got, want := gotActorRef, "team-a/"+testUUID; got != want { + t.Errorf("ResumeActor called with %q, want %q", got, want) + } + + mutation := res.Response.GetResponse().GetHeaderMutation() + gotMutations := map[string]string{} + for _, headerOption := range mutation.GetSetHeaders() { + gotMutations[strings.ToLower(headerOption.Header.Key)] = string(headerOption.Header.RawValue) + } + if got, want := gotMutations[strings.ToLower(atenet.TargetActorHeader)], "team-a/"+testUUID; got != want { + t.Errorf("actor target mutation = %q, want %q", got, want) + } +} + // dynamicMetadataTarget extracts the resolved worker address // HandleRequestHeaders reports via OriginalDstMetadataKey/OriginalDstAddressKey. func dynamicMetadataTarget(dynamicMetadata *structpb.Struct) string { @@ -396,11 +511,11 @@ func TestHandleRequestHeadersUsesRetainedConnectAuthorityForPort(t *testing.T) { "envoy.filters.http.ext_proc": { Fields: map[string]*structpb.Value{ extproc.ConnectAuthorityFilterStateAttribute: structpb.NewStringValue("unrelated.example:9090"), + extproc.TargetActorFilterStateAttribute: structpb.NewStringValue("team-a/" + testUUID), }, }, } md := extproc.NewRequestMetadata([]*corev3.HeaderValue{ - {Key: atenet.TargetActorHeader, Value: "team-a/" + testUUID}, {Key: ":authority", Value: "inner.example"}, }, attrs) @@ -411,6 +526,177 @@ func TestHandleRequestHeadersUsesRetainedConnectAuthorityForPort(t *testing.T) { if got, want := dynamicMetadataPort(res.DynamicMetadata), "9090"; got != want { t.Errorf("target port = %q, want %q", got, want) } + + mutation := res.Response.GetResponse().GetHeaderMutation() + gotMutations := map[string]string{} + for _, headerOption := range mutation.GetSetHeaders() { + gotMutations[strings.ToLower(headerOption.Header.Key)] = string(headerOption.Header.RawValue) + } + if got, want := gotMutations[strings.ToLower(atenet.TargetActorHeader)], "team-a/"+testUUID; got != want { + t.Errorf("actor target mutation = %q, want %q", got, want) + } +} + +func TestHandleRequestHeadersConnectRequiresOuterTargetActor(t *testing.T) { + const testUUID = "123e4567-e89b-12d3-a456-426614174000" + tests := []struct { + name string + headers []*corev3.HeaderValue + attrs map[string]*structpb.Struct + expectErr bool + wantStatusCode int + wantActorRef string + }{ + { + name: "connect request with no outer target actor and inner header is rejected", + headers: []*corev3.HeaderValue{ + {Key: atenet.TargetActorHeader, Value: "team-a/" + testUUID}, + {Key: ":authority", Value: "inner.example"}, + }, + attrs: map[string]*structpb.Struct{ + "envoy.filters.http.ext_proc": { + Fields: map[string]*structpb.Value{ + extproc.ConnectAuthorityFilterStateAttribute: structpb.NewStringValue("unrelated.example:9090"), + }, + }, + }, + expectErr: true, + wantStatusCode: int(envoy_type.StatusCode_NotFound), + }, + { + name: "connect request with empty outer target actor and inner header is rejected", + headers: []*corev3.HeaderValue{ + {Key: atenet.TargetActorHeader, Value: "team-a/" + testUUID}, + {Key: ":authority", Value: "inner.example"}, + }, + attrs: map[string]*structpb.Struct{ + "envoy.filters.http.ext_proc": { + Fields: map[string]*structpb.Value{ + extproc.ConnectAuthorityFilterStateAttribute: structpb.NewStringValue("unrelated.example:9090"), + extproc.TargetActorFilterStateAttribute: structpb.NewStringValue(""), + }, + }, + }, + expectErr: true, + wantStatusCode: int(envoy_type.StatusCode_NotFound), + }, + { + name: "connect request with no outer target actor and no inner header is rejected", + headers: []*corev3.HeaderValue{ + {Key: ":authority", Value: "inner.example"}, + }, + attrs: map[string]*structpb.Struct{ + "envoy.filters.http.ext_proc": { + Fields: map[string]*structpb.Value{ + extproc.ConnectAuthorityFilterStateAttribute: structpb.NewStringValue("unrelated.example:9090"), + }, + }, + }, + expectErr: true, + wantStatusCode: int(envoy_type.StatusCode_NotFound), + }, + { + name: "connect request with comma-separated outer target actor is rejected", + headers: []*corev3.HeaderValue{ + {Key: ":authority", Value: "inner.example"}, + }, + attrs: map[string]*structpb.Struct{ + "envoy.filters.http.ext_proc": { + Fields: map[string]*structpb.Value{ + extproc.ConnectAuthorityFilterStateAttribute: structpb.NewStringValue("unrelated.example:9090"), + extproc.TargetActorFilterStateAttribute: structpb.NewStringValue("team-a/a,team-b/b"), + }, + }, + }, + expectErr: true, + wantStatusCode: int(envoy_type.StatusCode_NotFound), + }, + { + name: "connect request with outer target actor resolves even if inner header is omitted", + headers: []*corev3.HeaderValue{ + {Key: ":authority", Value: "inner.example"}, + }, + attrs: map[string]*structpb.Struct{ + "envoy.filters.http.ext_proc": { + Fields: map[string]*structpb.Value{ + extproc.ConnectAuthorityFilterStateAttribute: structpb.NewStringValue("unrelated.example:9090"), + extproc.TargetActorFilterStateAttribute: structpb.NewStringValue("team-a/" + testUUID), + }, + }, + }, + expectErr: false, + wantActorRef: "team-a/" + testUUID, + }, + { + name: "connect request with outer target actor X overrides inner header Y", + headers: []*corev3.HeaderValue{ + {Key: atenet.TargetActorHeader, Value: "team-b/untrusted-actor"}, + {Key: ":authority", Value: "inner.example"}, + }, + attrs: map[string]*structpb.Struct{ + "envoy.filters.http.ext_proc": { + Fields: map[string]*structpb.Value{ + extproc.ConnectAuthorityFilterStateAttribute: structpb.NewStringValue("unrelated.example:9090"), + extproc.TargetActorFilterStateAttribute: structpb.NewStringValue("team-a/" + testUUID), + }, + }, + }, + expectErr: false, + wantActorRef: "team-a/" + testUUID, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var gotActorRef string + var resumeCalls int32 + clientMock := &mockClient{ + resumeFn: func(_ context.Context, in *ateapipb.ResumeActorRequest, _ ...grpc.CallOption) (*ateapipb.ResumeActorResponse, error) { + atomic.AddInt32(&resumeCalls, 1) + gotActorRef = in.GetActor().GetAtespace() + "/" + in.GetActor().GetName() + return &ateapipb.ResumeActorResponse{Actor: &ateapipb.Actor{ + Status: &ateapipb.ActorStatus{WorkerAssignment: &ateapipb.WorkerAssignment{WorkerPodIps: []string{"10.0.0.52"}}}, + }}, nil + }, + } + + h := New(clientMock, ParkedRequestConfig{}, nil) + md := extproc.NewRequestMetadata(tt.headers, tt.attrs) + + res, err := h.HandleRequestHeaders(context.Background(), md) + if tt.expectErr { + if err == nil { + t.Fatalf("HandleRequestHeaders() succeeded, want error") + } + var reqErr *extproc.ReqError + if !errors.As(err, &reqErr) { + t.Fatalf("error = %v, want *extproc.ReqError", err) + } + if reqErr.StatusCode != tt.wantStatusCode { + t.Errorf("StatusCode = %d, want %d", reqErr.StatusCode, tt.wantStatusCode) + } + if calls := atomic.LoadInt32(&resumeCalls); calls != 0 { + t.Errorf("ResumeActor called %d times, want 0", calls) + } + return + } + + if err != nil { + t.Fatalf("HandleRequestHeaders() error = %v", err) + } + if got, want := gotActorRef, tt.wantActorRef; got != want { + t.Errorf("ResumeActor called with %q, want %q", got, want) + } + mutation := res.Response.GetResponse().GetHeaderMutation() + gotMutations := map[string]string{} + for _, headerOption := range mutation.GetSetHeaders() { + gotMutations[strings.ToLower(headerOption.Header.Key)] = string(headerOption.Header.RawValue) + } + if got, want := gotMutations[strings.ToLower(atenet.TargetActorHeader)], tt.wantActorRef; got != want { + t.Errorf("actor target mutation = %q, want %q", got, want) + } + }) + } } // TestHandleRequestHeaders_FullLotServesRunningActor pins the design guarantee diff --git a/cmd/atenet/internal/router/xds.go b/cmd/atenet/internal/router/xds.go index e134bbf82c..8f0973b198 100644 --- a/cmd/atenet/internal/router/xds.go +++ b/cmd/atenet/internal/router/xds.go @@ -918,12 +918,12 @@ func (x *XdsServer) buildMainInternalListener() *listenerv3.Listener { // main_internal can read it across the CONNECT internal-listener hop. func actorRoutingFilterStateFilter(captureAuthority bool) *hcmv3.HttpFilter { values := make([]*setfilterstatecommonv3.FilterStateValue, 0, 2) - routingFields := []struct{ key, header string }{ - {extproc.TargetActorFilterStateKey, atenet.TargetActorHeader}, + routingFields := []struct{ key, format string }{ + {extproc.TargetActorFilterStateKey, "%CEL(request.headers['" + atenet.TargetActorHeader + "'])%"}, } if captureAuthority { - routingFields = append(routingFields, struct{ key, header string }{ - extproc.ConnectAuthorityFilterStateKey, extproc.AuthorityHeader, + routingFields = append(routingFields, struct{ key, format string }{ + extproc.ConnectAuthorityFilterStateKey, "%REQ(" + extproc.AuthorityHeader + ")%", }) } for _, routingField := range routingFields { @@ -937,7 +937,7 @@ func actorRoutingFilterStateFilter(captureAuthority bool) *hcmv3.HttpFilter { Format: &corev3.SubstitutionFormatString_TextFormatSource{ TextFormatSource: &corev3.DataSource{ Specifier: &corev3.DataSource_InlineString{ - InlineString: "%REQ(" + routingField.header + ")%", + InlineString: routingField.format, }, }, }, diff --git a/cmd/atenet/internal/router/xds_listen_test.go b/cmd/atenet/internal/router/xds_listen_test.go new file mode 100644 index 0000000000..0248374e59 --- /dev/null +++ b/cmd/atenet/internal/router/xds_listen_test.go @@ -0,0 +1,110 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package router + +import ( + "net" + "strconv" + "strings" + "testing" +) + +// TestXdsListenAddressIsLoopback guards #2276: the xDS server is plaintext +// and unauthenticated, and ADS/SDS hand out the whole dataplane topology and +// secret paths, so it must never bind a pod-IP-reachable address. +func TestXdsListenAddressIsLoopback(t *testing.T) { + for _, port := range []int{0, 18000} { + addr := xdsListenAddress(port) + host, p, err := net.SplitHostPort(addr) + if err != nil { + t.Fatalf("xdsListenAddress(%d) = %q, not host:port: %v", port, addr, err) + } + if ip := net.ParseIP(host); ip == nil || !ip.IsLoopback() { + t.Errorf("xdsListenAddress(%d) binds host %q; it must be a loopback IP", port, host) + } + if p != strconv.Itoa(port) { + t.Errorf("xdsListenAddress(%d) uses port %q", port, p) + } + } + + // The kernel must agree: listen on an ephemeral port and check what was + // actually bound, so a wildcard sneaking back in fails here. + lis, err := net.Listen("tcp", xdsListenAddress(0)) + if err != nil { + t.Fatalf("listening on %s: %v", xdsListenAddress(0), err) + } + defer lis.Close() + if ip := lis.Addr().(*net.TCPAddr).IP; !ip.IsLoopback() { + t.Errorf("xDS listener bound %v; it must be loopback", lis.Addr()) + } +} + +// TestXdsManifestIsLoopbackOnly checks that the shipped atenet-router manifest +// still works with a loopback-only xDS server, and does not advertise it: the +// Envoy bootstrap must dial a loopback IP on the --port-xds port, and no +// container in the pod may declare that port. +func TestXdsManifestIsLoopbackOnly(t *testing.T) { + pod := findDeployment(t, routerManifestPath, "atenet-router").Spec.Template.Spec + + xdsPort := 0 + for _, c := range pod.Containers { + if c.Name != "atenet-router" { + continue + } + for _, a := range c.Args { + if v, ok := strings.CutPrefix(a, "--port-xds="); ok { + n, err := strconv.Atoi(v) + if err != nil { + t.Fatalf("--port-xds=%q is not a number: %v", v, err) + } + xdsPort = n + } + } + } + if xdsPort == 0 { + t.Fatal("the atenet-router container sets no --port-xds; the manifest changed shape and this test is checking nothing") + } + + b := parseEnvoyAdminBootstrap(t, envoyConfigFrom(t, routerManifestPath, "atenet-router-envoy-config")) + found := false + for _, cl := range b.StaticResources.Clusters { + if cl.Name != "xds_cluster" { + continue + } + for _, ep := range cl.LoadAssignment.Endpoints { + for _, lb := range ep.LbEndpoints { + sa := lb.Endpoint.Address.SocketAddress + found = true + if ip := net.ParseIP(sa.Address); ip == nil || !ip.IsLoopback() { + t.Errorf("Envoy xds_cluster dials %q; the xDS server only listens on loopback", sa.Address) + } + if int(sa.PortValue) != xdsPort { + t.Errorf("Envoy xds_cluster dials port %d, but the router serves xDS on --port-xds=%d", sa.PortValue, xdsPort) + } + } + } + } + if !found { + t.Fatal("the Envoy bootstrap has no xds_cluster endpoint; the manifest changed shape and this test is checking nothing") + } + + for _, c := range append(pod.InitContainers, pod.Containers...) { + for _, p := range c.Ports { + if int(p.ContainerPort) == xdsPort { + t.Errorf("container %s declares port %d (%q), the loopback-only xDS port", c.Name, p.ContainerPort, p.Name) + } + } + } +} diff --git a/cmd/atenet/internal/router/xds_test.go b/cmd/atenet/internal/router/xds_test.go index 2bd5a64192..3794155743 100644 --- a/cmd/atenet/internal/router/xds_test.go +++ b/cmd/atenet/internal/router/xds_test.go @@ -62,14 +62,14 @@ func TestActorRoutingFilterStateFilter(t *testing.T) { { name: "ordinary ingress", want: map[string]string{ - extproc.TargetActorFilterStateKey: "%REQ(ate-target-actor)%", + extproc.TargetActorFilterStateKey: "%CEL(request.headers['ate-target-actor'])%", }, }, { name: "CONNECT termination", captureAuthority: true, want: map[string]string{ - extproc.TargetActorFilterStateKey: "%REQ(ate-target-actor)%", + extproc.TargetActorFilterStateKey: "%CEL(request.headers['ate-target-actor'])%", extproc.ConnectAuthorityFilterStateKey: "%REQ(:authority)%", }, }, diff --git a/cmd/ateom-gvisor/hosted.go b/cmd/ateom-gvisor/hosted.go index 890763ab9d..45b06e35e3 100644 --- a/cmd/ateom-gvisor/hosted.go +++ b/cmd/ateom-gvisor/hosted.go @@ -21,6 +21,7 @@ import ( "fmt" "log/slog" "net" + "sync/atomic" "time" "github.com/agent-substrate/substrate/internal/apierror" @@ -42,6 +43,8 @@ type hostedActor struct { session *workloadSession // usage is this activation's epoch, CPU baseline, and latest sample. usage *ateomstats.Activation + // deadReported keeps the usage sweep to one warning per activation. + deadReported atomic.Bool } // admitActor reserves capacity before network setup. An actor that is already diff --git a/cmd/ateom-gvisor/internal/cdiinject/inject.go b/cmd/ateom-gvisor/internal/cdiinject/inject.go index fadc35a7b9..a1306d02de 100644 --- a/cmd/ateom-gvisor/internal/cdiinject/inject.go +++ b/cmd/ateom-gvisor/internal/cdiinject/inject.go @@ -24,11 +24,13 @@ package cdiinject import ( "context" "encoding/json" + "errors" "fmt" "log/slog" "os" "path/filepath" "strings" + "syscall" specs "github.com/opencontainers/runtime-spec/specs-go" "golang.org/x/sys/unix" @@ -52,6 +54,11 @@ type Options struct { // binary the caller trusts rather than wherever the generator pointed. HookBinary string + // AllowedHostDirs are the host directories from which CDI mounts may be + // sourced. Every mount's HostPath must resolve inside one of these + // directories without escaping via symlinks. + AllowedHostDirs []string + // LibraryDirs are prepended to the container's LD_LIBRARY_PATH. LibraryDirs []string @@ -112,14 +119,20 @@ func IntoBundle(ctx context.Context, bundleDir string, spec *cdi.Spec, opts Opti }) } + cdiMounts := make([]specs.Mount, 0, len(edits.Mounts)) for _, m := range edits.Mounts { + if err := validateMount(m, opts.AllowedHostDirs); err != nil { + return err + } mType := m.Type if mType == "" { mType = "bind" // CDI omits type for its bind mounts; runsc's gofer needs it. } - ociSpec.Mounts = append(ociSpec.Mounts, specs.Mount{ + mount := specs.Mount{ Source: m.HostPath, Destination: m.ContainerPath, Type: mType, Options: m.Options, - }) + } + ociSpec.Mounts = append(ociSpec.Mounts, mount) + cdiMounts = append(cdiMounts, mount) } if ociSpec.Process != nil { @@ -160,7 +173,7 @@ func IntoBundle(ctx context.Context, bundleDir string, spec *cdi.Spec, opts Opti if !filepath.IsAbs(rootfs) { rootfs = filepath.Join(bundleDir, rootfs) } - if err := StageSonameSymlinks(ctx, rootfs, ociSpec.Mounts); err != nil { + if err := StageSonameSymlinks(ctx, rootfs, cdiMounts, opts.AllowedHostDirs); err != nil { return fmt.Errorf("staging SONAME symlinks: %w", err) } @@ -188,22 +201,101 @@ func hasInjectedDevice(l *specs.Linux, nodes []cdi.Dev) bool { return false } -// resolveDevNumbers fills a device node's major/minor from the host when the CDI -// spec omitted them. CDI delegates that to the OCI runtime, which stats the -// host; the merge happens here instead, so the stat happens here too. Otherwise -// the container gets bogus 0,0 char devices that reach no driver. +// devRoot is the host directory under which CDI device nodes must reside. A var +// so tests can point it at a fixture directory. +var devRoot = "/dev" + +// resolveDevNumbers validates that path is a clean device path confined to +// devRoot and fills its major/minor from the host when the CDI spec omitted +// them. CDI delegates that to the OCI runtime, which stats the host; the merge +// happens here instead, so the stat happens here too. Otherwise the container +// gets bogus 0,0 char devices that reach no driver. func resolveDevNumbers(path string, major, minor int64) (int64, int64, error) { + cleanDevRoot := filepath.Clean(devRoot) + if path == "" || !filepath.IsAbs(path) || filepath.Clean(path) != path { + return 0, 0, fmt.Errorf("CDI device path %q must be a clean absolute path", path) + } + rel, err := filepath.Rel(cleanDevRoot, path) + if err != nil || rel == "." || !filepath.IsLocal(rel) { + return 0, 0, fmt.Errorf("CDI device path %q is outside %s", path, cleanDevRoot) + } + root, err := os.OpenRoot(cleanDevRoot) + if err != nil { + return 0, 0, fmt.Errorf("opening device root %s: %w", cleanDevRoot, err) + } + defer root.Close() + + fi, err := root.Stat(rel) if major != 0 { + if err != nil && !errors.Is(err, os.ErrNotExist) { + return 0, 0, fmt.Errorf("stat device %s: %w", path, err) + } + if err == nil && fi.Mode()&os.ModeDevice == 0 { + return 0, 0, fmt.Errorf("CDI device path %s is not a device node", path) + } return major, minor, nil } - var st unix.Stat_t - if err := unix.Stat(path, &st); err != nil { + if err != nil { return 0, 0, fmt.Errorf("stat device %s: %w", path, err) } + if fi.Mode()&os.ModeDevice == 0 { + return 0, 0, fmt.Errorf("CDI device path %s is not a device node", path) + } + st, ok := fi.Sys().(*syscall.Stat_t) + if !ok { + return 0, 0, fmt.Errorf("unexpected stat type for %s", path) + } rdev := uint64(st.Rdev) return int64(unix.Major(rdev)), int64(unix.Minor(rdev)), nil } +// validateMount checks that a CDI mount has a clean, rootfs-local containerPath +// and a hostPath confined to one of allowedHostDirs. +func validateMount(m cdi.Mount, allowedHostDirs []string) error { + if m.ContainerPath == "" || !filepath.IsAbs(m.ContainerPath) || filepath.Clean(m.ContainerPath) != m.ContainerPath { + return fmt.Errorf("CDI mount containerPath %q must be a clean absolute path", m.ContainerPath) + } + relDst := strings.TrimPrefix(m.ContainerPath, "/") + if relDst == "" || !filepath.IsLocal(relDst) { + return fmt.Errorf("CDI mount containerPath %q escapes the rootfs", m.ContainerPath) + } + _, _, err := resolveAllowedHostPath(m.HostPath, allowedHostDirs) + return err +} + +// resolveAllowedHostPath verifies that hostPath is a clean absolute path inside +// one of allowedDirs and that resolving it through os.Root does not escape that +// directory via symlinks. It returns the matched root directory and relative path. +func resolveAllowedHostPath(hostPath string, allowedDirs []string) (string, string, error) { + if hostPath == "" || !filepath.IsAbs(hostPath) || filepath.Clean(hostPath) != hostPath { + return "", "", fmt.Errorf("CDI mount hostPath %q must be a clean absolute path", hostPath) + } + for _, dir := range allowedDirs { + if dir == "" { + continue + } + cleanDir := filepath.Clean(dir) + if !filepath.IsAbs(cleanDir) { + continue + } + rel, err := filepath.Rel(cleanDir, hostPath) + if err != nil || rel == "." || !filepath.IsLocal(rel) { + continue + } + root, err := os.OpenRoot(cleanDir) + if err != nil { + return "", "", fmt.Errorf("opening allowed host dir %q: %w", cleanDir, err) + } + _, statErr := root.Stat(rel) + root.Close() + if statErr != nil { + return "", "", fmt.Errorf("CDI mount hostPath %q invalid in allowed dir %q: %w", hostPath, cleanDir, statErr) + } + return cleanDir, rel, nil + } + return "", "", fmt.Errorf("CDI mount hostPath %q is not under any allowed host directory", hostPath) +} + // dropEnvVar returns env with every "KEY=..." entry for the given key removed. func dropEnvVar(env []string, key string) []string { prefix := key + "=" diff --git a/cmd/ateom-gvisor/internal/cdiinject/inject_test.go b/cmd/ateom-gvisor/internal/cdiinject/inject_test.go index 038d46fefc..3ce891d3e2 100644 --- a/cmd/ateom-gvisor/internal/cdiinject/inject_test.go +++ b/cmd/ateom-gvisor/internal/cdiinject/inject_test.go @@ -56,11 +56,12 @@ func injectFixture(t *testing.T, bundleDir, specDir string) error { return err } return IntoBundle(context.Background(), bundleDir, spec, Options{ - Devices: []string{"all"}, - AllowedHooks: map[string]bool{"create-symlinks": true, "enable-cuda-compat": true}, - HookBinary: testHookBinary, - LibraryDirs: []string{"/usr/local/nvidia/lib64"}, - DropEnv: []string{"NVIDIA_VISIBLE_DEVICES"}, + Devices: []string{"all"}, + AllowedHooks: map[string]bool{"create-symlinks": true, "enable-cuda-compat": true}, + HookBinary: testHookBinary, + AllowedHostDirs: []string{"/usr/local/nvidia"}, + LibraryDirs: []string{"/usr/local/nvidia/lib64"}, + DropEnv: []string{"NVIDIA_VISIBLE_DEVICES"}, }) } @@ -320,15 +321,24 @@ func TestStageSonameSymlinks_ConfinedToRootfs(t *testing.T) { } } + hostDir := filepath.Join(dir, "host") + if err := os.MkdirAll(hostDir, 0o755); err != nil { + t.Fatal(err) + } + hostLib := filepath.Join(hostDir, "libcuda.so.580.65.06") + if err := os.WriteFile(hostLib, []byte("stub"), 0o644); err != nil { + t.Fatal(err) + } + old := elfSonameFn - elfSonameFn = func(string) (string, error) { return tc.soname, nil } + elfSonameFn = func(string, string) (string, error) { return tc.soname, nil } defer func() { elfSonameFn = old }() // Refusing outright and skipping are both acceptable; writing outside is not. _ = StageSonameSymlinks(context.Background(), rootfs, []specs.Mount{{ - Source: "/host/libcuda.so.580.65.06", + Source: hostLib, Destination: tc.dest, - }}) + }}, []string{hostDir}) if b, err := os.ReadFile(victim); err != nil || string(b) != "do not delete" { t.Fatalf("a file outside the rootfs was modified: err=%v content=%q", err, b) @@ -355,3 +365,286 @@ func TestInjectGPUIntoBundle_MissingSpecFails(t *testing.T) { t.Fatal("expected error when the CDI spec is missing") } } + +func TestIntoBundle_HostMountsAllowlistAndConfinement(t *testing.T) { + dir := t.TempDir() + allowedDir := filepath.Join(dir, "allowed-driver") + outsideDir := filepath.Join(dir, "outside") + if err := os.MkdirAll(filepath.Join(allowedDir, "lib64"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(outsideDir, 0o755); err != nil { + t.Fatal(err) + } + + validLib := filepath.Join(allowedDir, "lib64", "libcuda.so.580.65.06") + if err := os.WriteFile(validLib, []byte("stub-lib"), 0o644); err != nil { + t.Fatal(err) + } + outsideSecret := filepath.Join(outsideDir, "libsecret.so.1.0") + if err := os.WriteFile(outsideSecret, []byte("secret"), 0o644); err != nil { + t.Fatal(err) + } + + escapeFileLink := filepath.Join(allowedDir, "lib64", "libescape.so.1.0") + if err := os.Symlink(outsideSecret, escapeFileLink); err != nil { + t.Fatal(err) + } + escapeRelLink := filepath.Join(allowedDir, "lib64", "librel-escape.so.1.0") + if err := os.Symlink("../../outside/libsecret.so.1.0", escapeRelLink); err != nil { + t.Fatal(err) + } + escapeDirLink := filepath.Join(allowedDir, "escaped-dir") + if err := os.Symlink(outsideDir, escapeDirLink); err != nil { + t.Fatal(err) + } + + old := elfSonameFn + elfSonameFn = func(string, string) (string, error) { return "libcuda.so.1", nil } + defer func() { elfSonameFn = old }() + + newBundle := func(t *testing.T) string { + t.Helper() + b := filepath.Join(t.TempDir(), "bundle") + if err := os.MkdirAll(filepath.Join(b, "rootfs"), 0o755); err != nil { + t.Fatal(err) + } + data, err := json.Marshal(&specs.Spec{Version: "1.0.0", Process: &specs.Process{Args: []string{"true"}}}) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(b, "config.json"), data, 0o644); err != nil { + t.Fatal(err) + } + return b + } + + specForMount := func(m cdi.Mount) *cdi.Spec { + return &cdi.Spec{ + Devices: []cdi.Device{{ + Name: "all", + ContainerEdits: cdi.Edits{ + DeviceNodes: []cdi.Dev{{Path: "/dev/nvidia0", Major: 195, Minor: 0, Type: "c"}}, + Mounts: []cdi.Mount{m}, + }, + }}, + } + } + + for _, tc := range []struct { + name string + mount cdi.Mount + allowedDirs []string + wantErr bool + }{ + { + name: "allowed regular file mount succeeds", + mount: cdi.Mount{ + HostPath: validLib, + ContainerPath: "/usr/local/nvidia/lib64/libcuda.so.580.65.06", + }, + allowedDirs: []string{allowedDir}, + wantErr: false, + }, + { + name: "empty allowlist rejects mount", + mount: cdi.Mount{ + HostPath: validLib, + ContainerPath: "/usr/local/nvidia/lib64/libcuda.so.580.65.06", + }, + allowedDirs: nil, + wantErr: true, + }, + { + name: "hostPath outside allowed directory is rejected", + mount: cdi.Mount{ + HostPath: outsideSecret, + ContainerPath: "/usr/local/nvidia/lib64/libsecret.so.1.0", + }, + allowedDirs: []string{allowedDir}, + wantErr: true, + }, + { + name: "unclean hostPath with dot-dot is rejected", + mount: cdi.Mount{ + HostPath: allowedDir + "/../outside/libsecret.so.1.0", + ContainerPath: "/usr/local/nvidia/lib64/libsecret.so.1.0", + }, + allowedDirs: []string{allowedDir}, + wantErr: true, + }, + { + name: "absolute symlink escaping allowed directory is rejected", + mount: cdi.Mount{ + HostPath: escapeFileLink, + ContainerPath: "/usr/local/nvidia/lib64/libescape.so.1.0", + }, + allowedDirs: []string{allowedDir}, + wantErr: true, + }, + { + name: "relative symlink escaping allowed directory is rejected", + mount: cdi.Mount{ + HostPath: escapeRelLink, + ContainerPath: "/usr/local/nvidia/lib64/librel-escape.so.1.0", + }, + allowedDirs: []string{allowedDir}, + wantErr: true, + }, + { + name: "parent directory symlink escaping allowed directory is rejected", + mount: cdi.Mount{ + HostPath: filepath.Join(escapeDirLink, "libsecret.so.1.0"), + ContainerPath: "/usr/local/nvidia/lib64/libsecret.so.1.0", + }, + allowedDirs: []string{allowedDir}, + wantErr: true, + }, + { + name: "relative containerPath is rejected", + mount: cdi.Mount{ + HostPath: validLib, + ContainerPath: "usr/local/nvidia/lib64/libcuda.so.580.65.06", + }, + allowedDirs: []string{allowedDir}, + wantErr: true, + }, + { + name: "unclean containerPath is rejected", + mount: cdi.Mount{ + HostPath: validLib, + ContainerPath: "/usr/local/../../escaped", + }, + allowedDirs: []string{allowedDir}, + wantErr: true, + }, + } { + t.Run(tc.name, func(t *testing.T) { + bundle := newBundle(t) + err := IntoBundle(context.Background(), bundle, specForMount(tc.mount), Options{ + Devices: []string{"all"}, + AllowedHostDirs: tc.allowedDirs, + }) + if (err != nil) != tc.wantErr { + t.Fatalf("IntoBundle() error = %v, wantErr %v", err, tc.wantErr) + } + if !tc.wantErr { + linkTarget, err := os.Readlink(filepath.Join(bundle, "rootfs/usr/local/nvidia/lib64/libcuda.so.1")) + if err != nil || linkTarget != "libcuda.so.580.65.06" { + t.Fatalf("SONAME symlink target = %q, err = %v; want libcuda.so.580.65.06", linkTarget, err) + } + } + }) + } +} + +func TestIntoBundle_DevicePathConfinement(t *testing.T) { + dir := t.TempDir() + fakeDev := filepath.Join(dir, "dev") + outside := filepath.Join(dir, "outside") + if err := os.MkdirAll(fakeDev, 0o755); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(outside, 0o755); err != nil { + t.Fatal(err) + } + + regularInDev := filepath.Join(fakeDev, "not-a-device") + if err := os.WriteFile(regularInDev, []byte("plain"), 0o644); err != nil { + t.Fatal(err) + } + escapeLink := filepath.Join(fakeDev, "escaped-dev") + if err := os.Symlink("/dev/null", escapeLink); err != nil { + t.Fatal(err) + } + + oldDevRoot := devRoot + devRoot = fakeDev + defer func() { devRoot = oldDevRoot }() + + for _, tc := range []struct { + name string + dev cdi.Dev + }{ + { + name: "device path outside devRoot is rejected", + dev: cdi.Dev{Path: "/etc/shadow", Major: 195, Minor: 0}, + }, + { + name: "unclean device path with dot-dot is rejected", + dev: cdi.Dev{Path: fakeDev + "/../outside/node", Major: 195, Minor: 0}, + }, + { + name: "symlink escaping devRoot is rejected when major is 0", + dev: cdi.Dev{Path: escapeLink}, + }, + { + name: "symlink escaping devRoot is rejected even when major is set", + dev: cdi.Dev{Path: escapeLink, Major: 1, Minor: 3}, + }, + { + name: "regular file inside devRoot is rejected", + dev: cdi.Dev{Path: regularInDev}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + bundle := filepath.Join(t.TempDir(), "bundle") + if err := os.MkdirAll(filepath.Join(bundle, "rootfs"), 0o755); err != nil { + t.Fatal(err) + } + data, _ := json.Marshal(&specs.Spec{Version: "1.0.0", Process: &specs.Process{Args: []string{"true"}}}) + if err := os.WriteFile(filepath.Join(bundle, "config.json"), data, 0o644); err != nil { + t.Fatal(err) + } + spec := &cdi.Spec{ + Devices: []cdi.Device{{ + Name: "all", + ContainerEdits: cdi.Edits{DeviceNodes: []cdi.Dev{tc.dev}}, + }}, + } + if err := IntoBundle(context.Background(), bundle, spec, Options{Devices: []string{"all"}}); err == nil { + t.Fatalf("expected error for device %+v, got nil", tc.dev) + } + }) + } +} + +func TestStageSonameSymlinks_HostConfinement(t *testing.T) { + dir := t.TempDir() + allowedDir := filepath.Join(dir, "allowed") + outsideDir := filepath.Join(dir, "outside") + rootfs := filepath.Join(dir, "rootfs") + for _, d := range []string{allowedDir, outsideDir, rootfs} { + if err := os.MkdirAll(d, 0o755); err != nil { + t.Fatal(err) + } + } + + outsideFile := filepath.Join(outsideDir, "liboutside.so.1.0") + if err := os.WriteFile(outsideFile, []byte("not-allowed"), 0o644); err != nil { + t.Fatal(err) + } + escapeLink := filepath.Join(allowedDir, "libescape.so.1.0") + if err := os.Symlink(outsideFile, escapeLink); err != nil { + t.Fatal(err) + } + + // Unallowed host path must fail StageSonameSymlinks. + if err := StageSonameSymlinks(context.Background(), rootfs, []specs.Mount{{ + Source: outsideFile, + Destination: "/usr/lib/liboutside.so.1.0", + }}, []string{allowedDir}); err == nil { + t.Fatal("expected error for mount source outside allowedHostDirs") + } + + // Symlink inside allowedDir pointing outside must fail both StageSonameSymlinks and elfSoname. + if err := StageSonameSymlinks(context.Background(), rootfs, []specs.Mount{{ + Source: escapeLink, + Destination: "/usr/lib/libescape.so.1.0", + }}, []string{allowedDir}); err == nil { + t.Fatal("expected error for symlink escaping allowedHostDirs") + } + if _, err := elfSoname(allowedDir, "libescape.so.1.0"); err == nil { + t.Fatal("expected elfSoname to reject symlink escaping allowedDir") + } +} diff --git a/cmd/ateom-gvisor/internal/cdiinject/soname.go b/cmd/ateom-gvisor/internal/cdiinject/soname.go index 281f9976d6..ea88c5c368 100644 --- a/cmd/ateom-gvisor/internal/cdiinject/soname.go +++ b/cmd/ateom-gvisor/internal/cdiinject/soname.go @@ -42,7 +42,7 @@ import ( // symlink out of the rootfs would otherwise have the kernel resolve it in ateom's // mount namespace, where the shared image cache and other actors' bundles are // mounted. Same treatment as createExtraDirs in internal/imagecache. -func StageSonameSymlinks(ctx context.Context, rootfs string, mounts []specs.Mount) error { +func StageSonameSymlinks(ctx context.Context, rootfs string, mounts []specs.Mount, allowedHostDirs []string) error { root, err := os.OpenRoot(rootfs) if err != nil { return fmt.Errorf("while opening rootfs %q: %w", rootfs, err) @@ -55,7 +55,11 @@ func StageSonameSymlinks(ctx context.Context, rootfs string, mounts []specs.Moun if !strings.Contains(base, ".so.") || m.Source == "" { continue } - soname, err := elfSonameFn(m.Source) + hostRoot, relSource, err := resolveAllowedHostPath(m.Source, allowedHostDirs) + if err != nil { + return err + } + soname, err := elfSonameFn(hostRoot, relSource) if err != nil { slog.WarnContext(ctx, "Skipping SONAME symlink for driver library", slog.String("library", m.Source), slog.Any("err", err)) @@ -91,18 +95,36 @@ func StageSonameSymlinks(ctx context.Context, rootfs string, mounts []specs.Moun // real ELF file on disk. var elfSonameFn = elfSoname -// elfSoname returns a shared library's DT_SONAME. It returns "" with a nil error for -// a library that simply carries no DT_SONAME entry, and an error when the file could -// not be opened or parsed as ELF. -func elfSoname(path string) (string, error) { - f, err := elf.Open(path) +// elfSoname returns a shared library's DT_SONAME, opening rel inside rootDir via +// os.Root so symlinks cannot escape the allowed host directory. It returns "" with +// a nil error for a library that simply carries no DT_SONAME entry, and an error +// when the file could not be opened or parsed as a regular ELF file. +func elfSoname(rootDir, rel string) (string, error) { + root, err := os.OpenRoot(rootDir) + if err != nil { + return "", fmt.Errorf("opening allowed host dir %s: %w", rootDir, err) + } + defer root.Close() + f, err := root.Open(rel) if err != nil { - return "", fmt.Errorf("opening ELF %s: %w", path, err) + return "", fmt.Errorf("opening ELF %s/%s: %w", rootDir, rel, err) } defer f.Close() - names, err := f.DynString(elf.DT_SONAME) + fi, err := f.Stat() + if err != nil { + return "", fmt.Errorf("stat ELF %s/%s: %w", rootDir, rel, err) + } + if !fi.Mode().IsRegular() { + return "", fmt.Errorf("ELF %s/%s is not a regular file", rootDir, rel) + } + ef, err := elf.NewFile(f) + if err != nil { + return "", fmt.Errorf("parsing ELF %s/%s: %w", rootDir, rel, err) + } + defer ef.Close() + names, err := ef.DynString(elf.DT_SONAME) if err != nil { - return "", fmt.Errorf("reading DT_SONAME from %s: %w", path, err) + return "", fmt.Errorf("reading DT_SONAME from %s/%s: %w", rootDir, rel, err) } if len(names) == 0 { return "", nil diff --git a/cmd/ateom-gvisor/internal/cgroupstats/cgroupstats.go b/cmd/ateom-gvisor/internal/cgroupstats/cgroupstats.go index 8ca586ce09..7ed8eb5124 100644 --- a/cmd/ateom-gvisor/internal/cgroupstats/cgroupstats.go +++ b/cmd/ateom-gvisor/internal/cgroupstats/cgroupstats.go @@ -21,11 +21,10 @@ // Every read is scoped to a caller-supplied directory rather than a hardcoded // /sys/fs/cgroup path, which keeps the parsing testable from a fixture tree // without root or a live sandbox. The package deliberately does not know what a -// sandbox is; it reads four numbers out of five files. +// sandbox is; it reads what the kernel reports in a handful of files. package cgroupstats import ( - "bufio" "bytes" "errors" "fmt" @@ -65,6 +64,16 @@ type Sample struct { // to this cgroup (see ateomcgroup.Delegate, which enables controllers // best-effort and carries on when one cannot be enabled). CPUUsageUsec uint64 + + // OOMKills is memory.events' oom_kill: processes in the cgroup the kernel's + // OOM killer has killed over the cgroup's lifetime. Zero if the file is + // absent. + OOMKills uint64 + + // Empty is true when cgroup.events reports populated 0: no process is left + // in the cgroup. It reads false when the file is absent, so a kernel that + // does not say is never taken for a dead sandbox. + Empty bool } // Read returns a Sample for the cgroup v2 directory at dir. @@ -80,6 +89,7 @@ type Sample struct { // memory controller's full accounting, and cpu.stat is absent when the cpu // controller was not delegated. Failing the sample for any of them would mean // reporting no memory numbers because the node could not report CPU. +// memory.events and cgroup.events are optional in the same way. func Read(dir string) (Sample, error) { current, ok, err := readUint(filepath.Join(dir, "memory.current")) if err != nil { @@ -95,6 +105,8 @@ func Read(dir string) (Sample, error) { peak, _, _ := readUint(filepath.Join(dir, "memory.peak")) inactiveFile, haveInactiveFile, _ := readKeyedUint(filepath.Join(dir, "memory.stat"), "inactive_file") cpuUsage, _, _ := readKeyedUint(filepath.Join(dir, "cpu.stat"), "usage_usec") + oomKills, _, _ := readKeyedUint(filepath.Join(dir, "memory.events"), "oom_kill") + populated, havePopulated, _ := readKeyedUint(filepath.Join(dir, "cgroup.events"), "populated") // Without inactive_file there is nothing to subtract, so the working set // collapses to memory.current. That over-reports by however much reclaimable @@ -117,6 +129,8 @@ func Read(dir string) (Sample, error) { MemoryPeakBytes: peak, MemoryWorkingSetBytes: workingSet, CPUUsageUsec: cpuUsage, + OOMKills: oomKills, + Empty: havePopulated && populated == 0, }, nil } @@ -150,23 +164,26 @@ func readKeyedUint(path, key string) (uint64, bool, error) { return 0, false, fmt.Errorf("reading %q: %w", path, err) } - sc := bufio.NewScanner(bytes.NewReader(b)) - for sc.Scan() { + // The files are a few hundred bytes and read for every actor on every + // sweep, so lines are walked in place, and only one that starts with the + // key is split. + for line := range bytes.Lines(b) { + line = bytes.TrimLeft(line, " \t") + if !bytes.HasPrefix(line, []byte(key)) { + continue + } // Fields, then a length check, rather than indexing straight into the // split: a blank or single-token line is not worth a panic in an RPC // handler, and these files are read on a timer for the life of a workload. - f := strings.Fields(sc.Text()) - if len(f) != 2 || f[0] != key { + f := bytes.Fields(line) + if len(f) != 2 || string(f[0]) != key { continue } - v, err := strconv.ParseUint(f[1], 10, 64) + v, err := strconv.ParseUint(string(f[1]), 10, 64) if err != nil { return 0, false, fmt.Errorf("parsing %q of %q: %w", key, path, err) } return v, true, nil } - if err := sc.Err(); err != nil { - return 0, false, fmt.Errorf("scanning %q: %w", path, err) - } return 0, false, nil } diff --git a/cmd/ateom-gvisor/internal/cgroupstats/cgroupstats_test.go b/cmd/ateom-gvisor/internal/cgroupstats/cgroupstats_test.go index 47d5dd365a..8b5198a295 100644 --- a/cmd/ateom-gvisor/internal/cgroupstats/cgroupstats_test.go +++ b/cmd/ateom-gvisor/internal/cgroupstats/cgroupstats_test.go @@ -167,6 +167,44 @@ func TestRead(t *testing.T) { }, want: Sample{}, }, + { + name: "live cgroup is not empty", + files: map[string]string{ + "memory.current": "1000\n", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0\n", + "cgroup.events": "populated 1\nfrozen 0\n", + }, + want: Sample{ + MemoryCurrentBytes: 1000, + MemoryWorkingSetBytes: 1000, + }, + }, + { + // What a sandbox leaf looked like on kind after the kernel OOM-killed + // the sentry: the leaf and its counters outlive every process in it. + name: "OOM-killed sandbox is empty", + files: map[string]string{ + "memory.current": "16384\n", + "memory.events": "low 0\nhigh 0\nmax 0\noom 5334\noom_kill 3\noom_group_kill 0\n", + "cgroup.events": "populated 0\nfrozen 0\n", + }, + want: Sample{ + MemoryCurrentBytes: 16384, + MemoryWorkingSetBytes: 16384, + OOMKills: 3, + Empty: true, + }, + }, + { + name: "no cgroup.events is not taken for empty", + files: map[string]string{ + "memory.current": "1000\n", + }, + want: Sample{ + MemoryCurrentBytes: 1000, + MemoryWorkingSetBytes: 1000, + }, + }, { // A short, blank, or over-long line must not panic: this runs in an RPC // handler on a timer, and grpc-go does not recover handler panics. @@ -190,6 +228,9 @@ func TestRead(t *testing.T) { "memory.current": "157286400\n", "memory.peak": "not-a-number\n", "cpu.stat": "usage_usec eleventy\n", + "memory.events": "oom_kill lots\n", + // Garbage here must not read as a dead sandbox. + "cgroup.events": "populated no\n", }, want: Sample{ MemoryCurrentBytes: 157286400, diff --git a/cmd/ateom-gvisor/main.go b/cmd/ateom-gvisor/main.go index ca92ec8374..3e21122300 100644 --- a/cmd/ateom-gvisor/main.go +++ b/cmd/ateom-gvisor/main.go @@ -40,6 +40,7 @@ import ( "github.com/agent-substrate/substrate/internal/ateom" "github.com/agent-substrate/substrate/internal/ateomcgroup" "github.com/agent-substrate/substrate/internal/ateomnet" + "github.com/agent-substrate/substrate/internal/ateomphaselog" "github.com/agent-substrate/substrate/internal/ateomstats" "github.com/agent-substrate/substrate/internal/ateomtunnel" "github.com/agent-substrate/substrate/internal/childreap" @@ -53,6 +54,7 @@ import ( "github.com/agent-substrate/substrate/internal/sizing" "github.com/agent-substrate/substrate/internal/version" "github.com/agent-substrate/substrate/internal/wakeupprobe" + atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" "github.com/spf13/pflag" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "google.golang.org/grpc" @@ -180,8 +182,8 @@ func do(ctx context.Context) error { // Create ateom dir ateomDir := nodepath.AteomPath(*podUID) - if err := resources.ValidateAteomUID(*podUID); err != nil { - return fmt.Errorf("in resources.ValidateAteomUID: %w", err) + if err := resources.ValidateWorkerPodUID(*podUID); err != nil { + return fmt.Errorf("in resources.ValidateWorkerPodUID: %w", err) } if err := os.MkdirAll(ateomDir, 0o700); err != nil { return fmt.Errorf("in os.MkdirAll(%q): %w", ateomDir, err) @@ -265,6 +267,7 @@ func do(ctx context.Context) error { TrustBundlePath: tunnelConfig.TrustBundle, AteletSPIFFEID: tunnelConfig.BrokerIdentity, Actors: *maxActors, + SandboxClass: string(atev1alpha1.SandboxClassGvisor), }) if err != nil && ctx.Err() == nil { serverboot.Fatal(ctx, "Failed to report worker capacity", err) @@ -544,6 +547,15 @@ func validateActorDirs(actorDirs *ateompb.ActorDirs) error { return nil } +// validateFidelity rejects a checkpoint or restore request whose fidelity +// this runtime cannot serve. +func validateFidelity(fidelity ateompb.SnapshotFidelity) error { + if errs := resources.ValidateSnapshotFidelity(fidelity, field.NewPath("fidelity")); len(errs) > 0 { + return apierror.InvalidArgument("%v", errs.ToAggregate()) + } + return nil +} + func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkloadRequest) (resp *ateompb.RunWorkloadResponse, retErr error) { if err := validateActorDirs(req.GetActorDirs()); err != nil { return nil, err @@ -593,6 +605,7 @@ func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkload actorDirs: req.GetActorDirs(), size: sizing.FromLimits(req.GetCpuMilli(), req.GetMemoryBytes()), durableVolumes: durableVolumeNames(req.GetSpec()), + containers: req.GetSpec().GetContainers(), } var containersToDelete []string defer func() { @@ -669,10 +682,15 @@ func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkload // Allow checkpointing even if the pod is shutting down. This will allow actors // (or the harness) to suspend on shutdown. -func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.CheckpointWorkloadRequest) (*ateompb.CheckpointWorkloadResponse, error) { +func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.CheckpointWorkloadRequest) (_ *ateompb.CheckpointWorkloadResponse, err error) { + // Taken before the lock, so prep covers the wait for it. + tStart := time.Now() if err := validateActorDirs(req.GetActorDirs()); err != nil { return nil, err } + if err := validateFidelity(req.GetFidelity()); err != nil { + return nil, err + } if err := validateRunscPath(req.GetRunscPath()); err != nil { return nil, err } @@ -686,13 +704,24 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec // Not cancelable: a checkpoint is saving the actor's state. defer s.inFlight.Add(req.GetActorUid(), rpcCheckpointWorkload, nil)() - if err := s.tunnel.Deactivate(ctx, ateomstats.ActorAttributionFromRequest(req)); err != nil { + // Per-phase timing, recorded on the way out so a failed checkpoint still + // reports the phases it completed, and the failing step its elapsed time. + attribution := ateomstats.ActorAttributionFromRequest(req) + fidelity := req.GetFidelity() + var timing checkpointTiming + tLast := tStart + defer func() { + timing.total = time.Since(tStart) + ateomphaselog.LogSnapshotPhases(ctx, "Checkpoint timing breakdown", attribution, fidelity, + ateomphaselog.CheckpointDurationKey, err, timing.phases()) + }() + + if err := s.tunnel.Deactivate(ctx, attribution); err != nil { return nil, err } - attribution := ateomstats.ActorAttributionFromRequest(req) s.actorLogger.EmitLifecycleLog(ctx, "Actor checkpointing", attribution) - // Read before the snapshot: a Full checkpoint stops the sandbox, and its + // Read before the snapshot: a MEMORY checkpoint stops the sandbox, and its // cgroup with it. The final record waits for the teardown. hosted := s.lookupActor(req.GetActorUid()) if hosted != nil { @@ -712,50 +741,60 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec } checkpointPath := req.GetActorDirs().GetCheckpointDir() - if err := os.MkdirAll(checkpointPath, 0o700); err != nil { + err = os.MkdirAll(checkpointPath, 0o700) + timing.prep = lap(&tLast) + if err != nil { return nil, fmt.Errorf("while creating checkpoint directory: %w", err) } - // durableFiles are the durable-dir tars written below: the DATA subset. + // durableFiles are the durable-dir tars written below: the VOLUMES subset. var durableFiles []string // Always take durable-dir snapshot if at least one container has a durable-dir volume mount. // TODO(dberkov): this is a temporary workaround until gVisor supports taking durable-dir snapshots in a single request with the process snapshot. - switch req.GetScope() { - case ateompb.SnapshotScope_SNAPSHOT_SCOPE_DATA: + switch fidelity { + case ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES: if !hasDurableVolumes(req.GetSpec().GetContainers()) { - return nil, fmt.Errorf("no durable-dir volumes found for DATA snapshot") + return nil, fmt.Errorf("no durable-dir volumes found for VOLUMES snapshot") } - if err := rcmd.cmdPause(ctx, ocispec.PauseContainer); err != nil { + err := rcmd.cmdPause(ctx, ocispec.PauseContainer) + timing.pause = lap(&tLast) + if err != nil { return nil, fmt.Errorf("while pausing pause container: %w", err) } var tarErr error durableFiles, tarErr = tarDurableVolumes(ctx, req.GetActorDirs().GetDurableDirVolumeMountsDir(), checkpointPath, durableVolumeNames(req.GetSpec())) + timing.durableDir = lap(&tLast) // Undoing our own pause must not depend on the caller's context: // tarutil does not check ctx, so a deadline expiring mid-tar would // fail the resume instantly and leave the sandbox paused forever. resumeCtx, cancelResume := context.WithTimeout(context.WithoutCancel(ctx), resumeTimeout) defer cancelResume() - if err := rcmd.cmdResume(resumeCtx, ocispec.PauseContainer); err != nil { + err = rcmd.cmdResume(resumeCtx, ocispec.PauseContainer) + timing.resume = lap(&tLast) + if err != nil { return nil, fmt.Errorf("while resuming pause container: %w", err) } if tarErr != nil { return nil, fmt.Errorf("while archiving durable-dir volumes: %w", tarErr) } - case ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL: + case ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY: // Checkpoint pause container (root of the sandbox) // TODO: Consider pause -> tar -> resume -> checkpoint order for better failure handling. - if err := rcmd.cmdCheckpoint(ctx, ocispec.PauseContainer, checkpointPath); err != nil { + err := rcmd.cmdCheckpoint(ctx, ocispec.PauseContainer, checkpointPath) + timing.checkpoint = lap(&tLast) + if err != nil { return nil, fmt.Errorf("while checkpointing pause: %w", err) } if hasDurableVolumes(req.GetSpec().GetContainers()) { var err error durableFiles, err = tarDurableVolumes(ctx, req.GetActorDirs().GetDurableDirVolumeMountsDir(), checkpointPath, durableVolumeNames(req.GetSpec())) + timing.durableDir = lap(&tLast) if err != nil { return nil, fmt.Errorf("while archiving durable-dir volumes: %w", err) } } default: - return nil, fmt.Errorf("unsupported snapshot scope: %v", req.GetScope()) + return nil, fmt.Errorf("unsupported snapshot fidelity: %v", fidelity) } // Cleanup the containers after checkpointing. This also unhosts the actor, @@ -767,6 +806,7 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec slog.String("actorUID", attribution.UID), slog.Any("err", err)) } + timing.teardown = lap(&tLast) s.recordFinalIfEnded(ctx, hosted) // Report exactly the files runsc wrote so atelet ships precisely this set @@ -856,9 +896,14 @@ func isContainerAlreadyGone(ctx context.Context, rcmd containerRuntime, name str } func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.RestoreWorkloadRequest) (resp *ateompb.RestoreWorkloadResponse, retErr error) { + // Taken before the lock, so prep covers the wait for it. + tStart := time.Now() if err := validateActorDirs(req.GetActorDirs()); err != nil { return nil, err } + if err := validateFidelity(req.GetFidelity()); err != nil { + return nil, err + } if err := validateRunscPath(req.GetRunscPath()); err != nil { return nil, err } @@ -868,6 +913,23 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore defer s.locks.Unlock(req.GetActorUid()) ctx, cancel := context.WithCancel(ctx) defer cancel() + + // Per-phase timing, recorded on the way out so a failed restore still + // reports the phases it completed, and the failing step its elapsed time. + // Registered ahead of the failure cleanup below, so on failure the total + // includes that cleanup; the record is marked by error.type either way. + attribution := ateomstats.ActorAttributionFromRequest(req) + fidelity := req.GetFidelity() + containers := req.GetSpec().GetContainers() + var timing restoreTiming + tLast := tStart + defer func() { + timing.total = time.Since(tStart) + attrs := ateomphaselog.SnapshotPhaseAttrs(attribution, fidelity, ateomphaselog.RestoreDurationKey, retErr, timing.phases()) + attrs = append(attrs, slog.Int(containerCountKey, len(containers))) + slog.LogAttrs(ctx, slog.LevelInfo, "Restore timing breakdown", attrs...) + }() + release, err := s.beginRPC(req.GetActorUid(), rpcRestoreWorkload, cancel) if err != nil { return nil, err @@ -877,11 +939,12 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore return nil, err } - if err := s.tunnel.Deactivate(ctx, ateomstats.ActorAttributionFromRequest(req)); err != nil { + err = s.tunnel.Deactivate(ctx, attribution) + timing.prep = lap(&tLast) + if err != nil { return nil, err } - attribution := ateomstats.ActorAttributionFromRequest(req) s.actorLogger.EmitLifecycleLog(ctx, "Actor restoring", attribution) // Contract with atelet: @@ -890,11 +953,13 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore // * All OCI bundles are set up, including for the pause container. // * Checkpoint downloaded and placed on disk - egress, err := s.tunnel.PrepareEgress(ctx, ateomstats.ActorAttributionFromRequest(req), req.GetEgressGateway()) + egress, err := s.tunnel.PrepareEgress(ctx, attribution, req.GetEgressGateway()) + timing.egressPrepare = lap(&tLast) if err != nil { return nil, err } hosted, err := s.hostActor(ctx, attribution, req.GetActorDirs()) + timing.netSetup = lap(&tLast) if err != nil { return nil, err } @@ -904,13 +969,14 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore actorDirs: req.GetActorDirs(), size: sizing.FromLimits(req.GetCpuMilli(), req.GetMemoryBytes()), durableVolumes: durableVolumeNames(req.GetSpec()), + containers: req.GetSpec().GetContainers(), } var containersToDelete []string defer func() { if retErr != nil { cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) defer cancel() - if err := s.tunnel.Deactivate(cleanupCtx, ateomstats.ActorAttributionFromRequest(req)); err != nil { + if err := s.tunnel.Deactivate(cleanupCtx, attribution); err != nil { slog.WarnContext(cleanupCtx, "Failed to deactivate actor networking after Restore failure", slog.Any("err", err)) } deleteContainers(cleanupCtx, rcmd, containersToDelete, "Restore") @@ -926,84 +992,106 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore }() checkpointDir := req.GetActorDirs().GetRestoreDir() - if hasDurableVolumes(req.GetSpec().GetContainers()) { - if err := untarDurableVolumes(req.GetActorDirs().GetDurableDirVolumeMountsDir(), checkpointDir, durableVolumeNames(req.GetSpec())); err != nil { + if hasDurableVolumes(containers) { + err := untarDurableVolumes(req.GetActorDirs().GetDurableDirVolumeMountsDir(), checkpointDir, durableVolumeNames(req.GetSpec())) + timing.durableDir = lap(&tLast) + if err != nil { return nil, fmt.Errorf("while restoring durable-dir volumes: %w", err) } } // Compose the pause rootfs before create (see RunWorkload). runsc restore // only needs the rootfs to hold the correct content; whether it came from // an untar or an overlay of cached layers is transparent to it. - if err := imagecache.SetupBundleRootfs(ociBundlePath(req.GetActorDirs(), ocispec.PauseContainer)); err != nil { + err = imagecache.SetupBundleRootfs(ociBundlePath(req.GetActorDirs(), ocispec.PauseContainer)) + timing.pauseRootfs = lap(&tLast) + if err != nil { return nil, fmt.Errorf("while composing pause rootfs: %w", err) } - switch req.GetScope() { - case ateompb.SnapshotScope_SNAPSHOT_SCOPE_DATA: + switch fidelity { + case ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES: // Create and start pause container (cold boot with durable-dir volumes restored) containersToDelete = append(containersToDelete, ocispec.PauseContainer) - if err := rcmd.cmdCreate(ctx, os.Stdout, ocispec.PauseContainer, nil); err != nil { + err := rcmd.cmdCreate(ctx, os.Stdout, ocispec.PauseContainer, nil) + timing.pauseCreate = lap(&tLast) + if err != nil { return nil, fmt.Errorf("while creating pause container: %w", err) } - if err := rcmd.cmdStart(ctx, os.Stdout, ocispec.PauseContainer); err != nil { + err = rcmd.cmdStart(ctx, os.Stdout, ocispec.PauseContainer) + timing.pauseRestore = lap(&tLast) + if err != nil { return nil, fmt.Errorf("while starting pause container: %w", err) } - case ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL: + case ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY: // Create and restore pause container containersToDelete = append(containersToDelete, ocispec.PauseContainer) - if err := rcmd.cmdCreate(ctx, os.Stdout, ocispec.PauseContainer, nil); err != nil { + err := rcmd.cmdCreate(ctx, os.Stdout, ocispec.PauseContainer, nil) + timing.pauseCreate = lap(&tLast) + if err != nil { return nil, fmt.Errorf("while creating pause container: %w", err) } - if err := rcmd.cmdRestore(ctx, os.Stdout, ocispec.PauseContainer, checkpointDir); err != nil { + err = rcmd.cmdRestore(ctx, os.Stdout, ocispec.PauseContainer, checkpointDir) + timing.pauseRestore = lap(&tLast) + if err != nil { return nil, fmt.Errorf("while restoring pause container: %w", err) } default: - return nil, fmt.Errorf("unexpected snapshot scope: %v", req.GetScope()) + return nil, fmt.Errorf("unexpected snapshot fidelity: %v", fidelity) } // Create and restore each application container, each with its own log pipe so // every line is tagged with the originating container (ate.actor.container.name). - for _, ac := range req.GetSpec().GetContainers() { + // app_create includes starting that pipe. + for _, ac := range containers { pw, err := s.actorLogger.StartJSONLogPipe(attribution, ac.GetName()) + timing.appCreate += lap(&tLast) if err != nil { return nil, fmt.Errorf("while starting json log pipe for %q: %w", ac.GetName(), err) } defer pw.Close() - if err := imagecache.SetupBundleRootfs(ociBundlePath(req.GetActorDirs(), ac.GetName())); err != nil { + err = imagecache.SetupBundleRootfs(ociBundlePath(req.GetActorDirs(), ac.GetName())) + timing.appRootfs += lap(&tLast) + if err != nil { return nil, fmt.Errorf("while composing %q rootfs: %w", ac.GetName(), err) } - switch req.GetScope() { - case ateompb.SnapshotScope_SNAPSHOT_SCOPE_DATA: - containersToDelete = append(containersToDelete, ac.GetName()) - if err := rcmd.cmdCreate(ctx, pw, ac.GetName(), nil); err != nil { - return nil, fmt.Errorf("while creating %q application container: %w", ac.GetName(), err) - } - if err := rcmd.cmdStart(ctx, pw, ac.GetName()); err != nil { + containersToDelete = append(containersToDelete, ac.GetName()) + err = rcmd.cmdCreate(ctx, pw, ac.GetName(), nil) + timing.appCreate += lap(&tLast) + if err != nil { + return nil, fmt.Errorf("while creating %q application container: %w", ac.GetName(), err) + } + switch fidelity { + case ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES: + err = rcmd.cmdStart(ctx, pw, ac.GetName()) + timing.appRestore += lap(&tLast) + if err != nil { return nil, fmt.Errorf("while starting %q application container: %w", ac.GetName(), err) } - case ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL: - containersToDelete = append(containersToDelete, ac.GetName()) - if err := rcmd.cmdCreate(ctx, pw, ac.GetName(), nil); err != nil { - return nil, fmt.Errorf("while creating %q application container: %w", ac.GetName(), err) - } - if err := rcmd.cmdRestore(ctx, pw, ac.GetName(), checkpointDir); err != nil { + case ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY: + err = rcmd.cmdRestore(ctx, pw, ac.GetName(), checkpointDir) + timing.appRestore += lap(&tLast) + if err != nil { return nil, fmt.Errorf("while restoring %q application container: %w", ac.GetName(), err) } default: - return nil, fmt.Errorf("unexpected snapshot scope: %v", req.GetScope()) + return nil, fmt.Errorf("unexpected snapshot fidelity: %v", fidelity) } } // Block until every wakeup-probe-enabled container reports 200. - if err := wakeupprobe.WaitAll(ctx, req.GetSpec().GetContainers(), ateomnet.ActorVethIP, wakeupprobe.DialFunc(s.sandboxDialer(req.GetActorUid()))); err != nil { + err = wakeupprobe.WaitAll(ctx, containers, ateomnet.ActorVethIP, wakeupprobe.DialFunc(s.sandboxDialer(req.GetActorUid()))) + timing.wakeupProbe = lap(&tLast) + if err != nil { return nil, fmt.Errorf("while waiting for container wakeup probe: %w", err) } - if err := s.tunnel.Activate(ateomstats.ActorAttributionFromRequest(req), s.sandboxDialer(req.GetActorUid()), egress); err != nil { + err = s.tunnel.Activate(attribution, s.sandboxDialer(req.GetActorUid()), egress) + timing.activate = lap(&tLast) + if err != nil { return nil, err } s.actorLogger.EmitLifecycleLog(ctx, "Actor restored", attribution) - s.setSession(req.GetActorUid(), &workloadSession{rcmd: rcmd, containers: containerNames(req.GetSpec().GetContainers())}) + s.setSession(req.GetActorUid(), &workloadSession{rcmd: rcmd, containers: containerNames(containers)}) s.recordInitial(ctx, hosted) return &ateompb.RestoreWorkloadResponse{}, nil diff --git a/cmd/ateom-gvisor/phaselog.go b/cmd/ateom-gvisor/phaselog.go new file mode 100644 index 0000000000..1509545447 --- /dev/null +++ b/cmd/ateom-gvisor/phaselog.go @@ -0,0 +1,115 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "time" + + "github.com/agent-substrate/substrate/internal/ateattr" + "github.com/agent-substrate/substrate/internal/ateomphaselog" +) + +// The phase names, suffixed onto the ateomphaselog duration keys. Kept out of +// ateattr on purpose: that package's SnapshotPhase* values are the +// ate.snapshot.phase metric enum, and these are not values of it. "total" is +// shared so the two layers' records agree on the denominator. +// +// Every phase of both operations is sequential, so the phases partition the +// total up to the few microseconds between them. The app_* phases are summed +// over the application containers, which are restored one after another; the +// container count rides along on the record so a reader can divide. +const ( + phasePrep = "prep" + phaseEgressPrepare = "egress_prepare" + phaseNetSetup = "net_setup" + phaseDurableDir = "durable_dir" + phasePauseRootfs = "pause_rootfs" + phasePauseCreate = "pause_create" + phasePauseRestore = "pause_restore" + phaseAppRootfs = "app_rootfs" + phaseAppCreate = "app_create" + phaseAppRestore = "app_restore" + phaseWakeupProbe = "wakeup_probe" + phaseActivate = "activate" + + phasePause = "pause" + phaseCheckpoint = "checkpoint" + phaseResume = "resume" + phaseTeardown = "teardown" + + phaseTotal = ateattr.SnapshotPhaseTotal +) + +// containerCountKey carries the number of application containers on the +// restore record, since the app_* phases are sums over them. +const containerCountKey = "ate.actor.container.count" + +// restoreTiming is the elapsed time of each RestoreWorkload phase. A phase +// left at zero never ran. Under a Data scope pauseRestore and appRestore time +// the cold start that stands in for the restore. +type restoreTiming struct { + prep, egressPrepare, netSetup, durableDir time.Duration + pauseRootfs, pauseCreate, pauseRestore time.Duration + appRootfs, appCreate, appRestore time.Duration + wakeupProbe, activate, total time.Duration +} + +func (t restoreTiming) phases() []ateomphaselog.Phase { + return []ateomphaselog.Phase{ + {Name: phasePrep, D: t.prep}, + {Name: phaseEgressPrepare, D: t.egressPrepare}, + {Name: phaseNetSetup, D: t.netSetup}, + {Name: phaseDurableDir, D: t.durableDir}, + {Name: phasePauseRootfs, D: t.pauseRootfs}, + {Name: phasePauseCreate, D: t.pauseCreate}, + {Name: phasePauseRestore, D: t.pauseRestore}, + {Name: phaseAppRootfs, D: t.appRootfs}, + {Name: phaseAppCreate, D: t.appCreate}, + {Name: phaseAppRestore, D: t.appRestore}, + {Name: phaseWakeupProbe, D: t.wakeupProbe}, + {Name: phaseActivate, D: t.activate}, + {Name: phaseTotal, D: t.total}, + } +} + +// checkpointTiming is the elapsed time of each CheckpointWorkload phase. A +// Full scope runs checkpoint; a Data scope runs pause and resume around the +// durable_dir capture instead. +type checkpointTiming struct { + prep, pause, checkpoint, durableDir, resume, teardown, total time.Duration +} + +func (t checkpointTiming) phases() []ateomphaselog.Phase { + return []ateomphaselog.Phase{ + {Name: phasePrep, D: t.prep}, + {Name: phasePause, D: t.pause}, + {Name: phaseCheckpoint, D: t.checkpoint}, + {Name: phaseDurableDir, D: t.durableDir}, + {Name: phaseResume, D: t.resume}, + {Name: phaseTeardown, D: t.teardown}, + {Name: phaseTotal, D: t.total}, + } +} + +// lap returns the time since *last and advances it, so consecutive calls +// measure consecutive phases without a timestamp per boundary. +func lap(last *time.Time) time.Duration { + now := time.Now() + d := now.Sub(*last) + *last = now + return d +} diff --git a/cmd/ateom-gvisor/phaselog_test.go b/cmd/ateom-gvisor/phaselog_test.go new file mode 100644 index 0000000000..a90f81cdd3 --- /dev/null +++ b/cmd/ateom-gvisor/phaselog_test.go @@ -0,0 +1,168 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "context" + "encoding/json" + "log/slog" + "testing" + "time" + + "github.com/agent-substrate/substrate/internal/ateomphaselog" + "github.com/agent-substrate/substrate/internal/proto/ateompb" + "github.com/agent-substrate/substrate/internal/resources" +) + +// renderPhaseRecord logs attrs through a JSON handler, so the test checks the +// record a collector would parse rather than the slice that built it. +func renderPhaseRecord(t *testing.T, attrs []slog.Attr) map[string]any { + t.Helper() + var buf bytes.Buffer + slog.New(slog.NewJSONHandler(&buf, nil)).LogAttrs(context.Background(), slog.LevelInfo, "Restore timing breakdown", attrs...) + var rec map[string]any + if err := json.Unmarshal(buf.Bytes(), &rec); err != nil { + t.Fatalf("unmarshal record %q: %v", buf.String(), err) + } + return rec +} + +// TestRestoreTimingPhases pins the keys a Full restore record carries, in +// the layout benchmarking tooling joins with atelet's record on, and that a +// phase which never ran stays off it. +func TestRestoreTimingPhases(t *testing.T) { + t.Parallel() + + timing := restoreTiming{ + prep: 10 * time.Millisecond, + egressPrepare: 120 * time.Millisecond, + netSetup: 30 * time.Millisecond, + // No durable volumes: the phase never ran. + durableDir: 0, + pauseRootfs: 5 * time.Millisecond, + pauseCreate: 200 * time.Millisecond, + pauseRestore: 900 * time.Millisecond, + appRootfs: 15 * time.Millisecond, + appCreate: 50 * time.Millisecond, + appRestore: 400 * time.Millisecond, + wakeupProbe: 250 * time.Millisecond, + activate: 2 * time.Millisecond, + total: 2 * time.Second, + } + attrs := ateomphaselog.SnapshotPhaseAttrs(resources.ActorAttribution{UID: "uid-abc"}, + ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, ateomphaselog.RestoreDurationKey, nil, timing.phases()) + attrs = append(attrs, slog.Int(containerCountKey, 2)) + + seen := make(map[string]bool, len(attrs)) + for _, attr := range attrs { + if seen[attr.Key] { + t.Errorf("key %s emitted twice", attr.Key) + } + seen[attr.Key] = true + } + + rec := renderPhaseRecord(t, attrs) + for k, want := range map[string]float64{ + "ateom.actor.restore.duration.prep": 0.01, + "ateom.actor.restore.duration.egress_prepare": 0.12, + "ateom.actor.restore.duration.net_setup": 0.03, + "ateom.actor.restore.duration.pause_rootfs": 0.005, + "ateom.actor.restore.duration.pause_create": 0.2, + "ateom.actor.restore.duration.pause_restore": 0.9, + "ateom.actor.restore.duration.app_rootfs": 0.015, + "ateom.actor.restore.duration.app_create": 0.05, + "ateom.actor.restore.duration.app_restore": 0.4, + "ateom.actor.restore.duration.wakeup_probe": 0.25, + "ateom.actor.restore.duration.activate": 0.002, + "ateom.actor.restore.duration.total": 2, + "ate.actor.container.count": 2, + } { + got, ok := rec[k].(float64) + if !ok { + t.Errorf("%s = %v (%T), want a number", k, rec[k], rec[k]) + } else if got != want { + t.Errorf("%s = %v, want %v", k, got, want) + } + } + if v, ok := rec["ateom.actor.restore.duration.durable_dir"]; ok { + t.Errorf("durable_dir present with %v, want absent", v) + } + if rec["ate.actor.uid"] != "uid-abc" { + t.Errorf("ate.actor.uid = %v, want uid-abc", rec["ate.actor.uid"]) + } + if rec["ate.snapshot.fidelity"] != "memory" { + t.Errorf("ate.snapshot.fidelity = %v, want memory", rec["ate.snapshot.fidelity"]) + } +} + +// TestCheckpointTimingPhases: a Data checkpoint pauses and resumes around the +// durable capture and never runs the runsc checkpoint, and the record says so. +func TestCheckpointTimingPhases(t *testing.T) { + t.Parallel() + + timing := checkpointTiming{ + prep: 20 * time.Millisecond, + pause: 3 * time.Millisecond, + durableDir: 300 * time.Millisecond, + resume: 4 * time.Millisecond, + teardown: 150 * time.Millisecond, + total: 500 * time.Millisecond, + } + rec := renderPhaseRecord(t, ateomphaselog.SnapshotPhaseAttrs(resources.ActorAttribution{UID: "uid-abc"}, + ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, ateomphaselog.CheckpointDurationKey, nil, timing.phases())) + + for k, want := range map[string]float64{ + "ateom.actor.checkpoint.duration.prep": 0.02, + "ateom.actor.checkpoint.duration.pause": 0.003, + "ateom.actor.checkpoint.duration.durable_dir": 0.3, + "ateom.actor.checkpoint.duration.resume": 0.004, + "ateom.actor.checkpoint.duration.teardown": 0.15, + "ateom.actor.checkpoint.duration.total": 0.5, + } { + got, ok := rec[k].(float64) + if !ok { + t.Errorf("%s = %v (%T), want a number", k, rec[k], rec[k]) + } else if got != want { + t.Errorf("%s = %v, want %v", k, got, want) + } + } + if v, ok := rec["ateom.actor.checkpoint.duration.checkpoint"]; ok { + t.Errorf("checkpoint present with %v, want absent", v) + } + if rec["ate.snapshot.fidelity"] != "volumes" { + t.Errorf("ate.snapshot.fidelity = %v, want volumes", rec["ate.snapshot.fidelity"]) + } +} + +// TestLap: consecutive laps measure consecutive intervals from one cursor. +func TestLap(t *testing.T) { + t.Parallel() + + last := time.Now().Add(-time.Second) + before := last + d := lap(&last) + if d < time.Second { + t.Errorf("lap = %v, want >= 1s", d) + } + if !last.After(before) { + t.Error("lap did not advance the cursor") + } + if d2 := lap(&last); d2 >= time.Second { + t.Errorf("second lap = %v, want far below 1s", d2) + } +} diff --git a/cmd/ateom-gvisor/runsc.go b/cmd/ateom-gvisor/runsc.go index 6c55706465..1745603333 100644 --- a/cmd/ateom-gvisor/runsc.go +++ b/cmd/ateom-gvisor/runsc.go @@ -43,6 +43,8 @@ type runsc struct { size sizing.SandboxSize // durableVolumes are the durable-dir volume names declared to the sandbox. durableVolumes []string + // containers are the workload's containers, from the request. + containers []*ateompb.Container } // durableVolumeNames returns the sorted, deduplicated durable-dir volume names @@ -58,13 +60,27 @@ func durableVolumeNames(spec *ateompb.WorkloadSpec) []string { return slices.Compact(names) } -// shapeSpec loads, shapes for gVisor, and saves the container's OCI spec. -func (r *runsc) shapeSpec(containerName string) error { - bundle := ociBundlePath(r.actorDirs, containerName) - spec, err := ocispec.Load(bundle) - if err != nil { - return err +// pauseContainer is the sandbox root container. It only reaps, so it gets no +// capabilities and mounts nothing. +func pauseContainer() *ateompb.Container { + return &ateompb.Container{ + Name: ocispec.PauseContainer, + ContainerSpec: &ateompb.ContainerSpec{Args: []string{"/pause"}}, + } +} + +// writeSpec builds the container's OCI spec, shapes it for gVisor, and saves it +// into the container's bundle. +func (r *runsc) writeSpec(containerName string) error { + container := pauseContainer() + if containerName != ocispec.PauseContainer { + i := slices.IndexFunc(r.containers, func(c *ateompb.Container) bool { return c.GetName() == containerName }) + if i < 0 { + return fmt.Errorf("container %q is not in the workload spec", containerName) + } + container = r.containers[i] } + spec := ocispec.Build(ocispec.Options{ActorUID: r.actorUID, ActorDirs: r.actorDirs, Container: container}) ocispec.ShapeGVisor(spec, ocispec.GVisorOptions{ ActorUID: r.actorUID, ContainerName: containerName, @@ -72,14 +88,14 @@ func (r *runsc) shapeSpec(containerName string) error { Size: r.size, ResolvConf: resolvConfPath(r.actorDirs), }) - return ocispec.Save(bundle, spec) + return ocispec.Save(ociBundlePath(r.actorDirs, containerName), spec) } func (r *runsc) cmdCreate(ctx context.Context, out io.Writer, containerName string, additionalArgs []string) error { slog.InfoContext(ctx, "About to run runsc create", slog.String("container", containerName)) - if err := r.shapeSpec(containerName); err != nil { - return fmt.Errorf("while shaping the OCI spec for %q: %w", containerName, err) + if err := r.writeSpec(containerName); err != nil { + return fmt.Errorf("while writing the OCI spec for %q: %w", containerName, err) } args := []string{ @@ -290,8 +306,8 @@ func (r *runsc) restoreArgs(containerName, checkpointPath string) []string { func (r *runsc) cmdRestore(ctx context.Context, out io.Writer, containerName, checkpointPath string) error { slog.InfoContext(ctx, "About to run runsc restore", slog.String("container", containerName)) - if err := r.shapeSpec(containerName); err != nil { - return fmt.Errorf("while shaping the OCI spec for %q: %w", containerName, err) + if err := r.writeSpec(containerName); err != nil { + return fmt.Errorf("while writing the OCI spec for %q: %w", containerName, err) } cmd := exec.CommandContext(ctx, r.path, r.restoreArgs(containerName, checkpointPath)...) diff --git a/cmd/ateom-gvisor/runsc_test.go b/cmd/ateom-gvisor/runsc_test.go index 58be691329..f9a8bcc709 100644 --- a/cmd/ateom-gvisor/runsc_test.go +++ b/cmd/ateom-gvisor/runsc_test.go @@ -18,9 +18,14 @@ package main import ( "context" + "encoding/json" + "os" + "path/filepath" "reflect" + "slices" "testing" + "github.com/opencontainers/runtime-spec/specs-go" "google.golang.org/grpc/codes" "github.com/agent-substrate/substrate/internal/actorlock" @@ -34,6 +39,50 @@ var testActorDirs = &ateompb.ActorDirs{ OciBundleDir: "/node/actors/test-actor-123/bundle", } +// writeSpec builds a workload container's spec from the request and the pause +// container's from ateom's own description. +func TestWriteSpec(t *testing.T) { + root := t.TempDir() + actorDirs := &ateompb.ActorDirs{RootDir: root, OciBundleDir: filepath.Join(root, "bundles")} + r := &runsc{ + actorUID: "uid-a", + actorDirs: actorDirs, + containers: []*ateompb.Container{{Name: "app", ContainerSpec: &ateompb.ContainerSpec{Args: []string{"/app", "serve"}}}}, + } + for name, wantArgs := range map[string][]string{ + "app": {"/app", "serve"}, + ocispec.PauseContainer: {"/pause"}, + } { + bundle := ociBundlePath(actorDirs, name) + if err := os.MkdirAll(bundle, 0o700); err != nil { + t.Fatal(err) + } + if err := r.writeSpec(name); err != nil { + t.Fatalf("writeSpec(%q) = %v", name, err) + } + b, err := os.ReadFile(filepath.Join(bundle, "config.json")) + if err != nil { + t.Fatal(err) + } + var spec specs.Spec + if err := json.Unmarshal(b, &spec); err != nil { + t.Fatal(err) + } + if !slices.Equal(spec.Process.Args, wantArgs) { + t.Errorf("%s args = %v, want %v", name, spec.Process.Args, wantArgs) + } + if got := spec.Annotations["io.kubernetes.cri.container-name"]; got != name { + t.Errorf("%s container-name annotation = %q, want the spec shaped for gVisor", name, got) + } + if name == ocispec.PauseContainer && len(spec.Process.Capabilities.Bounding) != 0 { + t.Errorf("pause capabilities = %v, want none", spec.Process.Capabilities.Bounding) + } + } + if err := r.writeSpec("missing"); err == nil { + t.Error("writeSpec() = nil for a container outside the workload spec") + } +} + func TestKillArgs(t *testing.T) { r := &runsc{ path: "/usr/bin/runsc", diff --git a/cmd/ateom-gvisor/stats.go b/cmd/ateom-gvisor/stats.go index b8c80285ac..c044cce546 100644 --- a/cmd/ateom-gvisor/stats.go +++ b/cmd/ateom-gvisor/stats.go @@ -104,7 +104,9 @@ func (s *AteomService) GetWorkloadStats(ctx context.Context, req *ateompb.GetWor if hosted == nil { return nil, apierror.NotFound("ateom is not executing actor %q", req.GetActorUid()) } - sample, err := s.measure(ctx, hosted) + // The raw reading is dropped: the dead-sandbox check runs in the usage + // sweep, which reads every hosted actor. + sample, _, err := s.measure(ctx, hosted) if err != nil { // The requested actor is the active one but its cgroup is not there. // Most often that is a poll landing in the boot: the ateom retains the @@ -162,10 +164,11 @@ func (s *AteomService) sweepUsage(ctx context.Context) { // An actor with no numbers is reported as pending, so a failed read for // one does not lose the others. Past the initial reading, a missing // cgroup means the sandbox is going away. - sample, err := s.measure(ctx, h) + sample, raw, err := s.measure(ctx, h) if err != nil { if !errors.Is(err, fs.ErrNotExist) { - slog.WarnContext(ctx, "Failed to read sandbox cgroup", slog.String("actorUID", h.attribution.UID), slog.Any("err", err)) + attrs := append(ateattr.ActorLogAttrs(h.attribution), slog.Any("err", err)) + slog.LogAttrs(ctx, slog.LevelWarn, "Failed to read sandbox cgroup", attrs...) } sample = h.usage.WithEpoch(pendingSample(&h.attribution)) } @@ -176,6 +179,9 @@ func (s *AteomService) sweepUsage(ctx context.Context) { if s.lookupActor(h.attribution.UID) != h { continue } + if raw.Empty { // a failed read returns a zero Sample + s.warnDeadSandbox(ctx, h, raw) + } h.usage.Periodic(sample, func() { s.usage.Emit(ctx, ateattr.StatsKindPeriodic, sample) }) } } @@ -183,7 +189,7 @@ func (s *AteomService) sweepUsage(ctx context.Context) { // recordInitial samples a new activation once its sandbox is up and writes its // initial record. func (s *AteomService) recordInitial(ctx context.Context, h *hostedActor) { - sample, err := s.measure(ctx, h) + sample, _, err := s.measure(ctx, h) if err != nil { slog.WarnContext(ctx, "No initial usage sample", slog.String(string(ateattr.ActorUIDKey), h.attribution.UID), slog.Any("err", err)) sample = nil @@ -195,7 +201,7 @@ func (s *AteomService) recordInitial(ctx context.Context, h *hostedActor) { // record a successful checkpoint or terminate writes. A failed read leaves the // newest measured sample to stand in. func (s *AteomService) readFinal(ctx context.Context, h *hostedActor) { - if sample, err := s.measure(ctx, h); err == nil { + if sample, _, err := s.measure(ctx, h); err == nil { h.usage.Store(sample) } } @@ -239,21 +245,26 @@ func pendingSample(active *resources.ActorAttribution) *ateompb.WorkloadStatsSam } } -// measure reads h's sandbox cgroup as a reading of its activation. -func (s *AteomService) measure(ctx context.Context, h *hostedActor) (*ateompb.WorkloadStatsSample, error) { - return h.usage.Measure(ctx, func() (*ateompb.WorkloadStatsSample, map[string]uint64, error) { - sample, err := s.sampleSandbox(&h.attribution) +// measure reads h's sandbox cgroup as a reading of its activation, returning +// the raw cgroup reading alongside it for the dead-sandbox check. +func (s *AteomService) measure(ctx context.Context, h *hostedActor) (*ateompb.WorkloadStatsSample, cgroupstats.Sample, error) { + var raw cgroupstats.Sample + sample, err := h.usage.Measure(ctx, func() (*ateompb.WorkloadStatsSample, map[string]uint64, error) { + sample, r, err := s.sampleSandbox(&h.attribution) + raw = r return sample, nil, err }) + return sample, raw, err } // sampleSandbox reads the sandbox cgroup and builds the sample attributed to -// active. Errors come back raw -- notably fs.ErrNotExist for a cgroup that is -// not there yet -- because the callers disagree on what that means: an error -// code for the keyed read, a pending sample for the sweep. +// active, returning the raw reading alongside it. Errors come back raw -- +// notably fs.ErrNotExist for a cgroup that is not there yet -- because the +// callers disagree on what that means: an error code for the keyed read, a +// pending sample for the sweep. // The read holds no lifecycle lock, so the keyed caller re-checks the actor // record it loaded after this returns. -func (s *AteomService) sampleSandbox(active *resources.ActorAttribution) (*ateompb.WorkloadStatsSample, error) { +func (s *AteomService) sampleSandbox(active *resources.ActorAttribution) (*ateompb.WorkloadStatsSample, cgroupstats.Sample, error) { read := s.readSandboxCgroup if read == nil { read = cgroupstats.Read @@ -261,7 +272,7 @@ func (s *AteomService) sampleSandbox(active *resources.ActorAttribution) (*ateom observedAt := time.Now() sample, err := read(filepath.Join(s.cgroupRoot, ocispec.GVisorCgroupLeaf(active.UID, sandboxCgroupContainer))) if err != nil { - return nil, err + return nil, cgroupstats.Sample{}, err } return &ateompb.WorkloadStatsSample{ @@ -280,5 +291,30 @@ func (s *AteomService) sampleSandbox(active *resources.ActorAttribution) (*ateom CpuUsageUsec: sample.CPUUsageUsec, ObservedAtUnixNano: observedAt.UnixNano(), - }, nil + }, sample, nil +} + +// warnDeadSandbox logs, once per activation, that h's sandbox cgroup has no +// process left. Nothing else notices this today: the actor stays RUNNING and +// its stats read like an idle actor's (#2211). +// +// A lifecycle RPC empties the cgroup on purpose when it tears the sandbox down, +// and holds the actor's lock while it does, so a busy actor is skipped. The +// cgroup was read before the lock was checked, and h is looked up again after, +// so a teardown that starts or finishes in between is never reported. +// +// oom_kill counts over the cgroup's lifetime. A leaf left behind by an earlier +// activation of the same actor carries its count forward. +func (s *AteomService) warnDeadSandbox(ctx context.Context, h *hostedActor, raw cgroupstats.Sample) { + if h.deadReported.Load() { + return + } + if s.locks.Busy(h.attribution.UID) || s.lookupActor(h.attribution.UID) != h { + return + } + if !h.deadReported.CompareAndSwap(false, true) { + return + } + attrs := append(ateattr.ActorLogAttrs(h.attribution), slog.Uint64(string(ateattr.SandboxOOMKillsKey), raw.OOMKills)) + slog.LogAttrs(ctx, slog.LevelWarn, "Sandbox has no processes left while the actor is hosted", attrs...) } diff --git a/cmd/ateom-gvisor/stats_test.go b/cmd/ateom-gvisor/stats_test.go index 4d0452f46a..42f80e92da 100644 --- a/cmd/ateom-gvisor/stats_test.go +++ b/cmd/ateom-gvisor/stats_test.go @@ -17,10 +17,14 @@ package main import ( + "bytes" "context" "github.com/agent-substrate/substrate/internal/actorlock" + "log/slog" "os" "path/filepath" + "strings" + "sync" "testing" "time" @@ -497,3 +501,176 @@ func sweepAndList(s *AteomService) (*ateompb.GetActiveWorkloadStatsResponse, err s.sweepUsage(context.Background()) return s.GetActiveWorkloadStats(context.Background(), &ateompb.GetActiveWorkloadStatsRequest{}) } + +// deadCgroup is a sandbox leaf after the kernel OOM-killed the sentry: the leaf +// and its counters remain, the processes do not. +var deadCgroup = map[string]string{ + "memory.current": "16384\n", + "memory.events": "oom 5334\noom_kill 3\n", + "cgroup.events": "populated 0\nfrozen 0\n", +} + +// captureWarnings routes the default logger to a buffer for the test. +func captureWarnings(t *testing.T) *bytes.Buffer { + t.Helper() + var buf bytes.Buffer + prev := slog.Default() + slog.SetDefault(slog.New(slog.NewJSONHandler(&buf, &slog.HandlerOptions{Level: slog.LevelWarn}))) + t.Cleanup(func() { slog.SetDefault(prev) }) + return &buf +} + +const deadSandboxMsg = "Sandbox has no processes left while the actor is hosted" + +// TestSweepUsageDeadSandbox: a hosted actor whose sandbox cgroup +// is empty is warned about once per activation, and not at all while a +// lifecycle RPC is tearing it down. +func TestSweepUsageDeadSandbox(t *testing.T) { + for _, tc := range []struct { + name string + files map[string]string + busy bool + wantWarns int + }{ + {name: "dead sandbox warns once", files: deadCgroup, wantWarns: 1}, + {name: "teardown in progress is skipped", files: deadCgroup, busy: true}, + {name: "live sandbox does not warn", files: healthyCgroup}, + {name: "booting sandbox with no cgroup yet does not warn", files: nil}, + } { + t.Run(tc.name, func(t *testing.T) { + logs := captureWarnings(t) + s := newStatsService(t, tc.files) + setHostedActor(s, &testActor) + if tc.busy { + if !s.locks.Lock(context.Background(), testActor.UID) { + t.Fatal("could not take the actor lock") + } + defer s.locks.Unlock(testActor.UID) + } + + // Two sweeps: the second must not repeat the warning. + for range 2 { + got, err := sweepAndList(s) + if err != nil { + t.Fatalf("sweepAndList() error = %v, want nil", err) + } + if len(got.GetSamples()) != 1 { + t.Fatalf("sweepAndList() = %v, want one sample", got) + } + } + + if n := strings.Count(logs.String(), deadSandboxMsg); n != tc.wantWarns { + t.Fatalf("got %d dead-sandbox warnings, want %d; logs:\n%s", n, tc.wantWarns, logs) + } + if tc.wantWarns > 0 { + for _, want := range []string{`"ate.actor.uid":"uid-a"`, `"ate.actor.name":"actor-a"`, `"ate.atespace":"space-a"`, `"ate.template.name":"template-a"`, `"ate.sandbox.oom_kills":3`} { + if !strings.Contains(logs.String(), want) { + t.Errorf("warning missing %s; logs:\n%s", want, logs) + } + } + } + }) + } +} + +// A record that is no longer the hosted one -- the actor was unhosted or +// re-hosted after the cgroup was read -- is never reported. +func TestWarnDeadSandboxStaleRecord(t *testing.T) { + logs := captureWarnings(t) + s := newStatsService(t, deadCgroup) + setHostedActor(s, &testActor) + stale := &hostedActor{attribution: testActor} + + s.warnDeadSandbox(context.Background(), stale, cgroupstats.Sample{Empty: true}) + + if strings.Contains(logs.String(), deadSandboxMsg) { + t.Fatalf("warned about a stale record; logs:\n%s", logs) + } + if stale.deadReported.Load() { + t.Error("stale record marked as reported") + } +} + +// The once-per-activation guarantee is per hostedActor: a new activation of +// the same actor that dies again is reported again. +func TestSweepUsageDeadSandboxNewActivation(t *testing.T) { + logs := captureWarnings(t) + s := newStatsService(t, deadCgroup) + for range 2 { + setHostedActor(s, &testActor) // a fresh hostedActor, as hostActor makes + s.sweepUsage(context.Background()) + } + if n := strings.Count(logs.String(), deadSandboxMsg); n != 2 { + t.Fatalf("got %d dead-sandbox warnings over two activations, want 2; logs:\n%s", n, logs) + } +} + +// Concurrent sweeps of one dead sandbox still warn once. +func TestSweepUsageDeadSandboxConcurrentSweeps(t *testing.T) { + logs := captureWarnings(t) + s := newStatsService(t, deadCgroup) + setHostedActor(s, &testActor) + + var wg sync.WaitGroup + for range 16 { + wg.Go(func() { + s.sweepUsage(context.Background()) + }) + } + wg.Wait() + if n := strings.Count(logs.String(), deadSandboxMsg); n != 1 { + t.Fatalf("got %d dead-sandbox warnings from concurrent sweeps, want 1; logs:\n%s", n, logs) + } +} + +// drainedCgroup is the sandbox leaf after a drain has killed the app +// containers: the sentry and gofers stay in the pause leaf, so it is still +// populated. +var drainedCgroup = map[string]string{ + "memory.current": "31457280\n", + "memory.events": "oom 0\noom_kill 0\n", + "cgroup.events": "populated 1\nfrozen 0\n", +} + +// TestDeadSandboxCheckDuringDrain pins what the check does while +// gracefulShutdown runs. The drain kills the app containers without taking +// actor locks or unhosting, but it never stops the sandbox, so the leaf stays +// populated and nothing is logged. A sandbox that dies during the drain is +// still reported. +func TestDeadSandboxCheckDuringDrain(t *testing.T) { + for _, tc := range []struct { + name string + files map[string]string + wantWarns int + }{ + {name: "app containers killed, sentry alive", files: drainedCgroup}, + {name: "sandbox died during the drain", files: deadCgroup, wantWarns: 1}, + } { + t.Run(tc.name, func(t *testing.T) { + logs := captureWarnings(t) + s := newStatsService(t, tc.files) + setHostedActor(s, &testActor) + s.shuttingDown.Store(true) + + s.sweepUsage(context.Background()) + if n := strings.Count(logs.String(), deadSandboxMsg); n != tc.wantWarns { + t.Fatalf("got %d dead-sandbox warnings, want %d; logs:\n%s", n, tc.wantWarns, logs) + } + }) + } +} + +// TestDrainNeverKillsPauseContainer: gracefulShutdown kills the names +// containerNames takes from the spec. atelet rejects a spec that names the +// pause container, so the drain cannot stop the sandbox itself. +func TestDrainNeverKillsPauseContainer(t *testing.T) { + if err := resources.ValidateContainerNames([]string{ocispec.PauseContainer}); err == nil { + t.Fatalf("ValidateContainerNames accepted %q; a spec could then put it on the drain's kill list", ocispec.PauseContainer) + } + spec := []*ateompb.Container{{Name: "app"}, {Name: "sidecar"}} + for _, name := range containerNames(spec) { + if name == ocispec.PauseContainer { + t.Fatalf("containerNames(%v) includes %q", spec, ocispec.PauseContainer) + } + } +} diff --git a/cmd/ateom-microvm/checkpoint.go b/cmd/ateom-microvm/checkpoint.go index 9fc10eff10..a282d2df34 100644 --- a/cmd/ateom-microvm/checkpoint.go +++ b/cmd/ateom-microvm/checkpoint.go @@ -30,6 +30,7 @@ import ( "github.com/agent-substrate/substrate/cmd/ateom-microvm/internal/ch" "github.com/agent-substrate/substrate/cmd/ateom-microvm/internal/kata" + "github.com/agent-substrate/substrate/internal/ateomphaselog" "github.com/agent-substrate/substrate/internal/ateomstats" "github.com/agent-substrate/substrate/internal/imagecache" "github.com/agent-substrate/substrate/internal/proto/ateompb" @@ -43,14 +44,14 @@ import ( // // What the snapshot holds depends on the requested scope: // -// - FULL: the whole guest. ateom drives the CH REST api-socket: pause -> snapshot +// - MEMORY: the whole guest. ateom drives the CH REST api-socket: pause -> snapshot // file:// (config.json + state.json + sparse memory-ranges) // -> tear the VMM down. Each container's rootfs is overlay(virtio-fs RO lower + // disk-backed upper): the upper is host-backed like the durable-dir volumes and // ships alongside as its own tar (see rootfsupper.go); process memory persists // via the memory snapshot. The RO lower is reconstructed from the OCI image at // restore, so it never ships. Durable-dir volumes ship alongside as per-volume tars. -// - DATA: the durable-dir volumes only, as those same tars. The guest is discarded, so +// - VOLUMES: the durable-dir volumes only, as those same tars. The guest is discarded, so // the actor cold-starts on restore with its volumes re-materialized. // // Either way the guest is paused first, which is what makes the tar coherent: the @@ -63,6 +64,9 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec if err := validateActorDirs(req.GetActorDirs()); err != nil { return nil, err } + if err := validateFidelity(req.GetFidelity()); err != nil { + return nil, err + } if !s.locks.Lock(ctx, req.GetActorUid()) { return nil, fmt.Errorf("gave up waiting for the actor's lock: %w", ctx.Err()) } @@ -80,17 +84,17 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec tStart := time.Now() var dPrep, dPause, dSnapshot, dDurable, dUpper, dTeardown time.Duration attribution := ateomstats.ActorAttributionFromRequest(req) - scope := req.GetScope() + scope := req.GetFidelity() defer func() { - logSnapshotPhases(ctx, "Checkpoint timing breakdown", attribution, scope, - checkpointDurationKey, err, []phase{ - {phasePrep, dPrep}, - {phasePause, dPause}, - {phaseSnapshot, dSnapshot}, - {phaseDurableDir, dDurable}, - {phaseRootfsUpper, dUpper}, - {phaseTeardown, dTeardown}, - {phaseTotal, time.Since(tStart)}, + ateomphaselog.LogSnapshotPhases(ctx, "Checkpoint timing breakdown", attribution, scope, + ateomphaselog.CheckpointDurationKey, err, []ateomphaselog.Phase{ + {Name: phasePrep, D: dPrep}, + {Name: phasePause, D: dPause}, + {Name: phaseSnapshot, D: dSnapshot}, + {Name: phaseDurableDir, D: dDurable}, + {Name: phaseRootfsUpper, D: dUpper}, + {Name: phaseTeardown, D: dTeardown}, + {Name: phaseTotal, D: time.Since(tStart)}, }) }() @@ -108,20 +112,20 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec // suspended mid-flight for a call that could never have succeeded. // // Durable-dir volumes are host-backed, so they are captured the same way - // under either scope — and are the ONLY thing a Data-scope snapshot + // under either fidelity — and are the ONLY thing a VOLUMES snapshot // captures. durable := hasDurableVolumes(req.GetSpec().GetContainers()) csi := hasCsiVolumes(req.GetSpec().GetContainers()) switch scope { - case ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL: - case ateompb.SnapshotScope_SNAPSHOT_SCOPE_DATA: + case ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY: + case ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES: // TODO: Revisit handling for CSI volumes since snapshots are currently quietly ignored. if !durable && !csi { return nil, apierror.FailedPrecondition( - "no durable-dir or CSI volumes found for a Data-scope snapshot") + "no durable-dir or CSI volumes found for a VOLUMES snapshot") } default: - return nil, apierror.InvalidArgument("unsupported snapshot scope: %v", scope) + return nil, apierror.InvalidArgument("unsupported snapshot fidelity: %v", scope) } // Captured now: the checkpoint unhosts the actor, and the final record @@ -167,18 +171,18 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec // window costs the slowest of them rather than their sum (the tars scale // with the actor's data; suspend latency is the metric that matters). // - // - CH snapshot (Full only): the guest memory + VM state. A Data snapshot + // - CH snapshot (MEMORY only): the guest memory + VM state. A VOLUMES snapshot // deliberately captures no VM state — no memory image, and no base-id, // since nothing will reattach to the frozen virtio-fs lower: at restore // the actor cold-boots from the OCI image. // - Durable-dir tars (any scope, when declared): host-backed, so pausing // the write-through share makes the tars coherent. - // - Rootfs upper tars (Full only): host-backed like the durable volumes — - // the memory snapshot does not carry rootfs writes. Under Data the + // - Rootfs upper tars (MEMORY only): host-backed like the durable volumes — + // the memory snapshot does not carry rootfs writes. Under VOLUMES the // workload cold-starts on restore, discarding rootfs state. var durableFiles []string g, gctx := errgroup.WithContext(ctx) - if scope == ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL { + if scope == ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY { g.Go(func() error { t := time.Now() d, err := s.snapshotVMState(gctx, client, ra, actorUID, checkpointDir) @@ -198,7 +202,7 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec return err }) } - if scope == ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL { + if scope == ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY { g.Go(func() error { t := time.Now() err := tarRootfsUpper(gctx, rootfsUpperDir(actorDirs), checkpointDir, containerNames(req.GetSpec().GetContainers())) @@ -211,8 +215,8 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec } // Report exactly the files we wrote so atelet ships precisely this snapshot: for - // Full, the CH snapshot (config.json + state.json + memory-ranges + base-id) plus - // any durable-dir tars; for Data, those tars alone. + // MEMORY, the CH snapshot (config.json + state.json + memory-ranges + base-id) + // plus any durable-dir tars; for VOLUMES, those tars alone. snapshotFiles, err := listFiles(checkpointDir) if err != nil { return nil, fmt.Errorf("while listing snapshot files: %w", err) diff --git a/cmd/ateom-microvm/durable.go b/cmd/ateom-microvm/durable.go index 76a990c892..c27a3f4e1d 100644 --- a/cmd/ateom-microvm/durable.go +++ b/cmd/ateom-microvm/durable.go @@ -19,7 +19,7 @@ package main // Durable-dir volumes for the micro-VM runtime. // // A durable-dir volume is a directory whose contents outlive the actor's process -// state: it survives suspend/resume and, under the Data snapshot scope, is the +// state: it survives suspend/resume and, under the VOLUMES snapshot fidelity, is the // ONLY thing captured (the workload cold-starts on restore). The host side is // owned by atelet, which creates one directory per volume under // ActorDirs.durable_dir_volume_mounts_dir and wipes them when the actor's diff --git a/cmd/ateom-microvm/main.go b/cmd/ateom-microvm/main.go index 9e46395802..443754be0c 100644 --- a/cmd/ateom-microvm/main.go +++ b/cmd/ateom-microvm/main.go @@ -52,6 +52,7 @@ import ( "github.com/agent-substrate/substrate/internal/resources" "github.com/agent-substrate/substrate/internal/serverboot" "github.com/agent-substrate/substrate/internal/version" + atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" "github.com/spf13/pflag" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "golang.org/x/sys/unix" @@ -171,8 +172,8 @@ func do(ctx context.Context) error { // Create ateom dir. ateomDir := nodepath.AteomPath(*podUID) - if err := resources.ValidateAteomUID(*podUID); err != nil { - return fmt.Errorf("in resources.ValidateAteomUID: %w", err) + if err := resources.ValidateWorkerPodUID(*podUID); err != nil { + return fmt.Errorf("in resources.ValidateWorkerPodUID: %w", err) } if err := os.MkdirAll(ateomDir, 0o700); err != nil { return fmt.Errorf("in os.MkdirAll(%q): %w", ateomDir, err) @@ -282,6 +283,7 @@ func do(ctx context.Context) error { TrustBundlePath: tunnelConfig.TrustBundle, AteletSPIFFEID: tunnelConfig.BrokerIdentity, Actors: *maxActors, + SandboxClass: string(atev1alpha1.SandboxClassMicroVM), }) if err != nil && ctx.Err() == nil { serverboot.Fatal(ctx, "Failed to report worker capacity", err) @@ -417,6 +419,15 @@ func validateActorDirs(actorDirs *ateompb.ActorDirs) error { return nil } +// validateFidelity rejects a checkpoint or restore request whose fidelity +// this runtime cannot serve. +func validateFidelity(fidelity ateompb.SnapshotFidelity) error { + if errs := resources.ValidateSnapshotFidelity(fidelity, field.NewPath("fidelity")); len(errs) > 0 { + return apierror.InvalidArgument("%v", errs.ToAggregate()) + } + return nil +} + // validateRuntimeAssetPaths ensures we only run assets from the static files dir func validateRuntimeAssetPaths(paths map[string]string) error { var errs field.ErrorList diff --git a/cmd/ateom-microvm/phaselog.go b/cmd/ateom-microvm/phaselog.go index bd41cfc728..bde1ee32ea 100644 --- a/cmd/ateom-microvm/phaselog.go +++ b/cmd/ateom-microvm/phaselog.go @@ -16,33 +16,12 @@ package main -import ( - "context" - "log/slog" - "time" +import "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/apierror" - "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/proto/ateompb" - "github.com/agent-substrate/substrate/internal/resources" - "google.golang.org/grpc/status" -) - -// The keys the per-phase durations are logged under. They are named like -// instruments but deliberately are not ones: the phases are implementation -// details of this binary, so they stay a developer-facing log record rather -// than becoming metric API. They decompose the ateom_restore / -// ateom_checkpoint phases of atelet's ate.actor.*.duration histograms, whose -// names they extend, and are joined per actor by benchmarking tooling. -const ( - restoreDurationKey = "ateom.actor.restore.duration" - checkpointDurationKey = "ateom.actor.checkpoint.duration" -) - -// The phase names, suffixed onto the duration keys. Kept out of ateattr on -// purpose: that package's SnapshotPhase* values are the ate.snapshot.phase -// metric enum, and these are not values of it. "total" is shared so the two -// layers' records agree on the denominator. +// The phase names, suffixed onto the ateomphaselog duration keys. Kept out of +// ateattr on purpose: that package's SnapshotPhase* values are the +// ate.snapshot.phase metric enum, and these are not values of it. "total" is +// shared so the two layers' records agree on the denominator. // // Checkpoint: snapshot, durable_dir and rootfs_upper run concurrently on the // paused guest, so the paused window costs their max, not their sum; prep, @@ -67,64 +46,3 @@ const ( phaseTotal = ateattr.SnapshotPhaseTotal ) - -// phase is one timed step of a snapshot operation. A zero duration means the -// phase never ran (a Data-scope checkpoint captures no guest) and is skipped -// rather than logged as instant. -type phase struct { - name string - d time.Duration -} - -// scopeLogValue maps the ateom wire enum onto the shared scope label values, -// the same way ateattr.SnapshotScopeValue does for the atelet enum. An -// unrecognized scope reports as unknown rather than stringified, so no wire -// value can widen the value set readers key on. -func scopeLogValue(scope ateompb.SnapshotScope) string { - switch scope { - case ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL: - return ateattr.SnapshotScopeFull - case ateompb.SnapshotScope_SNAPSHOT_SCOPE_DATA: - return ateattr.SnapshotScopeData - default: - return ateattr.SnapshotScopeUnknown - } -} - -// snapshotPhaseAttrs renders one joinable per-actor record for a checkpoint -// or restore, mirroring atelet's snapshotLogAttrs: full actor identity, the -// scope, and one float-seconds attr per non-zero phase under -// durationKey.. One record carries every phase of the operation, so a -// reader never joins two half-records that disagree about the same actor. -// -// err is the operation's outcome. A failed operation still records the phases -// it completed, marked with error.type (the gRPC code, context errors as -// DeadlineExceeded / Canceled) so a reader can leave it out of a latency -// distribution. Absence means success. -func snapshotPhaseAttrs(a resources.ActorAttribution, scope ateompb.SnapshotScope, durationKey string, err error, phases []phase) []slog.Attr { - attrs := ateattr.ActorLogAttrs(a) - attrs = append(attrs, slog.String(string(ateattr.SnapshotScopeKey), scopeLogValue(scope))) - if err != nil { - s, ok := apierror.FromError(err) - code := s.Code() - if !ok { - code = status.Code(err) - } - attrs = append(attrs, slog.String(string(ateattr.ErrorTypeKey), code.String())) - } - for _, p := range phases { - if p.d == 0 { - continue - } - // Seconds and not slog.Duration's nanoseconds: the values sit under - // the atelet histograms' unit (s), so readers compare them without a - // per-key unit table. - attrs = append(attrs, slog.Float64(durationKey+"."+p.name, p.d.Seconds())) - } - return attrs -} - -// logSnapshotPhases emits the snapshotPhaseAttrs record under msg. -func logSnapshotPhases(ctx context.Context, msg string, a resources.ActorAttribution, scope ateompb.SnapshotScope, durationKey string, err error, phases []phase) { - slog.LogAttrs(ctx, slog.LevelInfo, msg, snapshotPhaseAttrs(a, scope, durationKey, err, phases)...) -} diff --git a/cmd/ateom-microvm/restore.go b/cmd/ateom-microvm/restore.go index 4a326c3fba..ca1dbf642a 100644 --- a/cmd/ateom-microvm/restore.go +++ b/cmd/ateom-microvm/restore.go @@ -28,6 +28,7 @@ import ( "time" "github.com/agent-substrate/substrate/internal/apierror" + "github.com/agent-substrate/substrate/internal/ateomphaselog" "github.com/agent-substrate/substrate/internal/ateomstats" "github.com/agent-substrate/substrate/internal/ateomnet" @@ -66,7 +67,7 @@ func restoreMemMode(ctx context.Context, info ch.VMMInfo) string { } // reseedGuestCRNG mixes fresh, per-restore entropy into the restored guest's kernel -// CRNG through the kata-agent (see the call site in restoreFullScope for why a restore +// CRNG through the kata-agent (see the call site in restoreMemoryFidelity for why a restore // needs this). The nonce is a throwaway; its only job is to differ between restores so // that clones of one snapshot diverge instead of producing identical randomness. The // caller owns ac: it stays open for log forwarding and guest stats. @@ -90,12 +91,13 @@ func newReseedNonce() ([]byte, error) { } // RestoreWorkload brings the actor back from a snapshot, on a possibly different -// pod. What that means depends on the scope the snapshot was taken with: +// pod. What that means depends on the fidelity the snapshot was taken with: // -// - FULL: relaunch cloud-hypervisor from the snapshot and resume the guest -// (restoreFullScope). -// - DATA: there is no guest to resume — re-materialize the durable-dir volumes and -// cold-boot the actor, which starts its containers afresh from the OCI image. +// - MEMORY: relaunch cloud-hypervisor from the snapshot and resume the guest +// (restoreMemoryFidelity). +// - VOLUMES: there is no guest to resume — re-materialize the durable-dir +// volumes and cold-boot the actor, which starts its containers afresh from +// the OCI image. // // Contract with atelet: the snapshot's files are in ActorDirs.restore_dir, // and the durable-dir volume directories re-created (empty). @@ -103,6 +105,9 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore if err := validateActorDirs(req.GetActorDirs()); err != nil { return nil, err } + if err := validateFidelity(req.GetFidelity()); err != nil { + return nil, err + } if err := validateRuntimeAssetPaths(req.GetRuntimeAssetPaths()); err != nil { return nil, err } @@ -146,10 +151,10 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore return nil, err } // Publish attribution before restore, so a stats read during it is - // attributed. A Data scope cold-boots, so its CPU counts from zero; the other + // attributed. A VOLUMES restore cold-boots, so its CPU counts from zero; the other // scopes resume the guest's counters, so theirs counts from the first // reading. - resumesGuest := req.GetScope() != ateompb.SnapshotScope_SNAPSHOT_SCOPE_DATA + resumesGuest := req.GetFidelity() != ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES if _, err := s.hostActor(ctx, attribution, resumesGuest); err != nil { return nil, err } @@ -162,8 +167,8 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore } }() - // Restore the durable-dir volumes before anything can observe them: for Full - // that means before the share's virtiofsd starts, for Data before the workload + // Restore the durable-dir volumes before anything can observe them: for MEMORY + // that means before the share's virtiofsd starts, for VOLUMES before the workload // cold-starts. if hasDurableVolumes(p.containers) { if err := untarDurableVolumes(durableDir, restoreDir, durableVolumeNames(p.containers)); err != nil { @@ -171,13 +176,13 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore } } - switch scope := req.GetScope(); scope { - case ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL: - if err := s.restoreFullScope(ctx, p, scope, restoreDir, req.GetPreserveRestoreDir(), tStart); err != nil { + switch scope := req.GetFidelity(); scope { + case ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY: + if err := s.restoreMemoryFidelity(ctx, p, scope, restoreDir, req.GetPreserveRestoreDir(), tStart); err != nil { return nil, err } - case ateompb.SnapshotScope_SNAPSHOT_SCOPE_DATA: - // A Data snapshot holds no guest state, so this is a cold boot that + case ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES: + // A VOLUMES snapshot holds no guest state, so this is a cold boot that // happens to start with the volumes already populated. wakeup probe gating comes // with the cold-boot path, so the actor is serving when we return. if err := s.coldBootActorRetrying(ctx, p); err != nil { @@ -186,19 +191,19 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore dTotal := time.Since(tStart) slog.InfoContext(ctx, "Actor restored (durable-dir volumes, cold boot)", slog.String("id", p.actorUID), slog.Duration("total", dTotal)) - // A cold boot has none of the full-scope phases, so the total is the + // A cold boot has none of the memory-restore phases, so the total is the // only observation on its record. - logSnapshotPhases(ctx, "Restore timing breakdown", attribution, scope, - restoreDurationKey, nil, []phase{{phaseTotal, dTotal}}) + ateomphaselog.LogSnapshotPhases(ctx, "Restore timing breakdown", attribution, scope, + ateomphaselog.RestoreDurationKey, nil, []ateomphaselog.Phase{{Name: phaseTotal, D: dTotal}}) default: - return nil, apierror.InvalidArgument("unsupported snapshot scope: %v", scope) + return nil, apierror.InvalidArgument("unsupported snapshot fidelity: %v", scope) } s.actorLogger.EmitLifecycleLog(ctx, "Actor restored", attribution) return &ateompb.RestoreWorkloadResponse{}, nil } -// restoreFullScope restores a whole-guest snapshot: relaunch cloud-hypervisor +// restoreMemoryFidelity restores a whole-guest snapshot: relaunch cloud-hypervisor // directly from it and resume. // // Each container's rootfs is a host-merged overlay (image lower + host upper). Steps: @@ -210,7 +215,7 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore // and resume. Guest RAM — the actor's in-memory state and the frozen network config — // comes back from the memory snapshot; the durable-dir volumes were restored by the // caller from their tar. -func (s *AteomService) restoreFullScope(ctx context.Context, p actorBootParams, scope ateompb.SnapshotScope, restoreDir string, preserveRestoreDir bool, tStart time.Time) (retErr error) { +func (s *AteomService) restoreMemoryFidelity(ctx context.Context, p actorBootParams, scope ateompb.SnapshotFidelity, restoreDir string, preserveRestoreDir bool, tStart time.Time) (retErr error) { actorUID := p.actorUID rr := s.resolveRuntime(p.assetPaths) @@ -236,7 +241,7 @@ func (s *AteomService) restoreFullScope(ctx context.Context, p actorBootParams, } tPrep := time.Now() - // Full snapshots carry each container's upper as a tar (rootfsUpperTarFile). + // MEMORY snapshots carry each container's upper as a tar (rootfsUpperTarFile). // Start re-materializing the upper contents NOW, in the background: the // untar scales with the actor's data and is joined right before the host // overlay mounts need it, so it hides behind the bundle preparation below. @@ -269,7 +274,7 @@ func (s *AteomService) restoreFullScope(ctx context.Context, p actorBootParams, if len(containers) > maxActorContainers { return apierror.Unimplemented("ateom-microvm supports at most %d containers, got %d", maxActorContainers, len(containers)) } - ctrs, err := s.buildActorContainers(p.actorDirs, containers) + ctrs, err := s.buildActorContainers(p.actorUID, p.actorDirs, containers) if err != nil { return err } @@ -444,18 +449,18 @@ func (s *AteomService) restoreFullScope(ctx context.Context, p actorBootParams, // The joinable per-actor record the benchmarking tooling aggregates. The // durable delta is not carried: tDurable is pinned to tLowers today, so it // would always be the zero a record skips. - logSnapshotPhases(ctx, "Restore timing breakdown", p.actorAttribution(), scope, - restoreDurationKey, nil, []phase{ - {phasePrep, tPrep.Sub(tStart)}, - {phaseBundles, tBundles.Sub(tPrep)}, - {phaseUpperJoin, tUpper.Sub(tBundles)}, - {phaseLowers, tLowers.Sub(tUpper)}, - {phaseTap, tTap.Sub(tDurable)}, - {phaseVMMLaunch, tLaunch.Sub(tTap)}, - {phaseVMRestore, tVMRestore.Sub(tLaunch)}, - {phaseResume, tResume.Sub(tVMRestore)}, - {phaseWakeupProbe, dWakeupProbe}, - {phaseTotal, dTotal}, + ateomphaselog.LogSnapshotPhases(ctx, "Restore timing breakdown", p.actorAttribution(), scope, + ateomphaselog.RestoreDurationKey, nil, []ateomphaselog.Phase{ + {Name: phasePrep, D: tPrep.Sub(tStart)}, + {Name: phaseBundles, D: tBundles.Sub(tPrep)}, + {Name: phaseUpperJoin, D: tUpper.Sub(tBundles)}, + {Name: phaseLowers, D: tLowers.Sub(tUpper)}, + {Name: phaseTap, D: tTap.Sub(tDurable)}, + {Name: phaseVMMLaunch, D: tLaunch.Sub(tTap)}, + {Name: phaseVMRestore, D: tVMRestore.Sub(tLaunch)}, + {Name: phaseResume, D: tResume.Sub(tVMRestore)}, + {Name: phaseWakeupProbe, D: dWakeupProbe}, + {Name: phaseTotal, D: dTotal}, }) ra := &runningActor{ diff --git a/cmd/ateom-microvm/rootfsupper.go b/cmd/ateom-microvm/rootfsupper.go index 82f528a136..11b0bbaf01 100644 --- a/cmd/ateom-microvm/rootfsupper.go +++ b/cmd/ateom-microvm/rootfsupper.go @@ -36,13 +36,13 @@ package main // removed at teardown — after CleanupSandboxState has dropped the overlay // mounts that use it. // -// Snapshots: the upper does not ride in guest memory, so a FULL snapshot +// Snapshots: the upper does not ride in guest memory, so a MEMORY snapshot // ships it as one tar per container (rootfsUpperTarFile), taken while the // guest is paused (the share is write-through, so a paused guest's completed // writes are already in the upper). Each tar holds only the contents of that // container's upperdir: ateom creates the directory layout itself on restore, // so nothing in the snapshot decides a path ateom later mounts or wipes. A -// DATA snapshot deliberately excludes rootfs state: the workload cold-starts +// VOLUMES snapshot deliberately excludes rootfs state: the workload cold-starts // on restore. import ( diff --git a/cmd/ateom-microvm/run.go b/cmd/ateom-microvm/run.go index 50c8356104..1500968454 100644 --- a/cmd/ateom-microvm/run.go +++ b/cmd/ateom-microvm/run.go @@ -229,7 +229,8 @@ func (s *AteomService) resolveRuntime(paths map[string]string) resolvedRuntime { // Contract with atelet: // - The runtime assets (guest kernel, guest OS image, cloud-hypervisor, virtiofsd) // are on disk and passed as runtime asset paths. -// - The OCI bundle (config.json + populated rootfs/) is prepared per container. +// - Each container's bundle holds the overlay spec its rootfs/ is composed +// from. ateom builds the OCI spec itself. func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkloadRequest) (resp *ateompb.RunWorkloadResponse, retErr error) { if err := validateActorDirs(req.GetActorDirs()); err != nil { return nil, err @@ -294,7 +295,7 @@ func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkload } // actorBootParams is what a cold boot needs about an actor. It comes from a Run -// request, or from a Restore request whose snapshot scope covers only the +// request, or from a Restore request whose snapshot fidelity covers only the // durable-dir volumes (the workload itself cold-starts). type actorBootParams struct { actorRef resources.ActorRef @@ -423,8 +424,8 @@ func (s *AteomService) coldBootActor(ctx context.Context, p actorBootParams) (re // vCPUs round up; VM RAM reserves a fixed margin for the VMM + virtiofsd, which // share the pod cgroup with the guest RAM. A declared memory limit the reserve // leaves too small to boot is rejected (resolveGuestMemMiB) rather than silently - // falling back to the larger kata default. NB: a FULL-scope snapshot restore - // reuses the size baked into the snapshot (restoreFullScope), so resizing an + // falling back to the larger kata default. NB: a MEMORY snapshot restore + // reuses the size baked into the snapshot (restoreMemoryFidelity), so resizing an // existing actor takes effect on its next cold boot. sz := p.size if v := sz.VCPUs(); v > 0 { @@ -437,7 +438,7 @@ func (s *AteomService) coldBootActor(ctx context.Context, p actorBootParams) (re // Prepare each container's OCI spec + record its bundle rootfs (the overlay // lower the host merges under the container's writable upper). - ctrs, err := s.buildActorContainers(p.actorDirs, containers) + ctrs, err := s.buildActorContainers(p.actorUID, p.actorDirs, containers) if err != nil { return err } @@ -621,19 +622,16 @@ func (s *AteomService) coldBootActor(ctx context.Context, p actorBootParams) (re } // buildActorContainers prepares each of the actor's containers for the shared -// micro-VM: it loads the OCI spec from the per-container bundle, injects guest DNS, +// micro-VM: it builds the OCI spec from the request, injects guest DNS, // and records the bundle rootfs that backs the overlay's RO lower. No host disk is // mounted here — the merged overlays are assembled in stageMergedRootfs after the // sandbox state is clean. Both RunWorkload and RestoreWorkload go through here. -func (s *AteomService) buildActorContainers(actorDirs *ateompb.ActorDirs, containers []*ateompb.Container) ([]actorContainer, error) { +func (s *AteomService) buildActorContainers(actorUID string, actorDirs *ateompb.ActorDirs, containers []*ateompb.Container) ([]actorContainer, error) { ctrs := make([]actorContainer, len(containers)) for i, c := range containers { cn := c.GetName() bundle := ociBundlePath(actorDirs, cn) - spec, err := ocispec.Load(bundle) - if err != nil { - return nil, fmt.Errorf("while reading the OCI spec for %q: %w", cn, err) - } + spec := ocispec.Build(ocispec.Options{ActorUID: actorUID, ActorDirs: actorDirs, Container: c}) if err := ocispec.ShapeMicroVM(spec, ocispec.MicroVMOptions{ActorDirs: actorDirs, ContainerID: cn}); err != nil { return nil, fmt.Errorf("while shaping the OCI spec for %q: %w", cn, err) } diff --git a/cmd/benchmarking/isolate/fake-atelet/tlsconfig_test.go b/cmd/benchmarking/isolate/fake-atelet/tlsconfig_test.go index 477e1ff136..510cf1be15 100644 --- a/cmd/benchmarking/isolate/fake-atelet/tlsconfig_test.go +++ b/cmd/benchmarking/isolate/fake-atelet/tlsconfig_test.go @@ -71,7 +71,12 @@ func TestServerTLSConfigReloadsCACertsWithoutRestart(t *testing.T) { if err != nil { t.Fatalf("GetConfigForClient() first call error = %v", err) } - if err := os.WriteFile(path, testCertPEM(t), 0o600); err != nil { + // Replace the bundle like a projected-volume rotation, independent of mtime. + rotatedPath := path + ".rotated" + if err := os.WriteFile(rotatedPath, testCertPEM(t), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Rename(rotatedPath, path); err != nil { t.Fatal(err) } after, err := cfg.GetConfigForClient(nil) diff --git a/cmd/benchmarking/isolate/fake-workersync/controller.go b/cmd/benchmarking/isolate/fake-workersync/controller.go index 64c6a695d1..8920072373 100644 --- a/cmd/benchmarking/isolate/fake-workersync/controller.go +++ b/cmd/benchmarking/isolate/fake-workersync/controller.go @@ -326,7 +326,7 @@ func (c *controller) ensure(ctx context.Context, pod string, d desiredWorker, w index: d.index, node: fakeworker.Node(d.index, nodes), labels: maps.Clone(d.pool.GetLabels()), - sandboxClass: string(d.pool.Spec.SandboxClass), + sandboxClass: string(d.pool.Spec.DefaultSandboxClass()), } c.mu.Lock() c.workers[w.name] = w @@ -378,7 +378,10 @@ func (c *controller) ensure(ctx context.Context, pod string, d desiredWorker, w if err := c.relay.Report(ctx, addr, &ateapipb.RegisterWorkerRequest{ Worker: &ateapipb.ObjectRef{Name: name}, Capacity: fixed, - Hardware: hardware.ProbeHost(), + DefaultRuntime: &ateapipb.SandboxRuntime{ + SandboxClass: string(d.pool.Spec.DefaultSandboxClass()), + Version: hostCompat(), + }, }); err != nil { return fmt.Errorf("while reporting capacity for Worker %s: %w", name, err) } @@ -388,6 +391,15 @@ func (c *controller) ensure(ctx context.Context, pod string, d desiredWorker, w return nil } +func hostCompat() *ateapipb.VersionedSandboxCompat { + in := hardware.ProbeHost() + out := &ateapipb.VersionedSandboxCompat{SchemaVersion: in.GetSchemaVersion()} + for _, a := range in.GetAttributes() { + out.Attributes = append(out.Attributes, &ateapipb.AttributeEntry{Key: a.GetKey(), Value: a.GetValue()}) + } + return out +} + // create registers the Worker with the fields the worker syncer would copy // from the pool and its pod. Epoch stays 0: a rising epoch makes // ate-api-server crash every Actor on the Worker. @@ -401,7 +413,7 @@ func (c *controller) create(ctx context.Context, w *fakeWorker, wp *atev1alpha1. WorkerPodUid: name, NodeName: w.node, Ips: []string{fakeworker.IP(w.index)}, - SandboxClass: string(wp.Spec.SandboxClass), + SandboxClass: string(wp.Spec.DefaultSandboxClass()), Labels: maps.Clone(wp.GetLabels()), }}) if err != nil && status.Code(err) != codes.AlreadyExists { @@ -558,7 +570,7 @@ func templateOf(wp *atev1alpha1.WorkerPool) (template, error) { if err != nil { return template{}, err } - t := template{sandboxClass: string(wp.Spec.SandboxClass), actors: actors} + t := template{sandboxClass: string(wp.Spec.DefaultSandboxClass()), actors: actors} if wp.Spec.Template != nil && wp.Spec.Template.Resources != nil { limits := wp.Spec.Template.Resources.Limits t.cpuMilli = limits.Cpu().MilliValue() @@ -589,7 +601,7 @@ func (t template) capacity(allocatable corev1.ResourceList) *ateapipb.WorkerReso // down is replaced too. A held pool's Workers fit, since they are left as // they are. func fits(w *fakeWorker, wp *atev1alpha1.WorkerPool, allocatable map[string]corev1.ResourceList) bool { - if w.sandboxClass != string(wp.Spec.SandboxClass) { + if w.sandboxClass != string(wp.Spec.DefaultSandboxClass()) { return false } t, err := templateOf(wp) diff --git a/cmd/benchmarking/isolate/fake-workersync/controller_test.go b/cmd/benchmarking/isolate/fake-workersync/controller_test.go index 933ccbea9e..8d43bebaf2 100644 --- a/cmd/benchmarking/isolate/fake-workersync/controller_test.go +++ b/cmd/benchmarking/isolate/fake-workersync/controller_test.go @@ -25,7 +25,6 @@ import ( "time" "github.com/agent-substrate/substrate/cmd/benchmarking/isolate/internal/fakeworker" - "github.com/agent-substrate/substrate/internal/hardware" atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "google.golang.org/grpc" @@ -216,16 +215,16 @@ func (f *fakeControl) uid(pod string) string { type fakeRelay struct { control *fakeControl mu sync.Mutex - reported map[string]*ateapipb.WorkerResources // by Worker name - hardware map[string]*ateapipb.HardwareIdentity // by Worker name - via map[string]string // relay address, by Worker name + reported map[string]*ateapipb.WorkerResources // by Worker name + runtimes map[string]*ateapipb.SandboxRuntime // default runtime, by Worker name + via map[string]string // relay address, by Worker name reject map[string]codes.Code calls int live map[string]string // as last pruned to } func newFakeRelay() *fakeRelay { - return &fakeRelay{reported: map[string]*ateapipb.WorkerResources{}, hardware: map[string]*ateapipb.HardwareIdentity{}, via: map[string]string{}, reject: map[string]codes.Code{}} + return &fakeRelay{reported: map[string]*ateapipb.WorkerResources{}, runtimes: map[string]*ateapipb.SandboxRuntime{}, via: map[string]string{}, reject: map[string]codes.Code{}} } func (f *fakeRelay) Report(_ context.Context, addr string, req *ateapipb.RegisterWorkerRequest) error { @@ -237,12 +236,12 @@ func (f *fakeRelay) Report(_ context.Context, addr string, req *ateapipb.Registe return status.Error(code, "rejected") } f.reported[name] = req.GetCapacity() - f.hardware[name] = req.GetHardware() + f.runtimes[name] = req.GetDefaultRuntime() f.via[name] = addr if f.control != nil { f.control.mu.Lock() if w, ok := f.control.workers[name]; ok { - w.Status.Capacity, w.Status.Hardware = req.GetCapacity(), req.GetHardware() + w.Status.Capacity, w.Status.DefaultRuntime = req.GetCapacity(), req.GetDefaultRuntime() } f.control.mu.Unlock() } @@ -324,7 +323,7 @@ func (f *fakeCluster) setReplicas(pool string, n int32) { func pool(name string, replicas int32, limits corev1.ResourceList) *atev1alpha1.WorkerPool { wp := &atev1alpha1.WorkerPool{ ObjectMeta: metav1.ObjectMeta{Namespace: "benchmark-workloads", Name: name, Labels: map[string]string{"workload": name}}, - Spec: atev1alpha1.WorkerPoolSpec{Replicas: replicas, SandboxClass: atev1alpha1.SandboxClass("gvisor")}, + Spec: atev1alpha1.WorkerPoolSpec{Replicas: replicas, SandboxClasses: []atev1alpha1.WorkerPoolSandboxClass{{Name: atev1alpha1.SandboxClassGvisor}}}, } if limits != nil { wp.Spec.Template = &atev1alpha1.WorkerPoolPodTemplate{Resources: &corev1.ResourceRequirements{Limits: limits}} @@ -408,8 +407,9 @@ func TestReconcileHonorsReplicasAndLimits(t *testing.T) { if got, want := rel.reported[ctl.uid(name)], wantCapacity(1000, "1500m", "4Gi"); !proto.Equal(got, want) { t.Errorf("reported %v, want %v from the pool's limits", got, want) } - if got, want := rel.hardware[ctl.uid(name)], hardware.ProbeHost(); !proto.Equal(got, want) { - t.Errorf("hardware %v, want %v, which ate-api-server requires", got, want) + wantRuntime := &ateapipb.SandboxRuntime{SandboxClass: "gvisor", Version: hostCompat()} + if got := rel.runtimes[ctl.uid(name)]; !proto.Equal(got, wantRuntime) { + t.Errorf("default runtime %v, want %v: the pool's class on this host, which ate-api-server requires", got, wantRuntime) } if got, want := cl.statuses["bench"], (atev1alpha1.WorkerPoolStatus{Replicas: 3, ReadyReplicas: 3, Selector: "ate.dev/worker-pool=bench"}); got != want { t.Errorf("status = %+v, want %+v", got, want) @@ -878,7 +878,9 @@ func TestPoolSandboxClassChangeReplacesWorkers(t *testing.T) { oldBusy := ctl.uid(busy) ctl.assignments[oldBusy] = 1 - cl.editPool("bench", func(wp *atev1alpha1.WorkerPool) { wp.Spec.SandboxClass = "microvm" }) + cl.editPool("bench", func(wp *atev1alpha1.WorkerPool) { + wp.Spec.SandboxClasses = []atev1alpha1.WorkerPoolSandboxClass{{Name: atev1alpha1.SandboxClassMicroVM}} + }) if err := reconcile(t, c); err != nil { t.Fatalf("reconcile after the class change: %v", err) } @@ -918,7 +920,7 @@ func TestFailedDrainOfAReplacedWorkerIsNoReplica(t *testing.T) { old := ctl.worker(pod) ctl.drainErr = status.Error(codes.Unavailable, "connection reset") cl.editPool("bench", func(wp *atev1alpha1.WorkerPool) { - wp.Spec.SandboxClass = "microvm" + wp.Spec.SandboxClasses = []atev1alpha1.WorkerPoolSandboxClass{{Name: atev1alpha1.SandboxClassMicroVM}} wp.Labels = map[string]string{"workload": "v2"} }) if err := reconcile(t, c); err == nil { diff --git a/cmd/credential-provider/kubernetes-secrets/manifests_test.go b/cmd/credential-provider/kubernetes-secrets/manifests_test.go index 4778dd1bf9..f5153fabb9 100644 --- a/cmd/credential-provider/kubernetes-secrets/manifests_test.go +++ b/cmd/credential-provider/kubernetes-secrets/manifests_test.go @@ -330,7 +330,7 @@ func TestHelmAgentgatewayCredentialConfiguration(t *testing.T) { if listener.TLS.Mode != "dynamicCa" || listener.TLS.Cert != "/run/egress-mitm/tls.crt" || listener.TLS.Key != "/run/egress-mitm/tls.key" { t.Fatal("incorrect MITM configuration") } - if len(route.Backends) != 1 || route.Backends[0].Dynamic == nil || len(route.Backends[0].Dynamic) != 0 || route.Backends[0].Policies.BackendTLS == nil || len(route.Backends[0].Policies.BackendTLS) != 0 { + if len(route.Backends) != 1 || route.Backends[0].Dynamic == nil || len(route.Backends[0].Dynamic) != 0 || route.Backends[0].Policies.BackendTLS["root"] != "/etc/ssl/certs/ca-certificates.crt" { t.Fatal("HTTPS must use a dynamic destination with default public TLS trust") } } else if listener.Protocol != "HTTP" { diff --git a/cmd/dataplane/envoy/README.md b/cmd/dataplane/envoy/README.md index 6c7dfa646d..23727118fc 100644 --- a/cmd/dataplane/envoy/README.md +++ b/cmd/dataplane/envoy/README.md @@ -23,10 +23,10 @@ During a build from source, `ate-setup` builds the image from this directory via A pre-built install (`ate-setup deploy --image-repo REPO --image-tag TAG`) builds nothing and pins `REPO/envoy-dataplane:TAG` instead, so a release publishes it with the other images: ```bash -make build-release-images KO_DOCKER_REPO=REPO VERSION=TAG +VERSION=TAG go run ./cmd/ate-setup publish release-images --ko-docker-repo REPO ``` -`make build-envoy-dataplane` builds just this image. It targets `KO_DEFAULTPLATFORMS`, or `linux/amd64` when unset; set `DOCKERFILE_PLATFORMS` to override. Building another architecture compiles Rust under QEMU, which must be registered with binfmt (e.g. `docker run --privileged --rm tonistiigi/binfmt --install arm64`). +The image targets the same platforms as the ko images: `KO_DEFAULTPLATFORMS`, else `defaultPlatforms` in `.ko.yaml` (linux/amd64 and linux/arm64), else `linux/amd64`. Building another architecture compiles Rust under QEMU, which must be registered with binfmt (e.g. `docker run --privileged --rm tonistiigi/binfmt --install arm64`). To build the image locally without pushing it: diff --git a/cmd/kubectl-ate/README.md b/cmd/kubectl-ate/README.md index 55000782ec..6b3a0f2df5 100644 --- a/cmd/kubectl-ate/README.md +++ b/cmd/kubectl-ate/README.md @@ -298,6 +298,10 @@ kubectl ate logs actors my-actor -a -f # Show only one container's logs with -c/--container. kubectl ate logs actors my-actor -a -c my-container + +# Pick a class of lines with --source: containers (every container's output, +# no lifecycle events) or lifecycle (only the actor's lifecycle events). +kubectl ate logs actors my-actor -a --source=lifecycle ``` Logs are streamable only while the actor is bound to a worker (i.e., `ACTOR_STATE_RUNNING`). For history across worker migrations, route through a centralized log backend (Cloud Logging, Loki, etc.); see `docs/observability.md`. @@ -313,10 +317,19 @@ kubectl ate admin make-ca-pool \ --ca-id "1" # Generate a new JWT authority pool and push it to a Kubernetes Secret. The -# key is ES256 (--alg RS256 for relying parties that don't support ES256) and -# its ID defaults to the base64url SHA-256 of the public key's PKIX encoding -# (--key-id to override). +# key is ES256 (--alg RS256, a 4096-bit RSA key, for relying parties that don't +# support ES256) and its ID defaults to the base64url SHA-256 of the public +# key's PKIX encoding (--key-id to override). kubectl ate admin make-jwt-pool \ --name actor-id-jwt-pool \ --secret-namespace ate-system + +# Rotate the JWT signing key. The new key is published but signs nothing until +# activated; activate it once relying parties have refetched the key set, and +# remove the old key once every token it signed has expired. list-jwt-keys +# shows each key's ID and algorithm and marks the active one. +kubectl ate admin list-jwt-keys --name actor-id-jwt-pool --secret-namespace ate-system +kubectl ate admin add-jwt-key --name actor-id-jwt-pool --secret-namespace ate-system +kubectl ate admin activate-jwt-key --name actor-id-jwt-pool --secret-namespace ate-system --key-id +kubectl ate admin remove-jwt-key --name actor-id-jwt-pool --secret-namespace ate-system --key-id ``` diff --git a/cmd/kubectl-ate/internal/cmd/actor.go b/cmd/kubectl-ate/internal/cmd/actor.go index f063be1f60..ea74e334f6 100644 --- a/cmd/kubectl-ate/internal/cmd/actor.go +++ b/cmd/kubectl-ate/internal/cmd/actor.go @@ -55,6 +55,7 @@ var ( logsActorAtespaceFlag string logsActorFollowFlag bool logsActorContainerFlag string + logsActorSourceFlag string ) var getActorsCmd = &cobra.Command{ @@ -304,13 +305,14 @@ func (s *k8sPodLogsStreamer) StreamLogs(ctx context.Context, namespace, podName // LogsActorRunner executes the log printing or streaming. type LogsActorRunner struct { - apiClient AteAPIClient - streamer PodLogsStreamer - actorRef resources.ActorRef - stdout io.Writer - stderr io.Writer - follow bool - container string + apiClient AteAPIClient + streamer PodLogsStreamer + actorRef resources.ActorRef + stdout io.Writer + stderr io.Writer + follow bool + // filter selects the lines to display; see newLogLineFilter. + filter logLineFilter pollInterval time.Duration reconnectInterval time.Duration tickerInterval time.Duration @@ -318,6 +320,8 @@ type LogsActorRunner struct { // Run executes the logs command. func (r *LogsActorRunner) Run(ctx context.Context) error { + defer r.apiClient.Close() + if r.pollInterval <= 0 { r.pollInterval = 2 * time.Second } @@ -328,7 +332,6 @@ func (r *LogsActorRunner) Run(ctx context.Context) error { r.tickerInterval = 2 * time.Second } - defer r.apiClient.Close() if r.follow { return r.runFollow(ctx) } @@ -358,13 +361,12 @@ func (r *LogsActorRunner) runOneShot(ctx context.Context) error { } defer stream.Close() - filter := logLineFilter{target: r.actorRef, container: r.container} scanner := bufio.NewScanner(stream) buf := make([]byte, 0, 64*1024) scanner.Buffer(buf, 1024*1024) // Support up to 1MB lines for scanner.Scan() { line := scanner.Text() - filterAndDisplayLogLine(line, filter, r.stdout) + filterAndDisplayLogLine(line, r.filter, r.stdout) } if err := scanner.Err(); err != nil { return fmt.Errorf("error reading log stream: %w", err) @@ -374,6 +376,8 @@ func (r *LogsActorRunner) runOneShot(ctx context.Context) error { func (r *LogsActorRunner) runFollow(ctx context.Context) error { var lastWorkerPod string + // lastSeenTime is the kubelet's timestamp on the last line read from + // lastWorkerPod, the resume cursor for a reconnect to it. var lastSeenTime time.Time for { @@ -408,14 +412,21 @@ func (r *LogsActorRunner) runFollow(ctx context.Context) error { } } - // actor is resumed on anther worker - if podName != lastWorkerPod { + // The actor resumed on another worker. + switchedPod := podName != lastWorkerPod + if switchedPod { fmt.Fprintf(r.stderr, "Actor is currently running on pod %s/%s\n", namespace, podName) lastWorkerPod = podName } + // The kubelet's own timestamp on every line is the resume cursor. It + // is what SinceTime is compared against, and it does not depend on the + // line's content: an actor's record keeps a time the actor wrote, which + // can run ahead of the node's clock and would make a reconnect skip + // lines. opts := &corev1.PodLogOptions{ - Follow: true, + Follow: true, + Timestamps: true, } if !lastSeenTime.IsZero() { opts.SinceTime = &metav1.Time{Time: lastSeenTime} @@ -436,16 +447,22 @@ func (r *LogsActorRunner) runFollow(ctx context.Context) error { var wg sync.WaitGroup r.startMigrationMonitor(streamCtx, streamCancel, &wg, podName) - filter := logLineFilter{target: r.actorRef, container: r.container} scanner := bufio.NewScanner(stream) buf := make([]byte, 0, 64*1024) scanner.Buffer(buf, 1024*1024) // Support up to 1MB lines for scanner.Scan() { - line := scanner.Text() - logTime, _ := filterAndDisplayLogLine(line, filter, r.stdout) - if !logTime.IsZero() { - lastSeenTime = logTime + stamp, line, ok := splitKubeletTimestamp(scanner.Text()) + if ok { + // SinceTime has second precision, so a reconnect to the same pod + // re-reads the cursor's second; lines up to the cursor were + // already shown. Another pod's lines are all new, and its node's + // clock is not this one's, so they are not compared. + if !switchedPod && !stamp.After(lastSeenTime) { + continue + } + lastSeenTime = stamp } + filterAndDisplayLogLine(line, r.filter, r.stdout) } scanErr := scanner.Err() stream.Close() @@ -504,9 +521,33 @@ func (r *LogsActorRunner) startMigrationMonitor( }() } +// splitKubeletTimestamp splits the timestamp the kubelet prefixes to a log +// line when PodLogOptions.Timestamps is set (RFC 3339 with nanoseconds, then +// a space) from the line itself. ok is false for a line without one, which +// is then returned whole. +func splitKubeletTimestamp(line string) (stamp time.Time, rest string, ok bool) { + stampText, rest, found := strings.Cut(line, " ") + if !found { + return time.Time{}, line, false + } + stamp, err := time.Parse(time.RFC3339Nano, stampText) + if err != nil { + return time.Time{}, line, false + } + return stamp, rest, true +} + func runLogsActor(cmd *cobra.Command, args []string) error { ctx := cmd.Context() + // Before any connection: a bad flag combination is answered as such, not + // as an unreachable server. + actorRef := resources.ActorRef{Atespace: logsActorAtespaceFlag, Name: args[0]} + filter, err := newLogLineFilter(actorRef, logsActorSourceFlag, logsActorContainerFlag) + if err != nil { + return err + } + apiClient, err := ateclient.NewClient(ctx, kubeconfig, k8sContext, endpoint, tokenFile, traceEnabled) if err != nil { return fmt.Errorf("failed to connect to ate-api-server: %w", err) @@ -514,6 +555,7 @@ func runLogsActor(cmd *cobra.Command, args []string) error { config, err := ateclient.LoadKubeConfig(kubeconfig, k8sContext) if err != nil { + apiClient.Close() return fmt.Errorf("while loading kubeconfig: %w", err) } k8sClient, err := kubernetes.NewForConfig(config) @@ -525,11 +567,11 @@ func runLogsActor(cmd *cobra.Command, args []string) error { runner := &LogsActorRunner{ apiClient: apiClient, streamer: &k8sPodLogsStreamer{clientset: k8sClient}, - actorRef: resources.ActorRef{Atespace: logsActorAtespaceFlag, Name: args[0]}, + actorRef: actorRef, stdout: os.Stdout, stderr: os.Stderr, follow: logsActorFollowFlag, - container: logsActorContainerFlag, + filter: filter, pollInterval: 2 * time.Second, reconnectInterval: 1 * time.Second, tickerInterval: 2 * time.Second, @@ -538,13 +580,42 @@ func runLogsActor(cmd *cobra.Command, args []string) error { return runner.Run(ctx) } +// logSource names a class of lines in an actor's log stream, selected with +// --source. Container output carries the container's name; the lifecycle +// events ateom emits around a start, checkpoint, restore, or termination +// carry none, which is what tells the two apart. +type logSource string + +const ( + logSourceAll logSource = "all" + logSourceContainers logSource = "containers" + logSourceLifecycle logSource = "lifecycle" +) + // logLineFilter selects which of an actor's log lines are displayed: all of -// them by default, or only the named container's when container is set. +// them by default, one class of them when source is set, or only the named +// container's when container is set. type logLineFilter struct { target resources.ActorRef + source logSource container string } +// newLogLineFilter builds the filter for target from the --source and +// --container flag values, rejecting combinations that could never match. +func newLogLineFilter(target resources.ActorRef, source, container string) (logLineFilter, error) { + switch logSource(source) { + case logSourceAll, logSourceContainers: + case logSourceLifecycle: + if container != "" { + return logLineFilter{}, fmt.Errorf("--container cannot be combined with --source=lifecycle: lifecycle events are not emitted by a container") + } + default: + return logLineFilter{}, fmt.Errorf("invalid --source %q: must be one of %s, %s, %s", source, logSourceAll, logSourceContainers, logSourceLifecycle) + } + return logLineFilter{target: target, source: logSource(source), container: container}, nil +} + // matches reports whether a line emitted by emitter from containerName (empty // for lifecycle events) should be displayed. func (f logLineFilter) matches(emitter resources.ActorRef, containerName string) bool { @@ -553,24 +624,39 @@ func (f logLineFilter) matches(emitter resources.ActorRef, containerName string) if emitter != f.target || f.target.Atespace == "" || f.target.Name == "" { return false } + switch f.source { + case logSourceLifecycle: + // f.container is empty here: newLogLineFilter rejects it with lifecycle. + return containerName == "" + case logSourceContainers: + if containerName == "" { + return false + } + } return f.container == "" || containerName == f.container } -func filterAndDisplayLogLine(line string, filter logLineFilter, w io.Writer) (time.Time, bool) { +// containerNameKeyQuoted is the container-name label key as it appears in a +// JSON line. Every container line carries it (ateattr.ActorLogLabels) and +// no lifecycle line does; an actor cannot forge it, since actorlog strips +// the reserved keys from an actor's own labels, and the text inside a value +// only ever makes a container line skip early. +var containerNameKeyQuoted = `"` + string(ateattr.ActorContainerNameKey) + `"` + +// filterAndDisplayLogLine writes line to w if filter selects it, and +// reports whether it did. +func filterAndDisplayLogLine(line string, filter logLineFilter, w io.Writer) bool { + // Under lifecycle, nearly every line is a container's; a busy actor + // writes many, so skip them before decoding. + if filter.source == logSourceLifecycle && strings.Contains(line, containerNameKeyQuoted) { + return false + } + var m map[string]any dec := json.NewDecoder(strings.NewReader(line)) dec.UseNumber() if err := dec.Decode(&m); err != nil { - return time.Time{}, false - } - - var logTime time.Time - if tVal, ok := m["time"].(string); ok { - if t, err := time.Parse(time.RFC3339Nano, tVal); err == nil { - logTime = t - } else if t, err := time.Parse(time.RFC3339, tVal); err == nil { - logTime = t - } + return false } var emitter resources.ActorRef @@ -589,7 +675,7 @@ func filterAndDisplayLogLine(line string, filter logLineFilter, w io.Writer) (ti } if !filter.matches(emitter, emitterContainer) { - return time.Time{}, false + return false } // Remove substrate's labels from CLI output. Stripping the whole reserved @@ -619,7 +705,7 @@ func filterAndDisplayLogLine(line string, filter logLineFilter, w io.Writer) (ti enc := json.NewEncoder(&buf) enc.SetEscapeHTML(false) if err := enc.Encode(m); err != nil { - return time.Time{}, false + return false } encodedStr := strings.TrimSpace(buf.String()) @@ -636,7 +722,7 @@ func filterAndDisplayLogLine(line string, filter logLineFilter, w io.Writer) (ti fmt.Fprintln(w, encodedStr) } - return logTime, true + return true } func init() { @@ -672,5 +758,6 @@ func init() { logsActorsCmd.Flags().StringVarP(&logsActorAtespaceFlag, "atespace", "a", "", "Atespace the actor lives in") _ = logsActorsCmd.MarkFlagRequired("atespace") logsActorsCmd.Flags().StringVarP(&logsActorContainerFlag, "container", "c", "", "Show only logs from this container.") + logsActorsCmd.Flags().StringVar(&logsActorSourceFlag, "source", string(logSourceAll), "Which lines to show: all, containers (every container's output, no lifecycle events), or lifecycle (only the actor's lifecycle events).") logsCmd.AddCommand(logsActorsCmd) } diff --git a/cmd/kubectl-ate/internal/cmd/actor_test.go b/cmd/kubectl-ate/internal/cmd/actor_test.go index df93a5673d..2c3f36cf4a 100644 --- a/cmd/kubectl-ate/internal/cmd/actor_test.go +++ b/cmd/kubectl-ate/internal/cmd/actor_test.go @@ -32,6 +32,7 @@ import ( "google.golang.org/grpc/status" "google.golang.org/protobuf/testing/protocmp" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func TestActorCommandArgs(t *testing.T) { @@ -112,9 +113,9 @@ func TestFilterAndDisplayLogLine(t *testing.T) { name string line string target resources.ActorRef + source string container string wantMatched bool - wantTime string wantOutput string }{ { @@ -122,7 +123,6 @@ func TestFilterAndDisplayLogLine(t *testing.T) { line: `{"time":"2026-05-16T01:03:38.602878302Z","level":"info","msg":"Count","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-1"}}`, target: resources.ActorRef{Atespace: "space-1", Name: "act-1"}, wantMatched: true, - wantTime: "2026-05-16T01:03:38.602878302Z", wantOutput: `{"time":"2026-05-16T01:03:38.602878302Z","level":"info","msg":"Count"}`, }, { @@ -130,7 +130,6 @@ func TestFilterAndDisplayLogLine(t *testing.T) { line: `{"time":"2026-05-16T01:03:38Z","message":"Hello","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-1"}}`, target: resources.ActorRef{Atespace: "space-1", Name: "act-1"}, wantMatched: true, - wantTime: "2026-05-16T01:03:38Z", wantOutput: `{"time":"2026-05-16T01:03:38Z","message":"Hello"}`, }, { @@ -138,7 +137,6 @@ func TestFilterAndDisplayLogLine(t *testing.T) { line: `{"level":"error","msg":"Failed","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-1"}}`, target: resources.ActorRef{Atespace: "space-1", Name: "act-1"}, wantMatched: true, - wantTime: "", wantOutput: `{"level":"error","msg":"Failed"}`, }, { @@ -146,7 +144,6 @@ func TestFilterAndDisplayLogLine(t *testing.T) { line: `{"time":"2026-05-16T01:03:38.602878302Z","level":"info","msg":"Count","labels":{"ate.atespace":"space-1","ate.actor.name":"act-1"}}`, target: resources.ActorRef{Atespace: "space-1", Name: "act-1"}, wantMatched: true, - wantTime: "2026-05-16T01:03:38.602878302Z", wantOutput: `{"time":"2026-05-16T01:03:38.602878302Z","level":"info","msg":"Count"}`, }, { @@ -154,7 +151,6 @@ func TestFilterAndDisplayLogLine(t *testing.T) { line: `{"time":"2026-05-16T01:03:38Z","message":"Hello world","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-2"}}`, target: resources.ActorRef{Atespace: "space-1", Name: "act-1"}, wantMatched: false, - wantTime: "", // zero: another actor's line must not advance follow's resume cursor wantOutput: "", }, { @@ -162,7 +158,6 @@ func TestFilterAndDisplayLogLine(t *testing.T) { line: `{"time":"2026-05-16T01:03:38Z","message":"Hello world","logging.googleapis.com/labels":{"ate.atespace":"space-2","ate.actor.name":"act-1"}}`, target: resources.ActorRef{Atespace: "space-1", Name: "act-1"}, wantMatched: false, - wantTime: "", wantOutput: "", }, { @@ -170,7 +165,6 @@ func TestFilterAndDisplayLogLine(t *testing.T) { line: `{"time":"2026-05-16T01:03:38Z","message":"Hello world","logging.googleapis.com/labels":{"ate.actor.name":"act-1"}}`, target: resources.ActorRef{Atespace: "space-1", Name: "act-1"}, wantMatched: false, - wantTime: "", wantOutput: "", }, { @@ -178,7 +172,6 @@ func TestFilterAndDisplayLogLine(t *testing.T) { line: `{"time":"2026-05-16T01:03:38Z","message":"Hello world","logging.googleapis.com/labels":{"ate.atespace":"","ate.actor.name":"act-1"}}`, target: resources.ActorRef{Atespace: "", Name: "act-1"}, wantMatched: false, - wantTime: "", wantOutput: "", }, { @@ -186,7 +179,6 @@ func TestFilterAndDisplayLogLine(t *testing.T) { line: `{"time":"2026-05-16T01:03:38Z","message":"Hello world","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":""}}`, target: resources.ActorRef{Atespace: "space-1", Name: ""}, wantMatched: false, - wantTime: "", wantOutput: "", }, { @@ -194,7 +186,6 @@ func TestFilterAndDisplayLogLine(t *testing.T) { line: "not a json line", target: resources.ActorRef{Atespace: "space-1", Name: "act-1"}, wantMatched: false, - wantTime: "", wantOutput: "", }, { @@ -202,7 +193,6 @@ func TestFilterAndDisplayLogLine(t *testing.T) { line: `{"time":"2026-05-16T01:03:38Z","level":"info","msg":"Hello","traceID":"abc-123","err":"timeout","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-1"}}`, target: resources.ActorRef{Atespace: "space-1", Name: "act-1"}, wantMatched: true, - wantTime: "2026-05-16T01:03:38Z", wantOutput: `{"time":"2026-05-16T01:03:38Z","err":"timeout","level":"info","msg":"Hello","traceID":"abc-123"}`, }, { @@ -210,7 +200,6 @@ func TestFilterAndDisplayLogLine(t *testing.T) { line: `{"time":"2026-05-16T01:03:38Z","severity":"error","message":"Disk full","custom_tag":"alert","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-1"}}`, target: resources.ActorRef{Atespace: "space-1", Name: "act-1"}, wantMatched: true, - wantTime: "2026-05-16T01:03:38Z", wantOutput: `{"time":"2026-05-16T01:03:38Z","custom_tag":"alert","message":"Disk full","severity":"error"}`, }, { @@ -218,7 +207,6 @@ func TestFilterAndDisplayLogLine(t *testing.T) { line: `{"message":"login failed","code":401,"logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-1"}}`, target: resources.ActorRef{Atespace: "space-1", Name: "act-1"}, wantMatched: true, - wantTime: "", wantOutput: `{"code":401,"message":"login failed"}`, }, { @@ -226,7 +214,6 @@ func TestFilterAndDisplayLogLine(t *testing.T) { line: `{"time":"2026-05-16T01:03:38Z","level":"info","msg":"Hello","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-1","app":"my-app"}}`, target: resources.ActorRef{Atespace: "space-1", Name: "act-1"}, wantMatched: true, - wantTime: "2026-05-16T01:03:38Z", wantOutput: `{"time":"2026-05-16T01:03:38Z","level":"info","logging.googleapis.com/labels":{"app":"my-app"},"msg":"Hello"}`, }, { @@ -237,7 +224,6 @@ func TestFilterAndDisplayLogLine(t *testing.T) { line: `{"time":"2026-05-16T01:03:38Z","msg":"Hello","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-1","ate.tenant":"forged","app":"my-app"}}`, target: resources.ActorRef{Atespace: "space-1", Name: "act-1"}, wantMatched: true, - wantTime: "2026-05-16T01:03:38Z", wantOutput: `{"time":"2026-05-16T01:03:38Z","logging.googleapis.com/labels":{"app":"my-app"},"msg":"Hello"}`, }, { @@ -245,7 +231,6 @@ func TestFilterAndDisplayLogLine(t *testing.T) { line: `{"time":"2026-05-16T01:03:38Z","level":"info","msg":"hi","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-1","ate.actor.container.name":"counter"}}`, target: resources.ActorRef{Atespace: "space-1", Name: "act-1"}, wantMatched: true, - wantTime: "2026-05-16T01:03:38Z", wantOutput: `{"time":"2026-05-16T01:03:38Z","level":"info","msg":"hi"}`, }, { @@ -254,7 +239,6 @@ func TestFilterAndDisplayLogLine(t *testing.T) { target: resources.ActorRef{Atespace: "space-1", Name: "act-1"}, container: "counter", wantMatched: true, - wantTime: "2026-05-16T01:03:38Z", wantOutput: `{"time":"2026-05-16T01:03:38Z","level":"info","msg":"hi"}`, }, { @@ -263,7 +247,6 @@ func TestFilterAndDisplayLogLine(t *testing.T) { target: resources.ActorRef{Atespace: "space-1", Name: "act-1"}, container: "counter", wantMatched: false, - wantTime: "", // filtered out: only displayed lines may advance follow's resume cursor wantOutput: "", }, { @@ -272,7 +255,63 @@ func TestFilterAndDisplayLogLine(t *testing.T) { target: resources.ActorRef{Atespace: "space-1", Name: "act-1"}, container: "counter", wantMatched: false, - wantTime: "", + wantOutput: "", + }, + { + name: "source=all shows a lifecycle event", + line: `{"time":"2026-05-16T01:03:38Z","message":"Actor started","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-1"}}`, + target: resources.ActorRef{Atespace: "space-1", Name: "act-1"}, + source: "all", + wantMatched: true, + wantOutput: `{"time":"2026-05-16T01:03:38Z","message":"Actor started"}`, + }, + { + name: "source=lifecycle shows a lifecycle event", + line: `{"time":"2026-05-16T01:03:38Z","message":"Actor checkpointing","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-1"}}`, + target: resources.ActorRef{Atespace: "space-1", Name: "act-1"}, + source: "lifecycle", + wantMatched: true, + wantOutput: `{"time":"2026-05-16T01:03:38Z","message":"Actor checkpointing"}`, + }, + { + name: "source=lifecycle excludes a container line", + line: `{"time":"2026-05-16T01:03:38Z","level":"info","msg":"hi","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-1","ate.actor.container.name":"counter"}}`, + target: resources.ActorRef{Atespace: "space-1", Name: "act-1"}, + source: "lifecycle", + wantMatched: false, + wantOutput: "", + }, + { + name: "source=lifecycle excludes another actor's lifecycle event", + line: `{"time":"2026-05-16T01:03:38Z","message":"Actor started","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-2"}}`, + target: resources.ActorRef{Atespace: "space-1", Name: "act-1"}, + source: "lifecycle", + wantMatched: false, + wantOutput: "", + }, + { + name: "source=containers shows any container's line", + line: `{"time":"2026-05-16T01:03:38Z","level":"info","msg":"hi","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-1","ate.actor.container.name":"sidecar"}}`, + target: resources.ActorRef{Atespace: "space-1", Name: "act-1"}, + source: "containers", + wantMatched: true, + wantOutput: `{"time":"2026-05-16T01:03:38Z","level":"info","msg":"hi"}`, + }, + { + name: "source=containers excludes a lifecycle event", + line: `{"time":"2026-05-16T01:03:38Z","message":"Actor restored","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-1"}}`, + target: resources.ActorRef{Atespace: "space-1", Name: "act-1"}, + source: "containers", + wantMatched: false, + wantOutput: "", + }, + { + name: "source=containers with container filter keeps only the named container", + line: `{"time":"2026-05-16T01:03:38Z","level":"info","msg":"hi","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-1","ate.actor.container.name":"sidecar"}}`, + target: resources.ActorRef{Atespace: "space-1", Name: "act-1"}, + source: "containers", + container: "counter", + wantMatched: false, wantOutput: "", }, } @@ -280,26 +319,12 @@ func TestFilterAndDisplayLogLine(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { var buf bytes.Buffer - logTime, matched := filterAndDisplayLogLine(tc.line, logLineFilter{target: tc.target, container: tc.container}, &buf) + matched := filterAndDisplayLogLine(tc.line, logLineFilter{target: tc.target, source: logSource(tc.source), container: tc.container}, &buf) if matched != tc.wantMatched { t.Errorf("got matched = %v, want %v", matched, tc.wantMatched) } - if tc.wantTime != "" { - parsedTime, err := time.Parse(time.RFC3339Nano, tc.wantTime) - if err != nil { - parsedTime, _ = time.Parse(time.RFC3339, tc.wantTime) - } - if !logTime.Equal(parsedTime) { - t.Errorf("got logTime = %v, want %v", logTime, parsedTime) - } - } else { - if !logTime.IsZero() { - t.Errorf("got non-zero logTime = %v, want zero", logTime) - } - } - gotOutput := strings.TrimSpace(buf.String()) if gotOutput != tc.wantOutput { t.Errorf("got output %q, want %q", gotOutput, tc.wantOutput) @@ -308,6 +333,46 @@ func TestFilterAndDisplayLogLine(t *testing.T) { } } +func TestNewLogLineFilter(t *testing.T) { + target := resources.ActorRef{Atespace: "space-1", Name: "act-1"} + + tests := []struct { + name string + source string + container string + wantErr string + }{ + {name: "explicit empty source", source: "", wantErr: `invalid --source ""`}, + {name: "all", source: "all"}, + {name: "containers", source: "containers"}, + {name: "lifecycle", source: "lifecycle"}, + {name: "all with container", source: "all", container: "counter"}, + {name: "containers with container", source: "containers", container: "counter"}, + {name: "lifecycle with container", source: "lifecycle", container: "counter", wantErr: "--container cannot be combined with --source=lifecycle"}, + {name: "unknown source", source: "supervisor", wantErr: `invalid --source "supervisor"`}, + {name: "source is case-sensitive", source: "Lifecycle", wantErr: `invalid --source "Lifecycle"`}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got, err := newLogLineFilter(target, tc.source, tc.container) + if tc.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("got err %v, want it to contain %q", err, tc.wantErr) + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + want := logLineFilter{target: target, source: logSource(tc.source), container: tc.container} + if got != want { + t.Errorf("got %+v, want %+v", got, want) + } + }) + } +} + type mockAteAPIClient struct { GetActorFunc func(ctx context.Context, in *ateapipb.GetActorRequest, opts ...grpc.CallOption) (*ateapipb.Actor, error) CloseCalls int @@ -375,6 +440,7 @@ func TestLogsActorRunner_Run_OneShotSuccess(t *testing.T) { runner := &LogsActorRunner{ apiClient: mockAPI, actorRef: resources.ActorRef{Atespace: "space-1", Name: actorName}, + filter: logLineFilter{target: resources.ActorRef{Atespace: "space-1", Name: actorName}}, streamer: mockStreamer, stdout: &stdout, stderr: &stderr, @@ -397,30 +463,54 @@ func TestLogsActorRunner_Run_OneShotSuccess(t *testing.T) { } } -// TestLogsActorRunner_Run_OneShot_ContainerFilter exercises the --container -// selection against one stream that interleaves two containers and a -// lifecycle event. -func TestLogsActorRunner_Run_OneShot_ContainerFilter(t *testing.T) { +// TestLogsActorRunner_Run_OneShot_SourceFilter exercises the --source and +// --container selection against one stream that interleaves two containers +// and lifecycle events. +func TestLogsActorRunner_Run_OneShot_SourceFilter(t *testing.T) { actorName := "act-123" + startedLine := `{"time":"2026-05-16T01:03:37Z","message":"Actor started","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-123"}}` counterLine := `{"time":"2026-05-16T01:03:38Z","level":"info","msg":"from counter","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-123","ate.actor.container.name":"counter"}}` sidecarLine := `{"time":"2026-05-16T01:03:39Z","level":"info","msg":"from sidecar","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-123","ate.actor.container.name":"sidecar"}}` - lifecycleLine := `{"time":"2026-05-16T01:03:40Z","message":"Actor started","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-123"}}` + checkpointingLine := `{"time":"2026-05-16T01:03:40Z","message":"Actor checkpointing","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-123"}}` + stream := strings.Join([]string{startedLine, counterLine, sidecarLine, checkpointingLine}, "\n") + "\n" tests := []struct { name string + source string container string wantOutput []string }{ { name: "no filter shows everything", - wantOutput: []string{"from counter", "from sidecar", "Actor started"}, + wantOutput: []string{"Actor started", "from counter", "from sidecar", "Actor checkpointing"}, + }, + { + name: "source=all shows everything", + source: "all", + wantOutput: []string{"Actor started", "from counter", "from sidecar", "Actor checkpointing"}, }, { name: "container filter", container: "counter", wantOutput: []string{"from counter"}, }, + { + name: "source=containers drops lifecycle events", + source: "containers", + wantOutput: []string{"from counter", "from sidecar"}, + }, + { + name: "source=containers with container filter", + source: "containers", + container: "sidecar", + wantOutput: []string{"from sidecar"}, + }, + { + name: "source=lifecycle keeps only lifecycle events", + source: "lifecycle", + wantOutput: []string{"Actor started", "Actor checkpointing"}, + }, } for _, tc := range tests { @@ -439,9 +529,11 @@ func TestLogsActorRunner_Run_OneShot_ContainerFilter(t *testing.T) { }, nil }, } + streamCalls := 0 mockStreamer := &mockPodLogsStreamer{ StreamLogsFunc: func(ctx context.Context, ns, name string, opts *corev1.PodLogOptions) (io.ReadCloser, error) { - return io.NopCloser(strings.NewReader(counterLine + "\n" + sidecarLine + "\n" + lifecycleLine + "\n")), nil + streamCalls++ + return io.NopCloser(strings.NewReader(stream)), nil }, } @@ -452,12 +544,19 @@ func TestLogsActorRunner_Run_OneShot_ContainerFilter(t *testing.T) { streamer: mockStreamer, stdout: &stdout, stderr: &stderr, - container: tc.container, + filter: logLineFilter{target: resources.ActorRef{Atespace: "space-1", Name: actorName}, source: logSource(tc.source), container: tc.container}, } - if err := runner.Run(context.Background()); err != nil { + err := runner.Run(context.Background()) + if mockAPI.CloseCalls != 1 { + t.Errorf("api client Close called %d times, want 1", mockAPI.CloseCalls) + } + if err != nil { t.Fatalf("unexpected error: %v", err) } + if streamCalls != 1 { + t.Errorf("streamed %d times, want 1", streamCalls) + } var gotLines []string if got := strings.TrimSpace(stdout.String()); got != "" { @@ -497,6 +596,7 @@ func TestLogsActorRunner_Run_OneShot_ActorNotRunning(t *testing.T) { runner := &LogsActorRunner{ apiClient: mockAPI, actorRef: resources.ActorRef{Atespace: "space-1", Name: actorName}, + filter: logLineFilter{target: resources.ActorRef{Atespace: "space-1", Name: actorName}}, streamer: mockStreamer, stdout: &stdout, stderr: &stderr, @@ -554,7 +654,8 @@ func TestLogsActorRunner_Run_Follow_SuspendedToRunning(t *testing.T) { }, } - logLine := `{"time":"2026-05-16T01:03:38Z","level":"info","msg":"Follow hello","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-123"}}` + // As the kubelet streams it with Timestamps set. + logLine := `2026-05-16T01:03:38.500000000Z {"time":"2026-05-16T01:03:38Z","level":"info","msg":"Follow hello","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-123"}}` ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -564,8 +665,8 @@ func TestLogsActorRunner_Run_Follow_SuspendedToRunning(t *testing.T) { if ns != namespace || name != podName { return nil, fmt.Errorf("unexpected pod %s/%s", ns, name) } - if !opts.Follow { - return nil, fmt.Errorf("expected follow to be true in follow mode") + if !opts.Follow || !opts.Timestamps { + return nil, fmt.Errorf("expected follow and timestamps to be set in follow mode") } // Cancel main context soon to break the outer infinite loop @@ -586,6 +687,7 @@ func TestLogsActorRunner_Run_Follow_SuspendedToRunning(t *testing.T) { runner := &LogsActorRunner{ apiClient: mockAPI, actorRef: resources.ActorRef{Atespace: "space-1", Name: actorName}, + filter: logLineFilter{target: resources.ActorRef{Atespace: "space-1", Name: actorName}}, streamer: mockStreamer, stdout: &stdout, stderr: &stderr, @@ -636,6 +738,7 @@ func TestLogsActorRunner_Run_Follow_NotFoundActor(t *testing.T) { runner := &LogsActorRunner{ apiClient: mockAPI, actorRef: resources.ActorRef{Atespace: "space-1", Name: actorName}, + filter: logLineFilter{target: resources.ActorRef{Atespace: "space-1", Name: actorName}}, streamer: mockStreamer, stdout: &stdout, stderr: &stderr, @@ -751,6 +854,7 @@ func TestLogsActorRunner_Run_Follow_ActorMigration(t *testing.T) { runner := &LogsActorRunner{ apiClient: mockAPI, actorRef: resources.ActorRef{Atespace: "space-1", Name: actorName}, + filter: logLineFilter{target: resources.ActorRef{Atespace: "space-1", Name: actorName}}, streamer: mockStreamer, stdout: &stdout, stderr: &stderr, @@ -872,6 +976,7 @@ func TestLogsActorRunner_Run_Follow_ActorSuspendedMidStream(t *testing.T) { runner := &LogsActorRunner{ apiClient: mockAPI, actorRef: resources.ActorRef{Atespace: "space-1", Name: actorName}, + filter: logLineFilter{target: resources.ActorRef{Atespace: "space-1", Name: actorName}}, streamer: mockStreamer, stdout: &stdout, stderr: &stderr, @@ -895,10 +1000,14 @@ func TestLogsActorRunner_Run_Follow_ActorSuspendedMidStream(t *testing.T) { } } -// A worker pod's stream carries lines that are not displayed (other actors, -// filtered-out containers). Those must not advance the follow resume cursor -// (SinceTime), or a reconnect could skip the lines the user asked for. -func TestLogsActorRunner_Run_Follow_UndisplayedLineDoesNotAdvanceCursor(t *testing.T) { +// The follow resume cursor is the kubelet's timestamp on the last line read, +// displayed or not. Not the last displayed line: under a sparse filter such +// as --source=lifecycle a reconnect would replay everything since the last +// match. Not the line's own time either: an actor's record keeps a time the +// actor wrote, which can run ahead of the node's clock and would make a +// reconnect skip lines. On reconnect, lines up to the cursor are dropped, +// since SinceTime only has second precision. +func TestLogsActorRunner_Run_Follow_CursorIsKubeletTimestampOfLastLineRead(t *testing.T) { actorName := "act-123" mockAPI := &mockAteAPIClient{ @@ -916,45 +1025,54 @@ func TestLogsActorRunner_Run_Follow_UndisplayedLineDoesNotAdvanceCursor(t *testi }, } - // A different actor's line and a filtered-out sidecar line of our own actor: - // both carry parseable timestamps but neither is displayed. - foreignLine := `{"time":"2026-05-16T01:03:38Z","message":"not mine","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"other"}}` - sidecarLine := `{"time":"2026-05-16T01:03:39Z","level":"info","msg":"from sidecar","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-123","ate.actor.container.name":"sidecar"}}` + // Lines as the kubelet streams them with Timestamps set. Under + // --source=lifecycle only the first is displayed. The container line's + // own time is far ahead of the node's clock. + lifecycleLine := `2026-05-16T01:03:38.100000000Z {"time":"2026-05-16T01:03:38Z","message":"Actor started","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-123"}}` + foreignLine := `2026-05-16T01:03:39.200000000Z {"time":"2026-05-16T01:03:39Z","message":"not mine","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"other"}}` + counterLine := `2026-05-16T01:03:40.300000000Z {"time":"2099-01-01T00:00:00Z","level":"info","msg":"from counter","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-123","ate.actor.container.name":"counter"}}` + // After the reconnect the kubelet re-sends the cursor's second: the + // counter line again, then a new lifecycle event in the same second. + laterLine := `2026-05-16T01:03:40.900000000Z {"time":"2026-05-16T01:03:40Z","message":"Actor checkpointing","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-123"}}` + wantSince := time.Date(2026, 5, 16, 1, 3, 40, 300000000, time.UTC) ctx, cancel := context.WithCancel(context.Background()) defer cancel() var mu sync.Mutex var streamCalls int - var reconnectHadSinceTime bool - secondCall := make(chan struct{}) + var reconnectSince *metav1.Time + var reconnectTimestamps bool + thirdCall := make(chan struct{}) mockStreamer := &mockPodLogsStreamer{ StreamLogsFunc: func(streamCtx context.Context, ns, name string, opts *corev1.PodLogOptions) (io.ReadCloser, error) { mu.Lock() defer mu.Unlock() streamCalls++ - - if streamCalls == 1 { - // First connection: only undisplayed lines, then EOF to force a reconnect. - return io.NopCloser(strings.NewReader(foreignLine + "\n" + sidecarLine + "\n")), nil - } - if streamCalls == 2 { - reconnectHadSinceTime = opts.SinceTime != nil - close(secondCall) + switch streamCalls { + case 1: + return io.NopCloser(strings.NewReader(lifecycleLine + "\n" + foreignLine + "\n" + counterLine + "\n")), nil + case 2: + reconnectSince = opts.SinceTime + reconnectTimestamps = opts.Timestamps + return io.NopCloser(strings.NewReader(counterLine + "\n" + laterLine + "\n")), nil + case 3: + close(thirdCall) } return io.NopCloser(strings.NewReader("")), nil }, } + var stdout bytes.Buffer runner := &LogsActorRunner{ apiClient: mockAPI, actorRef: resources.ActorRef{Atespace: "space-1", Name: actorName}, streamer: mockStreamer, - stdout: &bytes.Buffer{}, + stdout: &stdout, stderr: &bytes.Buffer{}, follow: true, - container: "counter", + filter: logLineFilter{target: resources.ActorRef{Atespace: "space-1", Name: actorName}, source: logSourceLifecycle}, pollInterval: 1 * time.Millisecond, reconnectInterval: 1 * time.Millisecond, tickerInterval: time.Hour, // keep the migration monitor out of the way @@ -967,11 +1085,11 @@ func TestLogsActorRunner_Run_Follow_UndisplayedLineDoesNotAdvanceCursor(t *testi }() select { - case <-secondCall: + case <-thirdCall: case <-time.After(2 * time.Second): cancel() <-done - t.Fatal("reconnect (second StreamLogs call) never happened") + t.Fatal("the third StreamLogs call never happened") } cancel() @@ -979,7 +1097,62 @@ func TestLogsActorRunner_Run_Follow_UndisplayedLineDoesNotAdvanceCursor(t *testi mu.Lock() defer mu.Unlock() - if reconnectHadSinceTime { - t.Error("reconnect carried a SinceTime cursor; undisplayed lines must not advance it") + if !reconnectTimestamps { + t.Error("follow did not ask the kubelet for timestamps") + } + if reconnectSince == nil { + t.Fatal("reconnect carried no SinceTime cursor") + } + if !reconnectSince.Time.Equal(wantSince) { + t.Errorf("reconnect SinceTime = %v, want %v: the kubelet's timestamp on the last line read, not the last line displayed and not the line's own time", reconnectSince.Time, wantSince) + } + want := "{\"time\":\"2026-05-16T01:03:38Z\",\"message\":\"Actor started\"}\n" + + "{\"time\":\"2026-05-16T01:03:40Z\",\"message\":\"Actor checkpointing\"}\n" + if got := stdout.String(); got != want { + t.Errorf("output =\n%s\nwant\n%s", got, want) + } +} + +func TestSplitKubeletTimestamp(t *testing.T) { + tests := []struct { + name string + line string + wantOK bool + wantTime time.Time + wantRest string + }{ + { + name: "kubelet timestamp prefix", + line: `2026-05-16T01:03:40.300000000Z {"msg":"hi"}`, + wantOK: true, + wantTime: time.Date(2026, 5, 16, 1, 3, 40, 300000000, time.UTC), + wantRest: `{"msg":"hi"}`, + }, + { + name: "no prefix", + line: `{"msg":"hi there"}`, + wantOK: false, + wantRest: `{"msg":"hi there"}`, + }, + { + name: "no space at all", + line: `plain`, + wantOK: false, + wantRest: `plain`, + }, + { + name: "a space but not a timestamp", + line: `not-a-time {"msg":"hi"}`, + wantOK: false, + wantRest: `not-a-time {"msg":"hi"}`, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + stamp, rest, ok := splitKubeletTimestamp(tc.line) + if ok != tc.wantOK || rest != tc.wantRest || !stamp.Equal(tc.wantTime) { + t.Errorf("splitKubeletTimestamp(%q) = %v, %q, %v; want %v, %q, %v", tc.line, stamp, rest, ok, tc.wantTime, tc.wantRest, tc.wantOK) + } + }) } } diff --git a/cmd/kubectl-ate/internal/cmd/actortemplate_test.go b/cmd/kubectl-ate/internal/cmd/actortemplate_test.go index 32f5b0ea02..b4ffe73c56 100644 --- a/cmd/kubectl-ate/internal/cmd/actortemplate_test.go +++ b/cmd/kubectl-ate/internal/cmd/actortemplate_test.go @@ -57,7 +57,7 @@ resources: - name: memory quantity: 512Mi snapshotConfig: - onCommit: SNAPSHOT_CONTENT_SCOPE_FULL + preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://ate-snapshots/ate-demo-counter/ sandboxConfig: sandboxClass: SANDBOX_CLASS_GVISOR @@ -90,8 +90,8 @@ func TestActorTemplateFromManifest(t *testing.T) { {Name: "memory", Quantity: "512Mi"}, }}, SnapshotConfig: &ateapipb.SnapshotConfig{ - OnCommit: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, - StorageLocation: "gs://ate-snapshots/ate-demo-counter/", + PreferredFidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + StorageLocation: "gs://ate-snapshots/ate-demo-counter/", }, SandboxConfig: &ateapipb.SandboxConfig{ SandboxClass: ateapipb.SandboxClass_SANDBOX_CLASS_GVISOR, diff --git a/cmd/kubectl-ate/internal/cmd/admin.go b/cmd/kubectl-ate/internal/cmd/admin.go index 67d26831d8..ab27bcfbb6 100644 --- a/cmd/kubectl-ate/internal/cmd/admin.go +++ b/cmd/kubectl-ate/internal/cmd/admin.go @@ -15,7 +15,10 @@ package cmd import ( + "context" "fmt" + "io" + "text/tabwriter" "time" "github.com/agent-substrate/substrate/internal/ateclient" @@ -25,17 +28,19 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" + typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/client-go/util/retry" ) var ( - // Both pool commands write one secret, so they share the flags naming it. + // Each pool command writes one secret, so they share the flags naming it. poolSecretNamespaceFlag string poolSecretNameFlag string makeCaPoolIDFlag string makeCaPoolKeyTypeFlag string makeCaPoolValidityFlag time.Duration - makeJwtPoolAlgFlag string - makeJwtPoolKeyIDFlag string + jwtAlgFlag string + jwtKeyIDFlag string ) var adminCmd = &cobra.Command{ @@ -49,14 +54,9 @@ var makeCaPoolCmd = &cobra.Command{ RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() - kconfig, err := ateclient.LoadKubeConfig(kubeconfig, k8sContext) + kc, err := newKubeClient() if err != nil { - return fmt.Errorf("while reading kubeconfig: %w", err) - } - - kc, err := kubernetes.NewForConfig(kconfig) - if err != nil { - return fmt.Errorf("while creating Kubernetes client: %w", err) + return err } var keyType localca.KeyType @@ -125,31 +125,187 @@ var makeJwtPoolCmd = &cobra.Command{ RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() - kconfig, err := ateclient.LoadKubeConfig(kubeconfig, k8sContext) + kc, err := newKubeClient() if err != nil { - return fmt.Errorf("while reading kubeconfig: %w", err) + return err } - kc, err := kubernetes.NewForConfig(kconfig) + secret, keyID, err := newJWTPoolSecret(poolSecretNamespaceFlag, poolSecretNameFlag, jwtAlgFlag, jwtKeyIDFlag) if err != nil { - return fmt.Errorf("while creating Kubernetes client: %w", err) + return err } - secret, keyID, err := newJWTPoolSecret(poolSecretNamespaceFlag, poolSecretNameFlag, makeJwtPoolAlgFlag, makeJwtPoolKeyIDFlag) + _, err = kc.CoreV1().Secrets(poolSecretNamespaceFlag).Create(ctx, secret, metav1.CreateOptions{}) + if err != nil { + return fmt.Errorf("while uploading pool state to secret: %w", err) + } + + fmt.Printf("Successfully created JWT authority pool secret %s/%s with %s key %s\n", poolSecretNamespaceFlag, poolSecretNameFlag, jwtAlgFlag, keyID) + return nil + }, +} + +var listJwtKeysCmd = &cobra.Command{ + Use: "list-jwt-keys", + Short: "List the keys in a JWT authority pool secret", + RunE: func(cmd *cobra.Command, args []string) error { + kc, err := newKubeClient() if err != nil { return err } - _, err = kc.CoreV1().Secrets(poolSecretNamespaceFlag).Create(ctx, secret, metav1.CreateOptions{}) + _, pool, err := getJWTPool(cmd.Context(), kc.CoreV1().Secrets(poolSecretNamespaceFlag), poolSecretNameFlag) if err != nil { - return fmt.Errorf("while uploading pool state to secret: %w", err) + return err + } + return printJWTKeys(cmd.OutOrStdout(), pool) + }, +} + +var addJwtKeyCmd = &cobra.Command{ + Use: "add-jwt-key", + Short: "Add an inactive signing key to a JWT authority pool secret", + Long: `Add an inactive signing key to a JWT authority pool secret. + +The key is published to relying parties but signs nothing until activate-jwt-key +makes it the signing key. Activate it only once relying parties have refetched +the key set.`, + RunE: func(cmd *cobra.Command, args []string) error { + kc, err := newKubeClient() + if err != nil { + return err + } + + authority, err := localjwtauthority.GenerateAuthority(jwtAlgFlag, jwtKeyIDFlag) + if err != nil { + return fmt.Errorf("while generating JWT authority: %w", err) + } + if err := updateJWTPool(cmd.Context(), kc.CoreV1().Secrets(poolSecretNamespaceFlag), poolSecretNameFlag, func(pool *localjwtauthority.ConcretePool) error { + return pool.AddAuthority(authority) + }); err != nil { + return err + } + + fmt.Printf("Added inactive %s key %s to JWT authority pool secret %s/%s\n", jwtAlgFlag, authority.ID, poolSecretNamespaceFlag, poolSecretNameFlag) + return nil + }, +} + +var activateJwtKeyCmd = &cobra.Command{ + Use: "activate-jwt-key", + Short: "Make a key in a JWT authority pool secret the one that signs", + RunE: func(cmd *cobra.Command, args []string) error { + kc, err := newKubeClient() + if err != nil { + return err + } + + if err := updateJWTPool(cmd.Context(), kc.CoreV1().Secrets(poolSecretNamespaceFlag), poolSecretNameFlag, func(pool *localjwtauthority.ConcretePool) error { + return pool.Activate(jwtKeyIDFlag) + }); err != nil { + return err + } + + fmt.Printf("Activated key %s in JWT authority pool secret %s/%s\n", jwtKeyIDFlag, poolSecretNamespaceFlag, poolSecretNameFlag) + return nil + }, +} + +var removeJwtKeyCmd = &cobra.Command{ + Use: "remove-jwt-key", + Short: "Remove an inactive key from a JWT authority pool secret", + Long: `Remove an inactive key from a JWT authority pool secret. + +Tokens the key signed stop verifying once relying parties refetch the key set, +so remove it only after the last of them has expired.`, + RunE: func(cmd *cobra.Command, args []string) error { + kc, err := newKubeClient() + if err != nil { + return err + } + + if err := updateJWTPool(cmd.Context(), kc.CoreV1().Secrets(poolSecretNamespaceFlag), poolSecretNameFlag, func(pool *localjwtauthority.ConcretePool) error { + return pool.RemoveAuthority(jwtKeyIDFlag) + }); err != nil { + return err } - fmt.Printf("Successfully created JWT authority pool secret %s/%s with %s key %s\n", poolSecretNamespaceFlag, poolSecretNameFlag, makeJwtPoolAlgFlag, keyID) + fmt.Printf("Removed key %s from JWT authority pool secret %s/%s\n", jwtKeyIDFlag, poolSecretNamespaceFlag, poolSecretNameFlag) return nil }, } +// updateJWTPool applies change to the pool in the named secret and writes it +// back. The write is conditional on the secret's resourceVersion, and a +// conflict reruns change against the current pool. +func updateJWTPool(ctx context.Context, secrets typedcorev1.SecretInterface, name string, change func(*localjwtauthority.ConcretePool) error) error { + return retry.RetryOnConflict(retry.DefaultRetry, func() error { + secret, pool, err := getJWTPool(ctx, secrets, name) + if err != nil { + return err + } + + if err := change(pool); err != nil { + return err + } + + wire, err := localjwtauthority.Marshal(pool) + if err != nil { + return fmt.Errorf("while marshaling pool: %w", err) + } + secret.Data["pool"] = wire + if _, err := secrets.Update(ctx, secret, metav1.UpdateOptions{}); err != nil { + return fmt.Errorf("while writing pool secret: %w", err) + } + return nil + }) +} + +// getJWTPool reads the named secret and parses the pool it holds. +func getJWTPool(ctx context.Context, secrets typedcorev1.SecretInterface, name string) (*corev1.Secret, *localjwtauthority.ConcretePool, error) { + secret, err := secrets.Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return nil, nil, fmt.Errorf("while reading pool secret: %w", err) + } + wire, ok := secret.Data["pool"] + if !ok { + return nil, nil, fmt.Errorf("secret %s/%s has no \"pool\" key", secret.Namespace, secret.Name) + } + pool, err := localjwtauthority.Unmarshal(wire) + if err != nil { + return nil, nil, fmt.Errorf("while parsing pool: %w", err) + } + return secret, pool, nil +} + +// printJWTKeys prints the ID and algorithm of every key in the pool, marking +// the one that signs. It prints nothing about the private keys. +func printJWTKeys(out io.Writer, pool *localjwtauthority.ConcretePool) error { + w := tabwriter.NewWriter(out, 0, 0, 3, ' ', 0) + fmt.Fprintln(w, "ACTIVE\tKEY ID\tALGORITHM") + for _, authority := range pool.Authorities { + active := "" + if authority.ID == pool.ActiveID() { + active = "*" + } + fmt.Fprintf(w, "%s\t%s\t%s\n", active, authority.ID, authority.Algorithm) + } + return w.Flush() +} + +// newKubeClient builds a client from the --kubeconfig and --context flags. +func newKubeClient() (kubernetes.Interface, error) { + kconfig, err := ateclient.LoadKubeConfig(kubeconfig, k8sContext) + if err != nil { + return nil, fmt.Errorf("while reading kubeconfig: %w", err) + } + kc, err := kubernetes.NewForConfig(kconfig) + if err != nil { + return nil, fmt.Errorf("while creating Kubernetes client: %w", err) + } + return kc, nil +} + // newJWTPoolSecret builds a Secret holding a pool with one active authority, // and returns the authority's key ID. func newJWTPoolSecret(namespace, name, algorithm, keyID string) (*corev1.Secret, string, error) { @@ -180,10 +336,31 @@ func init() { _ = makeCaPoolCmd.MarkFlagRequired("name") adminCmd.AddCommand(makeCaPoolCmd) - makeJwtPoolCmd.Flags().StringVar(&makeJwtPoolAlgFlag, "alg", "ES256", "Signing algorithm of the initial key. One of [ES256, RS256]") - makeJwtPoolCmd.Flags().StringVar(&makeJwtPoolKeyIDFlag, "key-id", "", "The ID of the initial JWT signing key in the pool. Defaults to the base64url SHA-256 of the key's PKIX encoding") + makeJwtPoolCmd.Flags().StringVar(&jwtAlgFlag, "alg", "ES256", "Signing algorithm of the initial key. One of [ES256, RS256]; RS256 keys are 4096-bit RSA") + makeJwtPoolCmd.Flags().StringVar(&jwtKeyIDFlag, "key-id", "", "The ID of the initial JWT signing key in the pool. Defaults to the base64url SHA-256 of the key's PKIX encoding") makeJwtPoolCmd.Flags().StringVar(&poolSecretNamespaceFlag, "secret-namespace", "default", "Create the secret in this namespace") makeJwtPoolCmd.Flags().StringVar(&poolSecretNameFlag, "name", "", "Create the secret with this name") _ = makeJwtPoolCmd.MarkFlagRequired("name") adminCmd.AddCommand(makeJwtPoolCmd) + + listJwtKeysCmd.Flags().StringVar(&poolSecretNamespaceFlag, "secret-namespace", "default", "The namespace of the pool secret") + listJwtKeysCmd.Flags().StringVar(&poolSecretNameFlag, "name", "", "The name of the pool secret") + _ = listJwtKeysCmd.MarkFlagRequired("name") + adminCmd.AddCommand(listJwtKeysCmd) + + addJwtKeyCmd.Flags().StringVar(&jwtAlgFlag, "alg", "ES256", "Signing algorithm of the new key. One of [ES256, RS256]; RS256 keys are 4096-bit RSA") + addJwtKeyCmd.Flags().StringVar(&jwtKeyIDFlag, "key-id", "", "The ID of the new key. Defaults to the base64url SHA-256 of the key's PKIX encoding") + addJwtKeyCmd.Flags().StringVar(&poolSecretNamespaceFlag, "secret-namespace", "default", "The namespace of the pool secret") + addJwtKeyCmd.Flags().StringVar(&poolSecretNameFlag, "name", "", "The name of the pool secret") + _ = addJwtKeyCmd.MarkFlagRequired("name") + adminCmd.AddCommand(addJwtKeyCmd) + + for _, cmd := range []*cobra.Command{activateJwtKeyCmd, removeJwtKeyCmd} { + cmd.Flags().StringVar(&jwtKeyIDFlag, "key-id", "", "The ID of the key") + cmd.Flags().StringVar(&poolSecretNamespaceFlag, "secret-namespace", "default", "The namespace of the pool secret") + cmd.Flags().StringVar(&poolSecretNameFlag, "name", "", "The name of the pool secret") + _ = cmd.MarkFlagRequired("key-id") + _ = cmd.MarkFlagRequired("name") + adminCmd.AddCommand(cmd) + } } diff --git a/cmd/kubectl-ate/internal/cmd/admin_test.go b/cmd/kubectl-ate/internal/cmd/admin_test.go index 6060f641f5..143adab735 100644 --- a/cmd/kubectl-ate/internal/cmd/admin_test.go +++ b/cmd/kubectl-ate/internal/cmd/admin_test.go @@ -15,8 +15,19 @@ package cmd import ( + "context" + "strings" "testing" + "github.com/google/go-cmp/cmp" + corev1 "k8s.io/api/core/v1" + k8errors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" + "github.com/agent-substrate/substrate/internal/localjwtauthority" ) @@ -70,3 +81,179 @@ func TestNewJWTPoolSecretRejectsUnsupportedAlgorithm(t *testing.T) { t.Error("newJWTPoolSecret(HS256) returned nil error") } } + +// poolState reads back the key IDs and active key of the pool secret. +func poolState(t *testing.T, kc *fake.Clientset) ([]string, string) { + t.Helper() + secret, err := kc.CoreV1().Secrets("ate-system").Get(context.Background(), "actor-id-jwt-pool", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + pool, err := localjwtauthority.Unmarshal(secret.Data["pool"]) + if err != nil { + t.Fatal(err) + } + var ids []string + for _, a := range pool.Authorities { + ids = append(ids, a.ID) + } + return ids, pool.ActiveForSigning +} + +func newPoolClientset(t *testing.T) *fake.Clientset { + t.Helper() + secret, _, err := newJWTPoolSecret("ate-system", "actor-id-jwt-pool", "ES256", "old") + if err != nil { + t.Fatal(err) + } + return fake.NewClientset(secret) +} + +func TestUpdateJWTPoolRotation(t *testing.T) { + kc := newPoolClientset(t) + secrets := kc.CoreV1().Secrets("ate-system") + ctx := context.Background() + next, err := localjwtauthority.GenerateAuthority("RS256", "next") + if err != nil { + t.Fatal(err) + } + + for _, step := range []struct { + name string + change func(*localjwtauthority.ConcretePool) error + wantIDs []string + wantActive string + }{ + {"add", func(p *localjwtauthority.ConcretePool) error { return p.AddAuthority(next) }, []string{"old", "next"}, "old"}, + {"activate", func(p *localjwtauthority.ConcretePool) error { return p.Activate("next") }, []string{"old", "next"}, "next"}, + {"remove", func(p *localjwtauthority.ConcretePool) error { return p.RemoveAuthority("old") }, []string{"next"}, "next"}, + } { + if err := updateJWTPool(ctx, secrets, "actor-id-jwt-pool", step.change); err != nil { + t.Fatalf("%s: %v", step.name, err) + } + ids, active := poolState(t, kc) + if diff := cmp.Diff(step.wantIDs, ids); diff != "" || active != step.wantActive { + t.Errorf("after %s: active %q, want %q; key IDs (-want +got):\n%s", step.name, active, step.wantActive, diff) + } + } +} + +func TestUpdateJWTPoolRetriesConflict(t *testing.T) { + kc := newPoolClientset(t) + conflicted := false + kc.PrependReactor("update", "secrets", func(k8stesting.Action) (bool, runtime.Object, error) { + if conflicted { + return false, nil, nil + } + conflicted = true + // Another writer added a key between our read and write. + secret, _, err := newJWTPoolSecret("ate-system", "actor-id-jwt-pool", "ES256", "old") + if err != nil { + t.Fatal(err) + } + pool, err := localjwtauthority.Unmarshal(secret.Data["pool"]) + if err != nil { + t.Fatal(err) + } + other, err := localjwtauthority.GenerateAuthority("ES256", "other") + if err != nil { + t.Fatal(err) + } + if err := pool.AddAuthority(other); err != nil { + t.Fatal(err) + } + if secret.Data["pool"], err = localjwtauthority.Marshal(pool); err != nil { + t.Fatal(err) + } + if err := kc.Tracker().Update(schema.GroupVersionResource{Version: "v1", Resource: "secrets"}, secret, "ate-system"); err != nil { + t.Fatal(err) + } + return true, nil, k8errors.NewConflict(schema.GroupResource{Resource: "secrets"}, "actor-id-jwt-pool", nil) + }) + + next, err := localjwtauthority.GenerateAuthority("ES256", "next") + if err != nil { + t.Fatal(err) + } + if err := updateJWTPool(context.Background(), kc.CoreV1().Secrets("ate-system"), "actor-id-jwt-pool", func(p *localjwtauthority.ConcretePool) error { + return p.AddAuthority(next) + }); err != nil { + t.Fatalf("updateJWTPool: %v", err) + } + ids, _ := poolState(t, kc) + if diff := cmp.Diff([]string{"old", "other", "next"}, ids); diff != "" { + t.Errorf("key IDs (-want +got):\n%s", diff) + } +} + +func TestUpdateJWTPoolLeavesSecretOnError(t *testing.T) { + for _, tc := range []struct { + name string + secret *corev1.Secret + }{ + {name: "change refused"}, + {name: "no pool key", secret: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: "ate-system", Name: "actor-id-jwt-pool"}, + Data: map[string][]byte{"other": []byte("x")}, + }}, + } { + t.Run(tc.name, func(t *testing.T) { + kc := newPoolClientset(t) + if tc.secret != nil { + kc = fake.NewClientset(tc.secret) + } + err := updateJWTPool(context.Background(), kc.CoreV1().Secrets("ate-system"), "actor-id-jwt-pool", func(p *localjwtauthority.ConcretePool) error { + return p.RemoveAuthority("old") + }) + if err == nil { + t.Error("got nil error") + } + for _, action := range kc.Actions() { + if action.GetVerb() == "update" { + t.Errorf("secret was written: %v", action) + } + } + }) + } +} + +func TestUpdateJWTPoolMissingSecret(t *testing.T) { + kc := fake.NewClientset() + err := updateJWTPool(context.Background(), kc.CoreV1().Secrets("ate-system"), "actor-id-jwt-pool", func(*localjwtauthority.ConcretePool) error { + t.Error("change ran without a pool") + return nil + }) + if !k8errors.IsNotFound(err) { + t.Errorf("err = %v, want NotFound", err) + } +} + +func TestListJWTKeys(t *testing.T) { + kc := newPoolClientset(t) + secrets := kc.CoreV1().Secrets("ate-system") + next, err := localjwtauthority.GenerateAuthority("RS256", "next") + if err != nil { + t.Fatal(err) + } + if err := updateJWTPool(context.Background(), secrets, "actor-id-jwt-pool", func(p *localjwtauthority.ConcretePool) error { + return p.AddAuthority(next) + }); err != nil { + t.Fatal(err) + } + + _, pool, err := getJWTPool(context.Background(), secrets, "actor-id-jwt-pool") + if err != nil { + t.Fatal(err) + } + var out strings.Builder + if err := printJWTKeys(&out, pool); err != nil { + t.Fatal(err) + } + want := "" + + "ACTIVE KEY ID ALGORITHM\n" + + "* old ES256\n" + + " next RS256\n" + if diff := cmp.Diff(want, out.String()); diff != "" { + t.Errorf("output (-want +got):\n%s", diff) + } +} diff --git a/cmd/kubectl-ate/internal/cmd/tag_test.go b/cmd/kubectl-ate/internal/cmd/tag_test.go index b225a65c7a..1f9b769433 100644 --- a/cmd/kubectl-ate/internal/cmd/tag_test.go +++ b/cmd/kubectl-ate/internal/cmd/tag_test.go @@ -60,7 +60,14 @@ func TestUpdateTagScope(t *testing.T) { }, Scope: ateapipb.TagScope_TAG_SCOPE_ATESPACE, Status: &ateapipb.TagStatus{ - Snapshot: &ateapipb.ExternalSnapshot{SnapshotUri: "gs://private/atespaces/space-1/tags/tag-1"}, + Snapshot: &ateapipb.Snapshot{ + Storage: []*ateapipb.SnapshotStorage{{ + Durability: ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, + Status: ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED, + Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + Object: &ateapipb.ObjectSnapshot{SnapshotUri: "gs://private/atespaces/space-1/tags/tag-1"}, + }}, + }, }, } diff --git a/cmd/kubectl-ate/internal/printer/printer.go b/cmd/kubectl-ate/internal/printer/printer.go index 51d3ad0299..59c3e309fb 100644 --- a/cmd/kubectl-ate/internal/printer/printer.go +++ b/cmd/kubectl-ate/internal/printer/printer.go @@ -335,18 +335,18 @@ func PrintTagsTo(out io.Writer, tags []*ateapipb.Tag, format string) error { return printProto(out, &ateapipb.ListTagsResponse{Tags: tags}, format) case "table": w := tabwriter.NewWriter(out, 0, 0, 3, ' ', 0) - fmt.Fprintln(w, "ATESPACE\tNAME\tSCOPE\tSTATE\tSNAPSHOT\tCONTENT SCOPE\tAGE") + fmt.Fprintln(w, "ATESPACE\tNAME\tSCOPE\tSTATE\tSNAPSHOT\tFIDELITY\tAGE") for _, tag := range tags { // A pending tag has no snapshot yet, so neither its URI nor its - // content scope says anything. - snapshotURI, contentScope := "", "" - if snapshot := tag.GetStatus().GetSnapshot(); snapshot.GetSnapshotUri() != "" { - snapshotURI = snapshot.GetSnapshotUri() - contentScope = snapshot.GetContentScope().String() + // fidelity says anything. + snapshotURI, fidelity := "", "" + if st := durableSnapshotStorage(tag.GetStatus().GetSnapshot()); st != nil { + snapshotURI = st.GetObject().GetSnapshotUri() + fidelity = st.GetFidelity().String() } fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%s\n", tag.GetMetadata().GetAtespace(), tag.GetMetadata().GetName(), tag.GetScope(), - tagState(tag), snapshotURI, contentScope, + tagState(tag), snapshotURI, fidelity, formatAge(tag.GetMetadata().GetCreateTime())) } return w.Flush() @@ -355,11 +355,22 @@ func PrintTagsTo(out io.Writer, tags []*ateapipb.Tag, format string) error { } } +func durableSnapshotStorage(snapshot *ateapipb.Snapshot) *ateapipb.SnapshotStorage { + for _, st := range snapshot.GetStorage() { + if st.GetDurability() == ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE && + st.GetStatus() == ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED && + st.GetObject().GetSnapshotUri() != "" { + return st + } + } + return nil +} + // tagState reports whether a tag is usable. A tag is Pending until // the copy of its own snapshot lands; until then it names nothing an Actor can // be created from, and deleting it collects whatever the create stranded. func tagState(tag *ateapipb.Tag) string { - if tag.GetStatus().GetSnapshot().GetSnapshotUri() == "" { + if durableSnapshotStorage(tag.GetStatus().GetSnapshot()) == nil { return "Pending" } return "Ready" diff --git a/cmd/kubectl-ate/internal/printer/printer_test.go b/cmd/kubectl-ate/internal/printer/printer_test.go index a857277d37..70612bdeaa 100644 --- a/cmd/kubectl-ate/internal/printer/printer_test.go +++ b/cmd/kubectl-ate/internal/printer/printer_test.go @@ -599,7 +599,14 @@ func TestPrintTagsTo_Table(t *testing.T) { }, Scope: ateapipb.TagScope_TAG_SCOPE_PUBLISHED, Status: &ateapipb.TagStatus{ - Snapshot: &ateapipb.ExternalSnapshot{SnapshotUri: "gs://private/atespaces/team-a/tags/v2", ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL}, + Snapshot: &ateapipb.Snapshot{ + Storage: []*ateapipb.SnapshotStorage{{ + Durability: ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, + Status: ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED, + Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + Object: &ateapipb.ObjectSnapshot{SnapshotUri: "gs://private/atespaces/team-a/tags/v2"}, + }}, + }, }, }, { @@ -610,7 +617,14 @@ func TestPrintTagsTo_Table(t *testing.T) { }, Scope: ateapipb.TagScope_TAG_SCOPE_ATESPACE, Status: &ateapipb.TagStatus{ - Snapshot: &ateapipb.ExternalSnapshot{SnapshotUri: "gs://private/atespaces/team-a/tags/v1", ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL}, + Snapshot: &ateapipb.Snapshot{ + Storage: []*ateapipb.SnapshotStorage{{ + Durability: ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, + Status: ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED, + Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + Object: &ateapipb.ObjectSnapshot{SnapshotUri: "gs://private/atespaces/team-a/tags/v1"}, + }}, + }, }, }, { @@ -623,7 +637,14 @@ func TestPrintTagsTo_Table(t *testing.T) { }, Scope: ateapipb.TagScope_TAG_SCOPE_ATESPACE, Status: &ateapipb.TagStatus{ - StorageLocation: "gs://private", + Snapshot: &ateapipb.Snapshot{ + Storage: []*ateapipb.SnapshotStorage{{ + Durability: ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE, + Status: ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS, + Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + Object: &ateapipb.ObjectSnapshot{SnapshotUri: "gs://private/atespaces/team-a/tags/v3"}, + }}, + }, }, }, } @@ -633,10 +654,10 @@ func TestPrintTagsTo_Table(t *testing.T) { } // Sorted by atespace, then name. - expected := `ATESPACE NAME SCOPE STATE SNAPSHOT CONTENT SCOPE AGE -team-a v1 TAG_SCOPE_ATESPACE Ready gs://private/atespaces/team-a/tags/v1 SNAPSHOT_CONTENT_SCOPE_FULL 5h -team-a v2 TAG_SCOPE_PUBLISHED Ready gs://private/atespaces/team-a/tags/v2 SNAPSHOT_CONTENT_SCOPE_FULL 5m -team-a v3 TAG_SCOPE_ATESPACE Pending 30s + expected := `ATESPACE NAME SCOPE STATE SNAPSHOT FIDELITY AGE +team-a v1 TAG_SCOPE_ATESPACE Ready gs://private/atespaces/team-a/tags/v1 SNAPSHOT_FIDELITY_MEMORY 5h +team-a v2 TAG_SCOPE_PUBLISHED Ready gs://private/atespaces/team-a/tags/v2 SNAPSHOT_FIDELITY_MEMORY 5m +team-a v3 TAG_SCOPE_ATESPACE Pending 30s ` if diff := cmp.Diff(expected, buf.String()); diff != "" { t.Errorf("output mismatch (-want +got):\n%s", diff) diff --git a/cmd/podcertcontroller/internal/rendezvous/rendezvous.go b/cmd/podcertcontroller/internal/rendezvous/rendezvous.go index eb8ca35317..34bf8bad9a 100644 --- a/cmd/podcertcontroller/internal/rendezvous/rendezvous.go +++ b/cmd/podcertcontroller/internal/rendezvous/rendezvous.go @@ -44,7 +44,9 @@ import ( const labelKey = "rendezvous.ate.dev/application" var ( - leaseDuration = 15 * time.Second + // LeaseDuration is how long a replica's lease stays valid without renewal, + // and therefore the upper bound on how long a dead replica keeps its assignments. + LeaseDuration = 15 * time.Second leaseRenewalPeriod = 10 * time.Second ) @@ -170,7 +172,7 @@ func (h *Hasher) ensureLease(ctx context.Context) error { }, Spec: coordinationv1.LeaseSpec{ HolderIdentity: &h.replicaName, - LeaseDurationSeconds: ptr.To(int32(int64(leaseDuration) / 1_000_000_000)), + LeaseDurationSeconds: ptr.To(int32(int64(LeaseDuration) / 1_000_000_000)), AcquireTime: ptr.To(metav1.NewMicroTime(now)), RenewTime: ptr.To(metav1.NewMicroTime(now)), LeaseTransitions: ptr.To[int32](1), diff --git a/cmd/podcertcontroller/internal/signercontroller/signercontroller.go b/cmd/podcertcontroller/internal/signercontroller/signercontroller.go index fd844c670e..b768086247 100644 --- a/cmd/podcertcontroller/internal/signercontroller/signercontroller.go +++ b/cmd/podcertcontroller/internal/signercontroller/signercontroller.go @@ -149,7 +149,10 @@ func (c *Controller) processNextWorkItem(ctx context.Context) bool { err = c.handlePCR(ctx, pcr) if errors.Is(err, rendezvous.ErrNotAssigned) { - c.pcrQueue.AddRateLimited(key) + // Not an error: don't consume the failure rate limiter or inflate + // this item's backoff. Assignments only change when a replica joins + // or its lease expires, so re-examine after the lease duration. + c.pcrQueue.AddAfter(key, rendezvous.LeaseDuration) return true } if err != nil { diff --git a/cmd/snapshot-plugin/main.go b/cmd/snapshot-plugin/main.go new file mode 100644 index 0000000000..1d95e9e394 --- /dev/null +++ b/cmd/snapshot-plugin/main.go @@ -0,0 +1,160 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Command snapshot-plugin serves the snapshot plugin API on GCS or S3 over a +// Unix socket. It runs as a sidecar: "node" next to atelet, "control" next to +// ate-api-server. The backend is chosen by ATE_STORAGE_BACKEND ("s3", or GCS +// by default) with the ambient credentials, as atelet and ate-api-server do. +// "healthcheck" reports whether a running plugin is serving, for the sidecar's +// startup probe. +package main + +import ( + "context" + "fmt" + "log/slog" + "os" + "os/signal" + "syscall" + "time" + + "cloud.google.com/go/storage" + "github.com/agent-substrate/substrate/internal/nodepath" + "github.com/agent-substrate/substrate/internal/objectstore" + "github.com/agent-substrate/substrate/internal/objectstoreplugin" + "github.com/agent-substrate/substrate/internal/serverboot" + "github.com/agent-substrate/substrate/pkg/objectstorage" + objectstorev1 "github.com/agent-substrate/substrate/pkg/proto/objectstorepb/v1" + "github.com/spf13/pflag" + "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" + "google.golang.org/grpc" + "google.golang.org/grpc/health" + healthpb "google.golang.org/grpc/health/grpc_health_v1" +) + +const usage = `usage: snapshot-plugin [flags]` + +func main() { + if len(os.Args) < 2 { + fmt.Fprintln(os.Stderr, usage) + os.Exit(2) + } + mode := os.Args[1] + flags := pflag.NewFlagSet(mode, pflag.ExitOnError) + socket := flags.String("socket", "", "Unix socket to serve on, or to check in healthcheck mode") + root := flags.String("root", nodepath.BasePath, "node mode: the only directory tree local snapshot files may be read from or written to") + timeout := flags.Duration("timeout", 5*time.Second, "healthcheck mode: how long to wait for the plugin to report that it is serving") + _ = flags.Parse(os.Args[2:]) + + ctx := context.Background() + serverboot.InitLogger() + if *socket == "" { + serverboot.Fatal(ctx, "Missing --socket", fmt.Errorf("--socket is required")) + } + + // healthcheck is what the sidecar's startup probe runs: the image has no + // shell or grpc_health_probe, so the binary checks itself. + if mode == "healthcheck" { + if err := healthcheck(ctx, *socket, *timeout); err != nil { + serverboot.Fatal(ctx, "Snapshot plugin is not serving", err) + } + return + } + + tp, err := serverboot.InitTracing(ctx, serverboot.TracingOptions{ + ServiceName: "snapshot-plugin-" + mode, + Sampling: serverboot.ResolveTraceSampling(ctx, serverboot.ParentRatioSampling(serverboot.ControlPlaneTraceRatio)), + }) + if err != nil { + serverboot.Fatal(ctx, "Failed to initialize tracing", err) + } + defer serverboot.ShutdownProvider("TracerProvider", tp.Shutdown) + + srv := grpc.NewServer(grpc.StatsHandler(otelgrpc.NewServerHandler())) + switch mode { + case "node": + objects, err := objectstorage.NewFromEnv(ctx) + if err != nil { + serverboot.Fatal(ctx, "Failed to set up the object storage backend", err) + } + plugin, err := objectstoreplugin.NewNodePlugin(objects, *root) + if err != nil { + serverboot.Fatal(ctx, "Invalid --root", err) + } + objectstorev1.RegisterNodeProviderServer(srv, plugin) + case "control": + store, err := newObjectStore(ctx) + if err != nil { + serverboot.Fatal(ctx, "Failed to set up the object storage backend", err) + } + objectstorev1.RegisterControlProviderServer(srv, objectstoreplugin.NewControlPlugin(store)) + default: + fmt.Fprintln(os.Stderr, usage) + os.Exit(2) + } + healthpb.RegisterHealthServer(srv, health.NewServer()) + + lis, err := objectstoreplugin.Listen(*socket) + if err != nil { + serverboot.Fatal(ctx, "Failed to listen", err) + } + + // Finish in-flight transfers on SIGTERM. The pod's grace period bounds + // how long that may take. + sigCtx, stop := signal.NotifyContext(ctx, syscall.SIGTERM, os.Interrupt) + defer stop() + go func() { + <-sigCtx.Done() + slog.InfoContext(ctx, "Draining snapshot plugin") + srv.GracefulStop() + }() + + slog.InfoContext(ctx, "Serving snapshot plugin", slog.String("mode", mode), slog.String("socket", *socket)) + if err := srv.Serve(lis); err != nil { + serverboot.Fatal(ctx, "Serve failed", err) + } +} + +// newObjectStore builds the client the control plugin manages snapshot +// prefixes with, on the same backend objectstorage.NewFromEnv selects for the +// node plugin. It builds its own GCS client because objectstorage's carries a +// retry policy scoped to that package. +func newObjectStore(ctx context.Context) (objectstore.Store, error) { + if objectstorage.UsesS3() { + client, err := objectstorage.NewS3ClientFromEnv(ctx) + if err != nil { + return nil, err + } + return objectstore.NewS3(client), nil + } + // GCS is currently the default, TODO: we assume workload identity / ADC + client, err := storage.NewClient(ctx) + if err != nil { + return nil, fmt.Errorf("creating GCS client: %w", err) + } + return objectstore.NewGCS(client), nil +} + +// healthcheck dials the plugin at socket and waits up to timeout for it to +// report that it is serving. +func healthcheck(ctx context.Context, socket string, timeout time.Duration) error { + conn, err := objectstoreplugin.Dial(socket, objectstoreplugin.ReadyWait) + if err != nil { + return err + } + defer conn.Close() + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + return objectstoreplugin.WaitReady(ctx, conn) +} diff --git a/demos/autoscaled-workerpool/autoscaled-workerpool-template.yaml.tmpl b/demos/autoscaled-workerpool/autoscaled-workerpool-template.yaml.tmpl index 8502923d9a..dadd1e82ca 100644 --- a/demos/autoscaled-workerpool/autoscaled-workerpool-template.yaml.tmpl +++ b/demos/autoscaled-workerpool/autoscaled-workerpool-template.yaml.tmpl @@ -36,7 +36,7 @@ containers: - name: data mountPath: /home/counter snapshotConfig: - onCommit: SNAPSHOT_CONTENT_SCOPE_DATA + preferredFidelity: SNAPSHOT_FIDELITY_VOLUMES storageLocation: gs://${BUCKET_NAME}/ate-demo-autoscaled-workerpool/ sandboxConfig: sandboxClass: SANDBOX_CLASS_GVISOR diff --git a/demos/autoscaled-workerpool/autoscaled-workerpool.yaml.tmpl b/demos/autoscaled-workerpool/autoscaled-workerpool.yaml.tmpl index 6cde9b7938..000fd881b5 100644 --- a/demos/autoscaled-workerpool/autoscaled-workerpool.yaml.tmpl +++ b/demos/autoscaled-workerpool/autoscaled-workerpool.yaml.tmpl @@ -38,3 +38,5 @@ metadata: spec: replicas: 5 workerImage: ko://github.com/agent-substrate/substrate/cmd/ateom-gvisor + sandboxClasses: + - name: gvisor diff --git a/demos/claude-code-multiplex/agent-luna-template.yaml.tmpl b/demos/claude-code-multiplex/agent-luna-template.yaml.tmpl index 3d842a0bae..2eef82044c 100644 --- a/demos/claude-code-multiplex/agent-luna-template.yaml.tmpl +++ b/demos/claude-code-multiplex/agent-luna-template.yaml.tmpl @@ -41,7 +41,7 @@ containers: - name: ANTHROPIC_API_KEY value: "${ANTHROPIC_API_KEY}" snapshotConfig: - onCommit: SNAPSHOT_CONTENT_SCOPE_FULL + preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://${BUCKET_NAME}/claude-multiplex-demo/ sandboxConfig: sandboxClass: SANDBOX_CLASS_GVISOR diff --git a/demos/claude-code-multiplex/agent-mars-template.yaml.tmpl b/demos/claude-code-multiplex/agent-mars-template.yaml.tmpl index 46baedeae4..873cc4caec 100644 --- a/demos/claude-code-multiplex/agent-mars-template.yaml.tmpl +++ b/demos/claude-code-multiplex/agent-mars-template.yaml.tmpl @@ -34,7 +34,7 @@ containers: - name: ANTHROPIC_API_KEY value: "${ANTHROPIC_API_KEY}" snapshotConfig: - onCommit: SNAPSHOT_CONTENT_SCOPE_FULL + preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://${BUCKET_NAME}/claude-multiplex-demo/ sandboxConfig: sandboxClass: SANDBOX_CLASS_GVISOR diff --git a/demos/claude-code-multiplex/agent-orion-template.yaml.tmpl b/demos/claude-code-multiplex/agent-orion-template.yaml.tmpl index f6dc7ca890..2e6c84dcf3 100644 --- a/demos/claude-code-multiplex/agent-orion-template.yaml.tmpl +++ b/demos/claude-code-multiplex/agent-orion-template.yaml.tmpl @@ -34,7 +34,7 @@ containers: - name: ANTHROPIC_API_KEY value: "${ANTHROPIC_API_KEY}" snapshotConfig: - onCommit: SNAPSHOT_CONTENT_SCOPE_FULL + preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://${BUCKET_NAME}/claude-multiplex-demo/ sandboxConfig: sandboxClass: SANDBOX_CLASS_GVISOR diff --git a/demos/claude-code-multiplex/claude-code-multiplex.yaml.tmpl b/demos/claude-code-multiplex/claude-code-multiplex.yaml.tmpl index e76789be87..ef1360fdcc 100644 --- a/demos/claude-code-multiplex/claude-code-multiplex.yaml.tmpl +++ b/demos/claude-code-multiplex/claude-code-multiplex.yaml.tmpl @@ -35,3 +35,5 @@ metadata: spec: replicas: 2 workerImage: ko://github.com/agent-substrate/substrate/cmd/ateom-gvisor + sandboxClasses: + - name: gvisor diff --git a/demos/counter/README.md b/demos/counter/README.md index e62db12842..87696e5fd3 100644 --- a/demos/counter/README.md +++ b/demos/counter/README.md @@ -2,7 +2,7 @@ This directory contains a demo of a stateful counter application running on Agent Substrate. -It deploys a simple Go HTTP server (`counter.go`) that increments two counters on every request — one in process memory, one in a file on a durable volume — and preserves both across suspends and resumes: the template's `Full`-scope `onCommit` snapshot captures process memory alongside the durable volumes, so the in-memory count continues from where it left off. (With a `Data`-scope snapshot policy, only the durable-volume counter would survive and the in-memory counter would restart from a cold boot.) +It deploys a simple Go HTTP server (`counter.go`) that increments two counters on every request — one in process memory, one in a file on a durable volume — and preserves both across suspends and resumes: the template's `MEMORY` `preferredFidelity` snapshot captures process memory alongside the durable volumes, so the in-memory count continues from where it left off. (With a `VOLUMES` fidelity, only the durable-volume counter would survive and the in-memory counter would restart from a cold boot.) The demo uses two kinds of resources: the `WorkerPool` is a Kubernetes CRD, while the actor template is a Substrate `ActorTemplate` resource — an `ateapipb.ActorTemplate` living in an **atespace** rather than a Kubernetes namespace, managed through the ate API with `kubectl ate`. @@ -99,7 +99,7 @@ kubectl ate suspend actor my-counter-1 -a ate-demo-counter Repeat the `curl` from step 1 and the actor resumes from its snapshot — possibly on a different worker — with **both** counters continuing from where -they left off: the memory count comes back from the `Full` snapshot's process +they left off: the memory count comes back from the `MEMORY` snapshot's process memory, the file counter from the durable volume. 4. To permanently delete the suspended actor: diff --git a/demos/counter/counter-microvm-csi-test-template.yaml b/demos/counter/counter-microvm-csi-test-template.yaml index b82924cf3d..29d65c3d14 100644 --- a/demos/counter/counter-microvm-csi-test-template.yaml +++ b/demos/counter/counter-microvm-csi-test-template.yaml @@ -33,7 +33,7 @@ containers: - name: data mountPath: /home/counter snapshotConfig: - onCommit: SNAPSHOT_CONTENT_SCOPE_DATA + preferredFidelity: SNAPSHOT_FIDELITY_VOLUMES storageLocation: gs://ate-snapshots/ate-demo-counter-microvm-csi/ sandboxConfig: sandboxClass: SANDBOX_CLASS_MICROVM diff --git a/demos/counter/counter-microvm-csi-test.yaml b/demos/counter/counter-microvm-csi-test.yaml index e6e470a86a..3306226a75 100644 --- a/demos/counter/counter-microvm-csi-test.yaml +++ b/demos/counter/counter-microvm-csi-test.yaml @@ -32,5 +32,6 @@ metadata: workload: counter-microvm-csi spec: replicas: 1 - sandboxClass: microvm + sandboxClasses: + - name: microvm workerImage: ko://github.com/agent-substrate/substrate/cmd/ateom-microvm diff --git a/demos/counter/counter-microvm-template.yaml.tmpl b/demos/counter/counter-microvm-template.yaml.tmpl index 34e2ed76df..ad8f5ee949 100644 --- a/demos/counter/counter-microvm-template.yaml.tmpl +++ b/demos/counter/counter-microvm-template.yaml.tmpl @@ -60,10 +60,10 @@ resources: - name: memory quantity: 512Mi snapshotConfig: - # FULL on both triggers, matching the gVisor variant: a suspend captures + # MEMORY, matching the gVisor variant: a suspend captures # guest memory, so the in-RAM count continues from where it left off — the # round-trip this demo promises. - onCommit: SNAPSHOT_CONTENT_SCOPE_FULL + preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://${BUCKET_NAME}/ate-demo-counter-microvm/ sandboxConfig: sandboxClass: SANDBOX_CLASS_MICROVM diff --git a/demos/counter/counter-microvm.yaml.tmpl b/demos/counter/counter-microvm.yaml.tmpl index edacc2ab29..aa68bdd1f0 100644 --- a/demos/counter/counter-microvm.yaml.tmpl +++ b/demos/counter/counter-microvm.yaml.tmpl @@ -39,7 +39,10 @@ spec: # Two workers: one for the demo actor, one spare for the golden actor the # template reconciler runs while building the golden snapshot. replicas: 2 - sandboxClass: microvm + sandboxClasses: + - name: microvm + configRef: + name: microvm workerImage: ko://github.com/agent-substrate/substrate/cmd/ateom-microvm template: nodeSelector: diff --git a/demos/counter/counter-template.yaml.tmpl b/demos/counter/counter-template.yaml.tmpl index 61b7765dcb..66266e93bd 100644 --- a/demos/counter/counter-template.yaml.tmpl +++ b/demos/counter/counter-template.yaml.tmpl @@ -54,12 +54,12 @@ resources: - name: memory quantity: 512Mi snapshotConfig: - # FULL, so a suspend captures process memory as well as the durable + # MEMORY, so a suspend captures process memory as well as the durable # volumes and the tutorial's in-memory counter really does continue - # across suspend/resume. DATA-scope behavior (durable volumes only, + # across suspend/resume. VOLUMES behavior (durable volumes only, # memory cold-boots) is exercised by the e2e demo suite's own # parameterized templates, not by this one. - onCommit: SNAPSHOT_CONTENT_SCOPE_FULL + preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://${BUCKET_NAME}/ate-demo-counter/ sandboxConfig: sandboxClass: SANDBOX_CLASS_GVISOR diff --git a/demos/counter/counter.yaml.tmpl b/demos/counter/counter.yaml.tmpl index 5e53430f3f..61bf5e0ae3 100644 --- a/demos/counter/counter.yaml.tmpl +++ b/demos/counter/counter.yaml.tmpl @@ -38,6 +38,10 @@ spec: # deploy blocks on a rollout that never completes. replicas: 3 workerImage: ko://github.com/agent-substrate/substrate/cmd/ateom-gvisor + sandboxClasses: + - name: gvisor + configRef: + name: gvisor-default template: nodeSelector: ate.dev/substrate-version: "${SUBSTRATE_VERSION}" diff --git a/demos/egress/README.md b/demos/egress/README.md index c2e0d13a87..229f54bead 100644 --- a/demos/egress/README.md +++ b/demos/egress/README.md @@ -77,8 +77,9 @@ rejects that option with agentgateway rather than silently omitting it. ## Components -- **Egress app (`main.go`)** — the Actor: `POST /` with `{"url":"..."}` → fetches it → returns - status + body. It also serves `POST /grpc`, described below. +- **Egress app (`main.go`)** — the Actor: `POST /` fetches an HTTP(S) URL and returns the status, + body, and observed protocol. `POST /websocket` probes a `ws` or `wss` origin; `POST /grpc` makes + unary and streaming gRPC calls, described below. - **Egress gateway** — the `atenet-egress` Deployment. Envoy uses a co-located atenet `ext_proc` container started with `--mode=egress`; agentgateway uses its built-in `substrateEgress` policy and does not need that sidecar. The installer renders the matching configuration and container. @@ -132,6 +133,14 @@ then asserts: Add `--cleanup` to remove everything the script created. +### HTTPS and WebSocket E2E tests + +Both supported egress dataplanes intercept HTTPS policy traffic. The Actor +trusts the gateway through its projected CA bundle; the gateway verifies each +local test origin against the service-DNS CA. The [egress protocol test +runbook](../../docs/dev/mitm-egress-test-runbook.md) sets up this trust on Kind +and runs the HTTP, HTTPS, WebSocket, and delayed-restore tests. + ## Manual walkthrough ```bash diff --git a/demos/egress/egress-microvm-template.yaml.tmpl b/demos/egress/egress-microvm-template.yaml.tmpl index 9bd813e9e7..5d0cdd4d2f 100644 --- a/demos/egress/egress-microvm-template.yaml.tmpl +++ b/demos/egress/egress-microvm-template.yaml.tmpl @@ -77,5 +77,5 @@ sandboxConfig: # explicitly makes a missing or stale install fail loudly. configName: microvm snapshotConfig: - onCommit: SNAPSHOT_CONTENT_SCOPE_FULL + preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://${BUCKET_NAME}/ate-demo-egress-microvm/ diff --git a/demos/egress/egress-microvm.yaml.tmpl b/demos/egress/egress-microvm.yaml.tmpl index 3b0766bf54..3facf92508 100644 --- a/demos/egress/egress-microvm.yaml.tmpl +++ b/demos/egress/egress-microvm.yaml.tmpl @@ -36,7 +36,8 @@ metadata: workload: egress-microvm spec: replicas: 2 - sandboxClass: microvm + sandboxClasses: + - name: microvm workerImage: ko://github.com/agent-substrate/substrate/cmd/ateom-microvm # No template.resources, unlike the counter demo's micro-VM variant: an # unlimited ateom container reports no capacity, which the scheduler reads as diff --git a/demos/egress/egress-template.yaml.tmpl b/demos/egress/egress-template.yaml.tmpl index 90a86b4253..fc43b8a23e 100644 --- a/demos/egress/egress-template.yaml.tmpl +++ b/demos/egress/egress-template.yaml.tmpl @@ -57,5 +57,5 @@ sandboxConfig: sandboxClass: SANDBOX_CLASS_GVISOR configName: gvisor-default snapshotConfig: - onCommit: SNAPSHOT_CONTENT_SCOPE_FULL + preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://${BUCKET_NAME}/ate-demo-egress/ diff --git a/demos/egress/egress.yaml.tmpl b/demos/egress/egress.yaml.tmpl index 1d57a3e2b7..28265fda39 100644 --- a/demos/egress/egress.yaml.tmpl +++ b/demos/egress/egress.yaml.tmpl @@ -34,3 +34,5 @@ metadata: spec: replicas: 2 workerImage: ko://github.com/agent-substrate/substrate/cmd/ateom-gvisor + sandboxClasses: + - name: gvisor diff --git a/demos/egress/main.go b/demos/egress/main.go index 66607e054c..dcbc72daa1 100644 --- a/demos/egress/main.go +++ b/demos/egress/main.go @@ -19,6 +19,8 @@ package main import ( "context" + "crypto/tls" + "crypto/x509" "encoding/json" "errors" "fmt" @@ -31,6 +33,7 @@ import ( "strconv" "time" + "github.com/gorilla/websocket" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials/insecure" @@ -40,23 +43,50 @@ import ( ) const ( - listenAddress = ":80" - maxRequestBody = 64 << 10 - maxResponseBody = 1 << 20 - requestTimeout = 15 * time.Second + listenAddress = ":80" + maxRequestBody = 64 << 10 + maxResponseBody = 1 << 20 + requestTimeout = 15 * time.Second + maxWebSocketMessages = 16 + maxWebSocketMessageSize = 64 << 10 + maxWebSocketHandshakeBody = 4 << 10 ) type fetchRequest struct { - URL string `json:"url"` + URL string `json:"url"` + RootCA string `json:"rootCA,omitempty"` + HTTP1 bool `json:"http1,omitempty"` } type fetchResponse struct { StatusCode int `json:"statusCode,omitempty"` Body string `json:"body,omitempty"` ServerCert string `json:"serverCert,omitempty"` + Protocol string `json:"protocol,omitempty"` + TLS bool `json:"tls"` Error string `json:"error,omitempty"` } +type websocketRequest struct { + URL string `json:"url"` + RootCA string `json:"rootCA,omitempty"` + Messages []string `json:"messages"` +} + +type websocketMessage struct { + Type int `json:"type"` + Message string `json:"message"` +} + +type websocketResponse struct { + StatusCode int `json:"statusCode,omitempty"` + Protocol string `json:"protocol,omitempty"` + HandshakeBody string `json:"handshakeBody,omitempty"` + TLS bool `json:"tls"` + Messages []websocketMessage `json:"messages"` + Error string `json:"error,omitempty"` +} + // grpcRequest asks for one unary Echo against target, and additionally for a // server-stream or a bidirectional stream when the matching count is positive. type grpcRequest struct { @@ -136,7 +166,13 @@ func newHandler(client *http.Client) http.Handler { if traceparent := r.Header.Get("traceparent"); traceparent != "" { outbound.Header.Set("traceparent", traceparent) } - response, err := client.Do(outbound) + requestClient, cleanup, err := fetchClient(client, input) + if err != nil { + writeJSON(w, http.StatusBadRequest, fetchResponse{Error: fmt.Sprintf("configuring request: %v", err)}) + return + } + defer cleanup() + response, err := requestClient.Do(outbound) if err != nil { writeJSON(w, http.StatusBadGateway, fetchResponse{Error: fmt.Sprintf("request failed: %v", err)}) return @@ -149,15 +185,173 @@ func newHandler(client *http.Client) http.Handler { return } var serverCert string + verifiedTLS := response.TLS != nil && len(response.TLS.VerifiedChains) > 0 if response.TLS != nil && len(response.TLS.PeerCertificates) > 0 { serverCert = response.TLS.PeerCertificates[0].Issuer.String() } - writeJSON(w, response.StatusCode, fetchResponse{StatusCode: response.StatusCode, Body: string(body), ServerCert: serverCert}) + writeJSON(w, response.StatusCode, fetchResponse{ + StatusCode: response.StatusCode, + Body: string(body), + ServerCert: serverCert, + Protocol: response.Proto, + TLS: verifiedTLS, + }) }) + mux.HandleFunc("/websocket", handleWebSocket) mux.HandleFunc("/grpc", handleGRPC) return mux } +// fetchClient constructs a per-request client only when the request asks for +// request-scoped TLS or protocol behavior. The default path retains the +// caller's client, including its transport and redirect policy. +func fetchClient(base *http.Client, input fetchRequest) (*http.Client, func(), error) { + if input.RootCA == "" && !input.HTTP1 { + return base, func() {}, nil + } + if base == nil { + base = http.DefaultClient + } + // Start from the standard transport so request-scoped trust cannot inherit + // an insecure TLS callback or proxy from an injected/base client. + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.Proxy = nil + transport.TLSClientConfig = &tls.Config{MinVersion: tls.VersionTLS12} + if input.RootCA != "" { + roots := x509.NewCertPool() + if !roots.AppendCertsFromPEM([]byte(input.RootCA)) { + return nil, nil, errors.New("rootCA does not contain a valid certificate") + } + transport.TLSClientConfig.RootCAs = roots + } + if input.HTTP1 { + protocols := new(http.Protocols) + protocols.SetHTTP1(true) + transport.Protocols = protocols + transport.ForceAttemptHTTP2 = false + transport.TLSNextProto = nil + } + requestClient := *base + requestClient.Transport = transport + requestClient.Timeout = requestTimeout + requestClient.CheckRedirect = func(*http.Request, []*http.Request) error { + return http.ErrUseLastResponse + } + return &requestClient, transport.CloseIdleConnections, nil +} + +func handleWebSocket(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + w.Header().Set("Allow", http.MethodPost) + writeJSON(w, http.StatusMethodNotAllowed, websocketResponse{Messages: []websocketMessage{}, Error: "method must be POST"}) + return + } + + var input websocketRequest + decoder := json.NewDecoder(http.MaxBytesReader(w, r.Body, maxRequestBody)) + if err := decoder.Decode(&input); err != nil { + writeJSON(w, http.StatusBadRequest, websocketResponse{Messages: []websocketMessage{}, Error: fmt.Sprintf("invalid JSON payload: %v", err)}) + return + } + if err := validateWebSocketRequest(input); err != nil { + writeJSON(w, http.StatusBadRequest, websocketResponse{Messages: []websocketMessage{}, Error: err.Error()}) + return + } + + ctx, cancel := context.WithTimeout(r.Context(), requestTimeout) + defer cancel() + response, err := performWebSocket(ctx, input) + if err != nil { + response.Error = err.Error() + writeJSON(w, http.StatusBadGateway, response) + return + } + writeJSON(w, http.StatusOK, response) +} + +func validateWebSocketRequest(input websocketRequest) error { + parsed, err := url.Parse(input.URL) + if err != nil { + return fmt.Errorf("invalid WebSocket URL: %w", err) + } + if parsed.Scheme != "ws" && parsed.Scheme != "wss" { + return errors.New("WebSocket URL scheme must be ws or wss") + } + if parsed.Hostname() == "" { + return errors.New("WebSocket URL must include a hostname") + } + if len(input.Messages) == 0 { + return errors.New("messages must contain at least one message") + } + if len(input.Messages) > maxWebSocketMessages { + return fmt.Errorf("messages contains %d items, maximum is %d", len(input.Messages), maxWebSocketMessages) + } + for index, message := range input.Messages { + if len(message) > maxWebSocketMessageSize { + return fmt.Errorf("message %d is %d bytes, maximum is %d", index, len(message), maxWebSocketMessageSize) + } + } + return nil +} + +func performWebSocket(ctx context.Context, input websocketRequest) (websocketResponse, error) { + dialer := websocket.Dialer{HandshakeTimeout: requestTimeout, Proxy: nil} + if input.RootCA != "" { + roots := x509.NewCertPool() + if !roots.AppendCertsFromPEM([]byte(input.RootCA)) { + return websocketResponse{Messages: []websocketMessage{}}, errors.New("configuring TLS: rootCA does not contain a valid certificate") + } + dialer.TLSClientConfig = &tls.Config{RootCAs: roots, MinVersion: tls.VersionTLS12} + } + response := websocketResponse{Messages: []websocketMessage{}} + conn, handshakeResponse, err := dialer.DialContext(ctx, input.URL, nil) + if handshakeResponse != nil { + response.StatusCode = handshakeResponse.StatusCode + response.Protocol = handshakeResponse.Proto + if handshakeResponse.Body != nil { + body, _ := io.ReadAll(io.LimitReader(handshakeResponse.Body, maxWebSocketHandshakeBody)) + _ = handshakeResponse.Body.Close() + response.HandshakeBody = string(body) + } + } + if err != nil { + return response, fmt.Errorf("WebSocket handshake failed: %w", err) + } + if tlsConn, ok := conn.UnderlyingConn().(*tls.Conn); ok { + response.TLS = tlsConn.ConnectionState().VerifiedChains != nil + } + defer conn.Close() + conn.SetReadLimit(maxWebSocketMessageSize) + + stopCancellation := make(chan struct{}) + go func() { + select { + case <-ctx.Done(): + _ = conn.Close() + case <-stopCancellation: + } + }() + defer close(stopCancellation) + + for index, message := range input.Messages { + if err := conn.SetWriteDeadline(time.Now().Add(requestTimeout)); err != nil { + return response, fmt.Errorf("setting write deadline for message %d: %w", index, err) + } + if err := conn.WriteMessage(websocket.TextMessage, []byte(message)); err != nil { + return response, fmt.Errorf("writing message %d: %w", index, err) + } + if err := conn.SetReadDeadline(time.Now().Add(requestTimeout)); err != nil { + return response, fmt.Errorf("setting read deadline for message %d: %w", index, err) + } + messageType, received, err := conn.ReadMessage() + if err != nil { + return response, fmt.Errorf("reading response to message %d: %w", index, err) + } + response.Messages = append(response.Messages, websocketMessage{Type: messageType, Message: string(received)}) + } + return response, nil +} + // handleGRPC dials the requested target and echoes back what the RPCs returned. // It exists so an e2e can assert that gRPC survives the egress path: HTTP/2 // framing end to end, and a status delivered in trailers. diff --git a/demos/egress/main_test.go b/demos/egress/main_test.go index c164d3371f..c2182c471c 100644 --- a/demos/egress/main_test.go +++ b/demos/egress/main_test.go @@ -15,7 +15,9 @@ package main import ( + "bytes" "context" + "crypto/tls" "encoding/json" "errors" "fmt" @@ -26,10 +28,12 @@ import ( "strings" "testing" + "github.com/gorilla/websocket" "google.golang.org/grpc" "google.golang.org/grpc/codes" "github.com/agent-substrate/substrate/internal/proto/grpcechopb" + "github.com/agent-substrate/substrate/internal/testcert" ) func TestFetch(t *testing.T) { @@ -82,8 +86,9 @@ func TestFetchHTTPAndHTTPS(t *testing.T) { // The demo verifies origins with the roots its client is given: in the // cluster the projected trust bundle, here the test server's own. - handler := newHandler(httpsSrv.Client()) - + httpsClient := httpsSrv.Client() + httpsClient.Timeout = requestTimeout + handler := newHandler(httpsClient) for _, tc := range []struct { name string url string @@ -157,6 +162,271 @@ func TestOutboundFailure(t *testing.T) { } } +func TestFetchWithRequestScopedTLSAndHTTP1(t *testing.T) { + server, rootCA := startIPTLSServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Proto != "HTTP/1.1" { + t.Errorf("request protocol = %s, want HTTP/1.1", r.Proto) + } + _, _ = io.WriteString(w, "trusted") + })) + + recorder := postHandler(t, newHandler(http.DefaultClient), "/", fetchRequest{ + URL: server.URL, + RootCA: rootCA, + HTTP1: true, + }) + if recorder.Code != http.StatusOK { + t.Fatalf("status = %d, want %d; body = %s", recorder.Code, http.StatusOK, recorder.Body.String()) + } + var got fetchResponse + decodeRecorder(t, recorder, &got) + if got.StatusCode != http.StatusOK || got.Body != "trusted" || got.Protocol != "HTTP/1.1" || !got.TLS { + t.Errorf("response = %+v, want successful verified HTTP/1.1 TLS response", got) + } +} + +func TestFetchTLSVerificationFailure(t *testing.T) { + server, _ := startIPTLSServer(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, "should not arrive") + })) + + recorder := postHandler(t, newHandler(http.DefaultClient), "/", fetchRequest{URL: server.URL}) + if recorder.Code != http.StatusBadGateway { + t.Fatalf("status = %d, want %d; body = %s", recorder.Code, http.StatusBadGateway, recorder.Body.String()) + } + var got fetchResponse + decodeRecorder(t, recorder, &got) + if got.Error == "" || got.Body != "" || got.StatusCode != 0 { + t.Errorf("response = %+v, want TLS failure without upstream response", got) + } +} + +func TestFetchTLSWrongRootFails(t *testing.T) { + server, _ := startIPTLSServer(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, "should not arrive") + })) + _, wrongRoot := startIPTLSServer(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, "other server") + })) + + recorder := postHandler(t, newHandler(http.DefaultClient), "/", fetchRequest{URL: server.URL, RootCA: wrongRoot}) + if recorder.Code != http.StatusBadGateway { + t.Fatalf("status = %d, want %d; body = %s", recorder.Code, http.StatusBadGateway, recorder.Body.String()) + } +} + +func TestFetchDoesNotFollowRedirects(t *testing.T) { + redirected := false + target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + redirected = true + _, _ = io.WriteString(w, "redirected") + })) + t.Cleanup(target.Close) + source := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Location", target.URL) + w.WriteHeader(http.StatusFound) + })) + t.Cleanup(source.Close) + + recorder := postHandler(t, newHandler(http.DefaultClient), "/", fetchRequest{URL: source.URL, HTTP1: true}) + if recorder.Code != http.StatusFound { + t.Fatalf("status = %d, want %d; body = %s", recorder.Code, http.StatusFound, recorder.Body.String()) + } + if redirected { + t.Fatal("redirect target received a request") + } + var got fetchResponse + decodeRecorder(t, recorder, &got) + if got.StatusCode != http.StatusFound || got.Protocol == "" { + t.Errorf("response = %+v, want observed redirect response", got) + } +} + +func TestWebSocket(t *testing.T) { + server := startWebSocketServer(t, false) + messages := []string{"one", "two", "three"} + recorder := postHandler(t, newHandler(http.DefaultClient), "/websocket", websocketRequest{URL: toWebSocketURL(server), Messages: messages}) + if recorder.Code != http.StatusOK { + t.Fatalf("status = %d, want %d; body = %s", recorder.Code, http.StatusOK, recorder.Body.String()) + } + var got websocketResponse + decodeRecorder(t, recorder, &got) + if got.StatusCode != http.StatusSwitchingProtocols || got.Protocol != "HTTP/1.1" || got.TLS || got.Error != "" { + t.Errorf("response = %+v, want successful plaintext handshake", got) + } + if len(got.Messages) != len(messages) { + t.Fatalf("messages = %+v, want %d observations", got.Messages, len(messages)) + } + for i, message := range got.Messages { + if message.Type != websocket.TextMessage || message.Message != messages[i] { + t.Errorf("messages[%d] = %+v, want text %q", i, message, messages[i]) + } + } +} + +func TestWebSocketTLSAndVerificationFailure(t *testing.T) { + server, rootCA := startIPTLSServer(t, webSocketEchoHandler(false)) + messages := []string{"tls-one", "tls-two", "tls-three"} + recorder := postHandler(t, newHandler(http.DefaultClient), "/websocket", websocketRequest{URL: toWebSocketURL(server.URL), RootCA: rootCA, Messages: messages}) + if recorder.Code != http.StatusOK { + t.Fatalf("status = %d, want %d; body = %s", recorder.Code, http.StatusOK, recorder.Body.String()) + } + var got websocketResponse + decodeRecorder(t, recorder, &got) + if got.StatusCode != http.StatusSwitchingProtocols || got.Protocol != "HTTP/1.1" || !got.TLS || len(got.Messages) != len(messages) { + t.Errorf("response = %+v, want verified TLS observations", got) + } + + recorder = postHandler(t, newHandler(http.DefaultClient), "/websocket", websocketRequest{URL: toWebSocketURL(server.URL), Messages: messages}) + if recorder.Code != http.StatusBadGateway { + t.Fatalf("untrusted status = %d, want %d; body = %s", recorder.Code, http.StatusBadGateway, recorder.Body.String()) + } +} + +func TestWebSocketRejectsNonUpgrade(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = io.WriteString(w, "not websocket") + })) + t.Cleanup(server.Close) + + recorder := postHandler(t, newHandler(http.DefaultClient), "/websocket", websocketRequest{URL: toWebSocketURL(server.URL), Messages: []string{"one", "two", "three"}}) + if recorder.Code != http.StatusBadGateway { + t.Fatalf("status = %d, want %d; body = %s", recorder.Code, http.StatusBadGateway, recorder.Body.String()) + } + var got websocketResponse + decodeRecorder(t, recorder, &got) + if got.StatusCode != http.StatusOK || got.Protocol != "HTTP/1.1" || got.TLS || got.Error == "" { + t.Errorf("response = %+v, want preserved non-upgrade response and stage error", got) + } +} + +func TestWebSocketPreservesBoundedHandshakeBody(t *testing.T) { + for _, tc := range []struct { + name string + body string + want string + wantPrefix bool + }{ + {name: "exact denial body", body: "WebSocket egress is not supported", want: "WebSocket egress is not supported"}, + {name: "oversized body", body: strings.Repeat("x", maxWebSocketHandshakeBody+1), wantPrefix: true}, + } { + t.Run(tc.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusForbidden) + _, _ = io.WriteString(w, tc.body) + })) + t.Cleanup(server.Close) + + recorder := postHandler(t, newHandler(http.DefaultClient), "/websocket", websocketRequest{URL: toWebSocketURL(server.URL), Messages: []string{"probe"}}) + if recorder.Code != http.StatusBadGateway { + t.Fatalf("status = %d, want %d; body = %s", recorder.Code, http.StatusBadGateway, recorder.Body.String()) + } + var got websocketResponse + decodeRecorder(t, recorder, &got) + if got.StatusCode != http.StatusForbidden { + t.Errorf("handshake status = %d, want %d", got.StatusCode, http.StatusForbidden) + } + if tc.wantPrefix { + if len(got.HandshakeBody) == 0 || len(got.HandshakeBody) > maxWebSocketHandshakeBody || !strings.HasPrefix(tc.body, got.HandshakeBody) { + t.Errorf("handshake body length = %d, want a prefix no longer than %d bytes", len(got.HandshakeBody), maxWebSocketHandshakeBody) + } + } else if got.HandshakeBody != tc.want { + t.Errorf("handshake body = %q, want %q", got.HandshakeBody, tc.want) + } + }) + } +} + +func TestWebSocketPreservesPartialObservationsOnBadResponse(t *testing.T) { + server := startWebSocketServer(t, true) + recorder := postHandler(t, newHandler(http.DefaultClient), "/websocket", websocketRequest{URL: toWebSocketURL(server), Messages: []string{"one", "two", "three"}}) + if recorder.Code != http.StatusBadGateway { + t.Fatalf("status = %d, want %d; body = %s", recorder.Code, http.StatusBadGateway, recorder.Body.String()) + } + var got websocketResponse + decodeRecorder(t, recorder, &got) + if len(got.Messages) != 1 || got.Messages[0].Message != "one" || got.Messages[0].Type != websocket.TextMessage || got.Error == "" { + t.Errorf("response = %+v, want first observation and read failure", got) + } +} + +func postHandler(t *testing.T, handler http.Handler, path string, body any) *httptest.ResponseRecorder { + t.Helper() + payload, err := json.Marshal(body) + if err != nil { + t.Fatal(err) + } + recorder := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodPost, path, bytes.NewReader(payload)) + handler.ServeHTTP(recorder, request) + return recorder +} + +func decodeRecorder(t *testing.T, recorder *httptest.ResponseRecorder, dst any) { + t.Helper() + if err := json.NewDecoder(recorder.Body).Decode(dst); err != nil { + t.Fatalf("decoding response (HTTP %d): %v", recorder.Code, err) + } +} + +func startWebSocketServer(t *testing.T, badSecondResponse bool) string { + t.Helper() + server := httptest.NewServer(webSocketEchoHandler(badSecondResponse)) + t.Cleanup(server.Close) + return server.URL +} + +func webSocketEchoHandler(badSecondResponse bool) http.Handler { + upgrader := websocket.Upgrader{CheckOrigin: func(*http.Request) bool { return true }} + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := upgrader.Upgrade(w, r, nil) + if err != nil { + return + } + defer conn.Close() + for index := 0; ; index++ { + messageType, message, err := conn.ReadMessage() + if err != nil { + return + } + if badSecondResponse && index == 1 { + _ = conn.Close() + return + } + if err := conn.WriteMessage(messageType, message); err != nil { + return + } + } + }) +} + +func startIPTLSServer(t *testing.T, handler http.Handler) (*httptest.Server, string) { + t.Helper() + material := testcert.NewServerTLS(t, net.ParseIP("127.0.0.1")) + cert, err := tls.X509KeyPair(material.Certificate, material.PrivateKey) + if err != nil { + t.Fatalf("loading server certificate: %v", err) + } + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listening on loopback: %v", err) + } + server := httptest.NewUnstartedServer(handler) + server.Listener = listener + server.TLS = &tls.Config{Certificates: []tls.Certificate{cert}} + server.StartTLS() + t.Cleanup(server.Close) + return server, string(material.RootCA) +} + +func toWebSocketURL(raw string) string { + if strings.HasPrefix(raw, "https://") { + return strings.Replace(raw, "https://", "wss://", 1) + } + return strings.Replace(raw, "http://", "ws://", 1) +} + type roundTripFunc func(*http.Request) (*http.Response, error) func (f roundTripFunc) RoundTrip(request *http.Request) (*http.Response, error) { diff --git a/demos/jupyter/jupyter-template.yaml.tmpl b/demos/jupyter/jupyter-template.yaml.tmpl index ff7bcb3bb6..f9ede8217b 100644 --- a/demos/jupyter/jupyter-template.yaml.tmpl +++ b/demos/jupyter/jupyter-template.yaml.tmpl @@ -40,7 +40,7 @@ containers: path: /api port: 80 snapshotConfig: - onCommit: SNAPSHOT_CONTENT_SCOPE_FULL + preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://${BUCKET_NAME}/ate-demo-jupyter/ sandboxConfig: sandboxClass: SANDBOX_CLASS_GVISOR diff --git a/demos/jupyter/jupyter.yaml.tmpl b/demos/jupyter/jupyter.yaml.tmpl index e9494da0b0..9c49c176a6 100644 --- a/demos/jupyter/jupyter.yaml.tmpl +++ b/demos/jupyter/jupyter.yaml.tmpl @@ -33,6 +33,8 @@ metadata: spec: replicas: 1 workerImage: ko://github.com/agent-substrate/substrate/cmd/ateom-gvisor + sandboxClasses: + - name: gvisor --- apiVersion: v1 diff --git a/demos/multi-template/counter-template.yaml.tmpl b/demos/multi-template/counter-template.yaml.tmpl index 73b2b02141..d1d634f626 100644 --- a/demos/multi-template/counter-template.yaml.tmpl +++ b/demos/multi-template/counter-template.yaml.tmpl @@ -28,7 +28,7 @@ containers: - name: counter image: ko://github.com/agent-substrate/substrate/demos/counter snapshotConfig: - onCommit: SNAPSHOT_CONTENT_SCOPE_FULL + preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://${BUCKET_NAME}/ate-demo-multi-template-counter/ sandboxConfig: sandboxClass: SANDBOX_CLASS_GVISOR diff --git a/demos/multi-template/fspersist-template.yaml.tmpl b/demos/multi-template/fspersist-template.yaml.tmpl index 6d3d36083c..53f047eb5c 100644 --- a/demos/multi-template/fspersist-template.yaml.tmpl +++ b/demos/multi-template/fspersist-template.yaml.tmpl @@ -28,7 +28,7 @@ containers: - name: fspersist image: ko://github.com/agent-substrate/substrate/demos/multi-template/fspersist snapshotConfig: - onCommit: SNAPSHOT_CONTENT_SCOPE_FULL + preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://${BUCKET_NAME}/ate-demo-multi-template-fspersist/ sandboxConfig: sandboxClass: SANDBOX_CLASS_GVISOR diff --git a/demos/multi-template/multi-template.yaml.tmpl b/demos/multi-template/multi-template.yaml.tmpl index b9cd1ca3b4..1f8824ad75 100644 --- a/demos/multi-template/multi-template.yaml.tmpl +++ b/demos/multi-template/multi-template.yaml.tmpl @@ -37,3 +37,5 @@ metadata: spec: replicas: 3 workerImage: ko://github.com/agent-substrate/substrate/cmd/ateom-gvisor + sandboxClasses: + - name: gvisor diff --git a/demos/parking/parking-template.yaml.tmpl b/demos/parking/parking-template.yaml.tmpl index 212311d1c9..c3606c97fb 100644 --- a/demos/parking/parking-template.yaml.tmpl +++ b/demos/parking/parking-template.yaml.tmpl @@ -27,7 +27,7 @@ containers: image: ko://github.com/agent-substrate/substrate/demos/counter command: ["/ko-app/counter"] snapshotConfig: - onCommit: SNAPSHOT_CONTENT_SCOPE_FULL + preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://${BUCKET_NAME}/ate-demo-parking/ sandboxConfig: sandboxClass: SANDBOX_CLASS_GVISOR diff --git a/demos/parking/parking.yaml.tmpl b/demos/parking/parking.yaml.tmpl index b1ac48598e..bc09c25271 100644 --- a/demos/parking/parking.yaml.tmpl +++ b/demos/parking/parking.yaml.tmpl @@ -44,3 +44,5 @@ spec: # the pool and exercise the parking path. replicas: 2 workerImage: ko://github.com/agent-substrate/substrate/cmd/ateom-gvisor + sandboxClasses: + - name: gvisor diff --git a/demos/sandbox/manual-test-multi-template.yaml b/demos/sandbox/manual-test-multi-template.yaml index 71c9d80121..6476c31be0 100644 --- a/demos/sandbox/manual-test-multi-template.yaml +++ b/demos/sandbox/manual-test-multi-template.yaml @@ -35,7 +35,7 @@ containers: - name: vol-nfs mountPath: /nfs-data snapshotConfig: - onCommit: SNAPSHOT_CONTENT_SCOPE_FULL + preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://ate-snapshots/ate-manual-test-multi/ sandboxConfig: sandboxClass: SANDBOX_CLASS_GVISOR diff --git a/demos/sandbox/manual-test-multi.yaml b/demos/sandbox/manual-test-multi.yaml index 8cb3dbdbbb..74cab46198 100644 --- a/demos/sandbox/manual-test-multi.yaml +++ b/demos/sandbox/manual-test-multi.yaml @@ -31,3 +31,5 @@ metadata: spec: replicas: 1 workerImage: ko://github.com/agent-substrate/substrate/cmd/ateom-gvisor + sandboxClasses: + - name: gvisor diff --git a/demos/sandbox/sandbox-template.yaml.tmpl b/demos/sandbox/sandbox-template.yaml.tmpl index 04c5f8a299..48d99209cf 100644 --- a/demos/sandbox/sandbox-template.yaml.tmpl +++ b/demos/sandbox/sandbox-template.yaml.tmpl @@ -38,7 +38,7 @@ resources: - name: memory quantity: 1Gi snapshotConfig: - onCommit: SNAPSHOT_CONTENT_SCOPE_FULL + preferredFidelity: SNAPSHOT_FIDELITY_MEMORY storageLocation: gs://${BUCKET_NAME}/ate-demo-sandbox/ sandboxConfig: sandboxClass: SANDBOX_CLASS_GVISOR diff --git a/demos/sandbox/sandbox.yaml.tmpl b/demos/sandbox/sandbox.yaml.tmpl index 0d32300cdf..3f7614b424 100644 --- a/demos/sandbox/sandbox.yaml.tmpl +++ b/demos/sandbox/sandbox.yaml.tmpl @@ -31,6 +31,8 @@ metadata: spec: replicas: 2 workerImage: ko://github.com/agent-substrate/substrate/cmd/ateom-gvisor + sandboxClasses: + - name: gvisor template: # Per-worker resources: these size the worker POD and advertise its # scheduling capacity (the per-actor ceiling). The sandbox itself is sized by diff --git a/docs/api-guide.md b/docs/api-guide.md index 75c13b2e86..b2f6d956bc 100644 --- a/docs/api-guide.md +++ b/docs/api-guide.md @@ -12,7 +12,7 @@ The `WorkerPool` defines the pool of physical "warm" compute capacity. It manage | :--- | :--- | :--- | | `replicas` | `int32` | **Required.** Number of physical standby pods to maintain in the cluster. | | `workerImage` | `string` | **Required.** The container image for the `ateom` herder process (e.g. `ko://github.com/agent-substrate/substrate/cmd/ateom-gvisor`). | -| `sandboxClass` | `string` | Optional. The sandbox runtime family for the pool: `gvisor` (default) or `microvm`. Drives the worker pod shape (e.g. KVM device mounts, node placement). The sandbox binaries themselves come from the [`SandboxConfig`](#3-sandboxconfig-the-sandbox-itself) each `ActorTemplate` selects. | +| `sandboxClasses` | `[]WorkerPoolSandboxClass` | **Required.** The sandbox runtime families the pool runs; exactly one entry today. Each entry has `name` (**required**, `gvisor` or `microvm`), which drives the worker pod shape (e.g. KVM device mounts, node placement), and `configRef.name` (optional), which names a cluster-scoped [`SandboxConfig`](#3-sandboxconfig-the-sandbox-itself) of the same class. `configRef` is not consumed yet: the sandbox binaries still come from the `SandboxConfig` each `ActorTemplate` selects. | | `template` | `WorkerPoolPodTemplate` | **Optional.** Metadata, scheduling, and resource settings for worker workloads. | #### `WorkerPoolPodTemplate` (`spec.template`) @@ -81,13 +81,15 @@ metadata: spec: replicas: 10 workerImage: ko://github.com/agent-substrate/substrate/cmd/ateom-gvisor + sandboxClasses: + - name: gvisor + configRef: + name: gvisor-default template: labels: project: agent-platform annotations: policy.example.com/exemption: sandbox-host - # sandboxClass defaults to gvisor. The sandbox binaries come from the - # SandboxConfig each ActorTemplate selects, not from the pool. ``` ### Devices (GPUs) — temporarily unsupported @@ -119,15 +121,15 @@ The `ActorTemplate` defines the code, environment, and state-management policies | Field | Type | Description | | :--- | :--- | :--- | | `containers` | `[]Container` | **Required.** The workload definition — see [Container Fields](#container-fields) below. Each container may also declare an optional `wakeupProbe` HTTP probe — see [Container Wakeup Probe](#container-wakeup-probe-wakeupprobe). | -| `sandboxConfig` | `SandboxConfig` | **Required.** The sandbox runtime selection: `sandboxClass` (**required**, `SANDBOX_CLASS_GVISOR` or `SANDBOX_CLASS_MICROVM`) picks the runtime family this template's actors require — only `WorkerPool`s whose `sandboxClass` matches are eligible — and `configName` (**required**) names the cluster-scoped [`SandboxConfig`](#3-sandboxconfig-the-sandbox-itself) object supplying the sandbox binaries. It must reference an existing config of the matching class; `CreateActorTemplate` rejects the template otherwise. | +| `sandboxConfig` | `SandboxConfig` | **Required.** The sandbox runtime selection: `sandboxClass` (**required**, `SANDBOX_CLASS_GVISOR` or `SANDBOX_CLASS_MICROVM`) picks the runtime family this template's actors require — only `WorkerPool`s whose `sandboxClasses` entry matches are eligible — and `configName` (**required**) names the cluster-scoped [`SandboxConfig`](#3-sandboxconfig-the-sandbox-itself) object supplying the sandbox binaries. It must reference an existing config of the matching class; `CreateActorTemplate` rejects the template otherwise. | | `workerSelector` | `*Selector` | Optional. Gates which `WorkerPool`s actors from this template may use, by matching against each pool's labels (`matchLabels`). If unset, all pools are eligible (subject to the actor's own `worker_selector`). | -| `snapshotConfig` | `SnapshotConfig` | **Required.** The base object-storage location snapshots are written under, plus the snapshot scope (`onCommit`) used by both Pause and Suspend. See [Snapshot Storage Layout](#snapshot-storage-layout). | +| `snapshotConfig` | `SnapshotConfig` | **Required.** The base object-storage location snapshots are written under, plus the snapshot fidelity (`preferredFidelity`) every Pause and Suspend captures. See [Snapshot Storage Layout](#snapshot-storage-layout). | | `volumes` | `[]Volume` | Optional. Volumes the containers may mount, each a `durableDir`, an `externalVolumeTemplate` (see [CSI Volumes Guide](csi-volumes.md)), or a `systemInfo` volume (see [SystemInfo Volumes](#systeminfo-volumes)). Every declared volume must be mounted by at least one container. A `microvm` template may declare several `durableDir` volumes; a `gvisor` template is limited to one. | | `resources` | `*ResourceRequirements` | Optional. Declares each actor's compute size via `limits` — see [Sandbox Right-Sizing](#sandbox-right-sizing-resources). Immutable, like the rest of the template. | The sandbox itself — the binaries (e.g. the gVisor `runsc` binary) and the `pauseImage` holding the sandbox's namespaces — comes from the cluster-scoped [`SandboxConfig`](#3-sandboxconfig-the-sandbox-itself) object the template names via `sandboxConfig.configName`. An actor always resolves the config from its current template — repointing the actor at another template requires the same config. -Because a snapshot is not restorable across sandbox runtimes, `sandboxClass` is a **hard scheduling gate**: an actor is only ever placed on a `WorkerPool` of the matching class. It is AND'd with `workerSelector` (and the actor's `worker_selector`), which can only narrow the eligible pools further. It has no default — `sandboxConfig` is required and its `sandboxClass` must be set — and, like the rest of the template, is immutable, so each template's class is fixed at creation. +Because a snapshot is not restorable across sandbox runtimes, `sandboxClass` is a **hard scheduling gate**: an actor is only ever placed on a `WorkerPool` whose `sandboxClasses` entry matches. It is AND'd with `workerSelector` (and the actor's `worker_selector`), which can only narrow the eligible pools further. It has no default — `sandboxConfig` is required and its `sandboxClass` must be set — and, like the rest of the template, is immutable, so each template's class is fixed at creation. ### Sandbox Right-Sizing (`resources`) @@ -139,7 +141,7 @@ Unlike a Pod, an actor is sized by its **`limits`** (CPU and Memory): the size i 2. **Gate scheduling.** An actor is only placed on a `WorkerPool` whose [worker capacity](#worker-capacity-spectemplateresources) is `>=` these limits. 3. **Fall back to runtime defaults.** A zero or absent limit leaves that dimension at the runtime default: unlimited for gVisor, and 2 GiB / 1 vCPU for the micro-VM. -`requests` are not consulted today (an actor occupies its whole worker). Because the size is baked into snapshots, a **micro-VM FULL-scope restore reuses the size in the snapshot**; changing an actor's limits takes effect on its next cold boot. +`requests` are not consulted today (an actor occupies its whole worker). Because the size is baked into snapshots, a **micro-VM MEMORY-fidelity restore reuses the size in the snapshot**; changing an actor's limits takes effect on its next cold boot. Container environment variables support literal `value` entries only. Values are not interpolated (`$(VAR)` references are not expanded), and Kubernetes `envFrom`/`valueFrom` sources are not supported. @@ -183,28 +185,27 @@ To deliver identity information, including credentials, to a running actor, you Available information sources: #### actorMetadata -The actorMetadata data source projects the actor's identity fields to files, one per item, analogous to the [Kubernetes downwardAPI volume](https://kubernetes.io/docs/concepts/storage/volumes/#downwardapi). Each item selects a `field` — `name` (unique within an atespace), `atespace` (together with the name, the actor's full identity), or `uid` (server-generated, distinguishes incarnations of the same name) — and the `path` the value is written to, raw with no trailing newline. `path` is a clean relative path from the root of the volume (no leading `/`, no `.` or `..` segments, at most 16 segments) and must not repeat another path projected into the same volume. +The actorMetadata data source projects the actor's identity fields to files, one per item, analogous to the [Kubernetes downwardAPI volume](https://kubernetes.io/docs/concepts/storage/volumes/#downwardapi). Each item selects a `field` — `ACTOR_METADATA_FIELD_NAME` (unique within an atespace), `ACTOR_METADATA_FIELD_ATESPACE` (together with the name, the actor's full identity), or `ACTOR_METADATA_FIELD_UID` (server-generated, distinguishes incarnations of the same name) — and the `path` the value is written to, raw with no trailing newline. `path` is a clean relative path from the root of the volume (no leading `/`, no `.` or `..` segments, at most 16 segments) and must not repeat another path projected into the same volume. ```yaml -spec: - volumes: +volumes: +- name: system-info + systemInfo: + dataSources: + - actorMetadata: + items: + - field: ACTOR_METADATA_FIELD_NAME + path: actor-name + - field: ACTOR_METADATA_FIELD_ATESPACE + path: atespace + - field: ACTOR_METADATA_FIELD_UID + path: actor-uid +containers: +- name: main + # ... + volumeMounts: - name: system-info - systemInfo: - dataSources: - - actorMetadata: - items: - - field: name - path: actor-name - - field: atespace - path: atespace - - field: uid - path: actor-uid - containers: - - name: main - # ... - volumeMounts: - - name: system-info - mountPath: /run/ate # the actor reads e.g. /run/ate/actor-name + mountPath: /run/ate # the actor reads e.g. /run/ate/actor-name ``` The values are delivered as files on a read-only per-actor bind mount, not environment variables, precisely so they carry the correct values after a resume from a shared snapshot — an env var (or a file baked into the image) would be frozen at the snapshot-source actor's values, since it lives in the checkpointed process memory, and would therefore be identical for every actor restored from that snapshot. The metadata fields themselves are fixed for the actor's lifetime, so workloads may cache them; future data sources that rotate (identity tokens and certificates) must be re-read at time of use. @@ -220,21 +221,20 @@ Supported names are allowlisted: shipped on Debian (consumed via the distroless-static base image). ```yaml -spec: - volumes: +volumes: +- name: trust + systemInfo: + dataSources: + - trustBundle: + names: + - egress-mitm.ate.dev + path: ca.pem +containers: +- name: main + # ... + volumeMounts: - name: trust - systemInfo: - dataSources: - - trustBundle: - names: - - egress-mitm.ate.dev - path: ca.pem - containers: - - name: main - # ... - volumeMounts: - - name: trust - mountPath: /run/substrate/certs # the actor reads /run/substrate/certs/ca.pem + mountPath: /run/substrate/certs # the actor reads /run/substrate/certs/ca.pem ``` atelet resolves the bundle on the node when the actor starts, reading the backing object through a cluster-wide watch (the same informer that drives live refresh) and sanitizing it the way kubelet does for projections: only `CERTIFICATE` PEM blocks are kept, deduplicated across all the named bundles, with block headers stripped and the anchors deliberately shuffled, so consumers must not depend on their order. The actor itself never talks to any bundle backend. Starting the actor fails, with an error naming the bundle, if any name is not on the allowlist, the bundle's backend is unavailable in this deployment, or the resolved bundle is missing, empty, or contains no certificates. @@ -375,7 +375,7 @@ An actor takes a series of snapshots over its life, so it gets a prefix of its o An owner is collected by deleting everything under its prefix, and it can delete nothing else. That is what makes a borrowed snapshot safe: an actor created from a tag points at a URI under `tags/`, which its own prefix does not cover. See [Snapshot lifetime](#snapshot-lifetime). -An `Actor` reports its current snapshot in the server-managed `status.externalSnapshot` and a `Tag` in `status.snapshot`, each an `ExternalSnapshot` carrying `snapshotUri`, `contentScope`, and `actorTemplateUid`. The URI is recorded when the snapshot is written. `actorTemplateUid` records the `ActorTemplate` whose sandbox the guest state was captured from, which is not always the template the actor points at now: an actor may be repointed while `SUSPENDED`, and the snapshot on disk still came from the old one. A resume that finds the two disagree restores the durable data only and boots the guest fresh, because memory captured under one sandbox image cannot be resumed under another. An `ActorTemplate` references its golden tag with the `ObjectRef` in `status.goldenSnapshotStatus.goldenTag`. These status fields are server-owned and ignored on input. Parse a URI only against the scheme above. +An `Actor` reports its snapshots in the server-managed `status.snapshots` (and `status.lastAssignedGeneration`) and a `Tag` in `status.snapshot`, each a `Snapshot` carrying `generation`, `owner`, `actorTemplateUid`, and `storage`. The URI and `fidelity` are recorded in a `DURABLE` `SnapshotStorage` entry (`object.snapshotUri` and `fidelity`) when the snapshot is written. `actorTemplateUid` records the `ActorTemplate` whose sandbox the guest state was captured from, which is not always the template the actor points at now: an actor may be repointed while `SUSPENDED`, and the snapshot on disk still came from the old one. A resume that finds the two disagree restores the durable data only and boots the guest fresh, because memory captured under one sandbox image cannot be resumed under another. An `ActorTemplate` references its golden tag with the `ObjectRef` in `status.goldenSnapshotStatus.goldenTag`. These status fields are server-owned and ignored on input. Parse a URI only against the scheme above. An `ActorTemplate` belongs to one atespace, but one `storageLocation` still holds snapshots for many atespaces: the golden actor lives in the reserved `ate-golden` atespace, and a `PUBLISHED` snapshot may be cloned from other atespaces. The `` level exists so that access can be granted per tenant: an object-storage policy can only condition on an **object-name prefix**, and cannot read the identity recorded inside a snapshot's manifest. Binding a per-atespace grant on GCS looks like: @@ -396,7 +396,7 @@ One consequence worth planning for: **a published snapshot is read from the ates ## 3. SandboxConfig: The Sandbox Itself -`SandboxConfig` is a **cluster-scoped** resource that decouples the sandbox — its binaries (the gVisor `runsc` binary, or a micro-VM kernel/firmware/config) and the `pauseImage` that holds the sandbox's namespaces — from the workload definition in the `ActorTemplate`. An actor's cold boot resolves the sandbox binaries from the config its `ActorTemplate` names via `sandboxConfig.configName`. +`SandboxConfig` is a **cluster-scoped** resource that decouples the sandbox — its binaries (the gVisor `runsc` binary, or a micro-VM kernel/firmware/config) and the `pauseImage` that holds the sandbox's namespaces — from the workload definition in the `ActorTemplate`. An actor's cold boot resolves the sandbox binaries from the default version of the config its `ActorTemplate` names via `sandboxConfig.configName`. This means a single, cluster-managed config pins the sandbox runtime version for many templates: snapshots stay restorable because the version is recorded in each snapshot's manifest, and operators upgrade the runtime in one place. @@ -405,8 +405,17 @@ This means a single, cluster-managed config pins the sandbox runtime version for | Field | Type | Description | | :--- | :--- | :--- | | `sandboxClass` | `string` | **Required.** Runtime family this config applies to: `gvisor` (default) or `microvm`. An `ActorTemplate` only uses `SandboxConfig`s whose `sandboxClass` matches its own. | +| `defaultVersion` | `string` | **Required.** Name of the entry in `versions` that actors boot with. It must name an existing version that is not `Disabled`. | +| `versions` | `[]SandboxVersionConfig` | **Required.** 1–16 sandbox versions, unique by `name`. | + +Each entry of `versions` (`SandboxVersionConfig`): + +| Field | Type | Description | +| :--- | :--- | :--- | +| `name` | `string` | **Required.** Identifies the version within the config. A DNS label (lower-case alphanumerics and `-`, at most 63 characters). | +| `state` | `string` | Optional. `Enabled` (default) or `Disabled`. The version named by `defaultVersion` cannot be `Disabled`. | | `pauseImage` | `string` | **Required for `gvisor`; not allowed for `microvm`**, which runs no pause container. The image for the sandbox's root container (e.g. `registry.k8s.io/pause`, or `gcr.io/gke-release/pause` on GKE). Must include a digest (`...@sha256:...`) — it is recorded in each snapshot's manifest so a restore rebuilds the sandbox from the same image. | -| `assets` | `map[arch]map[name]AssetFile` | Optional. Content-addressed files atelet fetches, keyed by architecture (`amd64`, `arm64`) then asset name. gVisor expects a `gvisor` asset (the release's `gvisor.tar.zstd`), which atelet auto-extracts. A micro-VM backend expects several. Each `AssetFile` is a `{ url, sha256 }` pair. | +| `assets` | `map[arch]map[name]AssetFile` | Content-addressed files atelet fetches, keyed by architecture (`amd64`, `arm64`) then asset name. gVisor expects a `gvisor` asset (the release's `gvisor.tar.zstd`), which atelet auto-extracts. A micro-VM backend expects several. Each `AssetFile` is a `{ url, sha256 }` pair. A `ValidatingAdmissionPolicy` enforces each class's required assets on every version. | A cluster-wide gVisor `SandboxConfig` (`gvisor-default`) is installed with the platform, so gVisor templates can name it via `sandboxConfig.configName` without any extra setup. @@ -417,23 +426,29 @@ apiVersion: ate.dev/v1alpha1 kind: SandboxConfig metadata: name: gvisor-default + annotations: + sandboxconfig.ate.dev/is-class-default: "true" spec: sandboxClass: gvisor - pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" - assets: - amd64: - gvisor: - url: "gs://gvisor/releases/nightly/2026-09-02/x86_64/gvisor.tar.zstd" - sha256: "d547d81401461fd1c679c5c4fa0a6c2b8ef7dc3c22ce23c9e25dcc4c69cfd06f" - arm64: - gvisor: - url: "gs://gvisor/releases/nightly/2026-09-02/aarch64/gvisor.tar.zstd" - sha256: "a64916f9813ce7e4841a30480a599337f7dda07b421c6bf0123db2212aa7d1df" + defaultVersion: v1 + versions: + - name: v1 + state: Enabled + pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" + assets: + amd64: + gvisor: + url: "gs://gvisor/releases/nightly/2026-09-02/x86_64/gvisor.tar.zstd" + sha256: "d547d81401461fd1c679c5c4fa0a6c2b8ef7dc3c22ce23c9e25dcc4c69cfd06f" + arm64: + gvisor: + url: "gs://gvisor/releases/nightly/2026-09-02/aarch64/gvisor.tar.zstd" + sha256: "a64916f9813ce7e4841a30480a599337f7dda07b421c6bf0123db2212aa7d1df" ``` ### Micro-VM SandboxConfig -A `microvm` `SandboxConfig` supplies the [Kata Containers](https://katacontainers.io/) + [Cloud Hypervisor](https://www.cloudhypervisor.org/) toolchain instead of `runsc`. Each architecture must define the full asset set — `cloud-hypervisor`, `virtiofsd`, `kata-kernel`, and `kata-image` — which a `ValidatingAdmissionPolicy` enforces at apply time. Worker pods for a micro-VM pool require `/dev/kvm` and nested-virtualization-capable nodes. The controller requests those devices on the pod automatically, and atelet advertises them only where they exist, so placement follows the hardware rather than a node label. Clusters that reserve nested-virt nodes with an `ate.dev/sandboxClass=microvm` taint are still tolerated: advertising a device attracts these pods to capable nodes but repels nothing else from them. The same convention applies to every class: worker pods of a pool tolerate `ate.dev/sandboxClass=:NoSchedule`, so a cluster can reserve a node pool per sandbox class with that taint, and the atelet DaemonSet tolerates the key for any value. +A `microvm` `SandboxConfig` supplies the [Kata Containers](https://katacontainers.io/) + [Cloud Hypervisor](https://www.cloudhypervisor.org/) toolchain instead of `runsc`. Each architecture of each version must define the full asset set — `cloud-hypervisor`, `virtiofsd`, `kata-kernel`, and `kata-image` — which a `ValidatingAdmissionPolicy` enforces at apply time. Worker pods for a micro-VM pool require `/dev/kvm` and nested-virtualization-capable nodes. The controller requests those devices on the pod automatically, and atelet advertises them only where they exist, so placement follows the hardware rather than a node label. Clusters that reserve nested-virt nodes with an `ate.dev/sandboxClass=microvm` taint are still tolerated: advertising a device attracts these pods to capable nodes but repels nothing else from them. The same convention applies to every class: worker pods of a pool tolerate `ate.dev/sandboxClass=:NoSchedule`, so a cluster can reserve a node pool per sandbox class with that taint, and the atelet DaemonSet tolerates the key for any value. See [`hack/microvm-assets/`](../hack/microvm-assets/) for scripts that assemble and stage these assets, plus a worked counter demo (`demos/counter/counter-microvm.yaml.tmpl`) that suspends and resumes an in-RAM counter across worker pods. @@ -453,13 +468,30 @@ When an `ActorTemplate` is created: ### Resumption Lifecycle Once a template is `Ready`, creating an actor logically (via `kubectl ate create actor`) allows it to be resumed instantly on any free worker in the referenced `WorkerPool`. Substrate bypasses the standard container boot and restores the process directly from its last saved state. +### Eviction +A worker pod can go away while it hosts running actors: when its `WorkerPool` rolls to a new worker image or pod template, as in an [upgrade](upgrade.md), or scales down, when its node is drained, or when its Spot VM is reclaimed. + +When the pod gets `SIGTERM`, its worker stops accepting actors and sends `SIGTERM` to the main process of each container of every actor it hosts. Each actor then has 30 minutes, counted from the pod's `SIGTERM`, to be suspended. An actor suspended in that window keeps its state and resumes on another worker like any suspended actor. + +Substrate does not suspend the actor for you. Whatever drives the actor, usually its harness, has to call `SuspendActor` (`kubectl ate suspend`) within the 30 minutes. Only the actor gets the `SIGTERM`, so it has to pass that on, for example on an endpoint the harness polls. + +Every actor that must keep its state through an eviction needs a `SIGTERM` handler that: + +- **does not exit.** A process that has exited cannot be suspended. +- **runs in the container's main process.** Substrate signals only to the main process. +- **gets the actor suspended within the 30 minutes.** + +An actor still running when the 30 minutes are up is killed. Once the pod is gone, the actor moves to `ACTOR_STATE_CRASHED`, and everything since its last snapshot is lost. `RevertActor` (`kubectl ate revert`) returns a crashed actor to `ACTOR_STATE_SUSPENDED` at its last external snapshot. + +The node can cut the 30 minutes short. GKE node upgrades and cluster autoscaler scale-downs wait up to an hour for a pod, so they leave the full 30 minutes. A reclaimed GKE [Spot VM](https://docs.cloud.google.com/kubernetes-engine/docs/concepts/spot-vms) gives pods 15 seconds by default and at most 2 minutes, so an actor there has to be suspended in that time or it crashes. + --- ## 5. Best Practices * **Startup Logic:** Place expensive initialization (loading large models, establishing baseline connections) in your application's entry point. These will be captured in the Golden Snapshot and won't need to be repeated on every resumption. -* **Placement:** Ensure your `ActorTemplate`'s `sandboxClass` matches your `WorkerPool`'s, and use the template's `workerSelector` to target specific pools — pool selection is by label match, not by namespace or RBAC. +* **Placement:** Ensure your `ActorTemplate`'s `sandboxClass` matches your `WorkerPool`'s `sandboxClasses[].name`, and use the template's `workerSelector` to target specific pools — pool selection is by label match, not by namespace or RBAC. * **Version Management:** When updating code, create a new `ActorTemplate` (e.g. `v2`). Substrate treats each template as an immutable state root. -* **Eviction:** When its worker pod is evicted, an actor gets `SIGTERM` and 30 minutes to be suspended. After that it is killed and moves to `ACTOR_STATE_CRASHED`, and everything since its last snapshot is lost. So an actor that runs for more than 30 minutes without a suspend can lose data. A `CRASHED` actor can be recovered back to `ACTOR_STATE_SUSPENDED` at its last external snapshot using `RevertActor` (`kubectl ate revert`). +* **Eviction:** Give every actor a `SIGTERM` handler that does not exit, and have the actor suspended within 30 minutes of it. See [Eviction](#eviction). --- @@ -473,7 +505,7 @@ The Substrate Control Plane (`ate-api-server`) exposes a gRPC interface for mana Registers a new logical actor in the system. * **Request:** `CreateActorRequest` * `actor`: `Actor` — the actor to create. Its `metadata` carries the atespace and name (name must be a DNS-1123 label); the `actor_template` ref (atespace + name) selects the `ActorTemplate`. - * `actor.source_tag`: (Optional) `ObjectRef` of a `Tag` to seed the actor from. The tag must be taken under the same `ActorTemplate`, and either in the actor's own atespace or `PUBLISHED`. Nothing is copied: the new actor's `status.externalSnapshot` points at the tag's snapshot, under the tag's prefix, until its own first suspend. + * `actor.source_tag`: (Optional) `ObjectRef` of a `Tag` to seed the actor from. The tag must be taken under the same `ActorTemplate`, and either in the actor's own atespace or `PUBLISHED`. Nothing is copied: the new actor's `status.snapshots` points at the tag's snapshot, under the tag's prefix, until its own first suspend. * **Response:** the initialized `Actor`. #### `UpdateActor` @@ -496,7 +528,7 @@ Activates a suspended actor by restoring it onto a physical worker. Hibernate a running actor, capturing its current RAM and disk state into a snapshot. * **Request:** `SuspendActorRequest` * `actor`: `ObjectRef` of the actor to suspend. -* **Response:** `SuspendActorResponse` containing the `Actor` object in `ACTOR_STATE_SUSPENDED`, with its snapshot in `status.externalSnapshot`. +* **Response:** `SuspendActorResponse` containing the `Actor` object in `ACTOR_STATE_SUSPENDED`, with its snapshot in `status.snapshots`. * A successful suspend releases the actor's previous external snapshot: an actor keeps one, and only tags outlive it. To keep the snapshot a suspend just wrote, tag it with `CreateTag` while the actor is still suspended. #### Snapshot lifetime @@ -505,21 +537,21 @@ Every external snapshot has exactly one owner, and the control plane deletes it | Owner | Released when | | :--- | :--- | -| The actor that took it (`status.externalSnapshot`) | The actor's next successful suspend replaces it, or the actor is deleted. | +| The actor that took it (`status.snapshots`) | The actor's next successful suspend replaces it, or the actor is deleted. | | The tag that copied it (`status.snapshot`) | The tag is deleted. | An actor created from a tag borrows the tag's copy instead of taking one of its own. The borrowed URI sits under the tag's prefix, which the actor's own prefix does not cover, so neither suspending nor deleting the actor can reach it; its first own suspend writes a snapshot under the actor's prefix, and it owns its snapshots from then on. Deletion always runs before the database reference is dropped, and a failure fails the whole RPC. Clients are expected to retry with the same arguments: destinations are deterministic and every phase tolerates a partly-completed predecessor, so a retry resumes rather than duplicating work. The cost of that ordering is that a crash between the two can leave an external snapshot no row names; the reverse order would instead lose the handle needed to ever delete it. -> **Do not delete a tag while actors created from it exist.** A clone borrows the tag's snapshot rather than copying it, and only stops borrowing at its own first suspend (its `status.externalSnapshot.snapshotUri` still names the tag's prefix while it is). Deleting the tag leaves such a clone unable to resume. This is not prevented today. +> **Do not delete a tag while actors created from it exist.** A clone borrows the tag's snapshot rather than copying it, and only stops borrowing at its own first suspend (its `status.snapshots` entry still names the tag's prefix while it is). Deleting the tag leaves such a clone unable to resume. This is not prevented today. #### `RevertActor` -Discards an actor's live or crashed execution and transitions it to `ACTOR_STATE_SUSPENDED` at its last completed external snapshot (`status.externalSnapshot`). +Discards an actor's live or crashed execution and transitions it to `ACTOR_STATE_SUSPENDED` at its last completed external snapshot (`status.snapshots`). * **Request:** `RevertActorRequest` * `actor`: `ObjectRef` of the actor to revert. Accepted from `ACTOR_STATE_RUNNING`, `ACTOR_STATE_PAUSED`, and `ACTOR_STATE_CRASHED` (plus `ACTOR_STATE_REVERTING` for idempotent retries). Calling `RevertActor` on an already `ACTOR_STATE_SUSPENDED` actor returns `FAILED_PRECONDITION`. * **Response:** `RevertActorResponse` containing the reverted `Actor` in `ACTOR_STATE_SUSPENDED`. -* Reverting terminates any bound worker sandbox, clears node-local pause checkpoints (`localSnapshot`), and garbage-collects any partial external snapshot left by an interrupted suspend while preserving the last committed `externalSnapshot`. +* Reverting terminates any bound worker sandbox, clears node-local pause checkpoints, and garbage-collects any partial external snapshot left by an interrupted suspend while preserving the last committed durable snapshot in `status.snapshots`. * External volumes are not reverted. Their contents are never part of a snapshot, so a reverted actor comes back with its memory and root filesystem rewound but its volumes exactly as the discarded execution left them. #### `DeleteActor` diff --git a/docs/api-style-guide.md b/docs/api-style-guide.md index 8b8ca0ce69..6bfb643ccf 100644 --- a/docs/api-style-guide.md +++ b/docs/api-style-guide.md @@ -31,7 +31,7 @@ are **atespace-scoped**, whereas `Atespace` resources are naturally **global-sco ### 2.1 Identity and scope * Atespace-scoped resources belong to an atespace. Their identity is `(atespace, name)`, unique within the resource type. -* Global-scoped resources are global across the entire deployment and do not belong to any atespace. For these, the identity is `name` alone. +* Global-scoped resources are global across the entire Substrate installation and do not belong to any atespace. For these, the identity is `name` alone. In both cases, a `metadata` field contains both `atespace` and `name`. For global resources, the `atespace` must always be empty. @@ -119,6 +119,38 @@ Note that this assumes that ActorTemplates are also resources in the substrate g TODO: Decide the convention for cross-references that point to a resource in the *same* atespace as the referrer: whether the caller must fill in `atespace` explicitly, or leaves it empty and the server resolves/validates it against the referrer's atespace. Current leaning is to require it be filled in. +### 2.4 Subresources + +A **subresource** is a resource nested under a parent resource. It exists only in the context of that parent, is reached through the parent, and is deleted with it. Examples are `EgressPolicy` under `Actor`, `AccessPolicy` under `Atespace`, and `ActorAssignment` under `Worker`. + +A subresource's parent may also be **Global**, the implicit root that stands for the whole Substrate installation. The global `AccessPolicy` is a subresource of Global. + +A resource that only *lives in* an atespace is not a subresource of the `Atespace`. `Actor` is atespace-scoped but top-level, with its own standard methods and its own `ObjectRef`. A resource that only *references* another is not its subresource either. A `Tag` references its source `Actor` through `source_actor`, but outlives it. + +Rules: +- Each instance has exactly one parent instance, which never changes. A subresource type may allow more than one parent type, including Global. +- Deleting the parent **must** delete its subresources. +- A subresource has the same scope as its parent. `metadata.atespace` **must** equal the parent's atespace. For an `EgressPolicy` that is the Actor's atespace. For an `AccessPolicy` under an `Atespace` it is empty, because `Atespace` is global-scoped, and it is empty for every subresource of Global. +- `metadata.name` **must** be unique among the parent's subresources of that type. A singleton is named `default` (see section #2.5). +- The message **must** be named for the subresource itself, without the parent type. Use `EgressPolicy`, not `ActorEgressPolicy`. A subresource that allows more than one parent type uses the same message under each, and the RPC names tell the parents apart (`AccessPolicy` under Global and under `Atespace`). +- RPC names **must** be the standard verb, then the parent type, then the subresource type, as in `GetActorEgressPolicy`, `CreateAtespaceAccessPolicy` and `ListWorkerActorAssignments`. For a subresource of Global, the parent type is `Global`, as in `GetGlobalAccessPolicy`. +- The request **must** identify the parent with an `ObjectRef` field named after the parent type (`actor`, `atespace`, `worker`). Get and Delete requests **must** also carry a `name` field holding the subresource's name, and for a singleton `name` **must** be `default`. A request for a subresource of Global carries no parent reference, since Global has no identity. +- On Create and Update, the subresource is embedded in a field named after its snake_case type (`egress_policy`, `access_policy`). Its `metadata` **must** agree with the parent, and a mismatch returns `INVALID_ARGUMENT`. +- A subresource may be managed by the system, such as `ActorAssignment`. The system creates and deletes it as part of other operations, and the API exposes only List and, optionally, Get. +- Subresource's methods follow section #3. Update replaces the whole object with `uid` and `version` preconditions, Delete takes `DeleteOptions`, and List is paginated. + +TODO: The existing Get and Delete requests for singleton subresources do not carry `name` yet. They identify the subresource by its parent alone. + +### 2.5 Singletons + +A **singleton** is a resource type that allows at most one instance per parent. For example, each `Actor` has at most one `EgressPolicy`, and each `Atespace` has at most one `AccessPolicy`, as does Global. + +Rules: +- Clients **must** always set `metadata.name` to `default`. Any other value, including an empty one, fails with `INVALID_ARGUMENT`. +- A singleton **must not** have a List method. +- Get **must** return `NOT_FOUND` until the singleton has been created. +- Create **must** return `ALREADY_EXISTS` if the singleton exists. It is never an upsert. + --- ## 3. Standard Methods diff --git a/docs/architecture.md b/docs/architecture.md index effb2c23b0..45593ab924 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -327,15 +327,15 @@ The node-level subsystem manages the physical execution of sandboxes and the mov * **Lifecycle Management**: The `ateom` process invokes the sandbox runtime to checkpoint or restore processes within the physical pod boundaries — `runsc` for gVisor, or the Kata + Cloud Hypervisor stack for micro-VMs. (Note: the gVisor backend currently requires a `runsc` version with the `--allow-connected-on-save` flag to work around a bug in networking resumption during checkpointing.) - * **Storage Mover**: The `atelet` streams snapshots to and from GCS/S3, ensuring process state is persistent and portable across the cluster. + * **Storage Mover**: External snapshots move through a *snapshot plugin*, a sidecar that serves the `objectstore.v1` gRPC API over a Unix socket in its pod. `atelet`'s sidecar (`NodeProvider`) uploads and fetches a snapshot's files; `ate-api-server`'s sidecar (`ControlProvider`) copies and deletes whole snapshots. Neither main process talks to object storage for snapshots. The in-tree plugin, `cmd/snapshot-plugin`, backs the API with GCS or S3; `ATE_STORAGE_BACKEND` on the sidecar (unset or `gcs`, or `s3` with the `AWS_*` variables) selects which, see `manifests/ate-install/atelet.yaml`, `ate-api-server.yaml`, and the kind overlay for an S3 example. `atelet` keeps a storage client of its own only for sandbox asset downloads. ### Sandbox Classes -A `WorkerPool` selects a **sandbox class** (`spec.sandboxClass`), and each class has a matching `ateom` herder image. The sandbox binaries themselves are not baked into the worker image — they, and the pause image holding the sandbox's namespaces, come at runtime from a cluster-scoped [`SandboxConfig`](api-guide.md#3-sandboxconfig-the-sandbox-itself) the `ActorTemplate` names in its sandbox config (naming one is currently required; per-class cluster defaults are planned) and are pinned into each snapshot's manifest so restores stay reproducible across runtime upgrades. +A `WorkerPool` selects a **sandbox class** (`spec.sandboxClasses[].name`), and each class has a matching `ateom` herder image. The sandbox binaries themselves are not baked into the worker image — they, and the pause image holding the sandbox's namespaces, come at runtime from a cluster-scoped [`SandboxConfig`](api-guide.md#3-sandboxconfig-the-sandbox-itself) the `ActorTemplate` names in its sandbox config (naming one is currently required; per-class cluster defaults are planned) and are pinned into each snapshot's manifest so restores stay reproducible across runtime upgrades. * **gVisor** (`ateom-gvisor`, the default): Runs the workload under `runsc` for kernel-level sandboxing. Suspend and resume leverage gVisor's native checkpoint/restore of the sandboxed process tree. - * **micro-VM** (`ateom-microvm`): Runs the workload inside a [Kata Containers](https://katacontainers.io/) guest on the [Cloud Hypervisor](https://www.cloudhypervisor.org/) VMM. Suspend and resume capture a memory-only VM snapshot and restore it on-demand using `userfaultfd` memory demand-paging. Container rootfs writes are host-backed: the overlay is assembled on the host (read-only OCI image lower plus a per-actor writable upper) and served to the guest over the single virtio-fs share, so they cost reclaimable host page cache rather than guest RAM, and a `Full` snapshot ships the upper as its own tar. `DurableDir` volumes travel over that same share and are likewise shipped as a tar, so a `Data`-scope snapshot can capture them without any guest memory. Each volume is a subdirectory of the share, so an actor can have several at no extra cost in devices, which is why the micro-VM class lifts the single-`DurableDir` limit that still applies to gVisor. + * **micro-VM** (`ateom-microvm`): Runs the workload inside a [Kata Containers](https://katacontainers.io/) guest on the [Cloud Hypervisor](https://www.cloudhypervisor.org/) VMM. Suspend and resume capture a memory-only VM snapshot and restore it on-demand using `userfaultfd` memory demand-paging. Container rootfs writes are host-backed: the overlay is assembled on the host (read-only OCI image lower plus a per-actor writable upper) and served to the guest over the single virtio-fs share, so they cost reclaimable host page cache rather than guest RAM, and a `MEMORY` snapshot ships the upper as its own tar. `DurableDir` volumes travel over that same share and are likewise shipped as a tar, so a `VOLUMES` snapshot can capture them without any guest memory. Each volume is a subdirectory of the share, so an actor can have several at no extra cost in devices, which is why the micro-VM class lifts the single-`DurableDir` limit that still applies to gVisor. ### Networking Stack (`atenet` + `atunnel`) @@ -474,7 +474,7 @@ Triggered by an explicit `SuspendActor` call. borrowed from a tag is left alone, since the tag owns it. 5. **State**: State transitions back to `ACTOR_STATE_SUSPENDED`, and the Actor's - `status.externalSnapshot` names the external snapshot it resumes from. + `status.snapshots` records the external snapshot it resumes from. Snapshots may be given tags owned and addressed by an Atespace. The same tag name may exist in different Atespaces. A tag is an immutable alias and retention pin: diff --git a/docs/csi-volumes.md b/docs/csi-volumes.md index d33bb83c08..ae170b3f25 100644 --- a/docs/csi-volumes.md +++ b/docs/csi-volumes.md @@ -154,6 +154,8 @@ metadata: spec: replicas: 5 workerImage: ko://github.com/agent-substrate/substrate/cmd/ateom-gvisor + sandboxClasses: + - name: gvisor ``` The `ActorTemplate` is a protojson-shaped `ateapipb.ActorTemplate`, created diff --git a/docs/dev/best-practices/metrics.md b/docs/dev/best-practices/metrics.md index 5560b229e8..d39232f572 100644 --- a/docs/dev/best-practices/metrics.md +++ b/docs/dev/best-practices/metrics.md @@ -159,7 +159,7 @@ ones every new metric meets: * **Normalize at the producer.** When a value arrives from the wire or from a file nothing validated, map it onto the bounded set before recording: `ateattr.NormalizeSandboxClass`, `ateattr.NormalizeOperationName`, - `ateattr.SnapshotScopeValue`. An unrecognized value reports `unknown`, never + `ateattr.SnapshotFidelityValue`. An unrecognized value reports `unknown`, never the raw string. * **Omit a label you do not know rather than emit it empty.** A resume that failed before a worker was picked has no pool; the pool keys are absent, not diff --git a/docs/dev/mitm-egress-test-runbook.md b/docs/dev/mitm-egress-test-runbook.md new file mode 100644 index 0000000000..912e823104 --- /dev/null +++ b/docs/dev/mitm-egress-test-runbook.md @@ -0,0 +1,80 @@ +# Egress protocol tests on Kind + +The unified egress gateway intercepts TLS for HTTPS policy rules. It presents +an egress CA certificate to the Actor and verifies the origin certificate on +its upstream connection. The E2E networking tests use local origins signed by +the cluster service-DNS CA, so the gateway must trust that CA as well as its +normal public roots. + +The CI workflow installs both egress ActorTemplates, adds the service-DNS CA +to the gateway's upstream roots with `hack/setup-e2e-egress-tls-kind.sh`, and +runs the networking suite on gVisor and micro-VM. The commands below reproduce +that setup locally. + +## Prerequisites + +Follow the [development quickstart](../../README.md#quickstart-development) +for Go, Docker, `kubectl`, and Kind. The micro-VM lane also needs KVM and the +setup in [Running the microVM runtime locally](microvm-local.md). + +## Install the cluster and fixtures + +```bash +./hack/create-kind-cluster.sh +kubectl config use-context kind-kind +ATE_CREDENTIAL_PROVIDER='{"name":"k8s.io"}' ./hack/install-ate-kind.sh --deploy-ate-system +``` + +For micro-VM, install its assets before deploying the fixture: + +```bash +NO_DEV_ENV=true ATE_INSTALL_KIND=true KUBECTL_CONTEXT=kind-kind \ + ./hack/install-microvm-deps.sh --install +``` + +Deploy both egress ActorTemplates and add local-origin trust to the current +gateway: + +```bash +ATE_CREDENTIAL_PROVIDER='{"name":"k8s.io"}' ./hack/install-ate-kind.sh --deploy-demo-egress +ATE_CREDENTIAL_PROVIDER='{"name":"k8s.io"}' ./hack/install-ate-kind.sh --deploy-demo-egress-microvm +./hack/setup-e2e-egress-tls-kind.sh +``` + +The trust setup retains public roots, adds the service-DNS CA, and waits for +the egress deployment to roll out. The test origins use their Service DNS name +for SNI. The Actor verifies the gateway's minted certificate through its +projected egress trust bundle. + +## Run the networking tests + +Run the gVisor lane: + +```bash +hack/run-e2e-kind.sh ./internal/e2e/suites/networking -run '^TestActorEgress' -v -args --no-color +``` + +Run the micro-VM lane sequentially on the same cluster: + +```bash +E2E_SANDBOX_CLASS=microvm hack/run-e2e-kind.sh ./internal/e2e/suites/networking -run '^TestActorEgress|^TestMicroVMClockAfterDelayedRestore' -v -args --no-color +``` + +The HTTPS protocol test checks a verified HTTP/1.1 request to a local TLS +origin on port 8443. The WebSocket tests check that both `ws` and `wss` +upgrades are denied with HTTP 403 before the origin receives any messages. +The delayed-restore test issues a fresh origin certificate after the Actor is +suspended, then checks that the restored micro-VM can complete the TLS request. + +## Troubleshooting + +```bash +kubectl --context kind-kind get pods -n ate-system -o wide +kubectl --context kind-kind get workerpools -A +kubectl --context kind-kind get pods -A -l ate.dev/worker-pool -o wide +kubectl --context kind-kind -n ate-system logs deploy/atenet-egress -c envoy --tail=300 +``` + +For AgentGateway, use `-c agentgateway` for the last command. For a local TLS +failure, check that `setup-e2e-egress-tls-kind.sh` completed and that the test +URL uses the origin's Service DNS name. diff --git a/docs/dev/suspend-resume-phase-breakdown.md b/docs/dev/suspend-resume-phase-breakdown.md new file mode 100644 index 0000000000..8cc72e3b6a --- /dev/null +++ b/docs/dev/suspend-resume-phase-breakdown.md @@ -0,0 +1,167 @@ +# Suspend/resume phase breakdown: a runbook + +This guide walks through getting a per-phase breakdown of `SuspendActor` and +`ResumeActor` latency from a cluster you run yourself, from deploying the +workloads to reading the report. It strings together tools documented +elsewhere; follow the links when a step needs more than the command shown. + +The breakdown comes from two developer-facing log records, `Checkpoint timing +breakdown` and `Restore timing breakdown`, written once per operation by +atelet (the outer phases: `download`, `persist`, `ateom_*`, …) and by +ateom-microvm (the phases inside `ateom_*`: `pause`, `snapshot`, `vm_restore`, +`wakeup_probe`, …). `benchmarking/analysis/` collects the records from the pod +logs and aggregates them. See +[docs/observability.md](../observability.md#actor-attributed-component-logs) +for the record shape and +[benchmarking/analysis/README.md](../../benchmarking/analysis/README.md) for +the report. + +## What you need + +* A cluster with Agent Substrate installed from a tree that has the records + and the analysis tools: the + [Quickstart (Development)](../../README.md#quickstart-development) for kind, + or the [GKE Quickstart](../../README.md#gke-quickstart-development). For GKE + the scripts below read `.ate-dev-env.sh` from the repository root + (`collect_logs.sh` sources it itself; set `NO_DEV_ENV` to opt out). +* The **microVM** sandbox class, if you want the inner (ateom) phases: see + [microvm-local.md](microvm-local.md). ateom-gvisor writes no breakdown + record, so on a gVisor pool the report shows the atelet layer only. +* `kubectl`, `kubectl-ate` (`go install ./cmd/kubectl-ate`), and `python3` + (standard library only). + +## 1. Deploy the benchmark workloads + +```sh +./benchmarking/deploy_locust.sh --deploy \ + --sandbox-class microvm --worker-count 3 --actor-memory 1536Mi +``` + +This deploys the benchmark `WorkerPool` and `ActorTemplate`s into the +`benchmark-workloads` atespace, builds and pushes the locust image, and deploys +the locust master and workers (`benchmarking/README.md`, +[Deploy benchmarks](../../benchmarking/README.md#deploy-benchmarks)). On a kind +cluster with limited memory use smaller values, e.g. `--worker-count 1 +--actor-memory 512Mi`, and scale the load flags in step 2 down to match. + +Wait until every template has a golden snapshot — the column is a UUID once +the golden build (a full boot plus checkpoint) has finished: + +```sh +kubectl ate get actor-templates -a benchmark-workloads +``` + +Those golden builds are themselves checkpoints, so the worker pods already +carry `Checkpoint timing breakdown` records at this point: + +```sh +kubectl logs -n benchmark-workloads -l ate.dev/worker-pool --since=15m \ + | grep -c "timing breakdown" +``` + +A zero here means the running image predates the records; redeploy ate-system +from the current tree before going on. + +## 2. Run suspend/resume load + +The `glutton` user class creates an actor per user and cycles it through +suspend and resume. Give it a realistically sized working set so the memory +phases have something to measure: + +```sh +kubectl port-forward svc/locust -n benchmarking 8089:8089 +``` + +Open , pick `GluttonUser`, and set: + +| field | value | why | +|---|---|---| +| users / spawn rate | 2 / 1 | two concurrent cycles; keep it small so transfers do not contend for one NIC | +| `--mem-target` | `1Gi` | resident working set before the first suspend (below `--actor-memory`) | +| `--mem-churn` | `64Mi` | re-dirty part of it each cycle, so successive snapshots differ | +| `--mem-read` | `all` | walk the whole set after each resume, so a demand-paged restore pays for its pages | +| `--min-wait-time` / `--max-wait-time` | `1.0` / `1.0` | one cycle per second per user | + +The form fields are the flags documented in +`benchmarking/locust/common/memload_config.py`; the boomer workers fetch the +values from the master on each spawn. Let it run for about 90 seconds (≈10 +cycles per user), then stop the test. For a single operation without locust, +`kubectl ate suspend actor -a benchmark-workloads` and +`kubectl ate resume actor -a benchmark-workloads` on any actor in the +atespace produce one record pair each. + +## 3. Collect the node logs + +Collect **while the pods still exist**. `benchmarking/automation/ +orchestrator.py` deletes the workload and ate-system pods right after each +test and does not run this step yet, so on a cluster you manage yourself this +is the moment to do it: + +```sh +./benchmarking/analysis/collect_logs.sh --dest /tmp/run1 --since-time 2026-09-24T18:00:00Z +``` + +`--since-time` (or a relative `--since 30m`) should cover the run and nothing +before it. The script writes one file per atelet pod (`ate-system`) and per +worker pod (`benchmark-workloads`) under `--dest`; a pod it cannot read is +skipped with a warning. Use either these dumps or the Cloud Logging export +below for a run, not both. `kubectl logs` returns only a container's current log +file, so collect soon after the run, before kubelet rotates it. + +On GKE the same records are also in Cloud Logging, where they outlive both +rotation and teardown; an export is accepted by the report as-is: + +```sh +gcloud logging read '(jsonPayload.msg="Checkpoint timing breakdown" OR jsonPayload.msg="Restore timing breakdown") AND resource.labels.cluster_name="" AND timestamp>="2026-09-24T18:00:00Z"' \ + --project --format json > /tmp/run1/export.json +``` + +## 4. Produce the report + +```sh +python3 benchmarking/analysis/phase_report.py /tmp/run1/*.log --csv /tmp/run1/report +``` + +The first line says how many records it found; expect two per suspend and two +per resume (one per layer). If it is zero, the window in step 3 missed the run +or the images predate the records. + +## 5. Read it + +* **Phase percentiles** rank the phases. For a suspend, compare atelet's + `persist` against `ateom_checkpoint`, then the ateom rows (`snapshot`, + `teardown`, `pause`, …) against each other; for a resume, `download` against + `ateom_restore`, then `vm_restore` and `wakeup_probe`. The `unattributed` + row is the part of a checkpoint no logged phase names. +* **Waterfalls** show the slowest operations with the ateom record nested + under the atelet phase it decomposes; a tail that is always the same phase + is systematic, one that moves between phases is environmental. +* Two sanity checks on a healthy run: the atelet and ateom rows for one + operation have the same `n`, and atelet's `ateom_checkpoint` p50 is a few + tens of milliseconds above ateom's `total` p50 (the `(gap)` line in a + waterfall is that RPC overhead). + +`/tmp/run1/report/phase_percentiles.csv` holds the same table for comparing +runs, for example one at `--mem-target 1Gi` against one at `2Gi` to see which +phases scale with dirty memory. + +## 6. Tear down + +```sh +./benchmarking/deploy_locust.sh --delete +``` + +## Troubleshooting + +* **Only atelet rows, no ateom rows.** The pool is gVisor (ateom-gvisor emits + no record), or the worker image predates the records. Check + `kubectl get workerpools -A` and step 1's grep. +* **Counts differ between layers.** The collection window cut through a + cycle, or a worker pod restarted and took its log with it (the script dumps + a restarted container's previous log as `.previous.log`). On GKE the + Cloud Logging export has everything. +* **A record has `error.type`.** That operation failed; the report leaves it + out of the percentiles and the waterfalls. To see where it died, grep the + record itself: a `DeadlineExceeded` whose time sits in `download` or + `persist` with a normal-looking `ateom_*` phase points at object storage, + not the runtime. diff --git a/docs/glossary.md b/docs/glossary.md index 4f397ff885..ca6a17fe15 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -94,7 +94,7 @@ for etcd. ## Volumes - **DurableDir volume**: a directory mounted into one or more containers - whose contents are preserved by the [`Data` snapshot scope](#snapshots) + whose contents are preserved by the [`VOLUMES` snapshot fidelity](#snapshots) and therefore survive across Suspend/Resume independently of process memory or other rootfs writes. A volume may be mounted into multiple containers, potentially at different paths. This is the per-Actor @@ -108,27 +108,31 @@ for etcd. ## Snapshots -- **Snapshot scope**: what an `ActorTemplate`'s `SnapshotConfig` includes - in a given snapshot. Two scopes exist today: - - **`Full`**: process memory plus the rootfs delta on top of the OCI - image, and any attached `DurableDir` volumes. Used to capture - everything needed to resume hot. - - **`Data`**: only the contents of attached volumes that support - snapshots — currently `DurableDir` volumes. Process memory and the - rest of rootfs are discarded. Used to persist application data - cheaply without the cost of a full memory image. On Resume the - containers start afresh from the OCI image with the `DurableDir` - contents restored. - - Scopes describe only what a snapshot *captures*. A template sets one - scope, `onCommit`, and every snapshot of its actors uses it: the - node-local checkpoint a [Pause](#lifecycle) keeps on the node and the - snapshot a [Suspend](#lifecycle) uploads to snapshot storage. +- **Snapshot fidelity**: the state layers a snapshot holds, ordered lowest + to highest; each level includes the ones below it. + - **`VOLUMES`**: only the contents of attached volumes that support + snapshots — currently `DurableDir` volumes. Process memory and rootfs + changes are discarded. Used to persist application data cheaply + without the cost of a memory image. On Resume the containers start + afresh from the OCI image with the `DurableDir` contents restored. + - **`ROOTFS`**: volumes plus the root filesystem changes made since boot. + Defined in the API but not supported by any sandbox runtime yet; + templates that request it are rejected. + - **`MEMORY`**: volumes, rootfs changes, and process memory. Used to + capture everything needed to resume hot. + + Fidelity describes only what a snapshot *holds*. A template sets one + level, `preferredFidelity`, the ceiling for every snapshot of its actors: + the node-local checkpoint a [Pause](#lifecycle) keeps on the node and the + snapshot a [Suspend](#lifecycle) uploads to snapshot storage. It is best + effort: the system never packages more than that level, and may package + less when a layer cannot be captured or is too expensive to capture at + the time. A process resumed in place keeps its memory regardless. - **Golden Snapshot**: the initial checkpoint captured once, when an `ActorTemplate` is created, from a temporary "golden" boot of the workload. By default an Actor of that template is first restored from this shared - snapshot. It is always a `Full` capture. + snapshot. It is always a `MEMORY` capture. - **Last Snapshot**: the most recent per-Actor snapshot, written on Suspend and used to restore that specific Actor on the next Resume. diff --git a/docs/metrics/registry/events.yaml b/docs/metrics/registry/events.yaml index d649e2a34d..5b4a1ee694 100644 --- a/docs/metrics/registry/events.yaml +++ b/docs/metrics/registry/events.yaml @@ -108,7 +108,19 @@ groups: note: > The ateom that runs the actor writes one record per actor on a timer, plus an initial and a final record per activation. ate.stats.kind tells - them apart. + them apart. The records of an activation are written in that order: the + initial one, then the periodic ones, then the final one. Each is dated + when its sample was read, thus a final record that repeats an older + sample can carry an earlier timestamp than the records before it. The + initial record is absent when the first reading fails. The final record + is written when the ateom tears the activation down: a checkpoint, a + terminate, a graceful shutdown of the worker, or a re-host on a + micro-VM. It is read just before the teardown and repeats the newest + sample when that read fails or times out, or is pending when none was + measured. It is absent when the activation ends otherwise, as by a + crash, a lost worker, or a re-host on gVisor. Thus take the highest + ate.stats.cpu.time per ate.actor.uid and ate.actor.epoch rather than + wait for the final record. The timestamp is when the measurement was read. The measurements are absent, not zero, while the actor is not measurable: the source is @@ -125,7 +137,7 @@ groups: annotations: substrate: emitted_by: [ateom-gvisor, ateom-microvm] - code_anchor: internal/actorevent/actorevent.go + code_anchor: internal/ateomstats/usage.go cuj: How much CPU and memory does this actor use, and since when? attributes: - ref: ate.atespace diff --git a/docs/metrics/registry/metrics.yaml b/docs/metrics/registry/metrics.yaml index 70d2343141..b74d03a2c5 100644 --- a/docs/metrics/registry/metrics.yaml +++ b/docs/metrics/registry/metrics.yaml @@ -246,9 +246,9 @@ groups: - id: registry.ate.sandbox type: attribute_group brief: > - The class of the sandbox. The class comes from the template. Thus it adds - no series to the template labels. It lets a dashboard group the data by - class without a list of the template names. + The sandbox that hosts an actor. The class comes from the template. Thus + it adds no series to the template labels. It lets a dashboard group the + data by class without a list of the template names. attributes: - id: ate.sandbox.class stability: development @@ -272,6 +272,18 @@ groups: ateapi reports it for a Worker that CreateWorker accepted with no class, and for an ActorTemplate with no sandbox config. atelet reports it for a snapshot manifest with no class. + - id: ate.sandbox.oom_kills + stability: development + type: int + brief: > + The processes in the sandbox cgroup leaf that the kernel OOM killer + has killed, over the life of the leaf (memory.events oom_kill). + note: > + Logs only. It is only ever recorded beside actor identity, which no + metric may carry. ateom-gvisor sets it on the record it writes when a + hosted actor's sandbox has no process left. A leaf left behind by an + earlier activation of the same actor carries its count forward. + examples: [0, 3] - id: registry.ate.snapshot type: attribute_group @@ -301,19 +313,23 @@ groups: stability: development value: boot brief: A start from nothing. This is not a restore. It does not occur on the atelet histograms. - - id: ate.snapshot.scope + - id: ate.snapshot.fidelity stability: development - brief: The content of the snapshot. + brief: The state layers the snapshot holds. type: members: - - id: full + - id: memory + stability: development + value: memory + brief: Process memory, root filesystem changes, and volumes. + - id: volumes stability: development - value: full - brief: All the guest state and all the data. - - id: data + value: volumes + brief: Only the volumes of the actor. + - id: rootfs stability: development - value: data - brief: Only the data of the actor. + value: rootfs + brief: Volumes and root filesystem changes, without process memory. - id: unknown stability: development value: unknown @@ -599,10 +615,10 @@ groups: type: double brief: The CPU time used since the activation began, in seconds. note: > - Logs only. Restarts at zero with each ate.actor.epoch. The lifetime - figure is the sum over the epochs of the highest value in each. The - unit is that of the instrument ate.actor.stats.cpu.time, thus one - query serves both signals. + Logs only. Restarts at zero with each ate.actor.epoch and never + decreases within one. The lifetime figure is the sum over the epochs + of the highest value in each. The unit is that of the instrument + ate.actor.stats.cpu.time, thus one query serves both signals. examples: [1.5] - id: registry.ate.deviation @@ -793,9 +809,9 @@ groups: - ref: ate.snapshot.kind requirement_level: conditionally_required: The operation is a resume. A suspend and a pause do not restore, thus this key is absent. - - ref: ate.snapshot.scope + - ref: ate.snapshot.fidelity requirement_level: - conditionally_required: The scope is known. + conditionally_required: The fidelity is known. - ref: error.type requirement_level: conditionally_required: The operation failed. If this key is absent, the operation was correct. @@ -867,7 +883,7 @@ groups: requirement_level: required - ref: ate.template.name requirement_level: required - - ref: ate.snapshot.scope + - ref: ate.snapshot.fidelity requirement_level: required - ref: ate.snapshot.phase requirement_level: required @@ -901,7 +917,7 @@ groups: requirement_level: required - ref: ate.template.name requirement_level: required - - ref: ate.snapshot.scope + - ref: ate.snapshot.fidelity requirement_level: required - ref: ate.snapshot.phase requirement_level: required @@ -947,11 +963,21 @@ groups: stability: development brief: The total CPU time in seconds of the actors that are in operation. note: > - This is a usual counter. The other three stats instruments use a callback. - atelet adds the increase of each actor to this counter. Thus the value - always increases. If a template leaves the node, its series keeps its last - value. It does not disappear. This is the behaviour that a rate() query - needs. + This is a usual counter. The other three stats instruments use a + callback. atelet adds the increase of each activation to this counter. + Thus the value always increases. If a template leaves the node, its + series keeps its last value. It does not disappear. This is the + behaviour that a rate() query needs. + + It does not count usage twice. It can count less: the end of an + activation after the last sample atelet read of it, which is up to one + poll interval plus one ateom sample interval plus the sweep's own time + (up to 45s on a micro-VM), so an activation shorter than that can lose + most or all of its CPU here, while its usage records keep it; the time + around an atelet restart; and, in a + micro-VM restored from a full snapshot, a container's use before its + first successful reading. A node clock that steps back across an atelet + restart can count one activation in full a second time. annotations: substrate: emitted_by: [atelet] diff --git a/docs/metrics/substrate.yaml b/docs/metrics/substrate.yaml index 63a40343de..ffef2b775c 100644 --- a/docs/metrics/substrate.yaml +++ b/docs/metrics/substrate.yaml @@ -234,3 +234,12 @@ blind_spots: - area: router shutdown outcome code: cmd/atenet/internal/router/drain.go brief: The code has two TODOs. Nothing counts a drain that ends with an error. + - area: CPU of short activations + code: cmd/atelet/statspoller.go + brief: > + ate.actor.stats.cpu.time counts only the samples atelet reads while an + actor is hosted. It loses the end of each activation, up to one poll + interval plus one ateom sample interval. Thus an activation shorter than + that, as a checkpoint-when-idle agent, can show little or no CPU here. + Its ate.actor.usage_sampled records keep it. Issue #2362 moves the + counter to the ateoms. diff --git a/docs/observability.md b/docs/observability.md index 2e6bf639bc..93347ec10c 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -67,13 +67,25 @@ Actor is currently running on pod ate-demo-counter/counter-ab123-x4y5z {"time":"2026-05-22T21:50:02.123456789Z","count":2,"fshash":"mCY7...","level":"INFO","msg":"Count"} ``` -#### Example 4: Filtering by Container -An actor can run several containers. By default every line is shown, including the synthetic lifecycle events (`Actor started`, `Actor checkpointing`, ...). `--container` (short form `-c`) restricts the output to the named container's logs: +#### Example 4: Filtering by Container or Source +An actor can run several containers. By default every line is shown, including the synthetic lifecycle events (`Actor starting`, `Actor started`, `Actor checkpointing`, `Actor checkpointed`, `Actor restoring`, `Actor restored`, `Actor terminated`). `--container` (short form `-c`) restricts the output to the named container's logs: ```bash kubectl ate logs actors -a -c ``` +`--source` selects a class of lines instead of a container. `containers` shows every container's output with the lifecycle events removed; `lifecycle` shows only the lifecycle events, which is the quickest way to see how often an actor has been suspended and restored on its current worker (a migration moves the actor to another pod, whose log holds none of the earlier workers' events): + +```bash +$ kubectl ate logs actors test -a demo --source=lifecycle +{"time":"2026-10-07T21:31:48.648599369Z","message":"Actor checkpointing","span_id":"c475...","trace_flags":"01","trace_id":"e617..."} +{"time":"2026-10-07T21:31:48.791435032Z","message":"Actor checkpointed","span_id":"c475...","trace_flags":"01","trace_id":"e617..."} +{"time":"2026-10-07T21:31:59.772566601Z","message":"Actor restoring","span_id":"a68c...","trace_flags":"01","trace_id":"b387..."} +{"time":"2026-10-07T21:31:59.96129419Z","message":"Actor restored","span_id":"a68c...","trace_flags":"01","trace_id":"b387..."} +``` + +`--container` already implies container output, so it can be combined with `--source=containers` but not with `--source=lifecycle`: lifecycle events are not emitted by a container, and the CLI rejects that combination. + --- ### Centralized Logging Backends (Multi-Dimensional Aggregation) @@ -129,7 +141,7 @@ atelet's `Restore timing breakdown` and `Checkpoint timing breakdown` are the un {"time":"…","level":"INFO","msg":"Restore timing breakdown", "ate.atespace":"ate-demo-counter","ate.actor.name":"counter-1","ate.actor.uid":"8f2a…", "ate.template.atespace":"ate-demo-counter","ate.template.name":"counter", - "ate.snapshot.scope":"full","ate.snapshot.kind":"latest","ate.sandbox.class":"gvisor", + "ate.snapshot.fidelity":"memory","ate.snapshot.kind":"latest","ate.sandbox.class":"gvisor", "ate.actor.restore.duration.download":0.310, "ate.actor.restore.duration.oci_unpack":0.050, "ate.actor.restore.duration.ateom_restore":0.060, @@ -141,7 +153,7 @@ The duration keys are the [`ate.actor.restore.duration`](#the-metric-registry) i This is the record to use for a per-actor wake-up distribution. The histogram cannot answer that question at all, because actor identity is barred from metric labels; traces can, but the data plane is head-sampled at 1%. -ateom-microvm writes records with the same two messages, from inside the `ateom_restore` and `ateom_checkpoint` phases, under `ateom.actor.restore.duration.` and `ateom.actor.checkpoint.duration.` (for example `vm_restore`, `wakeup_probe`, `prep`, `pause`, `snapshot`, `teardown`). Those keys are not instruments: the phases are implementation details of one runtime, so they stay a developer-facing record. The same identity keys make the two layers' records joinable per actor. The checkpoint record is written on failure too, with `error.type` and the elapsed time of the step that failed; the restore record is written on success only. The restore phases are sequential and partition the total; a checkpoint's `snapshot`, `durable_dir` and `rootfs_upper` run concurrently on the paused guest, so the paused window costs their maximum, not their sum. +Both ateoms write records with the same two messages from inside the `ateom_restore` and `ateom_checkpoint` phases, under `ateom.actor.restore.duration.` and `ateom.actor.checkpoint.duration.`. The phases are each runtime's own and are listed in `cmd/ateom-gvisor/phaselog.go` and `cmd/ateom-microvm/phaselog.go`; `internal/ateomphaselog` renders the record. They are not instruments. The same identity keys make the two layers' records joinable per actor, and a failed operation's record carries `error.type`. ateapi's `Actor state changed` is written once per committed actor state transition. ateapi owns the state machine, so this is where an actor's state and the time it reached it come from: @@ -177,9 +189,20 @@ Creating an actor counts as a change. A new actor is born suspended, so it gets The counter carries no actor identity, so this record is the only way to attribute a crash to one agent. The decision-point line that precedes it (`Setting Actor to crashed due to error`) carries only `ate.atespace` and `ate.actor.name`: it is written before the Actor is loaded, so no uid exists yet. +ateom-gvisor's `Sandbox has no processes left while the actor is hosted` is a WARN written when a running actor's gVisor sandbox has died: the sentry crashed, was killed, or was OOM-killed. The ateom's usage sweep finds it, so it shows up within one sweep (`--usage-sample-interval`, a minute by default), once per activation. `ate.sandbox.oom_kills` is the cgroup's `oom_kill` count, so a non-zero value points at the kernel OOM killer: + +```json +{"time":"…","level":"WARN","msg":"Sandbox has no processes left while the actor is hosted", + "ate.atespace":"ate-demo-counter","ate.actor.name":"counter-1","ate.actor.uid":"8f2a…", + "ate.template.atespace":"ate-demo-counter","ate.template.name":"counter", + "ate.sandbox.oom_kills":3} +``` + +The actor still reports `RUNNING` after this record; nothing moves it to `CRASHED` yet ([#2211](https://github.com/agent-substrate/substrate/issues/2211)). It does not fire when an application container exits while the sentry is still up, or during a checkpoint or terminate. The count covers the life of the cgroup leaf, so a leaf left behind by an earlier activation of the same actor carries its count forward. + #### The same records over OTLP -With `OTEL_LOGS_EXPORTER=otlp` these records go out as OTLP log events instead of stdout, so a collector reads them without knowing substrate's stdout envelope. Unset or `none` keeps them on stdout. The Helm chart selects `otlp` when `otel.logs.enabled` is true and a logs endpoint resolves. `otlp,console` writes both, as kind does for debugging. `console` here is the component's own JSON log line, not the OTel SDK's console exporter format. The value is a comma-separated list: names are trimmed, lowercased, and counted once, and an unknown or empty item logs a warning and is skipped while the rest apply. `none` with an exporter, or a value that names no known exporter, logs a warning and keeps the default, `none`. The ateoms export through [the ateom relay](#the-ateom-otlp-relay), which carries logs, traces, and metrics. atecontroller copies `OTEL_LOGS_EXPORTER` into each worker pod, so the same selection applies to ateapi and the ateoms. +With `OTEL_LOGS_EXPORTER=otlp` these records go out as OTLP log events instead of stdout, so a collector reads them without knowing substrate's stdout envelope. Unset or `none` keeps them on stdout. The Helm chart selects `otlp` when `otel.logs.enabled` is true and a logs endpoint resolves. `otlp,console` writes both, as kind does for debugging. `console` here is the component's own JSON log line, not the OTel SDK's console exporter format. The value is a comma-separated list: names are trimmed, lowercased, and counted once, and an unknown or empty item logs a warning and is skipped while the rest apply. `none` with an exporter, or a value that names no known exporter, logs a warning and keeps the default, `none`. ateapi emits lifecycle events and the ateoms usage samples, on the same switch. The ateoms export through [the ateom relay](#the-ateom-otlp-relay), which carries logs, traces, and metrics. atecontroller copies `OTEL_LOGS_EXPORTER` into each worker pod, so the same selection applies to ateapi and the ateoms. Three `event.name` values, which is the OTLP LogRecord's own field rather than an attribute: @@ -189,9 +212,9 @@ Three `event.name` values, which is the OTLP LogRecord's own field rather than a | `ate.actor.crashed` | `Actor crashed` | 17 | the same keys | | `ate.actor.usage_sampled` | `Actor usage sampled` | 9 | the five identity keys, `ate.workerpool.*`, `ate.sandbox.class`, `ate.stats.*`, `ate.actor.epoch` | -`ate.actor.usage_sampled` is the ateoms' record: one per actor per sampling period, plus an `initial` and a `final` per activation, told apart by `ate.stats.kind`. Its timestamp is when the measurement was read. Its measurements are named after the `ate.actor.stats.*` instruments and share their units, so `ate.stats.cpu.time` is seconds. They are absent, not zero, while the actor is not measurable, which the record says with `ate.stats.source` unspecified. `ate.stats.cpu.time` restarts at zero with each `ate.actor.epoch`, the unix-nano time the activation began, so a lifetime figure is the sum over epochs of each epoch's highest value; `ate.stats.memory.usage` and `ate.stats.memory.working_set` are absolute, and `ate.stats.memory.peak` is as the source reports it. The same measurements ride `WorkloadStatsSample` on the stats RPCs, with the epoch beside them. +`ate.actor.usage_sampled` is the ateoms' record: one per actor per sampling interval, plus an `initial` record per activation and a `final` one when the ateom tears it down, told apart by `ate.stats.kind`. Its timestamp is when the measurement was read. Its measurements are named after the `ate.actor.stats.*` instruments and share their units, so `ate.stats.cpu.time` is seconds. They are absent, not zero, while the actor is not measurable, which the record says with `ate.stats.source` unspecified. `ate.stats.cpu.time` restarts at zero with each `ate.actor.epoch`, the unix-nano time the activation began, so a lifetime figure is the sum over epochs of each epoch's highest value; `ate.stats.memory.usage` and `ate.stats.memory.working_set` are absolute, and `ate.stats.memory.peak` is as the source reports it. The same measurements ride `WorkloadStatsSample` on the stats RPCs, with the epoch beside them. -A crash is its own name because an event name promises a set of attributes and a crash has a different severity and shape. There is no name per state: `ate.actor.state` already says which transition happened, so a consumer still selects on that one attribute and needs no map from a name to a state. All three names are in [`docs/metrics/registry/events.yaml`](metrics/registry/events.yaml), which `make verify` checks. +A crash is its own name because an event name promises a set of attributes and a crash has a different severity and shape. There is no name per state: `ate.actor.state` already says which transition happened, so a consumer still selects on that one attribute and needs no map from a name to a state. All three names are in [`docs/metrics/registry/events.yaml`](metrics/registry/events.yaml), which `hack/verify-all.sh` checks. The attributes are the same flat `ate.*` keys as the stdout record, so they arrive as real log attributes with no transform in front of them. Trace context is not among them: it goes on the record's own `TraceId` and `SpanId` fields, where the stdout record's top-level `trace_id`/`span_id` would be mapped to anyway. The instrumentation scope is `github.com/agent-substrate/substrate/internal/actorevent` for every actor event. Select or drop a stream by `event.name`: the lifecycle events must never be sampled, the usage samples may be. @@ -203,31 +226,31 @@ The attributes are the same flat `ate.*` keys as the stdout record, so they arri ### Per-Actor Usage Events -atelet emits one usage record per **executing** actor per sampling tick, from the same sweep that feeds the [`ate.actor.stats.*` metrics](#the-metric-registry). The two are the halves of one split: the metrics aggregate to the bounded template/pool label set a TSDB can hold, and everything carrying actor identity travels here, on the log stream, where cardinality is free. An idle fleet is silent by design — a worker with no executing actor emits nothing. +Each ateom samples its actors on its own timer and writes [`ate.actor.usage_sampled`](#the-same-records-over-otlp) for each one, in this order: an `initial` record when an activation becomes measurable, a `periodic` record per interval, and a `final` record when the ateom tears the activation down: a checkpoint, a terminate, a graceful shutdown, or a re-host on a micro-VM. The sampler reads an actor only after its initial reading and until its final record; until then the discovery read reports it as pending. A failed first reading writes no `initial` record, and periodic records start anyway, so an unreachable guest is still reported. These records and the [`ate.actor.stats.*` metrics](#the-metric-registry) are the halves of one split: the metrics aggregate to the bounded template and pool label set a TSDB can hold, and everything carrying actor identity travels here, on the log stream, where cardinality is free. + +The stdout record carries the same flat `ate.*` keys as the OTLP event, dated when the measurement was read: ```json -{"time":"…","level":"INFO","msg":"Actor usage sample", - "logging.googleapis.com/labels":{ - "ate.atespace":"ate-demo-counter","ate.actor.name":"counter-1","ate.actor.uid":"8f2a…", - "ate.template.atespace":"ate-demo-counter","ate.template.name":"counter", - "ate.workerpool.namespace":"ate-demo-counter","ate.workerpool.name":"counter-pool"}, - "kind":"periodic","sandbox_class":"gvisor","source":"cgroup", - "memory_current_bytes":39845888,"memory_peak_bytes":52428800, - "memory_working_set_bytes":31457280,"cpu_usage_usec":196776, - "observed_at_unix_nano":1788903245000000000} +{"time":"…","level":"INFO","msg":"Actor usage sampled", + "ate.atespace":"ate-demo-counter","ate.actor.name":"counter-1","ate.actor.uid":"8f2a…", + "ate.template.atespace":"ate-demo-counter","ate.template.name":"counter", + "ate.workerpool.namespace":"ate-demo-counter","ate.workerpool.name":"counter-pool", + "ate.sandbox.class":"gvisor","ate.stats.source":"cgroup","ate.stats.kind":"periodic", + "ate.actor.epoch":1788903185000000000, + "ate.stats.memory.usage":39845888,"ate.stats.memory.working_set":31457280, + "ate.stats.cpu.time":0.196776,"ate.stats.memory.peak":52428800} ``` -* **Identity** rides in the same label group as the actor lifecycle events and container logs (`labels` off GCE, `logging.googleapis.com/labels` on GCE, which Cloud Logging promotes into `LogEntry.labels`), so every [query dimension above](#centralized-logging-backends-multi-dimensional-aggregation) applies unchanged, and one filter returns an actor's output, transitions, and usage interleaved. The pool pair is present when the worker pod resolved to a WorkerPool and absent otherwise, the same rule as on the metrics. -* **Consumers filter on** `msg` plus `kind`. `kind` is `periodic` today; future kinds (lifecycle brackets) will join on the same record shape: +* **Consumers filter on** the message and the keys, which sit at the top level rather than in the label group: ```text -labels."ate.actor.uid"="8f2a…" AND jsonPayload.msg="Actor usage sample" +jsonPayload.msg="Actor usage sampled" AND jsonPayload."ate.actor.uid"="8f2a…" ``` -* **The measurements** mirror the wire sample. `memory_current_bytes` and `memory_working_set_bytes` are point-in-time (current includes reclaimable page cache; the working set does not, and is the figure to compare against a memory limit). `memory_peak_bytes` and `cpu_usage_usec` **accumulate within the current epoch** (peak reads the cgroup's `memory.peak`, so it is zero on kernels older than 5.19 — unmeasured, not measured-as-zero), whose boundary depends on `source`: the `cgroup` source (host cgroup, gVisor) restarts both at zero on every restore, while the `guest-agent` source (inside the micro-VM) keeps counting across restores. Either way, per-actor CPU over a window is the increase between two samples, never a sum of raw values — and a decrease means the counter reset, not negative usage. -* **Cadence and switch**: `--actor-stats-poll-interval` on atelet governs the sweep (default `1m`, floor `50s`, `0` disables the metrics and the events together). -* **No trace context**: the sweep serves no request, so there is no span to join — the [Joining Logs to Traces](#joining-logs-to-traces) fields are absent on these records. -* **Delivery is best-effort, decoupled from node health**: the records ride atelet's stdout behind a bounded queue, so a stalled log consumer costs events (with a `Usage events dropped` warning on recovery), never the sweep or the metrics. The feed does not honor atelet's `--log-level` — a node quieted to `warn` keeps emitting usage data; the interval flag is the only switch. +* **The measurements** follow [the rules above](#the-same-records-over-otlp): absent while the actor is pending, CPU in seconds since the activation began, memory usage and working set absolute. An activation that ends any other way, as by a crash, has no `final` record, so take the highest CPU per epoch rather than wait for one. The `final` reading is taken just before the teardown, bounded on a micro-VM to 10ms. When it fails or times out, the `final` record repeats the newest measured sample, dated when that sample was read, or is pending when none was measured. +* **Cadence**: `--usage-sample-interval` on the ateom sets the timer (default `1m`, at least `50s`). Each sweep writes one periodic record per actor, and `GetActiveWorkloadStats` serves the latest sample rather than reading again. +* **Trace context**: `initial` records come from the Run or Restore call (on a micro-VM, from a goroutine the call starts, so the record can land just before or after the call returns) and `final` records from the call that tears the activation down, a Checkpoint, a Terminate, or, on a micro-VM, the Run or Restore that re-hosts it; both carry that call's span, like any record in [Joining Logs to Traces](#joining-logs-to-traces). `periodic` records come from the sampler, and a shutdown's `final` records from the drain, which serve no request, so they carry none. +* **Delivery is best-effort**: the stdout record ignores `--log-level` and sits behind a bounded queue, so a stalled log consumer costs records, with a warning, never the sampler. **Do not put actor identity on a log-based metric.** Aggregating these events in the log store is what they are for, but a log-based metric built over them must label only by the bounded set (template, sandbox class, source, pool) — promoting `ate.actor.uid` or `ate.actor.name` into a metric label reintroduces exactly the per-actor cardinality this split keeps out of the TSDB. @@ -237,7 +260,7 @@ labels."ate.actor.uid"="8f2a…" AND jsonPayload.msg="Actor usage sample" Agent Substrate emits foundational OpenTelemetry system and server metrics to monitor the overall health and performance of the control plane services. Every metric below is emitted by a service binary over OTLP and is **independent of the deployment** — a Kind dev cluster gets the same instruments as production; only the backend differs (see [Where Telemetry Goes](#4-where-telemetry-goes)). -> [`docs/metrics/registry/metrics.yaml`](metrics/registry/metrics.yaml) defines each instrument. Read it when you need all the labels, the bucket limits, or the permitted values of a label. The table below does not have each instrument. The request-parking instruments and the actor resource-usage instruments (`ate.actor.stats.*`) are in the registry only. Refer to [The metric registry](#the-metric-registry); per-actor usage detail is the [events channel](#per-actor-usage-events)'s job. +> [`docs/metrics/registry/metrics.yaml`](metrics/registry/metrics.yaml) defines each instrument. Read it when you need all the labels, the bucket limits, or the permitted values of a label. The table below does not have each instrument. The request-parking instruments and the actor resource-usage instruments (`ate.actor.stats.*`) are in the registry only. Refer to [The metric registry](#the-metric-registry); per-actor usage detail is on the [usage records](#per-actor-usage-events). | Metric | Emitted by | Type | Measures | |--------|------------|------|----------| @@ -252,7 +275,7 @@ Agent Substrate emits foundational OpenTelemetry system and server metrics to mo | `ate.workerpool.workers` | ateapi | up/down counter | live worker count per pool, split by state (`idle`/`partial`/`at_capacity`/`unschedulable`) and sandbox class to provide fleet capacity and saturation at a glance | | `ate.actor.lifecycle.operation.duration` | ateapi | histogram | how long each actor operation (create/resume/suspend/pause/delete/revert) takes and whether it failed (`error.type` present = failure, absent = success); labeled by operation, template, pool (`ate.workerpool.namespace` + `ate.workerpool.name`), sandbox class, and snapshot kind and scope on resume; already-running resume no-ops are not recorded so the histogram tracks actual activations, not router traffic | | `ate.scheduler.assignment.duration` | ateapi | histogram | time of one attempt to assign a worker to an actor, with one record for each resume (ateapi retries an attempt that loses a race for a worker and records only the last attempt), with the outcome (`assigned` / `no_capacity` / `error`), the assigned pool (`ate.workerpool.namespace` + `ate.workerpool.name`) and sandbox class to catch scheduling latency and capacity starvation problems | -| `ate.actor.restore.duration` | atelet | histogram | how long each phase of a restore takes on the worker node, which is where cold-start latency actually goes once ateapi hands off (labels `ate.snapshot.phase`, `ate.snapshot.kind`, `ate.snapshot.scope`, `ate.template.atespace`, `ate.template.name`, `ate.sandbox.class`) | +| `ate.actor.restore.duration` | atelet | histogram | how long each phase of a restore takes on the worker node, which is where cold-start latency actually goes once ateapi hands off (labels `ate.snapshot.phase`, `ate.snapshot.kind`, `ate.snapshot.fidelity`, `ate.template.atespace`, `ate.template.name`, `ate.sandbox.class`) | | `ate.actor.checkpoint.duration` | atelet | histogram | the same phase breakdown for writing a snapshot, so a slow suspend can be attributed to ateom or to the upload (same labels as the restore histogram) | | `ate.imagecache.requests` | atelet | counter | image lookups in the node-local image cache, by outcome (`ate.imagecache.outcome`), with `error.type` on the `error` outcome. A miss pays for the pull and the unpack, so the hit ratio per node is a leading indicator of resume latency | @@ -277,7 +300,7 @@ For `ate.imagecache.requests`: The three snapshot labels are orthogonal and mean the same thing on every histogram that carries them: * `ate.snapshot.kind`: which snapshot the operation reads or writes. `local` (node-local, written by a pause), `latest` (the actor's own durable snapshot), `golden` (the template's image), or `boot` (from scratch, so it never appears on the atelet histograms). -* `ate.snapshot.scope`: what content it covers. `full` or `data`. +* `ate.snapshot.fidelity`: what state it holds. `volumes`, `rootfs`, or `memory`. * `ate.snapshot.phase`: which step was timed. `volume_mount`, `manifest_fetch`, `sandbox_assets`, `download`, `oci_unpack`, `ateom_restore` on restore; `sandbox_assets`, `ateom_checkpoint`, `persist` on checkpoint; `total` on both. **Phases overlap and do not sum to `total`.** The download runs concurrently with the asset fetch and OCI unpack, so each is an independent observation; use `total` as the denominator. A phase that never started is absent rather than zero. @@ -297,7 +320,7 @@ hack/verify/metrics.sh # the command that CI uses weaver registry check -r docs/metrics/registry # the same command, direct ``` -`make verify` runs the script. It uses a local `weaver` binary if there is one, and the official image if there is none. +`hack/verify-all.sh` runs the script. It uses a local `weaver` binary if there is one, and the official image if there is none. **To add or change an instrument:** follow [Metrics Best Practices](dev/best-practices/metrics.md), which covers the instrument choice, the labels, the Go, the test, and the registry entry; then change `metrics.yaml` and run the script. @@ -404,7 +427,7 @@ Telemetry is emitted the same way everywhere; only the backend differs between a | Path | service → in-cluster `opentelemetry-collector` | service → Google Managed Prometheus (GMP) | | Metrics | collector Prometheus exporter on `:8889` | Google Cloud Monitoring | | Traces | Jaeger UI | Google Cloud Trace | -| Logs | pod stdout; ateapi's [actor lifecycle events](#the-same-records-over-otlp) also to the collector's `debug` exporter | pod stdout. No OTLP logs unless `OTEL_LOGS_EXPORTER` is set | +| Logs | pod stdout; the [actor events](#the-same-records-over-otlp), ateapi's lifecycle records and the ateoms' usage samples, also to the collector's `debug` exporter | pod stdout. No OTLP logs unless `OTEL_LOGS_EXPORTER` is set | | Dashboards | Not supported | Google Cloud Monitoring (see [Dashboards](#5-dashboards)) | > In Kind, `ateapi`, `atelet`, `ate-controller`, and `atenet-router` are pointed at the in-cluster collector, and the controller propagates the endpoint to the ateom worker pods it creates, so all component telemetry lands locally. diff --git a/docs/operator-install.md b/docs/operator-install.md index a8b0b319ba..531dbecf67 100644 --- a/docs/operator-install.md +++ b/docs/operator-install.md @@ -267,6 +267,7 @@ command. A setting with no default is unset unless something supplies it. | Key | Environment variable | Flag | Default | Applies to | |----------------------------------------------|-------------------------------------------------|----------------------------------------------------|-----------------------|------------------------------------------| | `actorJWT.algorithm` | `ACTOR_JWT_ALGORITHM` | `--actor-jwt-algorithm` | `ES256` | (global) | +| `actorJWT.issuer` | `ACTOR_JWT_ISSUER` | `--actor-jwt-issuer` | — | (global) | | `ateapi.expectedJWTIssuer` | `EXPECTED_JWT_ISSUER` | `--ateapi-expected-jwt-issuer` | — | (global) | | `ateapi.postgres.cloudsql.gsa` | `ATE_API_POSTGRES_CLOUDSQL_GSA` | `--ateapi-postgres-cloudsql-gsa` | — | (global) | | `ateapi.postgres.cloudsql.iamAuth` | `ATE_API_POSTGRES_CLOUDSQL_IAM_AUTH` | `--ateapi-postgres-cloudsql-iam-auth` | — | (global) | @@ -279,6 +280,7 @@ command. A setting with no default is unset unless something supplies it. | `ateapi.postgres.readWrite.role` | `ATE_API_POSTGRES_READ_WRITE_ROLE` | `--ateapi-postgres-read-write-role` | `substrate_readwrite` | (global) | | `ateapi.postgres.schema` | `ATE_API_POSTGRES_SCHEMA` | `--ateapi-postgres-schema` | — | (global) | | `ateapi.postgres.serverCAFile` | `ATE_API_POSTGRES_SERVER_CA_FILE` | `--ateapi-postgres-server-ca-file` | — | (global) | +| `ateapi.postgres.storageClass` | `ATE_API_POSTGRES_STORAGE_CLASS` | `--ateapi-postgres-storage-class` | — | (global) | | `atenet.dataplane` | `ATE_ATENET_DATAPLANE` | `--atenet-dataplane` | `envoy` | (global) | | `atenet.egress.additionalExtprocService` | `ATE_ADDITIONAL_EGRESS_EXTPROC_SERVICE` | `--experimental-additional-egress-extproc-service` | — | (global) | | `atenet.egress.credentialProvider` | `ATE_CREDENTIAL_PROVIDER` | `--credential-provider` | — | (global) | diff --git a/docs/resource-model.md b/docs/resource-model.md new file mode 100644 index 0000000000..b4a90adf0b --- /dev/null +++ b/docs/resource-model.md @@ -0,0 +1,110 @@ +# Substrate Resource Model + +Substrate exposes resources through two APIs: + +- a **gRPC API** (`pkg/proto/ateapipb/ateapi.proto`), served by the + Substrate API server. Its resources follow the + [API Style Guide](api-style-guide.md). +- The **Kubernetes API**, through CRDs in `pkg/api/v1alpha1`, for + managing compute infrastructure. Its resources follow the + [Kubernetes API conventions](https://github.com/kubernetes/community/blob/main/contributors/devel/sig-architecture/api-conventions.md) + +## Scope and parent + +Every resource belongs either to the whole Substrate installation or to a +single atespace. A **global** resource, such as an `Atespace` or a `Worker`, +belongs to the whole Substrate installation. An **atespace-scoped** resource, +such as an `Actor` or a `Tag`, belongs to one atespace, and atespaces keep +their resources apart from each other. + +Some resources exist only as part of another resource. These are +**subresources**, and the resource they belong to is their **parent**. For +example, an `EgressPolicy` is part of an `Actor`. A subresource is always +reached through its parent and goes away when its parent does. + +A subresource's parent can also be **Global**, which stands for the whole +Substrate installation. Global is not a resource. It is the root of the +hierarchy below. + +## Hierarchy + +``` +Global +├── AccessPolicy "default" subresource (singleton) +├── Atespace +│ ├── AccessPolicy "default" subresource (singleton) +│ ├── ActorTemplate +│ ├── Actor +│ │ └── EgressPolicy "default" subresource (singleton) +│ └── Tag +└── Worker + └── ActorAssignment subresource (one per hosted Actor, managed by Substrate) +``` + +## Top-level resources + +| Resource | Scope | Cardinality | Written by | +| :--- | :--- | :--- | :--- | +| `Atespace` | Global | 0..N | Clients | +| `ActorTemplate` | Atespace | 0..N | Clients | +| `Actor` | Atespace | 0..N | Clients | +| `Tag` | Atespace | 0..N | Clients | +| `Worker` | Global | 0..N | Substrate | + +## Subresources + +| Subresource | Parent | Cardinality | Written by | +| :--- | :--- | :--- | :--- | +| `EgressPolicy` | `Actor` | 0..1, named `default` | Clients | +| `AccessPolicy` | Global | 0..1, named `default` | Clients | +| `AccessPolicy` | `Atespace` | 0..1, named `default` | Clients | +| `ActorAssignment` | `Worker` | 0..N | Substrate | + +## Kubernetes resources + +| CRD | Scope | +| :--- | :--- | +| `WorkerPool` | Namespaced | +| `SandboxConfig` | Cluster | +| `CSIDriverConfig` | Cluster | + +## Relationships + +```mermaid +classDiagram + namespace kube-apiserver { + class WorkerPool { + <> + } + class SandboxConfig { + <> + } + class WorkerPod + } + + namespace substrate-api-server { + class Atespace + class AtespaceAccessPolicy + class ActorTemplate + class Actor + class EgressPolicy + class Tag + class Worker + class ActorAssignment + } + + Atespace "1" *-- "0..1" AtespaceAccessPolicy + Atespace "1" o-- "*" ActorTemplate + Atespace "1" o-- "*" Actor + Atespace "1" o-- "*" Tag + Actor "1" *-- "0..1" EgressPolicy + Worker "1" *-- "*" ActorAssignment + Actor "*" --> "1" ActorTemplate : actor_template + Actor "*" --> "0..1" Tag : source_tag + Tag "*" --> "1" Actor : source_actor + ActorAssignment "0..1" --> "1" Actor : actor + ActorTemplate ..> WorkerPool : worker_selector + ActorTemplate ..> SandboxConfig : sandbox_config + WorkerPool "1" *-- "*" WorkerPod + Worker "1" --> "1" WorkerPod +``` diff --git a/docs/security.md b/docs/security.md new file mode 100644 index 0000000000..168bf42535 --- /dev/null +++ b/docs/security.md @@ -0,0 +1,92 @@ +# Substrate Security + +Agent Substrate executes untrusted actors at scale in a multi-tenant environment where workloads are dynamic and memory states migrate continuously across physical infrastructure. All security controls must operate within Substrate's sub-100ms cold-start and resume latency budget. This document defines the high-level security principles. More detailed security risks are documented [in the threat model](threat-model.md). + +## Security Scope + +Substrate provides the foundational base for agent security: + +- Sandboxing: built-in compute, storage, and networking separation. +- Identity: identity issuance is built-in and designed to federate with other identity systems. See the [Substrate Identity Critical User Journeys](https://docs.google.com/document/d/1seJNYQmsQYLCrYYzB9AmuxpSGSPCC3-u3tPNFsK2edc/edit?resourcekey=0-D6q8_0vqF6LPEk0BxWMTaA&tab=t.0#heading=h.6qjmtncwtihi). +- Gateway: built-in ingress and egress filtering that can do destination filtering and credential injection. + +We expect the application layer creating actors using substrate to be responsible for: + +- Handling human-to-actor authentication and authorization. +- Registering and maintaining information about agents in an agent registry. +- Maintaining agent hierarchy/state to be able to reconstruct agent lineage: who made this sub-sub-sub agent. + +## Security Extensibility + +More advanced security can be integrated using the ingress and egress extension points. This is where advanced prompt injection scanning, dynamic LLM-driven authorization decisions, data leak prevention (DLP), or cloud IAM policy enforcement can be added: + +- **Egress Extensibility**: The outbound path is a fully pluggable Policy Enforcement Point (PEP). Operators can either extend the existing egress proxy using Envoy's `ext_proc` callout mechanism to invoke external inspection and authorization services, or completely replace the egress gateway (`atenet-egress`) with an external solution. +- **Ingress Extensibility**: Ingress similarly supports `ext_proc` callouts for external authentication, authorization, and payload inspection. However, unlike egress, the ingress gateway (`atenet-router`) cannot be completely replaced from the outside: Substrate's core orchestration relies on ingress routing hooks to manage actor state, resolve worker assignments, and trigger on-demand cold starts or resume operations from snapshots to wake up agents. + +## Core Security Principles + +**Status Legend:** + +- Met: Fully implemented in the current codebase. +- In Progress: Partially implemented or under active development. +- Future: Planned architecture and roadmap capabilities not yet implemented. +- Out of Scope: Not substrate’s job, handled by a layer outside substrate. + +### Defensible Multi-Tenancy + +Substrate is designed to safely run mutually hostile, untrusted actors from disparate tenants side-by-side on the same physical host to maximize resource utilization and bin-packing efficiency. + +| Principle | Description | Status | +| :--- | :--- | :--- | +| **Always Sandboxed** | Every actor is required to run inside a strong compute sandbox. A linux container is not sufficient or supported. Our security design relies on strong containment. | Met (gVisor, CHV micro-VM) | +| **Per-Actor Storage Segregation** | Storage and persistent volume state must be strictly segregated per actor. Actors cannot traverse, enumerate, or access persistence belonging to other actors. Restoring a snapshot must re-validate the actor's current tenant authorization before execution resumes. | Met (Segregation)
Future (Per-Actor Encryption, Restore Authorization) | +| **Per-Actor Network Isolation** | Guest network stacks are segregated per actor. Lateral peer-to-peer network communication between co-located actors on the same host or across the cluster is blocked by default; inter-actor communication must route through authenticated, policy-enforced gateways. | Met (Private netns, atunnel) | +| **Delete Guarantee** | Reset, scrub, and sanitize local ephemeral volumes, cached state, and worker runtime environments between actor scheduling cycles to prevent cross-tenant data leakage. | In Progress (atelet resetActorDirs) | +| **Node Blast Radius Containment** | A compromise of an individual sandbox guest or node supervisor restricts impact to the actors and identities actively co-located on that node. | Future | +| **Atespace Separation** | The Substrate equivalent of a Kubernetes namespace that is used to group resources and partition object storage snapshot paths. It does not yet enforce runtime API authorization, inter-atespace network policies, or resource quotas, but an embedded OpenFGA relationship-based access control (ReBAC) model is actively being integrated. | In-Progress (OpenFGA) | + +Other forms of tenancy: + +- **[Possible, but not currently supported] Multiple Substrate installations inside a single Kubernetes cluster**, with nodes separated by taints/tolerations. This gets you Substrate API separation but gives up some node bin-packing efficiency. Care needs to be taken at the Kubernetes layer to keep everything separate. +- **[May be possible, but not currently supported] Attaching multiple Kubernetes clusters to a single Substrate installation**. This may be possible with some work today, but is not supported. We’re expecting a single large K8s cluster to be capable of running 100s of millions of mostly-idle agents. +- **[Out of Scope] Multi-tenancy inside an actor**. There is no meaningful security boundary inside an actor. Actors are designed to be cheap and efficient enough to run many, including one per human user, so that it’s possible to run them without handling content for multiple users inside the same actor. +- **[Not Supported] Per-Actor ingress/egress**. Actor-facing traffic proxies (`atenet-router` for ingress, `atenet-egress` for egress) are shared across all actors in the substrate installation by default. Due to the high cardinality and rapid lifecycle of actors, dedicated 1:1 proxies per actor are operationally and economically infeasible. While we may partition proxies further in the future to reduce sharing, by atespace for example, proxies will remain multi-tenant by default. + +### Identity + +| Principle | Description | Status | +| :--- | :--- | :--- | +| **Zero Ambient Credentials** | Default actor sandboxes execute completely credential-free. Actors may hold credential identifiers, encrypted credentials, or unprivileged tokens that need to be hydrated or decrypted out-of-band by Substrate infrastructure before accessing external services. | Met (no credentials inside sandbox) | +| **Distinct Workload Identity** | Every actor receives a short lifetime (1h) SPIFFE ID X.509 certificate. It is possible to distinguish an actor that was deleted and then re-created with the same name using their UID. | Met (X.509)
In-Progress (OIDC discovery) | +| **Universal Component mTLS** | All control plane and data plane components (`ate-api-server`, `atelet`, `ateom`, routers, tunnels) authenticate and communicate exclusively over mutual TLS using node- and service-attested identities. Unauthenticated traffic is rejected, exempting only local unauthenticated health/liveness probes (`/healthz`, `/readyz`). | Met (Mutual TLS)
In-Progress (Authz) | +| **Sub-Actor Provenance** | Delegated sub-actors inherit cryptographically signed lineage. Every tool invocation, data access, and egress transaction can be deterministically resolved to the root actor and execution chain. This provenance/hierarchy needs to be recorded in the system that calls substrate. | Out of Scope | + +### Storage + +| Principle | Description | Status | +| :--- | :--- | :--- | +| **Untrusted Storage** | Persistence layers are treated as untrusted environments. Remote storage is protected with encryption. Local snapshot caching on storage attached to the node is multi-tenant and protected from cross-tenant attacks with encryption and strict filesystem permissions. | Future (Storage is currently partitioned but is unencrypted) | +| **Dual-Party Snapshot Authorization** | Reading or writing snapshot storage requires dual-identity verification proving both the actor's identity and the node supervisor (`atelet`) operating on its behalf. Sandboxed guests cannot directly interact with snapshot storage. | Future | + +### Network + +| Principle | Description | Status | +| :--- | :--- | :--- | +| **No Access by Default** | Sandboxed actors and worker nodes launch with default-deny networking and have zero network or API access to control plane services (`ate-api-server`, Kubernetes API servers, PostgreSQL state store). DNS is currently allowed outbound by default, work on that is in progress. | Met (Loopback, private netns)
In-Progress (DNS filtering, see [#2075](https://github.com/agent-substrate/substrate/issues/2075)) | +| **Authenticated and Policy-Enforced Ingress** | Inbound invocations to actors terminate at authenticated edge routers (`atenet-router`), which authenticate callers and verify invocation policies before triggering on-demand actor cold-start or resume. | Met (atenet-router)
In-Progress (authentication) | +| **Fail-Closed Policy-Enforced Egress Gateway** | Outbound traffic is funneled through the multi-tenant egress proxy (`atenet-egress`), which validates actor identity, enforces destination host allowlists, executes TLS inspection or prompt filtering, and dynamically injects target credentials. If an egress policy enforcement point (PEP) is unconfigured/fails egress is dropped. Unsupported protocols are dropped, see [Egress Policy Table](https://docs.google.com/document/d/1XLCo9SMROddQzwsyEhdBNWuLYFtWWZWfYvGlIO1BcUI/edit?resourcekey=0-st2SqKp-O68nv5LeIUfA-g&tab=t.0#bookmark=id.pjnob1gppauk) for specifics. | Met (atenet-egress)
In-Progress (fail closed, UDP currently passthrough) | + +### Authorization + +| Principle | Description | Status | +| :--- | :--- | :--- | +| **Actor to Actor Authorization** | Built-in authorization policy for agent to agent communication will be added in the future. Before that exists users can install an authenticating reverse proxy, or implement authorization at the application level. | Future | +| **Atespace and Node-level containment** | Actors are grouped into logical Atespaces for declarative RBAC, resource quotas, and independent metadata/label authorization to prevent privilege escalation. | In Progress (embedded OpenFGA) | +| **Comprehensive Quotas & Anti-DoS Throttling** | The platform enforces strict limits across API call rates, memory/CPU allocations, and egress bandwidth to eliminate runaway actors and shared resource exhaustion. | Future | + +### Auditability and Response + +| Principle | Description | Status | +| :--- | :--- | :--- | +| **Structured Audit Trail** | All lifecycle transitions (start, suspend, resume, terminate), snapshot access, gateway routing events, and policy violations generate structured telemetry linked to cryptographic actor identity and parent lineage. Token log leaks are prevented with sanitization. | In-Progress (OTLP logging, shared principle)
Future (enhanced identity attribution, log leak protection) | +| **Dynamic Quarantine & Snapshot Tainting** | The control plane supports immediate revocation and network blackholing of misbehaving/compromised actors. Suspended states from compromised actors receive an immutable cryptographic taint, preventing resumption across the fleet. | Future | diff --git a/docs/upgrade.md b/docs/upgrade.md index 70b5e22388..6fc20f5fad 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -346,7 +346,7 @@ version did not change (see Checkout and environment) and the command rolled the running atelet in place. Last, build and push the new worker images. The refs print together -at the end. The one for your pool's `sandboxClass` goes into the clone +at the end. The one for your pool's `sandboxClasses[].name` goes into the clone in step 4: ```bash @@ -354,7 +354,7 @@ go run ./cmd/ate-setup publish worker-images ``` If you are using prebuilt images there is nothing to publish. The new worker image -is the release's `ateom-` image under the same repo and +is the release's `ateom-` image under the same repo and tag as the control plane, pinned by digest: ```bash @@ -376,15 +376,19 @@ else, including the `metadata.labels` the scheduler matches actors by, carries over as is. ```bash -NEW_IMAGE= +NEW_IMAGE= +# Only read when the old pool predates spec.sandboxClasses. +SANDBOX_CLASS= kubectl -n $NS get workerpool $OLD_WORKERPOOL -o json \ - | jq --arg name "$NEW_WORKERPOOL" --arg image "$NEW_IMAGE" --arg version "$NEW_VERSION" ' + | jq --arg name "$NEW_WORKERPOOL" --arg image "$NEW_IMAGE" --arg version "$NEW_VERSION" \ + --arg class "$SANDBOX_CLASS" ' {apiVersion, kind, metadata: {name: $name, namespace: .metadata.namespace, labels: .metadata.labels}, spec: (.spec + {workerImage: $image})} | .spec.template.nodeSelector["ate.dev/substrate-version"] = $version + | .spec.sandboxClasses //= [{name: $class}] ' \ | kubectl apply -f - ``` diff --git a/go.mod b/go.mod index 1b49790d18..cdc7a75d6d 100644 --- a/go.mod +++ b/go.mod @@ -62,6 +62,7 @@ require ( go.opentelemetry.io/otel/sdk/metric v1.45.0 go.opentelemetry.io/otel/trace v1.45.0 go.opentelemetry.io/proto/otlp v1.11.0 + golang.org/x/net v0.58.0 golang.org/x/sync v0.22.0 golang.org/x/sys v0.47.0 google.golang.org/api v0.274.0 @@ -241,7 +242,6 @@ require ( go.yaml.in/yaml/v3 v3.0.5 // indirect golang.org/x/crypto v0.56.0 // indirect golang.org/x/exp v0.0.0-20260718201538-764159d718ef // indirect - golang.org/x/net v0.58.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect golang.org/x/term v0.45.0 // indirect golang.org/x/text v0.41.0 // indirect diff --git a/hack/install-ate.sh b/hack/install-ate.sh index 8d046bce7b..529464f6ed 100755 --- a/hack/install-ate.sh +++ b/hack/install-ate.sh @@ -164,7 +164,9 @@ usage() { echo " (https://container.googleapis.com/v1/projects/.../clusters/...)," echo " else the cluster's OIDC discovery document" echo " ACTOR_JWT_ALGORITHM Signing algorithm of a newly created actor JWT pool: ES256 (default) | RS256." - echo " Use RS256 for relying parties that don't support ES256. Has no effect once the pool exists." + echo " Use RS256 (a 4096-bit RSA key) for relying parties that don't support ES256. Has no effect once the pool exists." + echo " ACTOR_JWT_ISSUER iss claim of actor JWTs, verbatim. Relying parties fetch /.well-known/openid-configuration." + echo " Default: https://idp..svc" echo "" echo "Benchmarks (see benchmarking/README.md for details and customization):" echo "" diff --git a/hack/render-manifests.sh b/hack/render-manifests.sh index a475d014ea..8215fcb31a 100755 --- a/hack/render-manifests.sh +++ b/hack/render-manifests.sh @@ -114,7 +114,7 @@ for src, docs in docs_by_source.items(): "# limitations under the License.\n" "\n" f"# DO NOT EDIT — generated from charts/{chart} by hack/render-manifests.sh.\n" - "# Run `make helm-template` to regenerate.\n" + "# Run `hack/render-manifests.sh` to regenerate.\n" "\n" ) with open(os.path.join(out_dir, src), "w") as out: @@ -134,7 +134,7 @@ if [ "${CHECK_MODE}" = "true" ]; then rm -f "${CHECK_TMP}/current/${file}" "${TMP_DIR}/out/${file}" done if ! diff -ruN "${CHECK_TMP}/current" "${TMP_DIR}/out" >/dev/null 2>&1; then - echo "manifests/ate-install/ is out of date. Run: make helm-template" >&2 + echo "manifests/ate-install/ is out of date. Run: hack/render-manifests.sh" >&2 diff -ruN "${CHECK_TMP}/current" "${TMP_DIR}/out" | head -60 >&2 || true exit 1 fi diff --git a/hack/setup-e2e-egress-tls-kind.sh b/hack/setup-e2e-egress-tls-kind.sh new file mode 100755 index 0000000000..47c36a160d --- /dev/null +++ b/hack/setup-e2e-egress-tls-kind.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit -o nounset -o pipefail + +# Test setup only: keep public origin verification and also trust local origins +# signed by the cluster's service-DNS CA. Run after installing the unified egress +# gateway and before its local-origin TLS tests. No CA signing key is mounted on +# the gateway. +context="${KUBECTL_CONTEXT:-kind-${KIND_CLUSTER_NAME:-kind}}" +kubectl_args=(--context "${context}" -n ate-system) +gateway="$(kubectl "${kubectl_args[@]}" get deployment atenet-egress -o jsonpath='{.spec.template.spec.containers[?(@.name=="envoy")].name}')" +if [[ -z "${gateway}" ]]; then + gateway="$(kubectl "${kubectl_args[@]}" get deployment atenet-egress -o jsonpath='{.spec.template.spec.containers[?(@.name=="agentgateway")].name}')" +fi +image="$(kubectl "${kubectl_args[@]}" get deployment atenet-egress -o jsonpath="{.spec.template.spec.containers[?(@.name==\"${gateway}\")].image}")" +sdsmint="$(kubectl "${kubectl_args[@]}" get deployment atenet-egress -o jsonpath='{.spec.template.spec.initContainers[?(@.name=="sdsmint")].name}')" +if [[ -z "${gateway}" || -z "${image}" ]]; then + echo "error: install the unified Envoy or AgentGateway egress before configuring E2E TLS origins" >&2 + exit 1 +fi +if [[ "${gateway}" == envoy && "${sdsmint}" != sdsmint ]]; then + echo "error: the Envoy egress is missing its sdsmint certificate service" >&2 + exit 1 +fi + +# AgentGateway is distroless, so use the pinned Envoy image as a shell-capable +# init helper. Its public roots plus the service-DNS CA are mounted only for +# this E2E test setup; production keeps the gateway's normal root behavior. +root_image="${image}" +if [[ "${gateway}" == agentgateway ]]; then + root_image="envoyproxy/envoy:v1.39-latest@sha256:57e14a549d7bd43c8d3f6d03e8cfa653e037d4b38e133acd9b54f38c524401b4" +fi + +# Use the helper image's public roots on every Pod start. The separate init +# container prevents re-runs from appending to an existing bundle. +kubectl "${kubectl_args[@]}" patch deployment atenet-egress --type=strategic --patch "$(cat < /run/e2e-egress-roots/ca-certificates.crt + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: [ALL] + volumeMounts: + - name: servicedns + mountPath: /run/servicedns.podcert.ate.dev + readOnly: true + - name: e2e-egress-upstream-roots + mountPath: /run/e2e-egress-roots + containers: + - name: ${gateway} + volumeMounts: + - name: e2e-egress-upstream-roots + mountPath: /etc/ssl/certs/ca-certificates.crt + subPath: ca-certificates.crt + readOnly: true + volumes: + - name: e2e-egress-upstream-roots + emptyDir: {} +EOF +)" +kubectl "${kubectl_args[@]}" rollout status deployment/atenet-egress --timeout=120s diff --git a/internal/actorlock/actorlock.go b/internal/actorlock/actorlock.go index 96312c6862..b822f6559e 100644 --- a/internal/actorlock/actorlock.go +++ b/internal/actorlock/actorlock.go @@ -83,6 +83,17 @@ func (l *Locks) Lock(ctx context.Context, actorUID string) bool { return false } +// Busy reports whether a lifecycle RPC holds or is waiting for the named +// actor's lock. It never blocks, so a reader that must not take the lock can +// still tell whether the actor is mid-operation. The answer may be stale by the +// time it returns: use it for advisory checks, not to decide lifecycle actions. +func (l *Locks) Busy(actorUID string) bool { + l.mu.Lock() + defer l.mu.Unlock() + _, ok := l.held[actorUID] + return ok +} + // Unlock releases the named actor's lock. func (l *Locks) Unlock(actorUID string) { l.release(actorUID, true) diff --git a/internal/actorlock/actorlock_test.go b/internal/actorlock/actorlock_test.go index 527a3c083a..726d13b8f4 100644 --- a/internal/actorlock/actorlock_test.go +++ b/internal/actorlock/actorlock_test.go @@ -60,6 +60,92 @@ func TestActorLocksSerializeTheSameActor(t *testing.T) { locks.Unlock("actor-a") } +// Busy is true from Lock to Unlock and only for the actor that holds it. +func TestActorLocksBusy(t *testing.T) { + locks := New() + if locks.Busy("actor-a") { + t.Fatal("Busy(actor-a) = true before any Lock") + } + if !locks.Lock(context.Background(), "actor-a") { + t.Fatal("could not take actor-a's lock") + } + if !locks.Busy("actor-a") { + t.Error("Busy(actor-a) = false while held") + } + if locks.Busy("actor-b") { + t.Error("Busy(actor-b) = true while only actor-a is held") + } + locks.Unlock("actor-a") + if locks.Busy("actor-a") { + t.Error("Busy(actor-a) = true after Unlock") + } +} + +// Busy counts waiters as well as the holder. A waiter that gives up, or that +// takes the lock over when the holder releases it, never leaves the actor +// looking idle while a lifecycle RPC still holds the lock. +func TestActorLocksBusyWaiter(t *testing.T) { + // startWaiter blocks a second Lock behind the holder and returns once it + // has registered, which is when the entry holds two references. + startWaiter := func(t *testing.T, locks *Locks, ctx context.Context) <-chan bool { + t.Helper() + got := make(chan bool, 1) + go func() { got <- locks.Lock(ctx, "actor-a") }() + deadline := time.Now().Add(5 * time.Second) + for { + locks.mu.Lock() + refs := locks.held["actor-a"].refs + locks.mu.Unlock() + if refs == 2 { + return got + } + if time.Now().After(deadline) { + t.Fatal("the second Lock never started waiting") + } + time.Sleep(time.Millisecond) + } + } + + t.Run("waiter gives up", func(t *testing.T) { + locks := New() + if !locks.Lock(context.Background(), "actor-a") { + t.Fatal("could not take actor-a's lock") + } + ctx, cancel := context.WithCancel(context.Background()) + got := startWaiter(t, locks, ctx) + cancel() + if <-got { + t.Fatal("the cancelled waiter took the lock") + } + if !locks.Busy("actor-a") { + t.Error("Busy(actor-a) = false after the waiter gave up, while the holder still holds it") + } + locks.Unlock("actor-a") + if locks.Busy("actor-a") { + t.Error("Busy(actor-a) = true after the holder released it") + } + }) + + t.Run("waiter takes over", func(t *testing.T) { + locks := New() + if !locks.Lock(context.Background(), "actor-a") { + t.Fatal("could not take actor-a's lock") + } + got := startWaiter(t, locks, context.Background()) + locks.Unlock("actor-a") + if !<-got { + t.Fatal("the waiter did not take the lock") + } + if !locks.Busy("actor-a") { + t.Error("Busy(actor-a) = false while the former waiter holds it") + } + locks.Unlock("actor-a") + if locks.Busy("actor-a") { + t.Error("Busy(actor-a) = true after the last holder released it") + } + }) +} + // The map must not grow by one entry per actor the worker has ever hosted. func TestActorLocksForgetIdleActors(t *testing.T) { locks := New() diff --git a/internal/actorlog/logger.go b/internal/actorlog/logger.go index 7ebc7f1a79..47885520c9 100644 --- a/internal/actorlog/logger.go +++ b/internal/actorlog/logger.go @@ -73,12 +73,9 @@ const ( labelsKeyGCE = "logging.googleapis.com/labels" ) -// LabelsKey returns the label group's spelling for this environment. Every -// emitter of the actor-identity label group (container logs, lifecycle -// events, usage events) must pick its key here, so they all promote into -// Cloud Logging the same way -- and so going vendor-neutral later means -// changing one function. -func LabelsKey(isOnGCE bool) string { +// labelsKeyFor returns the label group's spelling for this environment, so +// going vendor-neutral later means changing one function. +func labelsKeyFor(isOnGCE bool) string { if isOnGCE { return labelsKeyGCE } @@ -89,7 +86,7 @@ func LabelsKey(isOnGCE bool) string { func NewActorLogger(w io.Writer, isOnGCE bool) *ActorLogger { return &ActorLogger{ writer: w, - labelsKey: LabelsKey(isOnGCE), + labelsKey: labelsKeyFor(isOnGCE), } } diff --git a/internal/actorlog/logger_test.go b/internal/actorlog/logger_test.go index 7b6bf9d9a8..8d86caaa49 100644 --- a/internal/actorlog/logger_test.go +++ b/internal/actorlog/logger_test.go @@ -451,13 +451,13 @@ func mustSpanID(t *testing.T, s string) trace.SpanID { return id } -// TestLabelsKey pins the one place the label group's spelling is chosen: the +// TestLabelsKeyFor pins the one place the label group's spelling is chosen: the // GCE spelling is the key Cloud Logging promotes into LogEntry.labels. -func TestLabelsKey(t *testing.T) { - if got := LabelsKey(false); got != "labels" { - t.Errorf("LabelsKey(false) = %q, want labels", got) +func TestLabelsKeyFor(t *testing.T) { + if got := labelsKeyFor(false); got != "labels" { + t.Errorf("labelsKeyFor(false) = %q, want labels", got) } - if got := LabelsKey(true); got != "logging.googleapis.com/labels" { - t.Errorf("LabelsKey(true) = %q, want logging.googleapis.com/labels", got) + if got := labelsKeyFor(true); got != "logging.googleapis.com/labels" { + t.Errorf("labelsKeyFor(true) = %q, want logging.googleapis.com/labels", got) } } diff --git a/internal/ateattr/ateattr.go b/internal/ateattr/ateattr.go index 684889c18e..3d3531b864 100644 --- a/internal/ateattr/ateattr.go +++ b/internal/ateattr/ateattr.go @@ -164,7 +164,7 @@ const ( WorkerStateKey = attribute.Key("ate.worker.state") SandboxClassKey = attribute.Key("ate.sandbox.class") SnapshotKindKey = attribute.Key("ate.snapshot.kind") - SnapshotScopeKey = attribute.Key("ate.snapshot.scope") + SnapshotFidelityKey = attribute.Key("ate.snapshot.fidelity") SnapshotPhaseKey = attribute.Key("ate.snapshot.phase") ImageCacheOutcomeKey = attribute.Key("ate.imagecache.outcome") SchedulerOutcomeKey = attribute.Key("ate.scheduler.outcome") @@ -191,6 +191,11 @@ const ( StatsCPUTimeKey = attribute.Key("ate.stats.cpu.time") ) +// SandboxOOMKillsKey is memory.events' oom_kill for the sandbox cgroup leaf, on +// the record ateom-gvisor writes when a hosted actor's sandbox has no process +// left. Logs only: it is only ever recorded beside actor identity. +const SandboxOOMKillsKey = attribute.Key("ate.sandbox.oom_kills") + // Values for StatsKindKey. An initial or final sample brackets an activation; a // periodic one is the timer's. const ( @@ -324,25 +329,28 @@ const ( SnapshotKindBoot = "boot" ) -// Values for SnapshotScopeKey, mirroring ateletpb.SnapshotScope. +// Values for SnapshotFidelityKey, mirroring ateletpb.SnapshotFidelity. const ( - SnapshotScopeFull = "full" - SnapshotScopeData = "data" - SnapshotScopeUnknown = "unknown" + SnapshotFidelityVolumes = "volumes" + SnapshotFidelityRootfs = "rootfs" + SnapshotFidelityMemory = "memory" + SnapshotFidelityUnknown = "unknown" ) -// SnapshotScopeValue maps the wire enum onto its label value, shared so ateapi -// (which sets the scope) and atelet (which receives it) cannot drift. An -// unrecognized scope reports as unknown rather than stringified, so no wire -// value can widen the label set. -func SnapshotScopeValue(scope ateletpb.SnapshotScope) string { - switch scope { - case ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL: - return SnapshotScopeFull - case ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA: - return SnapshotScopeData +// SnapshotFidelityValue maps the wire enum onto its label value, shared so +// ateapi (which sets the fidelity) and atelet (which receives it) cannot +// drift. An unrecognized fidelity reports as unknown rather than stringified, +// so no wire value can widen the label set. +func SnapshotFidelityValue(fidelity ateletpb.SnapshotFidelity) string { + switch fidelity { + case ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES: + return SnapshotFidelityVolumes + case ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS: + return SnapshotFidelityRootfs + case ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY: + return SnapshotFidelityMemory default: - return SnapshotScopeUnknown + return SnapshotFidelityUnknown } } diff --git a/internal/ateattr/ateattr_test.go b/internal/ateattr/ateattr_test.go index 7531733924..f539f03fc9 100644 --- a/internal/ateattr/ateattr_test.go +++ b/internal/ateattr/ateattr_test.go @@ -160,7 +160,7 @@ func TestKeySpellings(t *testing.T) { {WorkerStateKey, "ate.worker.state"}, {SandboxClassKey, "ate.sandbox.class"}, {SnapshotKindKey, "ate.snapshot.kind"}, - {SnapshotScopeKey, "ate.snapshot.scope"}, + {SnapshotFidelityKey, "ate.snapshot.fidelity"}, {SnapshotPhaseKey, "ate.snapshot.phase"}, {ImageCacheOutcomeKey, "ate.imagecache.outcome"}, {SchedulerOutcomeKey, "ate.scheduler.outcome"}, @@ -396,9 +396,10 @@ func TestMetricLabelValues(t *testing.T) { {SnapshotKindLocal, "local"}, {SnapshotKindBoot, "boot"}, - {SnapshotScopeFull, "full"}, - {SnapshotScopeData, "data"}, - {SnapshotScopeUnknown, "unknown"}, + {SnapshotFidelityMemory, "memory"}, + {SnapshotFidelityVolumes, "volumes"}, + {SnapshotFidelityRootfs, "rootfs"}, + {SnapshotFidelityUnknown, "unknown"}, {SnapshotPhaseVolumeMount, "volume_mount"}, {SnapshotPhaseManifestFetch, "manifest_fetch"}, @@ -568,24 +569,25 @@ func TestWorkerPoolAttributes(t *testing.T) { }) } -// TestSnapshotScopeValue pins the enum-to-label mapping ateapi and atelet share. +// TestSnapshotFidelityValue pins the enum-to-label mapping ateapi and atelet share. // An unmapped enum value must report unknown rather than its stringified form, // which would let a wire value widen the label set. -func TestSnapshotScopeValue(t *testing.T) { +func TestSnapshotFidelityValue(t *testing.T) { tests := []struct { name string - scope ateletpb.SnapshotScope + scope ateletpb.SnapshotFidelity want string }{ - {name: "full", scope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL, want: SnapshotScopeFull}, - {name: "data", scope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA, want: SnapshotScopeData}, - {name: "unspecified", scope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_UNSPECIFIED, want: SnapshotScopeUnknown}, - {name: "value outside the enum", scope: ateletpb.SnapshotScope(9999), want: SnapshotScopeUnknown}, + {name: "memory", scope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, want: SnapshotFidelityMemory}, + {name: "volumes", scope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, want: SnapshotFidelityVolumes}, + {name: "rootfs", scope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS, want: SnapshotFidelityRootfs}, + {name: "unspecified", scope: ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED, want: SnapshotFidelityUnknown}, + {name: "value outside the enum", scope: ateletpb.SnapshotFidelity(9999), want: SnapshotFidelityUnknown}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := SnapshotScopeValue(tt.scope); got != tt.want { - t.Errorf("SnapshotScopeValue(%v) = %q, want %q", tt.scope, got, tt.want) + if got := SnapshotFidelityValue(tt.scope); got != tt.want { + t.Errorf("SnapshotFidelityValue(%v) = %q, want %q", tt.scope, got, tt.want) } }) } diff --git a/internal/atenet/headers_test.go b/internal/atenet/headers_test.go index b640ac8d1d..d984d1ad04 100644 --- a/internal/atenet/headers_test.go +++ b/internal/atenet/headers_test.go @@ -31,6 +31,8 @@ func TestParseTargetActor(t *testing.T) { {name: "empty actor", value: "team-a/", wantErr: true}, {name: "invalid atespace", value: "TEAM-A/actor-1", wantErr: true}, {name: "invalid actor", value: "team-a/ACTOR-1", wantErr: true}, + {name: "comma-joined actors", value: "demo/a,secret/x", wantErr: true}, + {name: "comma-separated with space", value: "demo/a, demo/b", wantErr: true}, } for _, tt := range tests { diff --git a/internal/ateom/register.go b/internal/ateom/register.go index 05bce5f288..4457fcba19 100644 --- a/internal/ateom/register.go +++ b/internal/ateom/register.go @@ -13,12 +13,14 @@ // limitations under the License. // Package ateom registers an ateom worker with the control plane through the -// node-local atelet, reporting its compute capacity and hardware identity. +// node-local atelet, reporting its compute capacity and the sandbox runtime +// its actors start under. package ateom import ( "context" "crypto/tls" + "errors" "fmt" "log/slog" "os" @@ -73,11 +75,16 @@ func fromDir(dir string, actors int) *ateletpb.WorkerResources { } } -// probeHardware returns the hardware identity that actors hosted by this ateom -// observe. -func probeHardware() *ateletpb.HardwareIdentity { - return &ateletpb.HardwareIdentity{ - Attributes: hardware.ProbeHost().GetAttributes(), +// defaultRuntime is the runtime actors hosted by this ateom start under: its +// sandbox class on this host's hardware. +// +// TODO: Report the pool's SandboxConfig so atelet can complete the identity +// with the digests of the default version's assets and list its other +// enabled versions as restorable_runtimes. +func defaultRuntime(sandboxClass string) *ateletpb.SandboxRuntime { + return &ateletpb.SandboxRuntime{ + SandboxClass: sandboxClass, + Version: hardware.ProbeHost(), } } @@ -112,7 +119,8 @@ func readLimit(path string) int64 { return value } -// ReportConfig is what an ateom needs to reach the atelet on its node. +// ReportConfig configures how an ateom reaches the node-local atelet and what +// capacity and sandbox runtime class it registers with. type ReportConfig struct { SocketPath string CredentialBundlePath string @@ -123,23 +131,30 @@ type ReportConfig struct { AteletSPIFFEID string // Actors is how many actors this ateom will host at once. Actors int + // SandboxClass is the class this ateom implements ("gvisor", "microvm"). + SandboxClass string } -// Report tells the node-local atelet what this ateom can supply and its -// hardware identity, retrying until it is accepted or ctx ends. +// Report tells the node-local atelet what this ateom can supply and the +// runtime its actors start under, retrying until it is accepted or ctx ends. // // Retrying is what makes a single report durable: atelet only accepts once the // control plane has recorded it, and the Worker record may not exist yet when // an ateom first comes up. Nothing else reports this, so giving up would leave // the Worker holding no capacity and hosting nothing. func Report(ctx context.Context, cfg ReportConfig) error { + if cfg.SandboxClass == "" { + // The control plane refuses a runtime without a class, and retrying + // would not change that. + return errors.New("capacity report: SandboxClass is required") + } tlsConfig, err := ateletdial.TLSConfig(cfg.CredentialBundlePath, cfg.TrustBundlePath, cfg.AteletSPIFFEID) if err != nil { return fmt.Errorf("capacity report: %w", err) } req := &ateletpb.RegisterWorkerRequest{ - Capacity: FromFiles(cfg.Actors), - Hardware: probeHardware(), + Capacity: FromFiles(cfg.Actors), + DefaultRuntime: defaultRuntime(cfg.SandboxClass), } err = retryReport(ctx, func() error { return reportOnce(ctx, cfg.SocketPath, tlsConfig, req) @@ -147,9 +162,9 @@ func Report(ctx context.Context, cfg ReportConfig) error { if err != nil { return err } - slog.InfoContext(ctx, "Registered worker capacity and hardware", + slog.InfoContext(ctx, "Registered worker capacity and sandbox runtime", slog.Any("capacity", req.GetCapacity()), - slog.Any("hardware", req.GetHardware())) + slog.Any("default_runtime", req.GetDefaultRuntime())) return nil } diff --git a/internal/ateom/register_test.go b/internal/ateom/register_test.go index ce39bd9b26..48b32bcaa0 100644 --- a/internal/ateom/register_test.go +++ b/internal/ateom/register_test.go @@ -72,13 +72,19 @@ func TestFromFilesMissing(t *testing.T) { } } -func TestProbeHardware(t *testing.T) { - got := probeHardware() - want := &ateletpb.HardwareIdentity{ - Attributes: map[string]string{hardware.AttrArchitecture: runtime.GOARCH}, +func TestDefaultRuntime(t *testing.T) { + got := defaultRuntime("gvisor") + // The class is the ateom's own; the compat identity is the host's, under + // the probe's schema version. + want := &ateletpb.SandboxRuntime{ + SandboxClass: "gvisor", + Version: &ateletpb.VersionedSandboxCompat{ + SchemaVersion: hardware.SchemaVersionV1, + Attributes: []*ateletpb.AttributeEntry{{Key: hardware.AttrArchitecture, Value: runtime.GOARCH}}, + }, } if diff := cmp.Diff(want, got, protocmp.Transform()); diff != "" { - t.Errorf("probeHardware() mismatch (-want +got):\n%s", diff) + t.Errorf("defaultRuntime() mismatch (-want +got):\n%s", diff) } } @@ -127,6 +133,7 @@ func TestReportFailsFastOnBadCredentials(t *testing.T) { SocketPath: filepath.Join(t.TempDir(), "atelet.sock"), CredentialBundlePath: filepath.Join(t.TempDir(), "does-not-exist.pem"), TrustBundlePath: filepath.Join(t.TempDir(), "also-missing.pem"), + SandboxClass: "gvisor", }) if err == nil { t.Fatal("Report() with unreadable credentials succeeded, want an error the caller can exit on") @@ -135,3 +142,18 @@ func TestReportFailsFastOnBadCredentials(t *testing.T) { t.Errorf("Report() retried a permanent failure until the deadline: %v", err) } } + +// A runtime without a class is refused by the control plane on every attempt, +// so an ateom that names none must exit rather than retry. +func TestReportFailsFastWithoutSandboxClass(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + err := Report(ctx, ReportConfig{SocketPath: filepath.Join(t.TempDir(), "atelet.sock")}) + if err == nil { + t.Fatal("Report() without a SandboxClass succeeded, want an error the caller can exit on") + } + if errors.Is(err, context.DeadlineExceeded) { + t.Errorf("Report() retried a permanent failure until the deadline: %v", err) + } +} diff --git a/internal/ateomnet/dns/dnshandler.go b/internal/ateomnet/dns/dnshandler.go new file mode 100644 index 0000000000..103b733b00 --- /dev/null +++ b/internal/ateomnet/dns/dnshandler.go @@ -0,0 +1,283 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dns + +import ( + "bytes" + "encoding/binary" + "log/slog" + "net" + "time" + + "github.com/agent-substrate/substrate/internal/ateomnet/dns/protocol" + "golang.org/x/net/dns/dnsmessage" +) + +// actionKind describes the transport-agnostic decision returned by the DNS +// protocol handler. +type actionKind string + +const ( + // actionDrop the DNS packet. + actionDrop actionKind = "Drop" + // actionReply immediately to the Actor request. + actionReply actionKind = "Reply" + // actionForward the packet upstream. + actionForward actionKind = "Forward" + // actionFailover to send the request to the next upstream. + actionFailover actionKind = "Failover" + // actionDeliver the DNS response to the Actor. + actionDeliver actionKind = "Deliver" +) + +// action is the decision returned by onUDPRequest, onUDPResponse, +// onTCPRequest, and onTCPResponse. +type action struct { + kind actionKind + payload []byte + upstreamID uint16 + clientAddr net.Addr + upstreamIdx int +} + +// queryPolicy decides whether a parsed DNS question is permitted. +type queryPolicy func(q dnsmessage.Question) bool + +// dnsHandler inspects DNS requests and responses and returns transport-agnostic +// routing decisions. +type dnsHandler struct { + pending *pendingRequests + limiter *limiter + allow queryPolicy +} + +func newDNSHandler(pending *pendingRequests, lim *limiter, allow queryPolicy) *dnsHandler { + if allow == nil { + allow = func(dnsmessage.Question) bool { return true } + } + return &dnsHandler{ + pending: pending, + limiter: lim, + allow: allow, + } +} + +// onRequestCommon checks packet structure and policy, returning actionDrop, +// actionReply (with a synthesized error response), or actionForward along with +// the parsed header and canonicalized question. +func (h *dnsHandler) onRequestCommon(raw []byte) (dnsmessage.Header, dnsmessage.Question, action) { + if !protocol.RequestSanityCheck(raw) { + return dnsmessage.Header{}, dnsmessage.Question{}, action{kind: actionDrop} + } + + var p dnsmessage.Parser + hdr, err := p.Start(raw) + if err != nil { + return hdr, dnsmessage.Question{}, errReplyAction(hdr, dnsmessage.RCodeFormatError, nil) + } + + qdCount := protocol.QDCount(raw) + + // OpCode must be QUERY (== 0). + if hdr.OpCode != 0 { + var qPtr *dnsmessage.Question + if qdCount == 1 { + if q, qErr := p.Question(); qErr == nil { + qPtr = &q + } + } + return hdr, dnsmessage.Question{}, errReplyAction(hdr, dnsmessage.RCodeNotImplemented, qPtr) + } + + // Must have one query. + if qdCount != 1 { + return hdr, dnsmessage.Question{}, errReplyAction(hdr, dnsmessage.RCodeFormatError, nil) + } + + // Question is valid. + q, err := p.Question() + if err != nil { + return hdr, dnsmessage.Question{}, errReplyAction(hdr, dnsmessage.RCodeFormatError, nil) + } + + // Question type must be supported. + if q.Type == dnsmessage.TypeAXFR || q.Type == protocol.TypeIXFR { + return hdr, dnsmessage.Question{}, errReplyAction(hdr, dnsmessage.RCodeNotImplemented, &q) + } + + // Check with query policy callout. + cq := canonicalizeQuestion(q) + if !h.allow(cq) { + return hdr, dnsmessage.Question{}, errReplyAction(hdr, dnsmessage.RCodeRefused, &q) + } + + return hdr, cq, action{ + kind: actionForward, + payload: raw, + } +} + +// onUDPRequest inspects an incoming UDP DNS query packet from the actor and +// returns whether to drop it, reply immediately with a synthesized error, or +// record and forward it to an upstream resolver. +func (h *dnsHandler) onUDPRequest(raw []byte, clientAddr net.Addr, upstreams []string) action { + if !h.limiter.inFlight.tryAcquire() { + slog.Debug("dns relay dropped query; too many in flight") + return action{kind: actionDrop} + } + + forwarded := false + defer func() { + if !forwarded { + h.limiter.inFlight.release() + } + }() + + hdr, cq, act := h.onRequestCommon(raw) + if act.kind != actionForward { + return act + } + + // Set to true before calling record, which takes ownership of the in-flight slot. + forwarded = true + upstreamID, ok := h.pending.record(raw, hdr.ID, cq, clientAddr, upstreams) + if !ok { + return action{kind: actionDrop} + } + + return action{ + kind: actionForward, + payload: raw, + upstreamID: upstreamID, + } +} + +// onUDPResponse validates an incoming UDP DNS response packet from an upstream +// resolver against pendingRequests and returns whether to drop it, fail over to +// the next upstream, or deliver it to the actor. +func (h *dnsHandler) onUDPResponse(raw []byte, from net.Addr) action { + if !protocol.ResponseSanityCheck(raw) { + return action{kind: actionDrop} + } + + // Parse response. + var p dnsmessage.Parser + hdr, err := p.Start(raw) + if err != nil { + return action{kind: actionDrop} + } + q, err := p.Question() + if err != nil { + return action{kind: actionDrop} + } + + h.pending.mu.Lock() + defer h.pending.mu.Unlock() + + // Look for request in the pending table. + entry, ok := h.pending.entries[hdr.ID] + cq := canonicalizeQuestion(q) + + if !ok || normalizeAddrString(entry.upstreams[entry.upstreamIdx]) != normalizeAddr(from) || entry.question != cq { + // Response does not match any pending request. + return action{kind: actionDrop} + } + + // Check if we need to failover to the next upstream AND there are more + // upstreams to try. + if isFailoverRCode(hdr.RCode) && entry.upstreamIdx+1 < len(entry.upstreams) { + entry.deferredResp = bytes.Clone(raw) + entry.upstreamIdx++ + entry.expiry = time.Now().Add(h.pending.timeoutForAttempt(entry.upstreamIdx, len(entry.upstreams))) + + return action{ + kind: actionFailover, + payload: bytes.Clone(entry.rawQuery), + upstreamIdx: entry.upstreamIdx, + } + } + + // Forward response to Actor. + clientID := entry.clientRequestID + clientAddr := entry.clientAddr + h.pending.deleteEntryLocked(hdr.ID) + + out := bytes.Clone(raw) + binary.BigEndian.PutUint16(out[0:2], clientID) + + return action{ + kind: actionDeliver, + payload: out, + clientAddr: clientAddr, + } +} + +// onTCPRequest inspects an incoming TCP DNS query frame from the actor and +// returns whether to drop it, reply immediately with a synthesized error, or +// forward it unmodified on the connection's dedicated upstream stream. +func (h *dnsHandler) onTCPRequest(raw []byte) action { + _, _, act := h.onRequestCommon(raw) + return act +} + +// onTCPResponse validates an incoming TCP DNS response frame from the +// connection's upstream stream and returns whether to drop or deliver it. +func (h *dnsHandler) onTCPResponse(raw []byte) action { + if !protocol.ResponseSanityCheck(raw) { + return action{kind: actionDrop} + } + var p dnsmessage.Parser + if _, err := p.Start(raw); err != nil { + return action{kind: actionDrop} + } + if _, err := p.Question(); err != nil { + return action{kind: actionDrop} + } + return action{ + kind: actionDeliver, + payload: raw, + } +} + +// errReplyAction returns an Action for an error situation. +func errReplyAction(hdr dnsmessage.Header, rcode dnsmessage.RCode, q *dnsmessage.Question) action { + return action{ + kind: actionReply, + payload: protocol.ErrorPacket(hdr, rcode, q), + } +} + +// isFailoverRCode returns true if the response error makes sense to try with +// the next DNS upstream server. +func isFailoverRCode(rcode dnsmessage.RCode) bool { + switch rcode { + case dnsmessage.RCodeServerFailure, dnsmessage.RCodeNotImplemented, dnsmessage.RCodeRefused: + return true + default: + return false + } +} + +// canonicalizeQuestion normalizes the contents of the Question. +func canonicalizeQuestion(q dnsmessage.Question) dnsmessage.Question { + for i := range int(q.Name.Length) { + c := q.Name.Data[i] + if 'A' <= c && c <= 'Z' { + q.Name.Data[i] = c + ('a' - 'A') + } + } + clear(q.Name.Data[q.Name.Length:]) + return q +} diff --git a/internal/ateomnet/dns/dnshandler_test.go b/internal/ateomnet/dns/dnshandler_test.go new file mode 100644 index 0000000000..cb7c052839 --- /dev/null +++ b/internal/ateomnet/dns/dnshandler_test.go @@ -0,0 +1,364 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dns + +import ( + "bytes" + "encoding/binary" + "net" + "testing" + + "github.com/agent-substrate/substrate/internal/ateomnet/dns/protocol" + "golang.org/x/net/dns/dnsmessage" +) + +func buildTestMessage(t *testing.T, hdr dnsmessage.Header, questions []dnsmessage.Question) []byte { + t.Helper() + b := dnsmessage.NewBuilder(nil, hdr) + if len(questions) > 0 { + if err := b.StartQuestions(); err != nil { + t.Fatal(err) + } + for _, q := range questions { + if err := b.Question(q); err != nil { + t.Fatal(err) + } + } + } + msg, err := b.Finish() + if err != nil { + t.Fatal(err) + } + return msg +} + +func TestDNSHandlerOnRequestDrop(t *testing.T) { + lim := &limiter{ + inFlight: &limiterSlots{max: 1}, + connections: &limiterSlots{max: 1}, + } + pending := newPendingRequests(lim) + h := newDNSHandler(pending, lim, nil) + upstreams := []string{"10.96.0.10:53"} + + // 1. Short packet (< 12 bytes). + if act := h.onUDPRequest([]byte{1, 2, 3, 4}, nil, upstreams); act.kind != actionDrop { + t.Errorf("short UDP packet action = %v, want actionDrop", act.kind) + } + if act := h.onTCPRequest([]byte{1, 2, 3, 4}); act.kind != actionDrop { + t.Errorf("short TCP packet action = %v, want actionDrop", act.kind) + } + + // 2. Packet with QR == 1 (response bit set). + resp := dnsAnswer(dnsQuery(0x1234), 0) + if act := h.onUDPRequest(resp, nil, upstreams); act.kind != actionDrop { + t.Errorf("QR=1 UDP packet action = %v, want actionDrop", act.kind) + } + if act := h.onTCPRequest(resp); act.kind != actionDrop { + t.Errorf("QR=1 TCP packet action = %v, want actionDrop", act.kind) + } + + // 3. Rate-limited (inFlight full) drops UDP requests. + lim.inFlight.tryAcquire() + defer lim.inFlight.release() + if act := h.onUDPRequest(dnsQuery(0x1234), nil, upstreams); act.kind != actionDrop { + t.Errorf("rate-limited UDP packet action = %v, want actionDrop", act.kind) + } +} + +func TestDNSHandlerOnRequestSynthesizedReplies(t *testing.T) { + lim := newLimiter() + pending := newPendingRequests(lim) + allow := func(q dnsmessage.Question) bool { + return q.Name.String() != "blocked.example.com." && q.Type != dnsmessage.TypeTXT + } + h := newDNSHandler(pending, lim, allow) + upstreams := []string{"10.96.0.10:53"} + + qExample := dnsmessage.Question{ + Name: dnsmessage.MustNewName("example.com."), + Type: dnsmessage.TypeA, + Class: dnsmessage.ClassINET, + } + + tests := []struct { + name string + raw []byte + wantRCode dnsmessage.RCode + }{ + { + name: "QDCOUNT == 0 -> FORMERR", + raw: buildTestMessage(t, dnsmessage.Header{ + ID: 0x1001, + RecursionDesired: true, + }, nil), + wantRCode: dnsmessage.RCodeFormatError, + }, + { + name: "QDCOUNT == 2 -> FORMERR", + raw: buildTestMessage(t, dnsmessage.Header{ + ID: 0x1002, + RecursionDesired: true, + }, []dnsmessage.Question{qExample, qExample}), + wantRCode: dnsmessage.RCodeFormatError, + }, + { + name: "malformed question name -> FORMERR", + raw: []byte{ + 0x10, 0x03, 0x01, 0x00, + 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x10, 'a', 'b', // label length 16 but truncated + }, + wantRCode: dnsmessage.RCodeFormatError, + }, + { + name: "non-zero OpCode -> NOTIMP", + raw: buildTestMessage(t, dnsmessage.Header{ + ID: 0x1004, + OpCode: 1, // IQUERY + RecursionDesired: true, + }, []dnsmessage.Question{qExample}), + wantRCode: dnsmessage.RCodeNotImplemented, + }, + { + name: "QTYPE AXFR -> NOTIMP", + raw: buildTestMessage(t, dnsmessage.Header{ + ID: 0x1005, + RecursionDesired: true, + }, []dnsmessage.Question{{ + Name: dnsmessage.MustNewName("example.com."), + Type: dnsmessage.TypeAXFR, + Class: dnsmessage.ClassINET, + }}), + wantRCode: dnsmessage.RCodeNotImplemented, + }, + { + name: "QTYPE IXFR -> NOTIMP", + raw: buildTestMessage(t, dnsmessage.Header{ + ID: 0x1006, + RecursionDesired: true, + }, []dnsmessage.Question{{ + Name: dnsmessage.MustNewName("example.com."), + Type: protocol.TypeIXFR, + Class: dnsmessage.ClassINET, + }}), + wantRCode: dnsmessage.RCodeNotImplemented, + }, + { + name: "disallowed domain -> REFUSED", + raw: buildTestMessage(t, dnsmessage.Header{ + ID: 0x1007, + RecursionDesired: true, + }, []dnsmessage.Question{{ + Name: dnsmessage.MustNewName("Blocked.Example.COM."), + Type: dnsmessage.TypeA, + Class: dnsmessage.ClassINET, + }}), + wantRCode: dnsmessage.RCodeRefused, + }, + { + name: "disallowed record type -> REFUSED", + raw: buildTestMessage(t, dnsmessage.Header{ + ID: 0x1008, + RecursionDesired: true, + }, []dnsmessage.Question{{ + Name: dnsmessage.MustNewName("example.com."), + Type: dnsmessage.TypeTXT, + Class: dnsmessage.ClassINET, + }}), + wantRCode: dnsmessage.RCodeRefused, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + wantID := binary.BigEndian.Uint16(tc.raw[0:2]) + for _, act := range []action{ + h.onUDPRequest(tc.raw, nil, upstreams), + h.onTCPRequest(tc.raw), + } { + if act.kind != actionReply { + t.Fatalf("action.kind = %v, want actionReply", act.kind) + } + var p dnsmessage.Parser + hdr, err := p.Start(act.payload) + if err != nil { + t.Fatalf("parsing synthesized reply: %v", err) + } + if hdr.ID != wantID { + t.Errorf("reply ID = %#x, want %#x", hdr.ID, wantID) + } + if !hdr.Response { + t.Error("reply QR = false, want true") + } + if hdr.RCode != tc.wantRCode { + t.Errorf("reply RCode = %v, want %v", hdr.RCode, tc.wantRCode) + } + if got := lim.inFlight.occupied(); got != 0 { + t.Errorf("inFlight slots = %d after synthesized reply, want 0", got) + } + } + }) + } +} + +func TestDNSHandlerUDPForwardAndResponseValidation(t *testing.T) { + lim := newLimiter() + pending := newPendingRequests(lim) + h := newDNSHandler(pending, lim, nil) + + upstream1 := &net.UDPAddr{IP: net.ParseIP("10.96.0.10"), Port: 53} + upstream2 := &net.UDPAddr{IP: net.ParseIP("10.96.0.11"), Port: 53} + wrongUpstream := &net.UDPAddr{IP: net.ParseIP("10.96.0.99"), Port: 53} + clientAddr := &net.UDPAddr{IP: net.ParseIP("169.254.0.2"), Port: 54321} + + // Send query with mixed-case QNAME to verify case-insensitive RFC 4343 matching. + query := buildTestMessage(t, dnsmessage.Header{ + ID: 0xbeef, + RecursionDesired: true, + }, []dnsmessage.Question{{ + Name: dnsmessage.MustNewName("ExAmPlE.CoM."), + Type: dnsmessage.TypeA, + Class: dnsmessage.ClassINET, + }}) + + act := h.onUDPRequest(bytes.Clone(query), clientAddr, []string{upstream1.String(), upstream2.String()}) + if act.kind != actionForward { + t.Fatalf("onUDPRequest kind = %v, want actionForward", act.kind) + } + rewritten := act.payload + upstreamID := act.upstreamID + if gotID := binary.BigEndian.Uint16(rewritten[0:2]); gotID != upstreamID { + t.Fatalf("payload ID = %#x, want upstreamID %#x", gotID, upstreamID) + } + + // 1. Response with QR == 0 is dropped. + if res := h.onUDPResponse(rewritten, upstream1); res.kind != actionDrop { + t.Errorf("QR=0 response kind = %v, want actionDrop", res.kind) + } + + // 2. Response from unexpected upstream address is dropped without evicting pending request. + validResp := dnsAnswer(rewritten, 0) + if res := h.onUDPResponse(validResp, wrongUpstream); res.kind != actionDrop { + t.Errorf("wrong upstream source kind = %v, want actionDrop", res.kind) + } + + // 3. Response with wrong transaction ID is dropped. + wrongIDResp := bytes.Clone(validResp) + binary.BigEndian.PutUint16(wrongIDResp[0:2], upstreamID^0xffff) + if res := h.onUDPResponse(wrongIDResp, upstream1); res.kind != actionDrop { + t.Errorf("wrong ID response kind = %v, want actionDrop", res.kind) + } + + // 4. Response with mismatched QNAME is dropped. + wrongQNameResp := buildTestMessage(t, dnsmessage.Header{ + ID: upstreamID, + Response: true, + }, []dnsmessage.Question{{ + Name: dnsmessage.MustNewName("other.com."), + Type: dnsmessage.TypeA, + Class: dnsmessage.ClassINET, + }}) + if res := h.onUDPResponse(wrongQNameResp, upstream1); res.kind != actionDrop { + t.Errorf("wrong QNAME response kind = %v, want actionDrop", res.kind) + } + + // 5. Response with mismatched QTYPE is dropped. + wrongQTypeResp := buildTestMessage(t, dnsmessage.Header{ + ID: upstreamID, + Response: true, + }, []dnsmessage.Question{{ + Name: dnsmessage.MustNewName("example.com."), + Type: dnsmessage.TypeAAAA, + Class: dnsmessage.ClassINET, + }}) + if res := h.onUDPResponse(wrongQTypeResp, upstream1); res.kind != actionDrop { + t.Errorf("wrong QTYPE response kind = %v, want actionDrop", res.kind) + } + + // 6. SERVFAIL from upstream1 triggers ActionFailover to upstream2. + servFailResp := dnsAnswer(rewritten, byte(dnsmessage.RCodeServerFailure)) + failoverAct := h.onUDPResponse(servFailResp, upstream1) + if failoverAct.kind != actionFailover { + t.Fatalf("SERVFAIL on first upstream kind = %v, want actionFailover", failoverAct.kind) + } + if failoverAct.upstreamIdx != 1 { + t.Errorf("failover upstreamIdx = %d, want 1", failoverAct.upstreamIdx) + } + if got := lim.inFlight.occupied(); got != 1 { + t.Errorf("inFlight slots during failover = %d, want 1", got) + } + + // 7. Valid response from upstream2 (with different QNAME casing) delivers and restores clientRequestID. + lowerResp := buildTestMessage(t, dnsmessage.Header{ + ID: upstreamID, + Response: true, + RCode: dnsmessage.RCodeSuccess, + }, []dnsmessage.Question{{ + Name: dnsmessage.MustNewName("example.com."), + Type: dnsmessage.TypeA, + Class: dnsmessage.ClassINET, + }}) + deliverAct := h.onUDPResponse(lowerResp, upstream2) + if deliverAct.kind != actionDeliver { + t.Fatalf("valid response on second upstream kind = %v, want actionDeliver", deliverAct.kind) + } + if gotID := binary.BigEndian.Uint16(deliverAct.payload[0:2]); gotID != 0xbeef { + t.Errorf("delivered ID = %#x, want 0xbeef", gotID) + } + if deliverAct.clientAddr != clientAddr { + t.Errorf("delivered clientAddr = %v, want %v", deliverAct.clientAddr, clientAddr) + } + if got := lim.inFlight.occupied(); got != 0 { + t.Errorf("inFlight slots after delivery = %d, want 0", got) + } +} + +func TestDNSHandlerTCPForwardAndResponseValidation(t *testing.T) { + lim := newLimiter() + pending := newPendingRequests(lim) + h := newDNSHandler(pending, lim, nil) + + query := dnsQuery(0xbeef) + act := h.onTCPRequest(bytes.Clone(query)) + if act.kind != actionForward { + t.Fatalf("onTCPRequest kind = %v, want actionForward", act.kind) + } + // TCP does not rewrite the transaction ID or record into pendingRequests. + if !bytes.Equal(act.payload, query) { + t.Errorf("onTCPRequest modified query payload") + } + if len(pending.entries) != 0 || lim.inFlight.occupied() != 0 { + t.Errorf("onTCPRequest touched pendingRequests (%d entries) or inFlight (%d slots)", len(pending.entries), lim.inFlight.occupied()) + } + + // Invalid responses are dropped. + if res := h.onTCPResponse([]byte{1, 2, 3}); res.kind != actionDrop { + t.Errorf("short TCP response kind = %v, want actionDrop", res.kind) + } + if res := h.onTCPResponse(query); res.kind != actionDrop { + t.Errorf("QR=0 TCP response kind = %v, want actionDrop", res.kind) + } + + // Valid response is delivered unmodified. + resp := dnsAnswer(query, 0) + deliverAct := h.onTCPResponse(resp) + if deliverAct.kind != actionDeliver { + t.Fatalf("valid TCP response kind = %v, want actionDeliver", deliverAct.kind) + } + if !bytes.Equal(deliverAct.payload, resp) { + t.Errorf("delivered TCP response differs from input") + } +} diff --git a/internal/ateomnet/dns/freemap/doc.go b/internal/ateomnet/dns/freemap/doc.go new file mode 100644 index 0000000000..36326a59cc --- /dev/null +++ b/internal/ateomnet/dns/freemap/doc.go @@ -0,0 +1,17 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package freemap implements an efficient mechanism for maintaining a map +// of free slots. +package freemap diff --git a/internal/ateomnet/dns/freemap/freemap64k.go b/internal/ateomnet/dns/freemap/freemap64k.go new file mode 100644 index 0000000000..1c5d7e5d56 --- /dev/null +++ b/internal/ateomnet/dns/freemap/freemap64k.go @@ -0,0 +1,204 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package freemap + +import ( + "math/bits" +) + +const ( + map64kBits = 65536 + wordSize = 64 + l0words = map64kBits / wordSize // 1024 words + l1words = l0words / wordSize // 16 words + l2Mask = (1 << l1words) - 1 // 0xFFFF +) + +// Map64k is a free map that supports 64k entries. This data structure +// uses ~8 KiB of memory. +// +// The zero value is an empty map ready to use. +// +// This data structure is not thread-safe. +// +// This is represented as a hierarchical bit-set. +type Map64k struct { + // Layer 0: Leaf-level storage (1 = occupied, 0 = empty) + l0 [l0words]uint64 + // Layer 1: Intermediate summary (1 = corresponding L0 word is fully occupied) + l1 [l1words]uint64 + // Layer 2: Root summary (1 = corresponding L1 word is fully occupied) + l2 uint64 + // count is the number of bits currently set in the map. + count int +} + +// Count returns the number of bits set in the map. +func (m *Map64k) Count() int { return m.count } + +// IsFull returns true if there is no more room in the map. +func (m *Map64k) IsFull() bool { return m.count == map64kBits } + +// bitsetIndex into the hierarchical bitset. +type bitsetIndex struct { + l0, l0Bit int + l1, l1Bit int +} + +// getIndex computes the index across all of the hierarchical bitsets. +func getIndex(index uint16) bitsetIndex { + return bitsetIndex{ + l0: int(index / wordSize), + l0Bit: int(index % wordSize), + l1: int(index / wordSize / wordSize), + l1Bit: int((index / wordSize) % wordSize), + } +} + +// Get returns true if the index is set. +func (m *Map64k) Get(index uint16) bool { + sidx := getIndex(index) + return m.l0[sidx.l0]&(1<= end { + return -1 + } + + // Fast path: nothing to find in a full map. + if m.IsFull() { + return -1 + } + + startIdx := getIndex(start) + + // 1. Scan remaining bits in the current Layer 0 word. + w0 := m.l0[startIdx.l0] | ((uint64(1) << startIdx.l0Bit) - 1) + if inv0 := ^w0; inv0 != 0 { + idx := (startIdx.l0 * wordSize) + bits.TrailingZeros64(inv0) + if idx < end { + return idx + } + return -1 + } + + // 2. Scan remaining Layer 0 words in the current Layer 1 word. + if nextL1Bit := startIdx.l1Bit + 1; nextL1Bit < wordSize && (startIdx.l0+1)*wordSize < end { + w1 := m.l1[startIdx.l1] | ((uint64(1) << nextL1Bit) - 1) + if inv1 := ^w1; inv1 != 0 { + l0WordInL1 := bits.TrailingZeros64(inv1) + targetL0 := (startIdx.l1 * wordSize) + l0WordInL1 + idx := (targetL0 * wordSize) + bits.TrailingZeros64(^m.l0[targetL0]) + if idx < end { + return idx + } + return -1 + } + } + + // 3. Scan remaining Layer 1 words in Layer 2 (Root). + if nextL1 := startIdx.l1 + 1; nextL1 < l1words && nextL1*wordSize*wordSize < end { + w2 := m.l2 | ((uint64(1) << nextL1) - 1) + if inv2 := (^w2) & l2Mask; inv2 != 0 { + targetL1 := bits.TrailingZeros64(inv2) + l0WordInL1 := bits.TrailingZeros64(^m.l1[targetL1]) + targetL0 := (targetL1 * wordSize) + l0WordInL1 + idx := (targetL0 * wordSize) + bits.TrailingZeros64(^m.l0[targetL0]) + if idx < end { + return idx + } + return -1 + } + } + + return -1 +} diff --git a/internal/ateomnet/dns/freemap/freemap64k_test.go b/internal/ateomnet/dns/freemap/freemap64k_test.go new file mode 100644 index 0000000000..4efb15136d --- /dev/null +++ b/internal/ateomnet/dns/freemap/freemap64k_test.go @@ -0,0 +1,624 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package freemap + +import ( + "math/rand/v2" + "testing" +) + +// asInt flattens a (index, ok) result into an int, with -1 for !ok, so tests +// can compare against a single expected value. +func asInt(idx uint16, ok bool) int { + if !ok { + return -1 + } + return int(idx) +} + +// fill sets every index in [start, end). +func fill(m *Map64k, start, end int) { + for i := start; i < end; i++ { + m.Set(uint16(i)) + } +} + +// refMap pairs a Map64k with a naive boolean-slice model of it. +type refMap struct { + m Map64k + occupied []bool + wantCount int +} + +func newRefMap() *refMap { + return &refMap{occupied: make([]bool, map64kBits)} +} + +func (r *refMap) set(idx uint16) { + if !r.occupied[idx] { + r.occupied[idx] = true + r.wantCount++ + } + r.m.Set(idx) +} + +func (r *refMap) clear(idx uint16) { + if r.occupied[idx] { + r.occupied[idx] = false + r.wantCount-- + } + r.m.Clear(idx) +} + +func (r *refMap) naiveFindFirstUnsetRange(start uint16, end int) int { + for idx := int(start); idx < end; idx++ { + if !r.occupied[idx] { + return idx + } + } + return -1 +} + +func (r *refMap) naiveFindFirstUnsetFrom(start uint16) int { + if idx := r.naiveFindFirstUnsetRange(start, map64kBits); idx != -1 { + return idx + } + return r.naiveFindFirstUnsetRange(0, int(start)) +} + +// checkRange compares findFirstUnsetRange against the model. +func (r *refMap) checkRange(t *testing.T, step int, start uint16, end int) { + t.Helper() + if got, want := r.m.findFirstUnsetRange(start, end), r.naiveFindFirstUnsetRange(start, end); got != want { + t.Fatalf("step %d: findFirstUnsetRange(%d, %d) = %d, want %d", step, start, end, got, want) + } +} + +// checkFrom compares FindFirstUnsetFrom and FindFirstUnset against the model. +func (r *refMap) checkFrom(t *testing.T, step int, start uint16) { + t.Helper() + if got, want := asInt(r.m.FindFirstUnsetFrom(start)), r.naiveFindFirstUnsetFrom(start); got != want { + t.Fatalf("step %d: FindFirstUnsetFrom(%d) = %d, want %d", step, start, got, want) + } + if got, want := asInt(r.m.FindFirstUnset()), r.naiveFindFirstUnsetRange(0, map64kBits); got != want { + t.Fatalf("step %d: FindFirstUnset() = %d, want %d", step, got, want) + } +} + +// checkInvariants compares Count and IsFull against the model. +func (r *refMap) checkInvariants(t *testing.T, step int) { + t.Helper() + if got := r.m.Count(); got != r.wantCount { + t.Fatalf("step %d: Count() = %d, want %d", step, got, r.wantCount) + } + if got, want := r.m.IsFull(), r.wantCount == map64kBits; got != want { + t.Fatalf("step %d: IsFull() = %v, want %v", step, got, want) + } +} + +func TestGetIndex(t *testing.T) { + tests := []struct { + index uint16 + want bitsetIndex + }{ + {index: 0, want: bitsetIndex{l0: 0, l0Bit: 0, l1: 0, l1Bit: 0}}, + {index: 63, want: bitsetIndex{l0: 0, l0Bit: 63, l1: 0, l1Bit: 0}}, + {index: 64, want: bitsetIndex{l0: 1, l0Bit: 0, l1: 0, l1Bit: 1}}, + {index: 127, want: bitsetIndex{l0: 1, l0Bit: 63, l1: 0, l1Bit: 1}}, + {index: 4095, want: bitsetIndex{l0: 63, l0Bit: 63, l1: 0, l1Bit: 63}}, + {index: 4096, want: bitsetIndex{l0: 64, l0Bit: 0, l1: 1, l1Bit: 0}}, + {index: 65535, want: bitsetIndex{l0: 1023, l0Bit: 63, l1: 15, l1Bit: 63}}, + } + + for _, tc := range tests { + got := getIndex(tc.index) + if got != tc.want { + t.Errorf("getIndex(%d) = %+v, want %+v", tc.index, got, tc.want) + } + } +} + +func TestMap64k_Basic(t *testing.T) { + var m Map64k + if got := m.Count(); got != 0 { + t.Errorf("expected count 0, got %d", got) + } + if m.IsFull() { + t.Errorf("expected IsFull() = false on empty map") + } + if got := asInt(m.FindFirstUnset()); got != 0 { + t.Errorf("expected first unset 0, got %d", got) + } + if m.Get(0) { + t.Errorf("expected Get(0) = false initially") + } + + m.Set(0) + if got := m.Count(); got != 1 { + t.Errorf("expected count 1, got %d", got) + } + if m.IsFull() { + t.Errorf("expected IsFull() = false with 1 bit set") + } + if !m.Get(0) { + t.Errorf("expected Get(0) = true after Set(0)") + } + if got := asInt(m.FindFirstUnset()); got != 1 { + t.Errorf("expected first unset 1, got %d", got) + } + + // Idempotent Set + m.Set(0) + if got := m.Count(); got != 1 { + t.Errorf("expected count 1 after redundant set, got %d", got) + } + + m.Clear(0) + if got := m.Count(); got != 0 { + t.Errorf("expected count 0, got %d", got) + } + if m.Get(0) { + t.Errorf("expected Get(0) = false after Clear(0)") + } + if got := asInt(m.FindFirstUnset()); got != 0 { + t.Errorf("expected first unset 0, got %d", got) + } + + // Idempotent Clear + m.Clear(0) + if got := m.Count(); got != 0 { + t.Errorf("expected count 0 after redundant clear, got %d", got) + } +} + +func TestMap64k_Get(t *testing.T) { + var m Map64k + testIndices := []uint16{0, 1, 63, 64, 127, 1024, 4095, 4096, 65534, 65535} + + for _, idx := range testIndices { + if m.Get(idx) { + t.Errorf("Get(%d) = true on empty map, want false", idx) + } + m.Set(idx) + if !m.Get(idx) { + t.Errorf("Get(%d) = false after Set, want true", idx) + } + m.Clear(idx) + if m.Get(idx) { + t.Errorf("Get(%d) = true after Clear, want false", idx) + } + } +} + +func TestMap64k_HierarchyPropagation(t *testing.T) { + var m Map64k + + // 1. Fill word 0 (bits 0..63). L1 bit 0 should be set, L2 should still be 0. + fill(&m, 0, 64) + if (m.l1[0] & 1) == 0 { + t.Errorf("expected l1[0] bit 0 to be set") + } + if m.l2 != 0 { + t.Errorf("expected l2 to be 0, got %x", m.l2) + } + + // Clearing one bit in word 0 should clear L1 bit 0. + m.Clear(10) + if (m.l1[0] & 1) != 0 { + t.Errorf("expected l1[0] bit 0 to be cleared after clearing bit 10") + } + + // Re-setting bit 10 should restore L1 bit 0. + m.Set(10) + if (m.l1[0] & 1) == 0 { + t.Errorf("expected l1[0] bit 0 to be set after re-setting bit 10") + } + + // 2. Fill all 4096 bits in L1 word 0 (words 0..63). L2 bit 0 should be set. + fill(&m, 64, 4096) + if (m.l2 & 1) == 0 { + t.Errorf("expected l2 bit 0 to be set after filling 4096 bits") + } + + // Clearing one bit should clear L1 bit and propagate to L2 bit 0. + m.Clear(2000) + if (m.l2 & 1) != 0 { + t.Errorf("expected l2 bit 0 to be cleared after clearing bit 2000") + } +} + +func TestMap64k_FindFirstUnsetFrom_Empty(t *testing.T) { + var m Map64k + testIndices := []uint16{0, 1, 63, 64, 100, 1023, 1024, 4095, 4096, 65535} + for _, idx := range testIndices { + if got := asInt(m.FindFirstUnsetFrom(idx)); got != int(idx) { + t.Errorf("FindFirstUnsetFrom(%d) on empty map = %d, want %d", idx, got, idx) + } + } +} + +func TestMap64k_FindFirstUnsetFrom_Full(t *testing.T) { + var m Map64k + fill(&m, 0, map64kBits) + if got := m.Count(); got != map64kBits { + t.Fatalf("expected count %d, got %d", map64kBits, got) + } + if !m.IsFull() { + t.Errorf("expected IsFull() = true on full map") + } + if _, ok := m.FindFirstUnset(); ok { + t.Errorf("FindFirstUnset on full map returned ok = true") + } + for _, idx := range []uint16{0, 1, 100, 4096, 65535} { + if _, ok := m.FindFirstUnsetFrom(idx); ok { + t.Errorf("FindFirstUnsetFrom(%d) on full map returned ok = true", idx) + } + } +} + +func TestMap64k_FindFirstUnsetFrom_SingleHole(t *testing.T) { + var m Map64k + fill(&m, 0, map64kBits) + + testHoles := []uint16{0, 1, 63, 64, 127, 128, 4095, 4096, 32768, 65534, 65535} + for _, hole := range testHoles { + m.Clear(hole) + if m.IsFull() { + t.Errorf("hole %d: expected IsFull() = false", hole) + } + if got := asInt(m.FindFirstUnset()); got != int(hole) { + t.Errorf("hole %d: FindFirstUnset = %d, want %d", hole, got, hole) + } + + // Starting from any index should find the only hole + for _, start := range []uint16{0, hole, hole + 1, 65535} { + if got := asInt(m.FindFirstUnsetFrom(start)); got != int(hole) { + t.Errorf("hole %d, start %d: FindFirstUnsetFrom = %d, want %d", hole, start, got, hole) + } + } + + m.Set(hole) + if !m.IsFull() { + t.Errorf("hole %d: expected IsFull() = true after re-setting hole", hole) + } + } +} + +func TestMap64k_FindFirstUnsetFrom_Levels(t *testing.T) { + check := func(t *testing.T, m *Map64k, start uint16, want int) { + t.Helper() + if got := asInt(m.FindFirstUnsetFrom(start)); got != want { + t.Errorf("FindFirstUnsetFrom(%d) = %d, want %d", start, got, want) + } + } + + t.Run("same L0 word", func(t *testing.T) { + var m Map64k + fill(&m, 0, 10) + for _, start := range []uint16{0, 5, 10} { + check(t, &m, start, 10) + } + }) + + t.Run("next L0 word in same L1 word", func(t *testing.T) { + var m Map64k + // Fill L0 word 0 (bits 0..63) and bits 0..4 of word 1 (bits 64..68) + fill(&m, 0, 69) + for _, start := range []uint16{0, 63, 64} { + check(t, &m, start, 69) + } + }) + + t.Run("next L1 word from L2", func(t *testing.T) { + var m Map64k + // Fill entire L1 word 0 (bits 0..4095) and bits 4096..4099 in L1 word 1. + fill(&m, 0, 4100) + for _, start := range []uint16{0, 2000, 4095, 4096} { + check(t, &m, start, 4100) + } + + // Fill the rest of L1 word 1 and bits 8192..8199 in L1 word 2. + fill(&m, 4100, 8200) + for _, start := range []uint16{0, 4095, 4096, 8191, 8192} { + check(t, &m, start, 8200) + } + }) + + t.Run("wrap around", func(t *testing.T) { + var m Map64k + fill(&m, 100, map64kBits) + fill(&m, 0, 10) + // Search from 100 should wrap around and find 10 + for _, start := range []uint16{100, 65535} { + check(t, &m, start, 10) + } + }) + + t.Run("wrap around with last bit set", func(t *testing.T) { + var m Map64k + m.Set(65535) + check(t, &m, 65535, 0) + }) +} + +func TestMap64k_FindFirstUnsetRange(t *testing.T) { + var m Map64k + + // Empty ranges + if got := m.findFirstUnsetRange(5, 5); got != -1 { + t.Errorf("range [5, 5) = %d, want -1", got) + } + if got := m.findFirstUnsetRange(10, 5); got != -1 { + t.Errorf("range [10, 5) = %d, want -1", got) + } + + // Range within single word + fill(&m, 10, 20) + if got := m.findFirstUnsetRange(10, 20); got != -1 { + t.Errorf("range [10, 20) fully set, got %d, want -1", got) + } + if got := m.findFirstUnsetRange(10, 25); got != 20 { + t.Errorf("range [10, 25), got %d, want 20", got) + } + if got := m.findFirstUnsetRange(5, 20); got != 5 { + t.Errorf("range [5, 20), got %d, want 5", got) + } + + // Range spanning word boundary + fill(&m, 60, 64) + // Bits 60..63 are set, 64 is unset + if got := m.findFirstUnsetRange(60, 64); got != -1 { + t.Errorf("range [60, 64) fully set, got %d, want -1", got) + } + if got := m.findFirstUnsetRange(60, 70); got != 64 { + t.Errorf("range [60, 70), got %d, want 64", got) + } + + // Range spanning L1 boundary (word 63 to word 64, bit 4095 to 4096) + fill(&m, 4090, 4096) + if got := m.findFirstUnsetRange(4090, 4096); got != -1 { + t.Errorf("range [4090, 4096) fully set, got %d, want -1", got) + } + if got := m.findFirstUnsetRange(4090, 4100); got != 4096 { + t.Errorf("range [4090, 4100), got %d, want 4096", got) + } + + // Full span + if got := m.findFirstUnsetRange(0, map64kBits); got != 0 { + t.Errorf("range [0, map64kBits), got %d, want 0", got) + } + + // Full map + fill(&m, 0, map64kBits) + if got := m.findFirstUnsetRange(0, map64kBits); got != -1 { + t.Errorf("range [0, map64kBits) on full map, got %d, want -1", got) + } + if got := m.findFirstUnsetRange(100, 200); got != -1 { + t.Errorf("range [100, 200) on full map, got %d, want -1", got) + } +} + +// TestMap64k_FindFirstUnsetRange_Cutoffs checks that a free bit found by the +// L1 (step 2) and L2 (step 3) scans is rejected when it lies past `end`. +func TestMap64k_FindFirstUnsetRange_Cutoffs(t *testing.T) { + t.Run("L1 scan", func(t *testing.T) { + var m Map64k + fill(&m, 0, 128) // L0 words 0 and 1 full; first free is 128. + if got := m.findFirstUnsetRange(0, 100); got != -1 { + t.Errorf("range [0, 100), got %d, want -1", got) + } + if got := m.findFirstUnsetRange(0, 128); got != -1 { + t.Errorf("range [0, 128), got %d, want -1", got) + } + if got := m.findFirstUnsetRange(0, 129); got != 128 { + t.Errorf("range [0, 129), got %d, want 128", got) + } + }) + + t.Run("L2 scan", func(t *testing.T) { + var m Map64k + fill(&m, 0, 8192) // L1 words 0 and 1 full; first free is 8192. + if got := m.findFirstUnsetRange(100, 5000); got != -1 { + t.Errorf("range [100, 5000), got %d, want -1", got) + } + if got := m.findFirstUnsetRange(100, 8192); got != -1 { + t.Errorf("range [100, 8192), got %d, want -1", got) + } + if got := m.findFirstUnsetRange(100, 9000); got != 8192 { + t.Errorf("range [100, 9000), got %d, want 8192", got) + } + }) +} + +// TestMap64k_Differential against a naive linear search on a boolean slice. +func TestMap64k_Differential(t *testing.T) { + ref := newRefMap() + r := rand.New(rand.NewPCG(42, 1337)) + + for step := range 5000 { + op := r.IntN(12) + switch { + case op < 3: // Set a random bit + idx := uint16(r.IntN(map64kBits)) + ref.set(idx) + if !ref.m.Get(idx) { + t.Fatalf("step %d: Get(%d) = false after Set", step, idx) + } + case op < 5: // Clear a random bit + idx := uint16(r.IntN(map64kBits)) + ref.clear(idx) + if ref.m.Get(idx) { + t.Fatalf("step %d: Get(%d) = true after Clear", step, idx) + } + case op == 5: // Set an entire 64-bit L0 word + base := uint16(r.IntN(l0words) * wordSize) + for i := range uint16(wordSize) { + ref.set(base + i) + } + case op == 6: // Clear an entire 64-bit L0 word + base := uint16(r.IntN(l0words) * wordSize) + for i := range uint16(wordSize) { + ref.clear(base + i) + } + case op == 7: // Set or clear an entire 4096-bit L1 block + base := uint16(r.IntN(l1words) * wordSize * wordSize) + set := r.IntN(2) == 0 + for i := range uint16(wordSize * wordSize) { + if set { + ref.set(base + i) + } else { + ref.clear(base + i) + } + } + case op < 10: // Query range + start := uint16(r.IntN(map64kBits)) + end := int(start) + r.IntN(map64kBits-int(start)+1) + ref.checkRange(t, step, start, end) + default: // Query from + ref.checkFrom(t, step, uint16(r.IntN(map64kBits))) + } + + ref.checkInvariants(t, step) + } +} + +func TestMap64k_Differential_Dense(t *testing.T) { + ref := newRefMap() + r := rand.New(rand.NewPCG(99, 12345)) + + // Start completely full so all L1 and L2 summary bits are set, then punch a few holes. + for i := range map64kBits { + ref.set(uint16(i)) + } + // holes records every cleared index. It may contain duplicates (clearing an + // already-free bit appends it again); that is harmless and keeps the + // property that an empty holes list means the map is completely full, so + // the test regularly drives the map back to full. + var holes []uint16 + for range 16 { + idx := uint16(r.IntN(map64kBits)) + ref.clear(idx) + holes = append(holes, idx) + } + + for step := range 5000 { + op := r.IntN(10) + switch { + case op < 3: // Plug an existing hole (or random bit) so L0/L1/L2 words become full again + if len(holes) > 0 { + hIdx := r.IntN(len(holes)) + ref.set(holes[hIdx]) + holes[hIdx] = holes[len(holes)-1] + holes = holes[:len(holes)-1] + } else { + ref.set(uint16(r.IntN(map64kBits))) + } + case op < 5: // Punch a new hole + idx := uint16(r.IntN(map64kBits)) + ref.clear(idx) + holes = append(holes, idx) + case op < 7: // Query range + start := uint16(r.IntN(map64kBits)) + end := int(start) + r.IntN(map64kBits-int(start)+1) + ref.checkRange(t, step, start, end) + default: // Query from and FindFirstUnset + ref.checkFrom(t, step, uint16(r.IntN(map64kBits))) + } + + ref.checkInvariants(t, step) + } +} + +// singleHoleMap returns a full map with only index 50000 free. +func singleHoleMap() *Map64k { + var m Map64k + fill(&m, 0, map64kBits) + m.Clear(50000) + return &m +} + +// sparseMap returns a map with ~37% of bits set at random. +func sparseMap() *Map64k { + var m Map64k + r := rand.New(rand.NewPCG(1, 2)) + for range 30000 { + m.Set(uint16(r.IntN(map64kBits))) + } + return &m +} + +func BenchmarkFindFirstUnsetFrom(b *testing.B) { + b.Run("sparse", func(b *testing.B) { + m := sparseMap() + var i uint16 + for b.Loop() { + m.FindFirstUnsetFrom(i) + i++ + } + }) + + b.Run("dense_single_hole", func(b *testing.B) { + m := singleHoleMap() + var i uint16 + for b.Loop() { + m.FindFirstUnsetFrom(i) + i++ + } + }) +} + +func BenchmarkFindFirstUnsetRange(b *testing.B) { + b.Run("sparse", func(b *testing.B) { + m := sparseMap() + var i int + for b.Loop() { + start := uint16(i % (map64kBits - 100)) + m.findFirstUnsetRange(start, int(start)+100) + i++ + } + }) + + b.Run("dense_single_hole", func(b *testing.B) { + m := singleHoleMap() + var i int + for b.Loop() { + start := uint16(i % (map64kBits - 100)) + m.findFirstUnsetRange(start, map64kBits) + i++ + } + }) +} + +// BenchmarkSetClear measures an allocate/free cycle, including the L1/L2 +// summary updates when a word transitions between full and not full. +func BenchmarkSetClear(b *testing.B) { + b.Run("sparse", func(b *testing.B) { + var m Map64k + var i uint16 + for b.Loop() { + m.Set(i) + m.Clear(i) + i++ + } + }) + + b.Run("dense_single_hole", func(b *testing.B) { + m := singleHoleMap() + for b.Loop() { + m.Set(50000) // Fills L0, L1 and L2. + m.Clear(50000) + } + }) +} diff --git a/internal/ateomnet/dns/limiter.go b/internal/ateomnet/dns/limiter.go new file mode 100644 index 0000000000..18f62d77fa --- /dev/null +++ b/internal/ateomnet/dns/limiter.go @@ -0,0 +1,74 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dns + +import "sync/atomic" + +const ( + // maxInFlight UDP queries. + maxInFlight = 64 + // maxConnections bounds open TCP connections. + maxConnections = 16 +) + +// limiter caps tracks DNS query limits. +type limiter struct { + inFlight *limiterSlots + connections *limiterSlots +} + +func newLimiter() *limiter { + return &limiter{ + inFlight: &limiterSlots{max: maxInFlight}, + connections: &limiterSlots{max: maxConnections}, + } +} + +type limiterSlots struct { + max int32 + used atomic.Int32 +} + +// tryAcquire a virtual slot. Returns false if this slot is at capacity. +func (s *limiterSlots) tryAcquire() bool { + // Disabled if zero. + if s == nil { + return true + } + + if s.used.Add(1) > s.max { + s.used.Add(-1) + return false + } + return true +} + +// release a use slot. MUST be paired with a successful tryAcquire. +func (s *limiterSlots) release() { + // Disabled if zero. + if s == nil { + return + } + + if s.used.Add(-1) < 0 { + panic("dns: slot released more times than acquired") + } +} + +// occupied returns the current slots occupied. Warning: this should only be +// used for testing as the count will not be accurate due to concurrency. +func (s *limiterSlots) occupied() int32 { + return s.used.Load() +} diff --git a/internal/ateomnet/dns/limiter_test.go b/internal/ateomnet/dns/limiter_test.go new file mode 100644 index 0000000000..4c77b1580f --- /dev/null +++ b/internal/ateomnet/dns/limiter_test.go @@ -0,0 +1,137 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dns + +import ( + "sync" + "sync/atomic" + "testing" +) + +func TestNewLimiter(t *testing.T) { + lim := newLimiter() + if lim.inFlight == nil || lim.inFlight.max != maxInFlight { + t.Errorf("inFlight = %+v, want max=%d", lim.inFlight, maxInFlight) + } + if lim.connections == nil || lim.connections.max != maxConnections { + t.Errorf("connections = %+v, want max=%d", lim.connections, maxConnections) + } + if got := lim.inFlight.occupied(); got != 0 { + t.Errorf("initial inFlight.occupied() = %d, want 0", got) + } + if got := lim.connections.occupied(); got != 0 { + t.Errorf("initial connections.occupied() = %d, want 0", got) + } +} + +func TestLimiterSlotsAcquireAndRelease(t *testing.T) { + const cap = 3 + s := &limiterSlots{max: cap} + + for i := int32(1); i <= cap; i++ { + if !s.tryAcquire() { + t.Fatalf("tryAcquire %d failed before reaching capacity %d", i, cap) + } + if got := s.occupied(); got != i { + t.Errorf("occupied() = %d after %d acquisitions, want %d", got, i, i) + } + } + + // At capacity: further acquisitions fail and do not change occupied count. + if s.tryAcquire() { + t.Fatal("tryAcquire succeeded at capacity") + } + if got := s.occupied(); got != cap { + t.Errorf("occupied() after rejected tryAcquire = %d, want %d", got, cap) + } + + // Releasing one slot allows one more acquisition. + s.release() + if got := s.occupied(); got != cap-1 { + t.Errorf("occupied() after release = %d, want %d", got, cap-1) + } + if !s.tryAcquire() { + t.Fatal("tryAcquire failed after releasing a slot") + } + + // Drain all slots. + for i := cap - 1; i >= 0; i-- { + s.release() + if got := s.occupied(); got != int32(i) { + t.Errorf("occupied() = %d, want %d", got, i) + } + } +} + +func TestLimiterSlotsNilReceiver(t *testing.T) { + var s *limiterSlots + if !s.tryAcquire() { + t.Error("nil limiterSlots.tryAcquire() = false, want true") + } + // Must not panic. + s.release() +} + +func TestLimiterSlotsReleaseUnderflowPanics(t *testing.T) { + s := &limiterSlots{max: 1} + defer func() { + if r := recover(); r == nil { + t.Fatal("expected panic when releasing an unacquired slot, got none") + } + }() + s.release() +} + +func TestLimiterSlotsConcurrent(t *testing.T) { + const ( + maxSlots = 8 + goroutines = 32 + iterations = 200 + ) + s := &limiterSlots{max: maxSlots} + + var active atomic.Int32 + var maxObserved atomic.Int32 + var wg sync.WaitGroup + + for range goroutines { + wg.Add(1) + go func() { + defer wg.Done() + for range iterations { + if !s.tryAcquire() { + continue + } + cur := active.Add(1) + for { + prev := maxObserved.Load() + if cur <= prev || maxObserved.CompareAndSwap(prev, cur) { + break + } + } + active.Add(-1) + s.release() + } + }() + } + wg.Wait() + + if got := maxObserved.Load(); got > maxSlots { + t.Errorf("max concurrent slots held = %d, exceeds limit %d", got, maxSlots) + } + if got := s.occupied(); got != 0 { + t.Errorf("occupied() after all goroutines finished = %d, want 0", got) + } +} diff --git a/internal/ateomnet/dns/parse.go b/internal/ateomnet/dns/parse.go deleted file mode 100644 index b3a7420155..0000000000 --- a/internal/ateomnet/dns/parse.go +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package dns - -import ( - "errors" - "fmt" -) - -// Response codes that say the resolver failed rather than answered. NXDOMAIN -// and NOERROR are answers and are passed back as they are. -const ( - rcodeServFail = 2 - rcodeNotImp = 4 - rcodeRefused = 5 -) - -// failoverRcode reports the response code when the relay should try the next -// upstream. Reads the 12-byte header only; anything shorter is passed through. -func failoverRcode(msg []byte) (byte, bool) { - if len(msg) < 12 { - return 0, false - } - rcode := msg[3] & 0x0f - switch rcode { - case rcodeServFail, rcodeNotImp, rcodeRefused: - return rcode, true - } - return 0, false -} - -func rcodeError(rcode byte) error { - switch rcode { - case rcodeServFail: - return errors.New("answered SERVFAIL") - case rcodeNotImp: - return errors.New("answered NOTIMP") - case rcodeRefused: - return errors.New("answered REFUSED") - } - return fmt.Errorf("answered rcode %d", rcode) -} diff --git a/internal/ateomnet/dns/pendingreq.go b/internal/ateomnet/dns/pendingreq.go new file mode 100644 index 0000000000..ba5146adeb --- /dev/null +++ b/internal/ateomnet/dns/pendingreq.go @@ -0,0 +1,242 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dns + +import ( + "bytes" + "math/rand/v2" + "net" + "sync" + "time" + + "github.com/agent-substrate/substrate/internal/ateomnet/dns/freemap" + "github.com/agent-substrate/substrate/internal/ateomnet/dns/protocol" + "golang.org/x/net/dns/dnsmessage" +) + +// pendingRequest holds the per-query state needed to validate UDP responses, +// restore the actor's transaction ID, and fail over across upstreams. +type pendingRequest struct { + // clientRequestID is the actor's original DNS transaction ID. + clientRequestID uint16 + // clientAddr identifies the actor UDP endpoint. + clientAddr net.Addr + // question is the canonicalized DNS question used to validate responses + // (RFC 5452). + question dnsmessage.Question + // rawQuery is a copy of the query with upstreamID written to bytes 0..1. + rawQuery []byte + // upstreams is the ordered list of upstream "host:port" addresses to try. + upstreams []string + // upstreamIdx is the index in upstreams of the current attempt. + upstreamIdx int + // deferredResp holds the last failover-eligible error response (e.g. + // SERVFAIL) to return if all remaining upstreams time out. + deferredResp []byte + // expiry is the deadline for the current upstream attempt. + expiry time.Time +} + +// pendingRequests maps upstreamID to in-flight UDP requests. +// +// TODO(bowei): move the response lookup and failover transition in +// dnsHandler.onUDPResponse into a method on pendingRequests so all lock and +// state transitions live in this file. +type pendingRequests struct { + // mu guards entries and inUse. + mu sync.Mutex + // entries maps active upstreamID keys to in-flight requests. + entries map[uint16]*pendingRequest + // inUse tracks allocated 16-bit upstream transaction IDs. + inUse freemap.Map64k + // limiter tracks in-flight request slots released when entries are removed. + limiter *limiter + + // exchangeTimeout bounds the total upstream exchange (defaults to + // dnsExchangeTimeout). + exchangeTimeout time.Duration + // attemptTimeout, if positive, overrides the per-attempt timeout (used in + // tests). + attemptTimeout time.Duration +} + +// newPendingRequests returns an empty pendingRequests table using lim for +// in-flight slot accounting. +func newPendingRequests(lim *limiter) *pendingRequests { + return &pendingRequests{ + entries: make(map[uint16]*pendingRequest), + limiter: lim, + exchangeTimeout: dnsExchangeTimeout, + } +} + +// timeoutForAttempt returns the deadline duration for attempt upstreamIdx out +// of numUpstreams. Attempts with fallback upstreams remaining get an equal +// share of exchangeTimeout capped at defaultUpstreamAttemptTimeout; the final +// upstream gets the full exchangeTimeout. +func (p *pendingRequests) timeoutForAttempt(upstreamIdx int, numUpstreams int) time.Duration { + if p.attemptTimeout > 0 { + return p.attemptTimeout + } + timeout := p.exchangeTimeout + if timeout <= 0 { + timeout = dnsExchangeTimeout + } + if numUpstreams > 1 && upstreamIdx+1 < numUpstreams { + return min(timeout/time.Duration(numUpstreams), defaultUpstreamAttemptTimeout) + } + return timeout +} + +// record allocates a unique upstreamID and stores a pendingRequest entry for +// upstreams[0]. It takes ownership of releasing the caller's in-flight slot on +// failure or entry removal. +func (p *pendingRequests) record( + raw []byte, + clientReqID uint16, + q dnsmessage.Question, + clientAddr net.Addr, + upstreams []string, +) (uint16, bool) { + p.mu.Lock() + defer p.mu.Unlock() + + // Probe from a random starting ID so upstream transaction IDs are not predictable. + start := uint16(rand.Uint32()) + upstreamID, ok := p.inUse.FindFirstUnsetFrom(start) + if !ok { + p.limiter.inFlight.release() + return 0, false + } + + // Rewrite the packet's transaction ID in place and clone it for failover retries. + protocol.SetTxnID(raw, upstreamID) + p.inUse.Set(upstreamID) + p.entries[upstreamID] = &pendingRequest{ + clientRequestID: clientReqID, + clientAddr: clientAddr, + question: q, + rawQuery: bytes.Clone(raw), + upstreams: upstreams, + upstreamIdx: 0, + expiry: time.Now().Add(p.timeoutForAttempt(0, len(upstreams))), + } + + return upstreamID, true +} + +// deleteEntryLocked removes upstreamID from entries, frees its ID in inUse, and +// releases its in-flight slot if upstreamID was present. p.mu must be held. +func (p *pendingRequests) deleteEntryLocked(upstreamID uint16) { + if _, ok := p.entries[upstreamID]; !ok { + return + } + delete(p.entries, upstreamID) + p.inUse.Clear(upstreamID) + p.limiter.inFlight.release() +} + +// failOverOnSendError advances an entry to the next upstream when sending to +// failedIdx fails synchronously, or removes the entry if no upstreams remain. +func (p *pendingRequests) failOverOnSendError( + upstreamID uint16, + failedIdx int, +) (nextIdx int, query []byte, ok bool) { + p.mu.Lock() + defer p.mu.Unlock() + + // Match both upstreamID and failedIdx so stale errors from earlier attempts + // are ignored. + entry, ok := p.entries[upstreamID] + if !ok || entry.upstreamIdx != failedIdx { + return 0, nil, false + } + // Drop the request if no fallback upstreams remain. + if entry.upstreamIdx+1 >= len(entry.upstreams) { + p.deleteEntryLocked(upstreamID) + return 0, nil, false + } + + // Advance the entry to the next upstream and reset its attempt deadline. + entry.upstreamIdx++ + entry.expiry = time.Now().Add(p.timeoutForAttempt(entry.upstreamIdx, len(entry.upstreams))) + return entry.upstreamIdx, entry.rawQuery, true +} + +// clearAll removes all pending entries and releases their in-flight slots. +func (p *pendingRequests) clearAll() { + p.mu.Lock() + defer p.mu.Unlock() + + for upstreamID := range p.entries { + p.deleteEntryLocked(upstreamID) + } +} + +// reqFailover describes a timed-out query to resend to the next upstream. +type reqFailover struct { + upstreamID uint16 + upstreamIdx int + query []byte +} + +// reqDelivery describes a deferred response to send back to the actor after all +// upstreams have been exhausted. +type reqDelivery struct { + clientAddr net.Addr + payload []byte +} + +// sweep inspects all pending entries at now, advancing expired multi-upstream +// entries to their next upstream and evicting entries that have exhausted all +// upstreams. +func (p *pendingRequests) sweep(now time.Time) ([]reqFailover, []reqDelivery) { + p.mu.Lock() + defer p.mu.Unlock() + + var failovers []reqFailover + var deliveries []reqDelivery + + for upstreamID, entry := range p.entries { + if entry.expiry.IsZero() || !now.After(entry.expiry) { + continue + } + // Advance to the next upstream if fallbacks remain. + if entry.upstreamIdx+1 < len(entry.upstreams) { + entry.upstreamIdx++ + entry.expiry = now.Add(p.timeoutForAttempt(entry.upstreamIdx, len(entry.upstreams))) + failovers = append(failovers, reqFailover{ + upstreamID: upstreamID, + upstreamIdx: entry.upstreamIdx, + query: entry.rawQuery, + }) + continue + } + + // All upstreams exhausted: evict the entry and deliver any held-back + // error response (e.g. SERVFAIL from an earlier upstream) with the + // actor's original transaction ID restored. + p.deleteEntryLocked(upstreamID) + if entry.deferredResp != nil && entry.clientAddr != nil { + resp := bytes.Clone(entry.deferredResp) + protocol.SetTxnID(resp, entry.clientRequestID) + deliveries = append(deliveries, reqDelivery{ + clientAddr: entry.clientAddr, + payload: resp, + }) + } + } + return failovers, deliveries +} diff --git a/internal/ateomnet/dns/pendingreq_test.go b/internal/ateomnet/dns/pendingreq_test.go new file mode 100644 index 0000000000..73ac25ccb0 --- /dev/null +++ b/internal/ateomnet/dns/pendingreq_test.go @@ -0,0 +1,367 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dns + +import ( + "bytes" + "encoding/binary" + "net" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "golang.org/x/net/dns/dnsmessage" +) + +// exampleQuestion is the question dnsQuery asks. +var exampleQuestion = dnsmessage.Question{ + Name: dnsmessage.MustNewName("example.com."), + Type: dnsmessage.TypeA, + Class: dnsmessage.ClassINET, +} + +// recordQuery records dnsQuery(clientID) holding an in-flight slot, as a +// forwarded query does, and returns its upstream ID and the rewritten query. +func recordQuery(t *testing.T, p *pendingRequests, clientID uint16, client net.Addr, upstreams []string) (uint16, []byte) { + t.Helper() + p.limiter.inFlight.tryAcquire() + raw := dnsQuery(clientID) + id, ok := p.record(raw, clientID, exampleQuestion, client, upstreams) + if !ok { + t.Fatalf("record(%#x) failed", clientID) + } + return id, raw +} + +// checkExpiry fails t unless expiry is timeout after an instant in [before, after]. +func checkExpiry(t *testing.T, expiry, before, after time.Time, timeout time.Duration) { + t.Helper() + if expiry.Before(before.Add(timeout)) || expiry.After(after.Add(timeout)) { + t.Errorf("expiry is %v after the call, want %v", expiry.Sub(before), timeout) + } +} + +func TestPendingRequestsTimeoutForAttempt(t *testing.T) { + // Long enough that splitting it two ways still exceeds the cap. + const exchange = 10 * defaultUpstreamAttemptTimeout + + for _, tc := range []struct { + name string + exchangeTimeout time.Duration + attemptTimeout time.Duration + upstreamIdx int + numUpstreams int + want time.Duration + }{ + { + name: "single upstream gets the whole exchange", + exchangeTimeout: exchange, + numUpstreams: 1, + want: exchange, + }, + { + name: "attempt with fallbacks left is capped", + exchangeTimeout: exchange, + numUpstreams: 2, + want: defaultUpstreamAttemptTimeout, + }, + { + name: "attempt with fallbacks left gets its share", + exchangeTimeout: defaultUpstreamAttemptTimeout, + upstreamIdx: 1, + numUpstreams: 4, + want: defaultUpstreamAttemptTimeout / 4, + }, + { + name: "last upstream gets the whole exchange", + exchangeTimeout: exchange, + upstreamIdx: 1, + numUpstreams: 2, + want: exchange, + }, + { + name: "unset exchange timeout uses the default", + numUpstreams: 1, + want: dnsExchangeTimeout, + }, + { + name: "attempt timeout overrides an attempt with fallbacks left", + exchangeTimeout: exchange, + attemptTimeout: 40 * time.Millisecond, + numUpstreams: 2, + want: 40 * time.Millisecond, + }, + { + name: "attempt timeout overrides the last attempt", + exchangeTimeout: exchange, + attemptTimeout: 40 * time.Millisecond, + upstreamIdx: 1, + numUpstreams: 2, + want: 40 * time.Millisecond, + }, + } { + t.Run(tc.name, func(t *testing.T) { + p := newPendingRequests(nil) + p.exchangeTimeout = tc.exchangeTimeout + p.attemptTimeout = tc.attemptTimeout + if got := p.timeoutForAttempt(tc.upstreamIdx, tc.numUpstreams); got != tc.want { + t.Errorf("timeoutForAttempt(%d, %d) = %v, want %v", tc.upstreamIdx, tc.numUpstreams, got, tc.want) + } + }) + } +} + +func TestPendingRequestsRecord(t *testing.T) { + lim := newLimiter() + p := newPendingRequests(lim) + client := &net.UDPAddr{IP: net.ParseIP("169.254.0.2"), Port: 54321} + // Responses are matched on their unmapped source address, so the key must + // be normalized. + upstreams := []string{"[::ffff:10.96.0.10]:53", "10.96.0.11:53"} + + lim.inFlight.tryAcquire() + raw := dnsQuery(0xbeef) + before := time.Now() + id, ok := p.record(raw, 0xbeef, exampleQuestion, client, upstreams) + after := time.Now() + if !ok { + t.Fatal("record failed") + } + if got := binary.BigEndian.Uint16(raw[0:2]); got != id { + t.Errorf("query ID = %#x, want the upstream ID %#x", got, id) + } + entry, ok := p.entries[id] + if !ok { + t.Fatal("no entry keyed by the upstream ID") + } + want := &pendingRequest{ + clientRequestID: 0xbeef, + clientAddr: client, + question: exampleQuestion, + rawQuery: raw, + upstreams: upstreams, + } + if diff := cmp.Diff(want, entry, cmp.AllowUnexported(pendingRequest{}), cmpopts.IgnoreFields(pendingRequest{}, "expiry")); diff != "" { + t.Errorf("entry mismatch (-want +got):\n%s", diff) + } + checkExpiry(t, entry.expiry, before, after, p.timeoutForAttempt(0, len(upstreams))) + if !p.inUse.Get(id) { + t.Errorf("upstream ID %#x is not marked in use", id) + } + if got := lim.inFlight.occupied(); got != 1 { + t.Errorf("%d in-flight slots held, want 1 until the request completes", got) + } + + // The entry keeps its own copy of the query to resend on failover. + raw[len(raw)-1] ^= 0xff + if bytes.Equal(entry.rawQuery, raw) { + t.Error("entry shares the caller's query buffer") + } +} + +// Responses are matched by upstream ID, so record must not hand out one still +// in flight. +func TestPendingRequestsRecordAllocatesFreeID(t *testing.T) { + lim := newLimiter() + p := newPendingRequests(lim) + upstreams := []string{"10.96.0.10:53"} + const free = 0xabcd + for id := range 1 << 16 { + if id != free { + p.inUse.Set(uint16(id)) + } + } + + if id, _ := recordQuery(t, p, 0x1234, nil, upstreams); id != free { + t.Fatalf("record allocated %#x, want the only free ID %#x", id, free) + } + + // With every ID in flight, record fails and frees the query's slot. + lim.inFlight.tryAcquire() + if _, ok := p.record(dnsQuery(0x5678), 0x5678, exampleQuestion, nil, upstreams); ok { + t.Fatal("record succeeded with every upstream ID in flight") + } + if got := lim.inFlight.occupied(); got != 1 { + t.Errorf("%d in-flight slots held, want 1 for the recorded query", got) + } +} + +func TestPendingRequestsDeleteEntryReleasesSlotOnce(t *testing.T) { + lim := newLimiter() + p := newPendingRequests(lim) + upstreams := []string{"10.96.0.10:53"} + lim.inFlight.tryAcquire() // held by another query + id, _ := recordQuery(t, p, 0x1234, nil, upstreams) + + p.mu.Lock() + p.deleteEntryLocked(id) + p.deleteEntryLocked(id) + p.mu.Unlock() + + if len(p.entries) != 0 || p.inUse.Count() != 0 { + t.Errorf("deleted entry left %d entries and %d IDs in use", len(p.entries), p.inUse.Count()) + } + if got := lim.inFlight.occupied(); got != 1 { + t.Errorf("%d in-flight slots held, want 1: deleting twice must free the slot once", got) + } +} + +func TestPendingRequestsFailOverOnSendError(t *testing.T) { + lim := newLimiter() + p := newPendingRequests(lim) + upstreams := []string{"10.96.0.10:53", "10.96.0.11:53"} + id, raw := recordQuery(t, p, 0x1234, nil, upstreams) + + // Errors for an unknown ID, or for an attempt not yet made, are ignored. + if _, _, ok := p.failOverOnSendError(id+1, 0); ok { + t.Error("failed over a request that was never recorded") + } + if _, _, ok := p.failOverOnSendError(id, 1); ok { + t.Error("failed over on an error for an attempt not yet made") + } + + // The first upstream failing moves the request to the second. + before := time.Now() + nextIdx, query, ok := p.failOverOnSendError(id, 0) + after := time.Now() + if !ok || nextIdx != 1 { + t.Fatalf("failOverOnSendError(%#x, 0) = %d, _, %v; want 1, _, true", id, nextIdx, ok) + } + if !bytes.Equal(query, raw) { + t.Error("failover query differs from the recorded query") + } + entry, ok := p.entries[id] + if !ok { + t.Fatal("entry missing after failover") + } + if entry.upstreamIdx != 1 { + t.Errorf("upstreamIdx = %d, want 1", entry.upstreamIdx) + } + checkExpiry(t, entry.expiry, before, after, p.timeoutForAttempt(1, len(upstreams))) + if got := lim.inFlight.occupied(); got != 1 { + t.Errorf("%d in-flight slots held during failover, want 1", got) + } + + // A repeated error for the first attempt is stale. + if _, _, ok := p.failOverOnSendError(id, 0); ok { + t.Error("failed over twice on errors for the same attempt") + } + + // The last upstream failing drops the request and frees its slot. + if _, _, ok := p.failOverOnSendError(id, 1); ok { + t.Error("failed over past the last upstream") + } + if len(p.entries) != 0 || p.inUse.Count() != 0 { + t.Errorf("dropped request left %d entries and %d IDs in use", len(p.entries), p.inUse.Count()) + } + if got := lim.inFlight.occupied(); got != 0 { + t.Errorf("%d in-flight slots held after the last upstream failed, want 0", got) + } +} + +func TestPendingRequestsClearAll(t *testing.T) { + lim := newLimiter() + p := newPendingRequests(lim) + upstreams := []string{"10.96.0.10:53", "10.96.0.11:53"} + for i := range 3 { + recordQuery(t, p, uint16(i), nil, upstreams) + } + + p.clearAll() + if len(p.entries) != 0 || p.inUse.Count() != 0 { + t.Errorf("clearAll left %d entries and %d IDs in use", len(p.entries), p.inUse.Count()) + } + if got := lim.inFlight.occupied(); got != 0 { + t.Errorf("%d in-flight slots held after clearAll, want 0", got) + } +} + +func TestPendingRequestsSweep(t *testing.T) { + lim := newLimiter() + p := newPendingRequests(lim) + upstreams := []string{"10.96.0.10:53", "10.96.0.11:53"} + id, raw := recordQuery(t, p, 0x1234, nil, upstreams) + entry := p.entries[id] + + if failovers, deliveries := p.sweep(entry.expiry); len(failovers) != 0 || len(deliveries) != 0 { + t.Fatalf("sweep at the expiry returned %d failovers and %d deliveries, want none until it has passed", len(failovers), len(deliveries)) + } + + // An expired attempt moves to the next upstream. + now := entry.expiry.Add(time.Nanosecond) + failovers, deliveries := p.sweep(now) + wantFailovers := []reqFailover{{upstreamID: id, upstreamIdx: 1, query: raw}} + if diff := cmp.Diff(wantFailovers, failovers, cmp.AllowUnexported(reqFailover{})); diff != "" { + t.Errorf("failovers mismatch (-want +got):\n%s", diff) + } + if len(deliveries) != 0 { + t.Errorf("sweep returned %d deliveries on failover, want 0", len(deliveries)) + } + if len(p.entries) != 1 || p.entries[id] != entry || entry.upstreamIdx != 1 { + t.Fatal("entry not advanced to the next upstream") + } + if want := now.Add(p.timeoutForAttempt(1, len(upstreams))); !entry.expiry.Equal(want) { + t.Errorf("expiry = %v, want %v", entry.expiry, want) + } + + // An expired last attempt with no answer held back is dropped silently. + failovers, deliveries = p.sweep(entry.expiry.Add(time.Nanosecond)) + if len(failovers) != 0 || len(deliveries) != 0 { + t.Errorf("sweep after the last attempt returned %d failovers and %d deliveries, want none", len(failovers), len(deliveries)) + } + if len(p.entries) != 0 || p.inUse.Count() != 0 { + t.Errorf("expired request left %d entries and %d IDs in use", len(p.entries), p.inUse.Count()) + } + if got := lim.inFlight.occupied(); got != 0 { + t.Errorf("%d in-flight slots held after expiry, want 0", got) + } +} + +// A SERVFAIL beats a timeout: when the last upstream never answers, the client +// gets the SERVFAIL held back from an earlier one. +func TestPendingRequestsSweepDeliversDeferredAnswer(t *testing.T) { + lim := newLimiter() + p := newPendingRequests(lim) + client := &net.UDPAddr{IP: net.ParseIP("169.254.0.2"), Port: 54321} + upstreams := []string{"10.96.0.10:53", "10.96.0.11:53"} + id, raw := recordQuery(t, p, 0x1234, client, upstreams) + + // Leave the request as a SERVFAIL from the first upstream does: waiting on + // the second, with the SERVFAIL held back. + if _, _, ok := p.failOverOnSendError(id, 0); !ok { + t.Fatal("failOverOnSendError failed") + } + entry := p.entries[id] + entry.deferredResp = dnsAnswer(raw, byte(dnsmessage.RCodeServerFailure)) + + failovers, deliveries := p.sweep(entry.expiry.Add(time.Nanosecond)) + if len(failovers) != 0 { + t.Errorf("sweep returned %d failovers past the last upstream, want 0", len(failovers)) + } + wantDeliveries := []reqDelivery{{ + clientAddr: client, + payload: dnsAnswer(dnsQuery(0x1234), byte(dnsmessage.RCodeServerFailure)), + }} + if diff := cmp.Diff(wantDeliveries, deliveries, cmp.AllowUnexported(reqDelivery{})); diff != "" { + t.Errorf("deliveries mismatch (-want +got):\n%s", diff) + } + if len(p.entries) != 0 || p.inUse.Count() != 0 { + t.Errorf("expired request left %d entries and %d IDs in use", len(p.entries), p.inUse.Count()) + } + if got := lim.inFlight.occupied(); got != 0 { + t.Errorf("%d in-flight slots held after expiry, want 0", got) + } +} diff --git a/internal/ateomnet/dns/protocol/doc.go b/internal/ateomnet/dns/protocol/doc.go new file mode 100644 index 0000000000..511fd5bb5b --- /dev/null +++ b/internal/ateomnet/dns/protocol/doc.go @@ -0,0 +1,17 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package protocol contains helpers to manage encoding/decoding DNS protocol +// to/from a byte array. +package protocol diff --git a/internal/ateomnet/dns/protocol/protocol.go b/internal/ateomnet/dns/protocol/protocol.go new file mode 100644 index 0000000000..d21c712842 --- /dev/null +++ b/internal/ateomnet/dns/protocol/protocol.go @@ -0,0 +1,117 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package protocol + +import ( + "encoding/binary" + + "golang.org/x/net/dns/dnsmessage" +) + +const ( + dnsHeaderLen = 12 + + // TypeIXFR is the incremental zone transfer QTYPE (RFC 1995). + TypeIXFR dnsmessage.Type = 251 + + flagQuery = 0 + flagResponse = 1 +) + +// qrFlag is the query (0) or response (1) bit +func qrFlag(raw []byte) int { + if raw[2]&0x80 == 0 { + return 0 + } + return 1 +} + +// RequestSanityCheck filters out obviously invalid packets. +func RequestSanityCheck(raw []byte) bool { + // DNS packets need to be at least 12 bytes. + if len(raw) < dnsHeaderLen { + return false + } + if qrFlag(raw) != flagQuery { + return false + } + return true +} + +// ResponseSanityCheck filters out obviously invalid packets. +func ResponseSanityCheck(raw []byte) bool { + // DNS packets need to be at least 12 bytes. + const dnsHeaderLen = 12 + if len(raw) < dnsHeaderLen { + return false + } + if qrFlag(raw) != flagResponse { + return false + } + if QDCount(raw) != 1 { + return false + } + return true +} + +// SetTxnID in the DNS packet. +func SetTxnID(raw []byte, id uint16) { + binary.BigEndian.PutUint16(raw[0:2], id) +} + +// qdCountFromRaw from the raw packet. +func QDCount(raw []byte) uint16 { + // +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ + // | QDCOUNT = 0 | bytes 4..5 + // +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ + return binary.BigEndian.Uint16(raw[4:6]) +} + +// ErrorPacket returns a serialized DNS packet signaling an error. +func ErrorPacket(hdr dnsmessage.Header, rcode dnsmessage.RCode, q *dnsmessage.Question) []byte { + respHdr := dnsmessage.Header{ + ID: hdr.ID, + Response: true, + OpCode: hdr.OpCode, + RecursionDesired: hdr.RecursionDesired, + RecursionAvailable: true, + RCode: rcode, + } + + b := dnsmessage.NewBuilder(nil, respHdr) + // Try to tack on the question, if possible. + if q != nil { + if err := b.StartQuestions(); err == nil { + _ = b.Question(*q) + } + } + + out, err := b.Finish() + if err == nil { + return out + } + + // Backstop reply: we had errors generating a response. Synthesize an empty + // DNS response that matches the request instead. + fallback := make([]byte, dnsHeaderLen) + SetTxnID(fallback, hdr.ID) + fallback[2] = 0x80 | (byte(hdr.OpCode)&0x0f)<<3 + if hdr.RecursionDesired { + fallback[2] |= 0x01 + } + fallback[3] = 0x80 | (byte(rcode) & 0x0f) + + return fallback +} diff --git a/internal/ateomnet/dns/protocol/protocol_test.go b/internal/ateomnet/dns/protocol/protocol_test.go new file mode 100644 index 0000000000..52c953dace --- /dev/null +++ b/internal/ateomnet/dns/protocol/protocol_test.go @@ -0,0 +1,287 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package protocol + +import ( + "encoding/binary" + "testing" + + "golang.org/x/net/dns/dnsmessage" +) + +func TestRequestSanityCheck(t *testing.T) { + tests := []struct { + name string + raw []byte + want bool + }{ + { + name: "nil slice", + raw: nil, + want: false, + }, + { + name: "empty slice", + raw: []byte{}, + want: false, + }, + { + name: "short packet (11 bytes)", + raw: make([]byte, 11), + want: false, + }, + { + name: "valid 12-byte query (QR=0)", + raw: make([]byte, 12), + want: true, + }, + { + name: "valid query with non-QR bits set in byte 2", + raw: []byte{ + 0x12, 0x34, 0x7f, 0x00, + 0x00, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + }, + want: true, + }, + { + name: "response packet (QR=1)", + raw: []byte{ + 0x12, 0x34, 0x80, 0x00, + 0x00, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + }, + want: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := RequestSanityCheck(tc.raw); got != tc.want { + t.Errorf("RequestSanityCheck(%x) = %v, want %v", tc.raw, got, tc.want) + } + }) + } +} + +func TestResponseSanityCheck(t *testing.T) { + tests := []struct { + name string + raw []byte + want bool + }{ + { + name: "nil slice", + raw: nil, + want: false, + }, + { + name: "short packet (11 bytes)", + raw: make([]byte, 11), + want: false, + }, + { + name: "query packet (QR=0) with QDCOUNT=1", + raw: []byte{ + 0x12, 0x34, 0x00, 0x00, + 0x00, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + }, + want: false, + }, + { + name: "response packet (QR=1) with QDCOUNT=0", + raw: []byte{ + 0x12, 0x34, 0x80, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + }, + want: false, + }, + { + name: "response packet (QR=1) with QDCOUNT=2", + raw: []byte{ + 0x12, 0x34, 0x80, 0x00, + 0x00, 0x02, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + }, + want: false, + }, + { + name: "valid response packet (QR=1, QDCOUNT=1)", + raw: []byte{ + 0x12, 0x34, 0x81, 0x80, + 0x00, 0x01, 0x00, 0x01, + 0x00, 0x00, 0x00, 0x00, + }, + want: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := ResponseSanityCheck(tc.raw); got != tc.want { + t.Errorf("ResponseSanityCheck(%x) = %v, want %v", tc.raw, got, tc.want) + } + }) + } +} + +func TestQDCount(t *testing.T) { + tests := []struct { + qdCount uint16 + }{ + {0}, + {1}, + {2}, + {0x1234}, + {0xffff}, + } + + for _, tc := range tests { + var raw [12]byte + binary.BigEndian.PutUint16(raw[4:6], tc.qdCount) + if got := QDCount(raw[:]); got != tc.qdCount { + t.Errorf("QDCount() = %#x, want %#x", got, tc.qdCount) + } + } +} + +func TestSetTxnID(t *testing.T) { + var raw [12]byte + SetTxnID(raw[:], 0xcafe) + if got := binary.BigEndian.Uint16(raw[0:2]); got != 0xcafe { + t.Errorf("SetTxnID wrote %#x, want 0xcafe", got) + } +} + +func TestErrorPacket(t *testing.T) { + validQ := dnsmessage.Question{ + Name: dnsmessage.MustNewName("example.com."), + Type: dnsmessage.TypeA, + Class: dnsmessage.ClassINET, + } + + t.Run("with valid question", func(t *testing.T) { + reqHdr := dnsmessage.Header{ + ID: 0xabcd, + OpCode: 0, + RecursionDesired: true, + } + pkt := ErrorPacket(reqHdr, dnsmessage.RCodeRefused, &validQ) + + var p dnsmessage.Parser + gotHdr, err := p.Start(pkt) + if err != nil { + t.Fatalf("p.Start() unexpected error: %v", err) + } + if gotHdr.ID != reqHdr.ID { + t.Errorf("ID = %#x, want %#x", gotHdr.ID, reqHdr.ID) + } + if !gotHdr.Response { + t.Error("Response = false, want true") + } + if gotHdr.OpCode != reqHdr.OpCode { + t.Errorf("OpCode = %v, want %v", gotHdr.OpCode, reqHdr.OpCode) + } + if !gotHdr.RecursionDesired { + t.Error("RecursionDesired = false, want true") + } + if !gotHdr.RecursionAvailable { + t.Error("RecursionAvailable = false, want true") + } + if gotHdr.RCode != dnsmessage.RCodeRefused { + t.Errorf("RCode = %v, want %v", gotHdr.RCode, dnsmessage.RCodeRefused) + } + + questions, err := p.AllQuestions() + if err != nil { + t.Fatalf("p.AllQuestions() unexpected error: %v", err) + } + if len(questions) != 1 || questions[0] != validQ { + t.Errorf("questions = %+v, want [%+v]", questions, validQ) + } + }) + + t.Run("with nil question", func(t *testing.T) { + reqHdr := dnsmessage.Header{ + ID: 0x1234, + OpCode: 1, + RecursionDesired: false, + } + pkt := ErrorPacket(reqHdr, dnsmessage.RCodeNotImplemented, nil) + + var p dnsmessage.Parser + gotHdr, err := p.Start(pkt) + if err != nil { + t.Fatalf("p.Start() unexpected error: %v", err) + } + if gotHdr.ID != reqHdr.ID { + t.Errorf("ID = %#x, want %#x", gotHdr.ID, reqHdr.ID) + } + if !gotHdr.Response { + t.Error("Response = false, want true") + } + if gotHdr.OpCode != reqHdr.OpCode { + t.Errorf("OpCode = %v, want %v", gotHdr.OpCode, reqHdr.OpCode) + } + if gotHdr.RecursionDesired { + t.Error("RecursionDesired = true, want false") + } + if !gotHdr.RecursionAvailable { + t.Error("RecursionAvailable = false, want true") + } + if gotHdr.RCode != dnsmessage.RCodeNotImplemented { + t.Errorf("RCode = %v, want %v", gotHdr.RCode, dnsmessage.RCodeNotImplemented) + } + + questions, err := p.AllQuestions() + if err != nil { + t.Fatalf("p.AllQuestions() unexpected error: %v", err) + } + if len(questions) != 0 { + t.Errorf("len(questions) = %d, want 0", len(questions)) + } + }) + + t.Run("with unencodable question falls back to empty question section", func(t *testing.T) { + // Zero-value Name (Length == 0, non-canonical) causes Builder.Question to fail, + // but Builder.Finish still succeeds with 0 questions. + badQ := dnsmessage.Question{ + Type: dnsmessage.TypeA, + Class: dnsmessage.ClassINET, + } + reqHdr := dnsmessage.Header{ + ID: 0x4321, + RecursionDesired: true, + } + pkt := ErrorPacket(reqHdr, dnsmessage.RCodeFormatError, &badQ) + + var p dnsmessage.Parser + gotHdr, err := p.Start(pkt) + if err != nil { + t.Fatalf("p.Start() unexpected error: %v", err) + } + if gotHdr.ID != reqHdr.ID { + t.Errorf("ID = %#x, want %#x", gotHdr.ID, reqHdr.ID) + } + if gotHdr.RCode != dnsmessage.RCodeFormatError { + t.Errorf("RCode = %v, want %v", gotHdr.RCode, dnsmessage.RCodeFormatError) + } + if qd := QDCount(pkt); qd != 0 { + t.Errorf("QDCount = %d, want 0", qd) + } + }) +} diff --git a/internal/ateomnet/dns/protocol/tcp.go b/internal/ateomnet/dns/protocol/tcp.go new file mode 100644 index 0000000000..c5c412f2c5 --- /dev/null +++ b/internal/ateomnet/dns/protocol/tcp.go @@ -0,0 +1,52 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package protocol + +import ( + "encoding/binary" + "errors" + "fmt" + "io" +) + +// ReadTCPFrame reads a single RFC 1035 §4.2.2 2-byte big-endian length-prefixed +// DNS message from r. +func ReadTCPFrame(r io.Reader) ([]byte, error) { + var lenBuf [2]byte + if _, err := io.ReadFull(r, lenBuf[:]); err != nil { + return nil, err + } + length := int(binary.BigEndian.Uint16(lenBuf[:])) + msg := make([]byte, length) + if _, err := io.ReadFull(r, msg); err != nil { + if errors.Is(err, io.EOF) { + return nil, io.ErrUnexpectedEOF + } + return nil, err + } + return msg, nil +} + +// WriteTCPFrame writes msg prefixed with its 2-byte big-endian length to `w“. +func WriteTCPFrame(w io.Writer, msg []byte) error { + if len(msg) > 0xffff { + return fmt.Errorf("dns: TCP message length %d exceeds uint16 maximum", len(msg)) + } + frame := make([]byte, 2+len(msg)) + binary.BigEndian.PutUint16(frame[0:2], uint16(len(msg))) + copy(frame[2:], msg) + _, err := w.Write(frame) + return err +} diff --git a/internal/ateomnet/dns/protocol/tcp_test.go b/internal/ateomnet/dns/protocol/tcp_test.go new file mode 100644 index 0000000000..4a05c43707 --- /dev/null +++ b/internal/ateomnet/dns/protocol/tcp_test.go @@ -0,0 +1,120 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package protocol + +import ( + "bytes" + "errors" + "io" + "testing" +) + +func TestTCPFrameRoundTrip(t *testing.T) { + payloads := [][]byte{ + {}, + {0x42}, + bytes.Repeat([]byte{0xab, 0xcd}, 256), + bytes.Repeat([]byte{0xff}, 0xffff), + } + + var buf bytes.Buffer + for i, p := range payloads { + if err := WriteTCPFrame(&buf, p); err != nil { + t.Fatalf("frame %d: WriteTCPFrame() unexpected error: %v", i, err) + } + } + + for i, want := range payloads { + got, err := ReadTCPFrame(&buf) + if err != nil { + t.Fatalf("frame %d: ReadTCPFrame() unexpected error: %v", i, err) + } + if !bytes.Equal(got, want) { + t.Errorf("frame %d: payload mismatch (got len %d, want len %d)", i, len(got), len(want)) + } + } + + // Subsequent read on drained buffer should return io.EOF. + if _, err := ReadTCPFrame(&buf); !errors.Is(err, io.EOF) { + t.Errorf("ReadTCPFrame on empty reader = %v, want io.EOF", err) + } +} + +func TestReadTCPFrameErrors(t *testing.T) { + tests := []struct { + name string + input []byte + wantErr error + }{ + { + name: "empty reader returns EOF", + input: nil, + wantErr: io.EOF, + }, + { + name: "1-byte truncated length prefix returns ErrUnexpectedEOF", + input: []byte{0x00}, + wantErr: io.ErrUnexpectedEOF, + }, + { + name: "length prefix with empty body returns ErrUnexpectedEOF", + input: []byte{0x00, 0x05}, + wantErr: io.ErrUnexpectedEOF, + }, + { + name: "length prefix with partial body returns ErrUnexpectedEOF", + input: []byte{0x00, 0x05, 1, 2, 3}, + wantErr: io.ErrUnexpectedEOF, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + _, err := ReadTCPFrame(bytes.NewReader(tc.input)) + if !errors.Is(err, tc.wantErr) { + t.Errorf("ReadTCPFrame(%x) error = %v, want %v", tc.input, err, tc.wantErr) + } + }) + } +} + +type failingWriter struct { + err error +} + +func (f failingWriter) Write([]byte) (int, error) { + return 0, f.err +} + +func TestWriteTCPFrameErrors(t *testing.T) { + t.Run("message exceeds uint16 max", func(t *testing.T) { + oversized := make([]byte, 0x10000) + var buf bytes.Buffer + if err := WriteTCPFrame(&buf, oversized); err == nil { + t.Error("WriteTCPFrame with 65536-byte message succeeded, want error") + } + if buf.Len() != 0 { + t.Errorf("buf.Len() = %d, want 0", buf.Len()) + } + }) + + t.Run("underlying writer error propagated", func(t *testing.T) { + wantErr := errors.New("write failed") + err := WriteTCPFrame(failingWriter{err: wantErr}, []byte{1, 2, 3}) + if !errors.Is(err, wantErr) { + t.Errorf("WriteTCPFrame error = %v, want %v", err, wantErr) + } + }) +} diff --git a/internal/ateomnet/dns/readme.md b/internal/ateomnet/dns/readme.md new file mode 100644 index 0000000000..ac742fe1ef --- /dev/null +++ b/internal/ateomnet/dns/readme.md @@ -0,0 +1,66 @@ +# Actor DNS + +Actor DNS relays DNS requests from the actor to the external world. + +## Code structure + +### Relay + +- `Relay` -- one per Ateom, holds upstream resolvers, and is used to create the per-Actor `Server`. +- `Relay.Serve()` -- creates a per-Actor `Server` that manages DNS requests for the Actor. Returns a `*Server`. + +### Server + +`Server` is the per-Actor state for the DNS traffic, including per-Actor concurrency limits (`maxInFlightDNS` and `maxDNSConnections`). + +- `Server.pendingRequests` map (UDP only): + - Allocates a rewritten `upstreamID` (`uint16`) per in-flight UDP query to avoid transaction ID collisions across different actor source ports on the shared egress socket (and prevent predictable upstream IDs). + - Maps `upstreamID` -> pending request entry: + - `clientRequestID` (`uint16`) and `clientAddr` (`net.Addr`) to restore the original ID and route the response back. + - Question metadata (`QNAME`, `QTYPE`, `QCLASS`) to validate that the response matches the query (RFC 5452). + - `upstreamIdx` and `deferredResp` to track multi-upstream failover (`SERVFAIL`, `NOTIMP`, `REFUSED`, or timeout) and return the last failure response if all upstreams fail. + - `expiry` timestamp for timing out stale entries and triggering failover via a periodic sweep ticker. +- `Server.Stop()` -- cancels in-flight work, closes sockets, and stops serving DNS requests. + +### Implementation details + +#### UDP handling + +- `udphandler.go`: `struct udpHandler` +- Two goroutines per `Server` (plus a sweep ticker for timeouts/failover): + - **Ingress reader**: reads datagrams from the actor-facing gateway `net.PacketConn`, runs `onUDPRequest()` (which validates the query, rewrites the transaction ID to `upstreamID`, and records the entry in `pendingRequests`), and sends the query on the egress socket. + - **Egress reader**: reads datagrams from an unconnected worker-namespace `net.PacketConn`, runs `onUDPResponse()` to match and validate against `pendingRequests`, either fails over to the next upstream or restores `clientRequestID` and writes the answer back to `clientAddr`. + +#### TCP handling + +- `tcphandler.go`: `struct tcpHandler` +- Framing: each DNS message on a TCP stream is prefixed with a 2-byte big-endian length (`uint16`, RFC 1035 §4.2.2). +- Dedicated upstream stream per connection (with pipelining and out-of-order response support per RFC 7766 §6.2.1.1): + - Two goroutines per accepted TCP connection sharing a per-connection downstream write mutex: + - **Downstream reader**: loops reading length-prefixed frames from the actor connection and calls `onTCPRequest()`. + - If forwarded: writes the unmodified frame directly to the dedicated upstream TCP connection. + - If rejected with a synthesized reply (`FORMERR`, `NOTIMP`, `REFUSED`): writes the framed error response directly to the actor connection under the write mutex without contacting upstream. + - On client `EOF`: half-closes the upstream write side (`CloseWrite()`) so in-flight pipelined responses can finish draining. + - **Upstream reader**: loops reading length-prefixed frames from the upstream TCP connection, validates each frame via `onTCPResponse()`, and writes valid responses back to the actor connection under the write mutex. + +#### DNS protocol layer + +`dnshandler.go`: `struct dnsHandler` + +UDP and TCP call up to the DNS protocol layer (`golang.org/x/net/dns/dnsmessage`), which inspects packets and returns transport-agnostic decisions: + +- Shared request validation (`validateRequest`): + - **Drop**: packet `< 12` bytes or `QR == 1` (response bit set on a query). + - **Reply** (synthesized response without upstream round-trip): + - `FORMERR` (`RCode = 1`): malformed header/question or `QDCOUNT != 1`. + - `NOTIMP` (`RCode = 4`): `OpCode != 0` (non-standard query) or zone transfer `QTYPE` (`AXFR`, `IXFR`). + - `REFUSED` (`RCode = 5`): disallowed by domain or record-type policy. +- `onUDPRequest(raw []byte, clientAddr net.Addr, upstreams []string)` -> `action`: + - Acquires an `inFlight` slot, runs `validateRequest()`, and on `actionForward` records the query in `pendingRequests` with a rewritten `upstreamID`. +- `onUDPResponse(raw []byte, from net.Addr)` -> `action`: + - Validates `QR == 1`, parses header + question, and matches against `pendingRequests` (verifying `upstreamID`, expected upstream source, and matching `QNAME`/`QTYPE`/`QCLASS`). + - Returns **Drop**, **Failover** (on `SERVFAIL`, `NOTIMP`, or `REFUSED` when fallback upstreams remain), or **Deliver** (restoring `clientRequestID` and returning `clientAddr`). +- `onTCPRequest(raw []byte)` -> `action`: + - Runs `validateRequest()` and returns **Drop**, **Reply**, or **Forward** (unmodified frame). +- `onTCPResponse(raw []byte)` -> `action`: + - Validates `QR == 1` and parses header + question, returning **Drop** or **Deliver**. diff --git a/internal/ateomnet/dns/relay.go b/internal/ateomnet/dns/relay.go index 804b5ff47b..81e2854399 100644 --- a/internal/ateomnet/dns/relay.go +++ b/internal/ateomnet/dns/relay.go @@ -19,6 +19,7 @@ import ( "fmt" "log/slog" "net" + "net/netip" "strconv" "time" @@ -29,38 +30,19 @@ const ( // dnsPort is the relay port on the sandbox's gateway. dnsPort = 53 - // Read complete UDP datagrams without truncating EDNS responses. - maxDNSDatagram = 65535 - - // Drop excess UDP queries to bound goroutines and upstream sockets. - maxInFlightDNS = 64 - - // maxDNSConnections bounds open TCP connections. - maxDNSConnections = 16 - - // dnsTCPTimeout limits connection lifetime, including idle clients. - dnsTCPTimeout = 30 * time.Second - // dnsExchangeTimeout bounds each upstream attempt. dnsExchangeTimeout = 5 * time.Second ) -// limiter caps UDP queries in flight and open TCP connections across every -// sandbox the relay serves, so each Server must release its slots on Stop. -type limiter struct { - inFlight chan struct{} - connections chan struct{} -} - -// Relay forwards UDP and TCP DNS unchanged to the worker pod's resolvers. -// It listens in the sandbox's gateway namespace and dials from the worker's. -// DNS bypasses the egress tunnel and is not checked against egress policy. +// Relay validates and forwards UDP and TCP DNS queries to the worker pod's +// resolvers. It listens in the sandbox's gateway namespace and dials from the +// worker's. DNS bypasses the egress tunnel and is not checked against egress +// policy. type Relay struct { upstreams []string // dialer reaches upstream resolvers from the worker namespace. - dialer *net.Dialer - limiter limiter + dialer *net.Dialer } // NewRelay reads nameservers from resolvConfPath and forwards to each on port 53. @@ -72,6 +54,29 @@ func NewRelay(resolvConfPath string) (*Relay, error) { return NewRelayForUpstreams(upstreams) } +func normalizeAddr(addr net.Addr) string { + if addr == nil { + return "" + } + switch a := addr.(type) { + case *net.UDPAddr: + ap := a.AddrPort() + return netip.AddrPortFrom(ap.Addr().Unmap(), ap.Port()).String() + case *net.TCPAddr: + ap := a.AddrPort() + return netip.AddrPortFrom(ap.Addr().Unmap(), ap.Port()).String() + default: + return normalizeAddrString(addr.String()) + } +} + +func normalizeAddrString(s string) string { + if ap, err := netip.ParseAddrPort(s); err == nil { + return netip.AddrPortFrom(ap.Addr().Unmap(), ap.Port()).String() + } + return s +} + // NewRelayForUpstreams forwards to upstreams, each "host:port", tried in order. func NewRelayForUpstreams(upstreams []string) (*Relay, error) { if len(upstreams) == 0 { @@ -81,6 +86,9 @@ func NewRelayForUpstreams(upstreams []string) (*Relay, error) { if _, _, err := net.SplitHostPort(u); err != nil { return nil, fmt.Errorf("dns: invalid upstream resolver %q: %w", u, err) } + if _, err := net.ResolveUDPAddr("udp", u); err != nil { + return nil, fmt.Errorf("dns: invalid upstream resolver %q: %w", u, err) + } } slog.Info("DNS relay configured", slog.Any("upstreams", upstreams)) @@ -88,10 +96,6 @@ func NewRelayForUpstreams(upstreams []string) (*Relay, error) { return &Relay{ upstreams: upstreams, dialer: &net.Dialer{Timeout: dnsExchangeTimeout}, - limiter: limiter{ - inFlight: make(chan struct{}, maxInFlightDNS), - connections: make(chan struct{}, maxDNSConnections), - }, }, nil } @@ -121,11 +125,19 @@ func (r *Relay) Serve(ctx context.Context, ns netns.Handle) (*Server, error) { return nil, err } - return r.serveOn(ctx, &netC), nil + egressUDP, err := net.ListenPacket("udp", ":0") + if err != nil { + _ = netC.udp.Close() + _ = netC.tcpListener.Close() + return nil, fmt.Errorf("while opening the worker DNS egress socket: %w", err) + } + netC.egressUDP = egressUDP + + return r.serveOn(ctx, &netC) } // serveOn serves DNS on netC's sockets, which the caller has already bound, // and dials upstreams with netC's dialer. -func (r *Relay) serveOn(ctx context.Context, netC *netConn) *Server { - return newServer(ctx, &serverConfig{upstreams: r.upstreams}, netC, &r.limiter) +func (r *Relay) serveOn(ctx context.Context, netC *netConn) (*Server, error) { + return newServer(ctx, &serverConfig{upstreams: r.upstreams}, netC) } diff --git a/internal/ateomnet/dns/relay_test.go b/internal/ateomnet/dns/relay_test.go index ef9a3333f1..104622c033 100644 --- a/internal/ateomnet/dns/relay_test.go +++ b/internal/ateomnet/dns/relay_test.go @@ -28,46 +28,54 @@ import ( "time" "github.com/google/go-cmp/cmp" + "golang.org/x/net/dns/dnsmessage" ) func TestRelayCancelsUDPExchange(t *testing.T) { // A resolver that receives the query and never answers, so the exchange is - // blocked on the read when the context is canceled. + // in flight when the server stops. silent, asked := newSilentResolver(t) - // Two upstreams: a canceled exchange must not move on to the second. - second := newFakeResolver(t, func(query []byte) []byte { return query }) + // Two upstreams: stopping the server must not move on to the second. + var secondAsked atomic.Bool + second := newFakeResolver(t, func(query []byte) []byte { + secondAsked.Store(true) + return dnsAnswer(query, 0) + }) relay, err := NewRelayForUpstreams([]string{silent, second}) if err != nil { t.Fatal(err) } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - done := make(chan error, 1) - go func() { - _, err := unstartedServer(relay).exchangeUDP(ctx, dnsQuery(0x1234)) - done <- err - }() + srv, udpAddr, _ := serveLoopback(t, relay) + client, err := net.Dial("udp", udpAddr.String()) + if err != nil { + t.Fatal(err) + } + defer client.Close() + + if _, err := client.Write(dnsQuery(0x1234)); err != nil { + t.Fatal(err) + } select { case <-asked: case <-time.After(5 * time.Second): t.Fatal("the resolver never saw the query") } - cancel() - select { - case err := <-done: - if !errors.Is(err, context.Canceled) { - t.Fatalf("exchange returned %v, want cancellation", err) - } - case <-time.After(5 * time.Second): - t.Fatal("canceled exchange remained blocked reading upstream") + + stopCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := srv.Stop(stopCtx); err != nil { + t.Fatalf("Stop: %v", err) + } + if secondAsked.Load() { + t.Error("stopping the server moved on to the second upstream") } } func TestRelayForwardsUDPVerbatim(t *testing.T) { upstream := newFakeResolver(t, func(query []byte) []byte { - return append([]byte{0xff}, query...) + return dnsAnswer(query, 0) }) relay, err := NewRelayForUpstreams([]string{upstream}) @@ -76,12 +84,12 @@ func TestRelayForwardsUDPVerbatim(t *testing.T) { } client := serveRelayUDP(t, relay) - query := []byte{0xab, 0xcd, 0x01, 0x00, 0x00, 0x01} + query := dnsQuery(0xabcd) if _, err := client.Write(query); err != nil { t.Fatal(err) } answer := readWithin(t, client) - if diff := cmp.Diff(append([]byte{0xff}, query...), answer); diff != "" { + if diff := cmp.Diff(dnsAnswer(query, 0), answer); diff != "" { t.Errorf("answer mismatch (-want +got):\n%s", diff) } } @@ -92,21 +100,22 @@ func TestRelayFallsBackToTheNextResolver(t *testing.T) { t.Fatal(err) } deadAddress := dead.LocalAddr().String() - // Closed, so the exchange fails rather than hanging to its deadline. + // Closed, so the exchange times out on the first upstream and fails over. dead.Close() - live := newFakeResolver(t, func(query []byte) []byte { return []byte("answered") }) + live := newFakeResolver(t, func(query []byte) []byte { return dnsAnswer(query, 0) }) relay, err := NewRelayForUpstreams([]string{deadAddress, live}) if err != nil { t.Fatal(err) } client := serveRelayUDP(t, relay) - if _, err := client.Write([]byte("query")); err != nil { + query := dnsQuery(0x2468) + if _, err := client.Write(query); err != nil { t.Fatal(err) } - if got := string(readWithin(t, client)); got != "answered" { - t.Errorf("answer = %q, want %q", got, "answered") + if diff := cmp.Diff(dnsAnswer(query, 0), readWithin(t, client)); diff != "" { + t.Errorf("answer mismatch (-want +got):\n%s", diff) } } @@ -135,7 +144,7 @@ func newFakeResolver(t *testing.T, respond func([]byte) []byte) string { } t.Cleanup(func() { pc.Close() }) go func() { - buf := make([]byte, maxDNSDatagram) + buf := make([]byte, maxDatagramSize) for { n, from, err := pc.ReadFrom(buf) if err != nil { @@ -160,7 +169,7 @@ func newSilentResolver(t *testing.T) (address string, asked <-chan struct{}) { t.Cleanup(func() { pc.Close() }) seen := make(chan struct{}, 1) go func() { - buf := make([]byte, maxDNSDatagram) + buf := make([]byte, maxDatagramSize) for { if _, _, err := pc.ReadFrom(buf); err != nil { return @@ -186,25 +195,19 @@ func serveLoopback(t *testing.T, relay *Relay) (*Server, net.Addr, net.Addr) { pc.Close() t.Fatal(err) } - srv := relay.serveOn(context.Background(), &netConn{ + srv, err := relay.serveOn(context.Background(), &netConn{ dialer: *relay.dialer, udp: pc, tcpListener: lis, }) + if err != nil { + pc.Close() + t.Fatal(err) + } t.Cleanup(func() { _ = srv.Stop(context.Background()) }) return srv, pc.LocalAddr(), lis.Addr() } -// unstartedServer builds a Server for relay without serving on any socket, so -// a test can call its methods directly. -func unstartedServer(relay *Relay) *Server { - return &Server{ - config: &serverConfig{upstreams: relay.upstreams}, - n: &netConn{dialer: *relay.dialer}, - limiter: &relay.limiter, - } -} - // serveRelayUDP runs the relay on loopback sockets and returns a connection to it. func serveRelayUDP(t *testing.T, relay *Relay) net.Conn { t.Helper() @@ -222,7 +225,7 @@ func readWithin(t *testing.T, conn net.Conn) []byte { if err := conn.SetReadDeadline(time.Now().Add(5 * time.Second)); err != nil { t.Fatal(err) } - buf := make([]byte, maxDNSDatagram) + buf := make([]byte, maxDatagramSize) n, err := conn.Read(buf) if err != nil { t.Fatalf("reading the relay's answer: %v", err) @@ -231,27 +234,32 @@ func readWithin(t *testing.T, conn net.Conn) []byte { } func TestRelayForwardsAnswersLargerThanTheCommonBuffer(t *testing.T) { - const size = 9000 - answer := make([]byte, size) - for i := range answer { - answer[i] = byte(i) - } - upstream := newFakeResolver(t, func([]byte) []byte { return answer }) + const size = maxDatagramSize + query := dnsQuery(0x1357) + baseAnswer := dnsAnswer(query, 0) + padding := make([]byte, size-len(baseAnswer)) + for i := range padding { + padding[i] = byte(i) + } + upstream := newFakeResolver(t, func(q []byte) []byte { + return append(dnsAnswer(q, 0), padding...) + }) relay, err := NewRelayForUpstreams([]string{upstream}) if err != nil { t.Fatal(err) } client := serveRelayUDP(t, relay) - if _, err := client.Write([]byte("query")); err != nil { + if _, err := client.Write(query); err != nil { t.Fatal(err) } + want := append(baseAnswer, padding...) got := readWithin(t, client) if len(got) != size { t.Fatalf("answer is %d bytes, want %d: it was cut down in the relay", len(got), size) } - if !bytes.Equal(got, answer) { + if !bytes.Equal(got, want) { t.Error("answer differs from what the resolver sent") } } @@ -267,14 +275,13 @@ func TestRelayDropsQueriesBeyondItsInFlightLimit(t *testing.T) { t.Cleanup(func() { close(release) }) var inFlight atomic.Int64 go func() { - buf := make([]byte, maxDNSDatagram) + buf := make([]byte, maxDatagramSize) for { n, from, err := pc.ReadFrom(buf) if err != nil { return } - answer := make([]byte, n) - copy(answer, buf[:n]) + answer := dnsAnswer(buf[:n], 0) go func() { inFlight.Add(1) <-release @@ -289,8 +296,8 @@ func TestRelayDropsQueriesBeyondItsInFlightLimit(t *testing.T) { } client := serveRelayUDP(t, relay) - for range maxInFlightDNS * 4 { - if _, err := client.Write([]byte("query")); err != nil { + for i := range maxInFlight * 4 { + if _, err := client.Write(dnsQuery(uint16(i))); err != nil { t.Fatal(err) } } @@ -305,8 +312,8 @@ func TestRelayDropsQueriesBeyondItsInFlightLimit(t *testing.T) { last = n } } - if got := inFlight.Load(); got > maxInFlightDNS { - t.Errorf("the relay had %d queries in flight, want at most %d", got, maxInFlightDNS) + if got := inFlight.Load(); got > maxInFlight { + t.Errorf("the relay had %d queries in flight, want at most %d", got, maxInFlight) } } @@ -366,14 +373,14 @@ func TestRelayRefusesTCPConnectionsBeyondItsLimit(t *testing.T) { } _, _, relayAddr := serveLoopback(t, relay) - for range maxDNSConnections { + for range maxConnections { conn, err := net.Dial("tcp", relayAddr.String()) if err != nil { t.Fatal(err) } t.Cleanup(func() { conn.Close() }) } - waitFor(t, "the relay to fill up", func() bool { return held.Load() == maxDNSConnections }) + waitFor(t, "the relay to fill up", func() bool { return held.Load() == maxConnections }) // The relay should close a connection accepted beyond its limit. extra, err := net.Dial("tcp", relayAddr.String()) @@ -387,8 +394,8 @@ func TestRelayRefusesTCPConnectionsBeyondItsLimit(t *testing.T) { if _, err := extra.Read(make([]byte, 1)); !errors.Is(err, io.EOF) { t.Errorf("reading the refused connection = %v, want %v", err, io.EOF) } - if got := held.Load(); got != maxDNSConnections { - t.Errorf("the relay holds %d upstream connections, want %d", got, maxDNSConnections) + if got := held.Load(); got != maxConnections { + t.Errorf("the relay holds %d upstream connections, want %d", got, maxConnections) } } @@ -406,7 +413,7 @@ func TestRelayClosesTCPConnectionsWhenServingEnds(t *testing.T) { } defer conn.Close() waitFor(t, "the connection to reach an upstream", func() bool { return held.Load() == 1 }) - if got := len(relay.limiter.connections); got != 1 { + if got := srv.limiter.connections.occupied(); got != 1 { t.Fatalf("%d connection slots held before Stop, want 1", got) } @@ -415,7 +422,7 @@ func TestRelayClosesTCPConnectionsWhenServingEnds(t *testing.T) { if err := srv.Stop(stopCtx); err != nil { t.Fatalf("Stop: %v", err) } - if got := len(relay.limiter.connections); got != 0 { + if got := srv.limiter.connections.occupied(); got != 0 { t.Errorf("%d connection slots still held after Stop, want 0", got) } @@ -432,7 +439,7 @@ func TestRelayClosesTCPConnectionsWhenServingEnds(t *testing.T) { } // Stop must cancel queries still being resolved rather than wait out -// dnsExchangeTimeout, and return their slots to the worker's limiter. +// dnsExchangeTimeout, and return their slots to the server's limiter. func TestStopCancelsUDPQueriesInFlight(t *testing.T) { silent, asked := newSilentResolver(t) relay, err := NewRelayForUpstreams([]string{silent}) @@ -454,7 +461,7 @@ func TestStopCancelsUDPQueriesInFlight(t *testing.T) { case <-time.After(5 * time.Second): t.Fatal("the resolver never saw the query") } - if got := len(relay.limiter.inFlight); got != 1 { + if got := srv.limiter.inFlight.occupied(); got != 1 { t.Fatalf("%d queries in flight before Stop, want 1", got) } @@ -464,11 +471,49 @@ func TestStopCancelsUDPQueriesInFlight(t *testing.T) { if err := srv.Stop(stopCtx); err != nil { t.Fatalf("Stop: %v", err) } - if got := len(relay.limiter.inFlight); got != 0 { + if got := srv.limiter.inFlight.occupied(); got != 0 { t.Errorf("%d in-flight slots still held after Stop, want 0", got) } } +func TestRelayLimitsArePerActor(t *testing.T) { + upstream, held := newHeldTCPResolver(t) + relay, err := NewRelayForUpstreams([]string{upstream}) + if err != nil { + t.Fatal(err) + } + + srvA, _, addrA := serveLoopback(t, relay) + srvB, _, addrB := serveLoopback(t, relay) + + // Saturate Actor A's TCP connection limit. + for range maxConnections { + conn, err := net.Dial("tcp", addrA.String()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { conn.Close() }) + } + waitFor(t, "actor A to fill up", func() bool { return held.Load() == maxConnections }) + if got := srvA.limiter.connections.occupied(); got != maxConnections { + t.Fatalf("actor A connection slots = %d, want %d", got, maxConnections) + } + if got := srvB.limiter.connections.occupied(); got != 0 { + t.Fatalf("actor B connection slots = %d, want 0", got) + } + + // Actor B must still accept a TCP connection even though Actor A is full. + connB, err := net.Dial("tcp", addrB.String()) + if err != nil { + t.Fatal(err) + } + defer connB.Close() + waitFor(t, "actor B connection to reach upstream", func() bool { return held.Load() == maxConnections+1 }) + if got := srvB.limiter.connections.occupied(); got != 1 { + t.Errorf("actor B connection slots = %d, want 1", got) + } +} + // dnsQuery builds a real query for example.com A, so tests exercise messages a // resolver would actually send rather than an arbitrary string. func dnsQuery(id uint16) []byte { @@ -496,7 +541,7 @@ func TestRelayFailsOverOnServerFailure(t *testing.T) { var sickCalls, healthyCalls atomic.Int32 sick := newFakeResolver(t, func(query []byte) []byte { sickCalls.Add(1) - return dnsAnswer(query, rcodeServFail) + return dnsAnswer(query, byte(dnsmessage.RCodeServerFailure)) }) healthy := newFakeResolver(t, func(query []byte) []byte { healthyCalls.Add(1) @@ -507,10 +552,11 @@ func TestRelayFailsOverOnServerFailure(t *testing.T) { if err != nil { t.Fatal(err) } - answer, err := unstartedServer(relay).exchangeUDP(context.Background(), dnsQuery(0x1234)) - if err != nil { - t.Fatalf("exchange: %v", err) + client := serveRelayUDP(t, relay) + if _, err := client.Write(dnsQuery(0x1234)); err != nil { + t.Fatal(err) } + answer := readWithin(t, client) if got := answer[3] & 0x0f; got != 0 { t.Errorf("answer rcode = %d, want 0: the relay returned the failing resolver's answer", got) } @@ -522,28 +568,28 @@ func TestRelayFailsOverOnServerFailure(t *testing.T) { // With every resolver failing there is nothing better to return, and a real // SERVFAIL beats a timeout: the sandbox's resolver can act on it. func TestRelayReturnsServerFailureWhenAllFail(t *testing.T) { - first := newFakeResolver(t, func(query []byte) []byte { return dnsAnswer(query, rcodeServFail) }) - second := newFakeResolver(t, func(query []byte) []byte { return dnsAnswer(query, rcodeRefused) }) + first := newFakeResolver(t, func(query []byte) []byte { return dnsAnswer(query, byte(dnsmessage.RCodeServerFailure)) }) + second := newFakeResolver(t, func(query []byte) []byte { return dnsAnswer(query, byte(dnsmessage.RCodeRefused)) }) relay, err := NewRelayForUpstreams([]string{first, second}) if err != nil { t.Fatal(err) } - answer, err := unstartedServer(relay).exchangeUDP(context.Background(), dnsQuery(0x2345)) - if err != nil { - t.Fatalf("exchange: %v", err) + client := serveRelayUDP(t, relay) + if _, err := client.Write(dnsQuery(0x2345)); err != nil { + t.Fatal(err) } - if got := answer[3] & 0x0f; got != rcodeRefused { - t.Errorf("answer rcode = %d, want the last resolver's %d", got, rcodeRefused) + answer := readWithin(t, client) + if got := dnsmessage.RCode(answer[3] & 0x0f); got != dnsmessage.RCodeRefused { + t.Errorf("answer rcode = %v, want the last resolver's %v", got, dnsmessage.RCodeRefused) } } // NXDOMAIN is an answer, not a failure: failing over would ask every resolver // about a name that does not exist. func TestRelayReturnsNXDomainWithoutFailover(t *testing.T) { - const rcodeNXDomain = 3 var secondCalls atomic.Int32 - first := newFakeResolver(t, func(query []byte) []byte { return dnsAnswer(query, rcodeNXDomain) }) + first := newFakeResolver(t, func(query []byte) []byte { return dnsAnswer(query, byte(dnsmessage.RCodeNameError)) }) second := newFakeResolver(t, func(query []byte) []byte { secondCalls.Add(1) return dnsAnswer(query, 0) @@ -553,12 +599,13 @@ func TestRelayReturnsNXDomainWithoutFailover(t *testing.T) { if err != nil { t.Fatal(err) } - answer, err := unstartedServer(relay).exchangeUDP(context.Background(), dnsQuery(0x3456)) - if err != nil { - t.Fatalf("exchange: %v", err) + client := serveRelayUDP(t, relay) + if _, err := client.Write(dnsQuery(0x3456)); err != nil { + t.Fatal(err) } - if got := answer[3] & 0x0f; got != rcodeNXDomain { - t.Errorf("answer rcode = %d, want NXDOMAIN %d", got, rcodeNXDomain) + answer := readWithin(t, client) + if got := dnsmessage.RCode(answer[3] & 0x0f); got != dnsmessage.RCodeNameError { + t.Errorf("answer rcode = %v, want NXDOMAIN %v", got, dnsmessage.RCodeNameError) } if secondCalls.Load() != 0 { t.Error("the relay failed over on NXDOMAIN, which is a valid answer") diff --git a/internal/ateomnet/dns/server.go b/internal/ateomnet/dns/server.go index dab647d3f1..23987fbb54 100644 --- a/internal/ateomnet/dns/server.go +++ b/internal/ateomnet/dns/server.go @@ -22,7 +22,6 @@ import ( "log/slog" "net" "sync" - "time" ) type serverConfig struct { @@ -32,15 +31,19 @@ type serverConfig struct { type netConn struct { dialer net.Dialer udp net.PacketConn + egressUDP net.PacketConn tcpListener net.Listener } // Server is one sandbox's running DNS relay, returned by [Relay.Serve]. type Server struct { - config *serverConfig + n *netConn - n *netConn - limiter *limiter + limiter *limiter + pendingRequests *pendingRequests + dns *dnsHandler + udp *udpHandler + tcp *tcpHandler stopServing context.CancelFunc closeOnce sync.Once @@ -48,36 +51,58 @@ type Server struct { serving sync.WaitGroup } +// newServer binds the sockets and starts serving goroutines. func newServer( ctx context.Context, config *serverConfig, netC *netConn, - limiter *limiter, -) *Server { - // Detached from the activation RPC's context but cancelable: the relay's - // capacity is the worker's, so teardown must drop queries still in flight. +) (*Server, error) { + if netC.egressUDP == nil { + egressUDP, err := net.ListenPacket("udp", ":0") + if err != nil { + slog.WarnContext(ctx, "Failed to open worker DNS egress socket", slog.Any("err", err)) + return nil, fmt.Errorf("DNS egress socket ListenPacket: %w", err) + } + netC.egressUDP = egressUDP + } + + // Detached from the activation RPC's context but cancelable so teardown + // drops queries still in flight and releases the actor's sockets. serveCtx, stopServing := context.WithCancel(context.WithoutCancel(ctx)) + + udpAddrs := make([]*net.UDPAddr, 0, len(config.upstreams)) + for _, u := range config.upstreams { + if addr, err := net.ResolveUDPAddr("udp", u); err == nil { + udpAddrs = append(udpAddrs, addr) + } + } + s := &Server{ - config: config, stopServing: stopServing, n: netC, - limiter: limiter, + limiter: newLimiter(), } + + s.pendingRequests = newPendingRequests(s.limiter) + s.dns = newDNSHandler(s.pendingRequests, s.limiter, nil) + s.udp = newUDPHandler(netC.udp, netC.egressUDP, udpAddrs, s.dns, s.pendingRequests) + s.tcp = newTCPHandler(config.upstreams, netC.tcpListener, netC.dialer, s.dns, s.limiter) + s.serving.Add(2) go func() { defer s.serving.Done() - if err := s.servePacket(serveCtx); err != nil { + if err := s.udp.serve(serveCtx); err != nil { slog.WarnContext(ctx, "Actor DNS socket stopped", slog.Any("err", err)) } }() go func() { defer s.serving.Done() - if err := s.serveTCP(serveCtx); err != nil { + if err := s.tcp.serve(serveCtx); err != nil { slog.WarnContext(ctx, "Actor DNS listener stopped", slog.Any("err", err)) } }() - return s + return s, nil } // Stop cancels in-flight queries and connections, closes the sockets, and @@ -87,7 +112,10 @@ func (s *Server) Stop(ctx context.Context) error { // Cancel first: closing the sockets alone leaves the queries already // being resolved holding the relay. s.stopServing() - for _, c := range []io.Closer{s.n.udp, s.n.tcpListener} { + for _, c := range []io.Closer{s.n.udp, s.n.egressUDP, s.n.tcpListener} { + if c == nil { + continue + } if err := c.Close(); err != nil && !errors.Is(err, net.ErrClosed) { s.closeErr = errors.Join(s.closeErr, err) } @@ -105,183 +133,3 @@ func (s *Server) Stop(ctx context.Context) error { return errors.Join(s.closeErr, fmt.Errorf("dns: waiting for relay to stop: %w", ctx.Err())) } } - -// servePacket answers UDP queries until ctx is canceled or the socket fails. -func (s *Server) servePacket(ctx context.Context) error { - var wg sync.WaitGroup - defer wg.Wait() - - buf := make([]byte, maxDNSDatagram) - for { - n, from, err := s.n.udp.ReadFrom(buf) - if err != nil { - if ctx.Err() != nil || errors.Is(err, net.ErrClosed) { - return nil - } - return fmt.Errorf("dns: reading actor DNS query: %w", err) - } - // Copied: the buffer is reused by the next read. - query := make([]byte, n) - copy(query, buf[:n]) - - select { - case s.limiter.inFlight <- struct{}{}: - default: - slog.DebugContext(ctx, "dns relay dropped a DNS query; too many in flight") - continue - } - wg.Add(1) - go func() { - defer wg.Done() - defer func() { <-s.limiter.inFlight }() - answer, err := s.exchangeUDP(ctx, query) - if err != nil { - slog.WarnContext(ctx, "dns relay could not resolve an actor DNS query", slog.Any("err", err)) - return - } - if _, err := s.n.udp.WriteTo(answer, from); err != nil && ctx.Err() == nil { - slog.WarnContext(ctx, "dns relay could not return a DNS answer", slog.Any("err", err)) - } - }() - } -} - -// serveTCP relays TCP DNS connections until ctx is canceled or the listener closes. -func (s *Server) serveTCP(ctx context.Context) error { - // Wait for the relays to drain before returning, so a closed listener - // leaves no goroutine still holding a connection slot. - var wg sync.WaitGroup - defer wg.Wait() - - for { - conn, err := s.n.tcpListener.Accept() - if err != nil { - if ctx.Err() != nil || errors.Is(err, net.ErrClosed) { - return nil - } - return fmt.Errorf("dns: accepting actor DNS connection: %w", err) - } - select { - case s.limiter.connections <- struct{}{}: - default: - slog.DebugContext(ctx, "dns relay refused a DNS connection; too many open") - _ = conn.Close() - continue - } - wg.Add(1) - go func() { - defer wg.Done() - defer func() { <-s.limiter.connections }() - s.relayTCP(ctx, conn) - }() - } -} - -func (s *Server) exchangeUDP(ctx context.Context, query []byte) ([]byte, error) { - var errs error - // deferred holds a server-failure answer to fall back on, see below. - var deferred []byte - for _, upstream := range s.config.upstreams { - if err := ctx.Err(); err != nil { - return nil, err - } - conn, err := s.n.dialer.DialContext(ctx, "udp", upstream) - if err != nil { - errs = errors.Join(errs, err) - continue - } - answer, err := func() ([]byte, error) { - defer conn.Close() - stop := context.AfterFunc(ctx, func() { _ = conn.Close() }) - defer stop() - if err := conn.SetDeadline(time.Now().Add(dnsExchangeTimeout)); err != nil { - return nil, err - } - if _, err := conn.Write(query); err != nil { - return nil, err - } - buf := make([]byte, maxDNSDatagram) - n, err := conn.Read(buf) - if err != nil { - return nil, err - } - return buf[:n], nil - }() - if err != nil { - errs = errors.Join(errs, fmt.Errorf("upstream %s: %w", upstream, err)) - continue - } - // SERVFAIL is not an answer, and the sandbox sees only the gateway, so - // it cannot try the pod's other resolvers itself. Keep the last one to - // return if none does better: a real response beats a timeout. - if rcode, ok := failoverRcode(answer); ok { - errs = errors.Join(errs, fmt.Errorf("upstream %s: %w", upstream, rcodeError(rcode))) - deferred = answer - continue - } - return answer, nil - } - if deferred != nil { - return deferred, nil - } - return nil, fmt.Errorf("dns: no upstream resolver answered: %w", errs) -} - -// relayTCP copies a DNS stream without parsing its length-prefixed messages. -func (s *Server) relayTCP(ctx context.Context, downstream net.Conn) { - defer downstream.Close() - - var upstream net.Conn - var errs error - for _, address := range s.config.upstreams { - conn, err := s.n.dialer.DialContext(ctx, "tcp", address) - if err != nil { - errs = errors.Join(errs, err) - continue - } - upstream = conn - break - } - if upstream == nil { - slog.WarnContext(ctx, "dns relay could not reach any resolver for an actor DNS connection", slog.Any("err", errs)) - return - } - defer upstream.Close() - - // Cancel active copies on teardown to release the worker's connection slots. - relayDone := make(chan struct{}) - defer close(relayDone) - go func() { - select { - case <-ctx.Done(): - _ = downstream.Close() - _ = upstream.Close() - case <-relayDone: - } - }() - - deadline := time.Now().Add(dnsTCPTimeout) - if ctxDeadline, ok := ctx.Deadline(); ok && ctxDeadline.Before(deadline) { - deadline = ctxDeadline - } - _ = downstream.SetDeadline(deadline) - _ = upstream.SetDeadline(deadline) - - var wg sync.WaitGroup - wg.Add(2) - go func() { - defer wg.Done() - _, _ = io.Copy(upstream, downstream) - if c, ok := upstream.(*net.TCPConn); ok { - _ = c.CloseWrite() - } - }() - go func() { - defer wg.Done() - _, _ = io.Copy(downstream, upstream) - if c, ok := downstream.(*net.TCPConn); ok { - _ = c.CloseWrite() - } - }() - wg.Wait() -} diff --git a/internal/ateomnet/dns/tcphandler.go b/internal/ateomnet/dns/tcphandler.go new file mode 100644 index 0000000000..93ca9883dd --- /dev/null +++ b/internal/ateomnet/dns/tcphandler.go @@ -0,0 +1,254 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dns + +import ( + "context" + "errors" + "fmt" + "io" + "log/slog" + "net" + "sync" + "time" + + "github.com/agent-substrate/substrate/internal/ateomnet/dns/protocol" +) + +const ( + // defaultConnectionTimeout limits idle connection time between reads/writes. + defaultConnectionTimeout = 30 * time.Second +) + +// tcpClientConn wraps a downstream actor TCP connection with a mutex to +// serialize concurrent frame writes from replies and out-of-order +// upstream responses. +type tcpClientConn struct { + conn net.Conn + writeMu sync.Mutex +} + +// tcpHandler accepts actor TCP DNS connections and relays length-prefixed DNS +// frames with pipelining and out-of-order response support. +type tcpHandler struct { + upstreams []string + + listener net.Listener + dialer net.Dialer + dns *dnsHandler + limiter *limiter + + tcpTimeout time.Duration +} + +func newTCPHandler( + upstreams []string, + listener net.Listener, + dialer net.Dialer, + dns *dnsHandler, + lim *limiter, +) *tcpHandler { + return &tcpHandler{ + upstreams: upstreams, + listener: listener, + dialer: dialer, + dns: dns, + limiter: lim, + tcpTimeout: defaultConnectionTimeout, + } +} + +// serve accepts and relays actor TCP DNS connections until ctx is canceled or +// the listener closes. +func (h *tcpHandler) serve(ctx context.Context) error { + var wg sync.WaitGroup + defer wg.Wait() + + for { + conn, err := h.listener.Accept() + if err != nil { + if ctx.Err() != nil || errors.Is(err, net.ErrClosed) { + return nil + } + return fmt.Errorf("dns: accepting actor DNS connection: %w", err) + } + if !h.limiter.connections.tryAcquire() { + slog.DebugContext(ctx, "dns relay: refusing connection, at limit") + _ = conn.Close() + continue + + } + wg.Add(1) + go func() { + defer wg.Done() + defer func() { h.limiter.connections.release() }() + + h.onConnect(ctx, conn) + }() + } +} + +// refreshDeadline for extending the TCP connection closure on idle. +func (h *tcpHandler) refreshDeadline(ctx context.Context, downstream, upstream net.Conn) { + deadline := time.Now().Add(h.tcpTimeout) + if ctxDeadline, ok := ctx.Deadline(); ok && ctxDeadline.Before(deadline) { + deadline = ctxDeadline + } + _ = downstream.SetDeadline(deadline) + _ = upstream.SetDeadline(deadline) +} + +func (h *tcpHandler) onConnect(ctx context.Context, downstream net.Conn) { + defer downstream.Close() + + var upstream net.Conn + var errs error + + // Get a connection to a healthy upstream. + for _, address := range h.upstreams { + conn, err := h.dialer.DialContext(ctx, "tcp", address) + if err != nil { + errs = errors.Join(errs, err) + continue + } + upstream = conn + break + } + if upstream == nil { + slog.WarnContext(ctx, "dns relay: could not reach any upstream", slog.Any("err", errs)) + return + } + defer upstream.Close() + + stop := context.AfterFunc(ctx, func() { + _ = downstream.Close() + _ = upstream.Close() + }) + defer stop() + + // Set initial idle close deadline for the connections. + h.refreshDeadline(ctx, downstream, upstream) + + client := &tcpClientConn{conn: downstream} + + // Forward request/responses. + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + h.processRequests(ctx, client, upstream) + }() + go func() { + defer wg.Done() + h.processResponses(ctx, client, upstream) + }() + wg.Wait() +} + +func (h *tcpHandler) processRequests( + ctx context.Context, + client *tcpClientConn, + upstream net.Conn, +) { + for { + raw, err := protocol.ReadTCPFrame(client.conn) + if err != nil { + if errors.Is(err, io.EOF) { + if c, ok := upstream.(*net.TCPConn); ok { + _ = c.CloseWrite() + } + return + } + _ = upstream.Close() + _ = client.conn.Close() + return + } + h.refreshDeadline(ctx, client.conn, upstream) + + act := h.dns.onTCPRequest(raw) + + switch act.kind { + case actionDrop: + slog.DebugContext(ctx, "dns relay: dropping request and closing stream") + _ = upstream.Close() + _ = client.conn.Close() + return + case actionReply: + if err := client.writeFrame(act.payload); err != nil { + if ctx.Err() == nil && !errors.Is(err, net.ErrClosed) { + slog.DebugContext(ctx, "dns relay: error on TCP reply", slog.Any("err", err)) + } + _ = upstream.Close() + _ = client.conn.Close() + return + } + h.refreshDeadline(ctx, client.conn, upstream) + case actionForward: + if err := protocol.WriteTCPFrame(upstream, act.payload); err != nil { + if ctx.Err() == nil && !errors.Is(err, net.ErrClosed) { + slog.WarnContext(ctx, "dns relay: error in forward", slog.Any("err", err)) + } + _ = upstream.Close() + _ = client.conn.Close() + return + } + h.refreshDeadline(ctx, client.conn, upstream) + } + } +} + +func (h *tcpHandler) processResponses( + ctx context.Context, + client *tcpClientConn, + upstream net.Conn, +) { + for { + raw, err := protocol.ReadTCPFrame(upstream) + if err != nil { + if errors.Is(err, io.EOF) { + if c, ok := client.conn.(*net.TCPConn); ok { + _ = c.CloseWrite() + } + } + _ = client.conn.Close() + return + } + h.refreshDeadline(ctx, client.conn, upstream) + + act := h.dns.onTCPResponse(raw) + switch act.kind { + case actionDrop: + _ = upstream.Close() + _ = client.conn.Close() + return + case actionDeliver: + if err := client.writeFrame(act.payload); err != nil { + if ctx.Err() == nil && !errors.Is(err, net.ErrClosed) { + slog.WarnContext(ctx, "dns relay could not return TCP DNS answer", slog.Any("err", err)) + } + _ = upstream.Close() + _ = client.conn.Close() + return + } + h.refreshDeadline(ctx, client.conn, upstream) + } + } +} + +func (c *tcpClientConn) writeFrame(msg []byte) error { + c.writeMu.Lock() + defer c.writeMu.Unlock() + return protocol.WriteTCPFrame(c.conn, msg) +} diff --git a/internal/ateomnet/dns/tcphandler_test.go b/internal/ateomnet/dns/tcphandler_test.go new file mode 100644 index 0000000000..6322342bdd --- /dev/null +++ b/internal/ateomnet/dns/tcphandler_test.go @@ -0,0 +1,293 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dns + +import ( + "context" + "encoding/binary" + "net" + "sync" + "testing" + "time" + + "github.com/agent-substrate/substrate/internal/ateomnet/dns/protocol" + "golang.org/x/net/dns/dnsmessage" +) + +func startTestTCPHandler(t *testing.T, upstreams []string, configure func(*tcpHandler, *dnsHandler)) net.Addr { + t.Helper() + lis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + + lim := newLimiter() + dnsH := newDNSHandler(nil, lim, nil) + h := newTCPHandler(upstreams, lis, net.Dialer{Timeout: 5 * time.Second}, dnsH, lim) + if configure != nil { + configure(h, dnsH) + } + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + defer close(done) + _ = h.serve(ctx) + }() + t.Cleanup(func() { + cancel() + _ = lis.Close() + <-done + }) + + return lis.Addr() +} + +// newOutOfOrderTCPResolver reads 2 queries on a single TCP connection and +// answers the second query before the first to exercise RFC 7766 pipelining. +func newOutOfOrderTCPResolver(t *testing.T) string { + t.Helper() + lis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = lis.Close() }) + + go func() { + for { + conn, err := lis.Accept() + if err != nil { + return + } + go func() { + defer conn.Close() + q1, err := protocol.ReadTCPFrame(conn) + if err != nil { + return + } + q2, err := protocol.ReadTCPFrame(conn) + if err != nil { + return + } + // Respond to q2 first, then q1. + _ = protocol.WriteTCPFrame(conn, dnsAnswer(q2, 0)) + _ = protocol.WriteTCPFrame(conn, dnsAnswer(q1, 0)) + }() + } + }() + return lis.Addr().String() +} + +func TestTCPHandlerPipeliningAndOutOfOrderResponses(t *testing.T) { + upstream := newOutOfOrderTCPResolver(t) + addr := startTestTCPHandler(t, []string{upstream}, nil) + + conn, err := net.Dial("tcp", addr.String()) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + _ = conn.SetDeadline(time.Now().Add(5 * time.Second)) + + if err := protocol.WriteTCPFrame(conn, dnsQuery(0x1111)); err != nil { + t.Fatal(err) + } + if err := protocol.WriteTCPFrame(conn, dnsQuery(0x2222)); err != nil { + t.Fatal(err) + } + + resp1, err := protocol.ReadTCPFrame(conn) + if err != nil { + t.Fatalf("reading first response frame: %v", err) + } + resp2, err := protocol.ReadTCPFrame(conn) + if err != nil { + t.Fatalf("reading second response frame: %v", err) + } + + if got1 := binary.BigEndian.Uint16(resp1[0:2]); got1 != 0x2222 { + t.Errorf("first response ID = %#x, want 0x2222 (out-of-order)", got1) + } + if got2 := binary.BigEndian.Uint16(resp2[0:2]); got2 != 0x1111 { + t.Errorf("second response ID = %#x, want 0x1111", got2) + } +} + +func TestTCPHandlerSynthesizedReplyAndHalfClose(t *testing.T) { + lis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = lis.Close() }) + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + conn, err := lis.Accept() + if err != nil { + return + } + defer conn.Close() + q, err := protocol.ReadTCPFrame(conn) + if err != nil { + return + } + _ = protocol.WriteTCPFrame(conn, dnsAnswer(q, 0)) + }() + + addr := startTestTCPHandler(t, []string{lis.Addr().String()}, func(_ *tcpHandler, dnsH *dnsHandler) { + dnsH.allow = func(q dnsmessage.Question) bool { + return q.Name.String() != "blocked.example.com." + } + }) + + conn, err := net.Dial("tcp", addr.String()) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + _ = conn.SetDeadline(time.Now().Add(5 * time.Second)) + + blockedQuery := buildTestMessage(t, dnsmessage.Header{ + ID: 0xbad0, + RecursionDesired: true, + }, []dnsmessage.Question{{ + Name: dnsmessage.MustNewName("blocked.example.com."), + Type: dnsmessage.TypeA, + Class: dnsmessage.ClassINET, + }}) + + // First frame triggers a synthesized reply; second frame is forwarded, and + // then client half-closes its write side. + if err := protocol.WriteTCPFrame(conn, blockedQuery); err != nil { + t.Fatal(err) + } + if err := protocol.WriteTCPFrame(conn, dnsQuery(0x1234)); err != nil { + t.Fatal(err) + } + if tc, ok := conn.(*net.TCPConn); ok { + _ = tc.CloseWrite() + } + + r1, err := protocol.ReadTCPFrame(conn) + if err != nil { + t.Fatalf("reading first frame: %v", err) + } + r2, err := protocol.ReadTCPFrame(conn) + if err != nil { + t.Fatalf("reading second frame: %v", err) + } + + got := map[uint16]dnsmessage.RCode{ + binary.BigEndian.Uint16(r1[0:2]): dnsmessage.RCode(r1[3] & 0x0f), + binary.BigEndian.Uint16(r2[0:2]): dnsmessage.RCode(r2[3] & 0x0f), + } + if got[0xbad0] != dnsmessage.RCodeRefused { + t.Errorf("0xbad0 rcode = %v, want %v", got[0xbad0], dnsmessage.RCodeRefused) + } + if got[0x1234] != dnsmessage.RCodeSuccess { + t.Errorf("0x1234 rcode = %v, want %v", got[0x1234], dnsmessage.RCodeSuccess) + } + wg.Wait() +} + +func TestTCPHandlerSlidingIdleTimeout(t *testing.T) { + lis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = lis.Close() }) + + go func() { + for { + conn, err := lis.Accept() + if err != nil { + return + } + go func() { + defer conn.Close() + for { + q, err := protocol.ReadTCPFrame(conn) + if err != nil { + return + } + if err := protocol.WriteTCPFrame(conn, dnsAnswer(q, 0)); err != nil { + return + } + } + }() + } + }() + + const idleTimeout = 80 * time.Millisecond + addr := startTestTCPHandler(t, []string{lis.Addr().String()}, func(h *tcpHandler, _ *dnsHandler) { + h.tcpTimeout = idleTimeout + }) + + conn, err := net.Dial("tcp", addr.String()) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + _ = conn.SetDeadline(time.Now().Add(5 * time.Second)) + + // Send 3 queries spaced by half the idleTimeout (total elapsed ~1.5x idleTimeout). + // Because each read/write refreshes the deadline, the connection stays alive. + for i := range 3 { + if i > 0 { + time.Sleep(idleTimeout / 2) + } + id := uint16(0x3000 + i) + if err := protocol.WriteTCPFrame(conn, dnsQuery(id)); err != nil { + t.Fatalf("query %d write failed: %v", i, err) + } + resp, err := protocol.ReadTCPFrame(conn) + if err != nil { + t.Fatalf("query %d read failed: %v", i, err) + } + if got := binary.BigEndian.Uint16(resp[0:2]); got != id { + t.Errorf("query %d response ID = %#x, want %#x", i, got, id) + } + } + + // Now leave the connection idle past idleTimeout; the handler should close it. + var buf [1]byte + if _, err := conn.Read(buf[:]); err == nil { + t.Error("expected idle connection to be closed after timeout, but Read succeeded") + } +} + +func TestTCPHandlerClosesOnDroppedFrame(t *testing.T) { + upstream, _ := newHeldTCPResolver(t) + addr := startTestTCPHandler(t, []string{upstream}, nil) + + conn, err := net.Dial("tcp", addr.String()) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + _ = conn.SetDeadline(time.Now().Add(5 * time.Second)) + + // Send an invalid (< 12 byte) DNS frame that triggers actionDrop. + if err := protocol.WriteTCPFrame(conn, []byte{1, 2, 3}); err != nil { + t.Fatal(err) + } + + var buf [1]byte + if _, err := conn.Read(buf[:]); err == nil { + t.Error("expected connection to be closed after dropped frame, but Read succeeded") + } +} diff --git a/internal/ateomnet/dns/udphandler.go b/internal/ateomnet/dns/udphandler.go new file mode 100644 index 0000000000..495e079382 --- /dev/null +++ b/internal/ateomnet/dns/udphandler.go @@ -0,0 +1,220 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dns + +import ( + "bytes" + "context" + "encoding/binary" + "errors" + "fmt" + "log/slog" + "net" + "sync" + "time" +) + +const ( + // defaultSweepInterval is how often pendingRequests is checked for expired + // UDP queries. + defaultSweepInterval = 50 * time.Millisecond + // defaultUpstreamAttemptTimeout bounds a single UDP upstream attempt + // when fallback upstreams remain. + defaultUpstreamAttemptTimeout = 1 * time.Second + // maxDatagramSize that we accept. Raising this consumes more memory while + // processing packets. + // + // References: + // - 4096: https://www.rfc-editor.org/info/rfc6891/#section-6.2.5 + // - 1232: CoreDNS default, https://www.dnsflagday.net/2020/#dns-flag-day-2020 + maxDatagramSize = 4 * 1024 +) + +// udpHandler manages actor UDP DNS traffic using an actorSock from +// the Actor and upstreamSock out to the upstream DNS server. +type udpHandler struct { + // actorSock that receives DNS requests from the Actor. + actorSock net.PacketConn + // upstreamSock for the outbound DNS requests. + upstreamSock net.PacketConn + + upstreams []*net.UDPAddr + upstreamStrs []string + + dns *dnsHandler + pending *pendingRequests + sweepInterval time.Duration +} + +func newUDPHandler( + actorSock net.PacketConn, + upstreamSock net.PacketConn, + upstreams []*net.UDPAddr, + dns *dnsHandler, + pending *pendingRequests, +) *udpHandler { + strs := make([]string, len(upstreams)) + for i, u := range upstreams { + strs[i] = normalizeAddr(u) + } + return &udpHandler{ + actorSock: actorSock, + upstreamSock: upstreamSock, + upstreams: upstreams, + upstreamStrs: strs, + dns: dns, + pending: pending, + sweepInterval: defaultSweepInterval, + } +} + +// serve runs the readers, and timeout sweep ticker until ctx is canceled +// or the sockets close. +func (h *udpHandler) serve(ctx context.Context) error { + var wg sync.WaitGroup + errCh := make(chan error, 2) + + wg.Add(3) + go func() { + defer wg.Done() + if err := h.processRequests(ctx); err != nil { + errCh <- err + } + }() + go func() { + defer wg.Done() + if err := h.processResponses(ctx); err != nil { + errCh <- err + } + }() + go func() { + defer wg.Done() + h.sweepLoop(ctx) + }() + + wg.Wait() + h.pending.clearAll() + close(errCh) + + var errs error + for err := range errCh { + errs = errors.Join(errs, err) + } + + return errs +} + +func (h *udpHandler) processRequests(ctx context.Context) error { + buf := make([]byte, maxDatagramSize) + for { + n, from, err := h.actorSock.ReadFrom(buf) + if err != nil { + if ctx.Err() != nil || errors.Is(err, net.ErrClosed) { + return nil + } + return fmt.Errorf("dns relay: reading actor DNS query: %w", err) + } + raw := bytes.Clone(buf[:n]) + act := h.dns.onUDPRequest(raw, from, h.upstreamStrs) + switch act.kind { + case actionDrop: + continue + case actionReply: + if _, err := h.actorSock.WriteTo(act.payload, from); err != nil && ctx.Err() == nil && !errors.Is(err, net.ErrClosed) { + slog.WarnContext(ctx, "dns relay: could not return a synthesized DNS reply", slog.Any("err", err)) + } + case actionForward: + h.sendToUpstream(ctx, act.upstreamID, 0, act.payload) + } + } +} + +func (h *udpHandler) sendToUpstream(ctx context.Context, upstreamID uint16, idx int, query []byte) { + for idx < len(h.upstreams) { + _, err := h.upstreamSock.WriteTo(query, h.upstreams[idx]) + if err == nil { + return + } + if ctx.Err() != nil || errors.Is(err, net.ErrClosed) { + slog.ErrorContext(ctx, "dns relay: upstream UDP socket closed, stopping forwarding") + h.pending.clearAll() + return + } + slog.WarnContext(ctx, "dns relay: could not send query to upstream", slog.String("upstream", h.upstreamStrs[idx]), slog.Any("err", err)) + nextIdx, nextQuery, ok := h.pending.failOverOnSendError(upstreamID, idx) + if !ok { + return + } + idx = nextIdx + query = nextQuery + } +} + +func (h *udpHandler) processResponses(ctx context.Context) error { + buf := make([]byte, maxDatagramSize) + for { + n, from, err := h.upstreamSock.ReadFrom(buf) + if err != nil { + if ctx.Err() != nil || errors.Is(err, net.ErrClosed) { + return nil + } + slog.DebugContext(ctx, "dns relay: upstream socket read error", slog.Any("err", err)) + continue + } + raw := bytes.Clone(buf[:n]) + act := h.dns.onUDPResponse(raw, from) + switch act.kind { + case actionDrop: + continue + case actionFailover: + if act.upstreamIdx >= 0 && act.upstreamIdx < len(h.upstreams) && len(act.payload) >= 2 { + upstreamID := binary.BigEndian.Uint16(act.payload[0:2]) + h.sendToUpstream(ctx, upstreamID, act.upstreamIdx, act.payload) + } + case actionDeliver: + if act.clientAddr != nil { + if _, err := h.actorSock.WriteTo(act.payload, act.clientAddr); err != nil && ctx.Err() == nil && !errors.Is(err, net.ErrClosed) { + slog.WarnContext(ctx, "dns relay: could not return a DNS answer", slog.Any("err", err)) + } + } + } + } +} + +func (h *udpHandler) sweepLoop(ctx context.Context) { + interval := h.sweepInterval + if interval <= 0 { + interval = defaultSweepInterval + } + ticker := time.NewTicker(interval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case now := <-ticker.C: + failovers, deliveries := h.pending.sweep(now) + for _, f := range failovers { + h.sendToUpstream(ctx, f.upstreamID, f.upstreamIdx, f.query) + } + for _, d := range deliveries { + if _, err := h.actorSock.WriteTo(d.payload, d.clientAddr); err != nil && ctx.Err() == nil && !errors.Is(err, net.ErrClosed) { + slog.WarnContext(ctx, "dns relay: could not return a deferred DNS answer", slog.Any("err", err)) + } + } + } + } +} diff --git a/internal/ateomnet/dns/udphandler_test.go b/internal/ateomnet/dns/udphandler_test.go new file mode 100644 index 0000000000..ca7854a260 --- /dev/null +++ b/internal/ateomnet/dns/udphandler_test.go @@ -0,0 +1,169 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dns + +import ( + "context" + "encoding/binary" + "net" + "sync/atomic" + "testing" + "time" + + "golang.org/x/net/dns/dnsmessage" +) + +func startTestUDPHandler(t *testing.T, upstreams []string, configure func(*udpHandler, *dnsHandler)) net.Conn { + t.Helper() + ingress, err := net.ListenPacket("udp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + egress, err := net.ListenPacket("udp", "127.0.0.1:0") + if err != nil { + ingress.Close() + t.Fatal(err) + } + + udpAddrs := make([]*net.UDPAddr, len(upstreams)) + for i, u := range upstreams { + addr, err := net.ResolveUDPAddr("udp", u) + if err != nil { + ingress.Close() + egress.Close() + t.Fatal(err) + } + udpAddrs[i] = addr + } + + lim := newLimiter() + pending := newPendingRequests(lim) + dnsH := newDNSHandler(pending, lim, nil) + h := newUDPHandler(ingress, egress, udpAddrs, dnsH, pending) + if configure != nil { + configure(h, dnsH) + } + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + defer close(done) + _ = h.serve(ctx) + }() + t.Cleanup(func() { + cancel() + _ = ingress.Close() + _ = egress.Close() + <-done + }) + + client, err := net.Dial("udp", ingress.LocalAddr().String()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = client.Close() }) + return client +} + +func TestUDPHandlerForwardsAndRestoresID(t *testing.T) { + var seenUpstreamID atomic.Uint32 + upstream := newFakeResolver(t, func(query []byte) []byte { + seenUpstreamID.Store(uint32(binary.BigEndian.Uint16(query[0:2]))) + return dnsAnswer(query, 0) + }) + + client := startTestUDPHandler(t, []string{upstream}, nil) + const clientID = 0x4242 + query := dnsQuery(clientID) + if _, err := client.Write(query); err != nil { + t.Fatal(err) + } + got := readWithin(t, client) + if gotID := binary.BigEndian.Uint16(got[0:2]); gotID != clientID { + t.Errorf("response ID = %#x, want %#x", gotID, clientID) + } + if got[2]&0x80 == 0 { + t.Error("expected response bit QR=1 to be set") + } +} + +func TestUDPHandlerSynthesizedReply(t *testing.T) { + var upstreamCalls atomic.Int32 + upstream := newFakeResolver(t, func(query []byte) []byte { + upstreamCalls.Add(1) + return dnsAnswer(query, 0) + }) + + client := startTestUDPHandler(t, []string{upstream}, func(_ *udpHandler, dnsH *dnsHandler) { + dnsH.allow = func(dnsmessage.Question) bool { return false } + }) + + query := dnsQuery(0x9999) + if _, err := client.Write(query); err != nil { + t.Fatal(err) + } + got := readWithin(t, client) + if rcode := dnsmessage.RCode(got[3] & 0x0f); rcode != dnsmessage.RCodeRefused { + t.Errorf("synthesized rcode = %v, want %v", rcode, dnsmessage.RCodeRefused) + } + if upstreamCalls.Load() != 0 { + t.Errorf("upstream was called %d times on synthesized reply, want 0", upstreamCalls.Load()) + } +} + +func TestUDPHandlerFailoverOnSweepTimeoutAndDeferredDelivery(t *testing.T) { + silent1, _ := newSilentResolver(t) + healthy := newFakeResolver(t, func(query []byte) []byte { + return dnsAnswer(query, 0) + }) + + client := startTestUDPHandler(t, []string{silent1, healthy}, func(h *udpHandler, _ *dnsHandler) { + h.pending.attemptTimeout = 40 * time.Millisecond + h.sweepInterval = 10 * time.Millisecond + }) + + if _, err := client.Write(dnsQuery(0x7777)); err != nil { + t.Fatal(err) + } + got := readWithin(t, client) + if gotID := binary.BigEndian.Uint16(got[0:2]); gotID != 0x7777 { + t.Errorf("response ID = %#x, want 0x7777", gotID) + } + if rcode := got[3] & 0x0f; rcode != 0 { + t.Errorf("rcode = %d, want 0 after timeout failover", rcode) + } + + // Also verify that if the first upstream returns SERVFAIL and the second + // upstream times out, the deferred SERVFAIL is delivered on expiry. + sick := newFakeResolver(t, func(query []byte) []byte { + return dnsAnswer(query, byte(dnsmessage.RCodeServerFailure)) + }) + silent2, _ := newSilentResolver(t) + client2 := startTestUDPHandler(t, []string{sick, silent2}, func(h *udpHandler, _ *dnsHandler) { + h.pending.attemptTimeout = 40 * time.Millisecond + h.sweepInterval = 10 * time.Millisecond + }) + + if _, err := client2.Write(dnsQuery(0x8888)); err != nil { + t.Fatal(err) + } + got2 := readWithin(t, client2) + if gotID := binary.BigEndian.Uint16(got2[0:2]); gotID != 0x8888 { + t.Errorf("deferred response ID = %#x, want 0x8888", gotID) + } + if rcode := dnsmessage.RCode(got2[3] & 0x0f); rcode != dnsmessage.RCodeServerFailure { + t.Errorf("deferred rcode = %v, want %v", rcode, dnsmessage.RCodeServerFailure) + } +} diff --git a/internal/ateomphaselog/phaselog.go b/internal/ateomphaselog/phaselog.go new file mode 100644 index 0000000000..58929f7675 --- /dev/null +++ b/internal/ateomphaselog/phaselog.go @@ -0,0 +1,108 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package ateomphaselog renders the per-actor "timing breakdown" log record an +// ateom emits for each checkpoint and restore. Every ateom binary times its +// own phases, which are implementation details of that runtime; this package +// holds the shape they share so the records of all ateoms carry the same +// identity, fidelity and key layout, and atelet's own record joins to any of them. +package ateomphaselog + +import ( + "context" + "log/slog" + "time" + + "github.com/agent-substrate/substrate/internal/apierror" + "github.com/agent-substrate/substrate/internal/ateattr" + "github.com/agent-substrate/substrate/internal/proto/ateompb" + "github.com/agent-substrate/substrate/internal/resources" + "google.golang.org/grpc/status" +) + +// The keys the per-phase durations are logged under. They are named like +// instruments but deliberately are not ones: the phases are implementation +// details of each ateom, so they stay a developer-facing log record rather +// than becoming metric API. They decompose the ateom_restore / +// ateom_checkpoint phases of atelet's ate.actor.*.duration histograms, whose +// names they extend, and are joined per actor by benchmarking tooling. +const ( + RestoreDurationKey = "ateom.actor.restore.duration" + CheckpointDurationKey = "ateom.actor.checkpoint.duration" +) + +// Phase is one timed step of a snapshot operation. A zero duration means the +// phase never ran (a VOLUMES checkpoint captures no guest) and is skipped +// rather than logged as instant. Name is suffixed onto the duration key; each +// ateom defines its own names, with ateattr.SnapshotPhaseTotal shared so the +// two layers' records agree on the denominator. +type Phase struct { + Name string + D time.Duration +} + +// FidelityLogValue maps the ateom wire enum onto the shared fidelity label +// values, the same way ateattr.SnapshotFidelityValue does for the atelet +// enum. An unrecognized fidelity reports as unknown rather than stringified, +// so no wire value can widen the value set readers key on. +func FidelityLogValue(fidelity ateompb.SnapshotFidelity) string { + switch fidelity { + case ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES: + return ateattr.SnapshotFidelityVolumes + case ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS: + return ateattr.SnapshotFidelityRootfs + case ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY: + return ateattr.SnapshotFidelityMemory + default: + return ateattr.SnapshotFidelityUnknown + } +} + +// SnapshotPhaseAttrs renders one joinable per-actor record for a checkpoint +// or restore, mirroring atelet's snapshotLogAttrs: full actor identity, the +// scope, and one float-seconds attr per non-zero phase under +// durationKey.. One record carries every phase of the operation, so a +// reader never joins two half-records that disagree about the same actor. +// +// err is the operation's outcome. A failed operation still records the phases +// it completed, marked with error.type (the gRPC code, context errors as +// DeadlineExceeded / Canceled) so a reader can leave it out of a latency +// distribution. Absence means success. +func SnapshotPhaseAttrs(a resources.ActorAttribution, fidelity ateompb.SnapshotFidelity, durationKey string, err error, phases []Phase) []slog.Attr { + attrs := ateattr.ActorLogAttrs(a) + attrs = append(attrs, slog.String(string(ateattr.SnapshotFidelityKey), FidelityLogValue(fidelity))) + if err != nil { + s, ok := apierror.FromError(err) + code := s.Code() + if !ok { + code = status.Code(err) + } + attrs = append(attrs, slog.String(string(ateattr.ErrorTypeKey), code.String())) + } + for _, p := range phases { + if p.D == 0 { + continue + } + // Seconds and not slog.Duration's nanoseconds: the values sit under + // the atelet histograms' unit (s), so readers compare them without a + // per-key unit table. + attrs = append(attrs, slog.Float64(durationKey+"."+p.Name, p.D.Seconds())) + } + return attrs +} + +// LogSnapshotPhases emits the SnapshotPhaseAttrs record under msg. +func LogSnapshotPhases(ctx context.Context, msg string, a resources.ActorAttribution, fidelity ateompb.SnapshotFidelity, durationKey string, err error, phases []Phase) { + slog.LogAttrs(ctx, slog.LevelInfo, msg, SnapshotPhaseAttrs(a, fidelity, durationKey, err, phases)...) +} diff --git a/cmd/ateom-microvm/phaselog_test.go b/internal/ateomphaselog/phaselog_test.go similarity index 63% rename from cmd/ateom-microvm/phaselog_test.go rename to internal/ateomphaselog/phaselog_test.go index cddd1dd54d..7df3d0c6ca 100644 --- a/cmd/ateom-microvm/phaselog_test.go +++ b/internal/ateomphaselog/phaselog_test.go @@ -1,5 +1,3 @@ -//go:build linux - // Copyright 2026 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package ateomphaselog import ( "bytes" @@ -55,15 +53,15 @@ func renderPhaseRecord(t *testing.T, attrs []slog.Attr) map[string]any { func TestSnapshotPhaseAttrs(t *testing.T) { t.Parallel() - attrs := snapshotPhaseAttrs(phaseLogAttribution(), ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL, - checkpointDurationKey, nil, []phase{ - {phasePrep, 40 * time.Millisecond}, - {phasePause, 3 * time.Millisecond}, - {phaseSnapshot, 850 * time.Millisecond}, + attrs := SnapshotPhaseAttrs(phaseLogAttribution(), ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + CheckpointDurationKey, nil, []Phase{ + {Name: "prep", D: 40 * time.Millisecond}, + {Name: "pause", D: 3 * time.Millisecond}, + {Name: "snapshot", D: 850 * time.Millisecond}, // A capture that did not run stays off the record. - {phaseDurableDir, 0}, - {phaseTeardown, 230 * time.Millisecond}, - {phaseTotal, 1250 * time.Millisecond}, + {Name: "durable_dir", D: 0}, + {Name: "teardown", D: 230 * time.Millisecond}, + {Name: ateattr.SnapshotPhaseTotal, D: 1250 * time.Millisecond}, }) // json.Unmarshal keeps the last of a repeated key, so a duplicate is @@ -83,7 +81,7 @@ func TestSnapshotPhaseAttrs(t *testing.T) { "ate.actor.uid": "uid-abc", "ate.template.atespace": "templates", "ate.template.name": "support-agent", - "ate.snapshot.scope": ateattr.SnapshotScopeFull, + "ate.snapshot.fidelity": ateattr.SnapshotFidelityMemory, } { if got, ok := rec[k]; !ok { t.Errorf("missing %s", k) @@ -141,9 +139,13 @@ func TestSnapshotPhaseAttrsFailure(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - rec := renderPhaseRecord(t, snapshotPhaseAttrs(phaseLogAttribution(), - ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL, checkpointDurationKey, tt.err, - []phase{{phasePrep, 40 * time.Millisecond}, {phasePause, 3 * time.Millisecond}, {phaseTotal, 30 * time.Second}})) + rec := renderPhaseRecord(t, SnapshotPhaseAttrs(phaseLogAttribution(), + ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, CheckpointDurationKey, tt.err, + []Phase{ + {Name: "prep", D: 40 * time.Millisecond}, + {Name: "pause", D: 3 * time.Millisecond}, + {Name: ateattr.SnapshotPhaseTotal, D: 30 * time.Second}, + })) if got := rec["error.type"]; got != tt.want { t.Errorf("error.type = %v, want %q", got, tt.want) } @@ -159,23 +161,54 @@ func TestSnapshotPhaseAttrsFailure(t *testing.T) { } } -// TestScopeLogValue pins the mapping onto the shared scope values, so the -// ateom and atelet records of one operation agree on the scope they carry. -func TestScopeLogValue(t *testing.T) { +// TestFidelityLogValue pins the mapping onto the shared fidelity values, so the +// ateom and atelet records of one operation agree on the fidelity they carry. +func TestFidelityLogValue(t *testing.T) { t.Parallel() tests := []struct { - scope ateompb.SnapshotScope + scope ateompb.SnapshotFidelity want string }{ - {ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL, ateattr.SnapshotScopeFull}, - {ateompb.SnapshotScope_SNAPSHOT_SCOPE_DATA, ateattr.SnapshotScopeData}, - {ateompb.SnapshotScope_SNAPSHOT_SCOPE_UNSPECIFIED, ateattr.SnapshotScopeUnknown}, - {ateompb.SnapshotScope(99), ateattr.SnapshotScopeUnknown}, + {ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, ateattr.SnapshotFidelityMemory}, + {ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, ateattr.SnapshotFidelityVolumes}, + {ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS, ateattr.SnapshotFidelityRootfs}, + {ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED, ateattr.SnapshotFidelityUnknown}, + {ateompb.SnapshotFidelity(99), ateattr.SnapshotFidelityUnknown}, } for _, tt := range tests { - if got := scopeLogValue(tt.scope); got != tt.want { - t.Errorf("scopeLogValue(%v) = %q, want %q", tt.scope, got, tt.want) + if got := FidelityLogValue(tt.scope); got != tt.want { + t.Errorf("FidelityLogValue(%v) = %q, want %q", tt.scope, got, tt.want) } } } + +// TestLogSnapshotPhases: the record goes out under the caller's message and +// at Info, through the default logger the ateoms write to. +func TestLogSnapshotPhases(t *testing.T) { + var buf bytes.Buffer + prev := slog.Default() + slog.SetDefault(slog.New(slog.NewJSONHandler(&buf, nil))) + t.Cleanup(func() { slog.SetDefault(prev) }) + + LogSnapshotPhases(context.Background(), "Restore timing breakdown", phaseLogAttribution(), + ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, RestoreDurationKey, nil, + []Phase{{Name: ateattr.SnapshotPhaseTotal, D: 2 * time.Second}}) + + var rec map[string]any + if err := json.Unmarshal(buf.Bytes(), &rec); err != nil { + t.Fatalf("unmarshal record %q: %v", buf.String(), err) + } + if rec["msg"] != "Restore timing breakdown" { + t.Errorf("msg = %v, want %q", rec["msg"], "Restore timing breakdown") + } + if rec["level"] != "INFO" { + t.Errorf("level = %v, want INFO", rec["level"]) + } + if rec["ate.snapshot.fidelity"] != ateattr.SnapshotFidelityVolumes { + t.Errorf("ate.snapshot.fidelity = %v, want %q", rec["ate.snapshot.fidelity"], ateattr.SnapshotFidelityVolumes) + } + if got := rec["ateom.actor.restore.duration.total"]; got != 2.0 { + t.Errorf("ateom.actor.restore.duration.total = %v, want 2", got) + } +} diff --git a/internal/ateomstats/usage.go b/internal/ateomstats/usage.go index 17bd09c08f..b3801fd961 100644 --- a/internal/ateomstats/usage.go +++ b/internal/ateomstats/usage.go @@ -31,10 +31,6 @@ import ( "github.com/agent-substrate/substrate/internal/resources" ) -// finalFlushTimeout bounds the flush after a final record. The ateom stays up -// after a checkpoint, so the flush buys promptness, not delivery. -const finalFlushTimeout = 500 * time.Millisecond - // Pool is the WorkerPool of the ateom's pod. type Pool struct { Namespace string @@ -46,9 +42,6 @@ type Pool struct { type UsageEmitter struct { pool Pool events *actorevent.Emitter - // flush pushes queued OTLP records out after a final record; nil when OTLP - // is off. - flush func(context.Context) error } // NewUsageEmitter writes the records over OTLP through lp, the provider @@ -57,7 +50,7 @@ func NewUsageEmitter(lp *sdklog.LoggerProvider, stdout slog.Handler, pool Pool) if lp == nil { return &UsageEmitter{pool: pool, events: actorevent.NewEmitterTo(noop.NewLoggerProvider(), stdout)} } - return &UsageEmitter{pool: pool, events: actorevent.NewEmitterTo(lp, stdout), flush: lp.ForceFlush} + return &UsageEmitter{pool: pool, events: actorevent.NewEmitterTo(lp, stdout)} } // Emit writes one record for s, dated when s was read. kind is one of the @@ -70,25 +63,6 @@ func (e *UsageEmitter) Emit(ctx context.Context, kind string, s *ateompb.Workloa e.events.LogAt(ctx, actorevent.UsageSampled, at, UsageAttrs(e.pool, kind, s)) } -// EmitFinal writes the final record of an activation and flushes it in the -// background, off the checkpoint's path. -func (e *UsageEmitter) EmitFinal(ctx context.Context, s *ateompb.WorkloadStatsSample) { - if e == nil { - return - } - e.Emit(ctx, ateattr.StatsKindFinal, s) - if e.flush == nil { - return - } - go func() { - flushCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), finalFlushTimeout) - defer cancel() - if err := e.flush(flushCtx); err != nil { - slog.DebugContext(ctx, "Usage records not flushed after the final sample", slog.Any("err", err)) - } - }() -} - // UsageAttrs is the attribute set of ate.actor.usage_sampled for s. The // measurements are absent while s has no source, and the peak also when the // source reports none. diff --git a/internal/ateomstats/usage_test.go b/internal/ateomstats/usage_test.go index 339a3cc6ed..a6efc61d7c 100644 --- a/internal/ateomstats/usage_test.go +++ b/internal/ateomstats/usage_test.go @@ -145,16 +145,14 @@ func (r *recorder) len() int { return len(r.records) } -func newTestEmitter(flush func(context.Context) error) (*UsageEmitter, *recorder) { +func newTestEmitter() (*UsageEmitter, *recorder) { rec := &recorder{} - e := NewUsageEmitter(nil, rec, testPool) - e.flush = flush - return e, rec + return NewUsageEmitter(nil, rec, testPool), rec } func TestEmitDatesRecordAtReadTime(t *testing.T) { t.Parallel() - e, rec := newTestEmitter(nil) + e, rec := newTestEmitter() s := measuredSample() e.Emit(context.Background(), ateattr.StatsKindInitial, s) if rec.len() != 1 { @@ -169,39 +167,6 @@ func TestNilEmitterWritesNothing(t *testing.T) { t.Parallel() var e *UsageEmitter e.Emit(context.Background(), ateattr.StatsKindInitial, measuredSample()) - e.EmitFinal(context.Background(), measuredSample()) -} - -func TestEmitFinalFlushes(t *testing.T) { - t.Parallel() - deadlines := make(chan time.Time, 1) - e, rec := newTestEmitter(func(ctx context.Context) error { - d, _ := ctx.Deadline() - deadlines <- d - return nil - }) - start := time.Now() - e.EmitFinal(context.Background(), measuredSample()) - if rec.len() != 1 { - t.Fatalf("wrote %d records, want 1", rec.len()) - } - if kind := attrMap(recordAttrs(rec.records[0]))[string(ateattr.StatsKindKey)].String(); kind != ateattr.StatsKindFinal { - t.Errorf("kind = %q, want final", kind) - } - select { - case d := <-deadlines: - if d.IsZero() || d.Sub(start) > finalFlushTimeout+time.Second { - t.Errorf("flush deadline = %v, want about %v after the record", d, finalFlushTimeout) - } - case <-time.After(5 * time.Second): - t.Fatal("final record was not flushed") - } -} - -func recordAttrs(r slog.Record) []slog.Attr { - var attrs []slog.Attr - r.Attrs(func(a slog.Attr) bool { attrs = append(attrs, a); return true }) - return attrs } // TestStartSamplerSweepsOnSchedule runs six intervals and stops: six sweeps, diff --git a/internal/atunnel/ingress.go b/internal/atunnel/ingress.go index fdd3e56d38..2a135a8025 100644 --- a/internal/atunnel/ingress.go +++ b/internal/atunnel/ingress.go @@ -532,7 +532,11 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { } func (s *Server) authorize(r *http.Request) (*activation, context.Context, func(), bool) { - ref, err := atenet.ParseTargetActor(r.Header.Get(atenet.TargetActorHeader)) + vals := r.Header.Values(atenet.TargetActorHeader) + if len(vals) != 1 { + return nil, nil, nil, false + } + ref, err := atenet.ParseTargetActor(vals[0]) if err != nil { return nil, nil, nil, false } diff --git a/internal/atunnel/ingress_test.go b/internal/atunnel/ingress_test.go index 983ff7f106..ffeeb3889e 100644 --- a/internal/atunnel/ingress_test.go +++ b/internal/atunnel/ingress_test.go @@ -290,6 +290,73 @@ func TestServeHTTP(t *testing.T) { } } +func TestServerRejectsDuplicateOrCommaTargetActorHeaders(t *testing.T) { + upstreamURL, err := url.Parse("http://actor.internal:80") + if err != nil { + t.Fatal(err) + } + + s := newTestServer(t, upstreamURL) + var upstreamCalled bool + actorTransport := roundTripFunc(func(r *http.Request) (*http.Response, error) { + upstreamCalled = true + return &http.Response{ + StatusCode: http.StatusNoContent, + Header: make(http.Header), + Body: http.NoBody, + }, nil + }) + if err := s.Activate("team-a", "actor-1", "uid-actor-1", testDial); err != nil { + t.Fatal(err) + } + setActorTransport(t, s, "team-a", "actor-1", actorTransport) + + tests := []struct { + name string + host string + headers []string + }{ + { + name: "duplicate different actors", + host: "actor-1.team-a.actors.resources.substrate.ate.dev", + headers: []string{"team-a/actor-1", "secret/x"}, + }, + { + name: "duplicate identical actors", + host: "actor-1.team-a.actors.resources.substrate.ate.dev", + headers: []string{"team-a/actor-1", "team-a/actor-1"}, + }, + { + name: "comma-joined actors", + host: "actor-1.team-a.actors.resources.substrate.ate.dev", + headers: []string{"team-a/actor-1,secret/x"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + upstreamCalled = false + req := httptest.NewRequest(http.MethodGet, "https://worker/hello", nil) + req.Host = tt.host + for _, h := range tt.headers { + req.Header.Add(atenet.TargetActorHeader, h) + } + rec := httptest.NewRecorder() + s.ServeHTTP(rec, req) + + if rec.Code != http.StatusMisdirectedRequest { + t.Fatalf("status = %d, want %d", rec.Code, http.StatusMisdirectedRequest) + } + if rec.Header().Get(StaleAssignmentHeader) != "true" { + t.Errorf("missing %s response header", StaleAssignmentHeader) + } + if upstreamCalled { + t.Errorf("upstream was dialed, expected request to be rejected") + } + }) + } +} + func TestServeHTTPHonorsTargetPortHeader(t *testing.T) { upstreamURL, err := url.Parse("http://actor.internal:80") if err != nil { diff --git a/internal/e2e/atenet_dataplane.go b/internal/e2e/atenet_dataplane.go index 25dc80ce3e..50ca3f86a2 100644 --- a/internal/e2e/atenet_dataplane.go +++ b/internal/e2e/atenet_dataplane.go @@ -35,6 +35,7 @@ type AtenetDataplane interface { IsRetryableParkingBudgetExhaustion(status int, body string) bool ParkingBudgetStatus() int IsEgressPolicyDenied(status int, body string) bool + EgressUpgradeDeniedBody() string PlatformMetricPrefixes([]string) []string RouteDurationSeen(context.Context, string) (bool, error) } @@ -66,6 +67,10 @@ func (envoyAtenetDataplane) IsRetryableParkingBudgetExhaustion(status int, body func (envoyAtenetDataplane) ParkingBudgetStatus() int { return http.StatusServiceUnavailable } +func (envoyAtenetDataplane) EgressUpgradeDeniedBody() string { + return "WebSocket egress is not supported" +} + func (envoyAtenetDataplane) IsEgressPolicyDenied(status int, body string) bool { return (status == http.StatusForbidden && strings.Contains(body, "egress denied")) || (status == http.StatusBadGateway && strings.Contains(body, "request failed")) @@ -89,6 +94,10 @@ func (agentGatewayAtenetDataplane) IsRetryableParkingBudgetExhaustion(status int func (agentGatewayAtenetDataplane) ParkingBudgetStatus() int { return http.StatusGatewayTimeout } +func (agentGatewayAtenetDataplane) EgressUpgradeDeniedBody() string { + return "HTTP upgrades, including CONNECT, are denied for actor egress" +} + func (agentGatewayAtenetDataplane) IsEgressPolicyDenied(status int, body string) bool { return status == http.StatusForbidden && strings.Contains(body, "actor egress policy denied") } diff --git a/internal/e2e/fixtures/security/atespace-a-pool.yaml.tmpl b/internal/e2e/fixtures/security/atespace-a-pool.yaml.tmpl new file mode 100644 index 0000000000..32fd9bc1b1 --- /dev/null +++ b/internal/e2e/fixtures/security/atespace-a-pool.yaml.tmpl @@ -0,0 +1,50 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Atespace A of the cross-atespace security fixture: a namespace plus the only +# WorkerPool. Atespace B deploys atespace-b-namespace.yaml.tmpl, a namespace +# with no pool, so its actors run on this one too. +# +# replicas is 1 on purpose. Pools are not scoped to an atespace: the scheduler +# matches on sandbox class, labels and capacity, never on the actor's +# atespace. The test runs the victim, suspends it, then starts the attacker. +# With one worker the attacker has to land on the worker the victim just +# left, which is where the victim's per-actor host state is. With more +# workers the attacker could boot on a fresh one and the host-state checks +# would prove nothing. +# +# The workload label is a fixed string rather than ${FIXTURE_SUFFIX}-suffixed +# because both atespaces' templates must select this one pool and they render +# with different suffixes. Sharing the label across the gVisor and micro-VM +# lanes is safe: they run one after the other, and sandbox class is itself a +# scheduling constraint. + +apiVersion: v1 +kind: Namespace +metadata: + name: ate-e2e${FIXTURE_SUFFIX} + +--- + +apiVersion: ate.dev/v1alpha1 +kind: WorkerPool +metadata: + name: shared + namespace: ate-e2e${FIXTURE_SUFFIX} + labels: + workload: ate-e2e-security +spec: + replicas: 1 + workerImage: ${ATEOM_IMAGE} +${WORKERPOOL_RUNTIME} diff --git a/internal/e2e/fixtures/security/atespace-b-namespace.yaml.tmpl b/internal/e2e/fixtures/security/atespace-b-namespace.yaml.tmpl new file mode 100644 index 0000000000..6dfd9038ce --- /dev/null +++ b/internal/e2e/fixtures/security/atespace-b-namespace.yaml.tmpl @@ -0,0 +1,29 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Atespace B of the cross-atespace security fixture: a namespace only. +# +# Atespace B deliberately has no WorkerPool. Pools are not scoped to an +# atespace, so B's actors are placed on the single-worker pool that +# atespace-a-pool.yaml.tmpl deploys in atespace A's namespace. That forces +# A's and B's actors to take turns on one worker, which is the reuse boundary +# the suite tests. +# +# The namespace is still needed: Substrate reconciles an atespace's +# namespaced objects (network policies, for example) into it. + +apiVersion: v1 +kind: Namespace +metadata: + name: ate-e2e${FIXTURE_SUFFIX} diff --git a/internal/e2e/fixtures/security/atespace-template.yaml.tmpl b/internal/e2e/fixtures/security/atespace-template.yaml.tmpl new file mode 100644 index 0000000000..a54d995349 --- /dev/null +++ b/internal/e2e/fixtures/security/atespace-template.yaml.tmpl @@ -0,0 +1,76 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The ActorTemplate of the cross-atespace security fixture: a +# protojson-shaped ateapipb.ActorTemplate created through the ate API by +# e2e.DeploySubstrateFixture. +# +# BOTH atespaces deploy this same file. They differ only in the ${FIXTURE_SUFFIX} +# they render with, which is what gives each its own atespace and its own +# snapshot prefix, so the suite compares two actors that are identical in +# every respect except the boundary under test. +# +# The container is the shared e2e probe image. Its /readfile and /writefile +# endpoints let the test try reads and writes from inside the running actor, +# which is the only place the isolation claim can be checked: what the actor +# actually sees at /data, not what atelet's mount config says it should see. +# +# The workerSelector names the single shared pool (see atespace-a-pool.yaml.tmpl), +# not a per-atespace one. + +metadata: + atespace: ate-e2e${FIXTURE_SUFFIX} + name: probe +workerSelector: + matchLabels: + workload: ate-e2e-security +containers: +- name: probe + image: ko://github.com/agent-substrate/substrate/internal/e2e/fixtures/probe + command: ["/ko-app/probe"] + volumeMounts: + # The probe expects its identity files under /run/ate. The mount also makes + # atelet write a per-actor identity file on the worker host, which is one + # of the host paths the attacker is checked against. + - name: system-info + mountPath: /run/ate + # The per-actor durable directory whose isolation is under test. + - name: data + mountPath: /data + # Gate the golden snapshot on the probe actually serving, so a read or write + # right after a resume cannot race a sandbox checkpointed before the + # listener came up. + wakeupProbe: + httpGet: + path: /healthz + port: 80 + timeoutSeconds: 60 +volumes: +- name: system-info + systemInfo: + dataSources: + - actorMetadata: + items: + - field: ACTOR_METADATA_FIELD_NAME + path: actor-id + - field: ACTOR_METADATA_FIELD_ATESPACE + path: atespace + - field: ACTOR_METADATA_FIELD_UID + path: actor-uid +- name: data + durableDir: {} +${TEMPLATE_RESOURCES} +${TEMPLATE_SANDBOX_CONFIG} +snapshotConfig: + storageLocation: gs://${BUCKET_NAME}/ate-e2e${FIXTURE_SUFFIX}/ diff --git a/internal/e2e/fixtures/testserver/common_test.go b/internal/e2e/fixtures/testserver/common_test.go new file mode 100644 index 0000000000..d6773c08f7 --- /dev/null +++ b/internal/e2e/fixtures/testserver/common_test.go @@ -0,0 +1,96 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "errors" + "net" + "net/http" + "os" + "path/filepath" + "testing" + "time" + + "github.com/agent-substrate/substrate/internal/testcert" +) + +type originCertificate struct { + caPEM []byte + certFile string + keyFile string +} + +type originServer struct { + httpURL string + httpsURL string + wsURL string + wssURL string +} + +func writeOriginCertificate(t *testing.T) originCertificate { + t.Helper() + material := testcert.NewServerTLS(t, net.ParseIP("127.0.0.1")) + + certFile := filepath.Join(t.TempDir(), "server.crt") + keyFile := filepath.Join(t.TempDir(), "server.key") + if err := os.WriteFile(certFile, material.Certificate, 0o600); err != nil { + t.Fatalf("write server certificate: %v", err) + } + if err := os.WriteFile(keyFile, material.PrivateKey, 0o600); err != nil { + t.Fatalf("write server key: %v", err) + } + return originCertificate{caPEM: material.RootCA, certFile: certFile, keyFile: keyFile} +} + +func startOriginServer(t *testing.T, handler http.Handler, certFile, keyFile string) originServer { + t.Helper() + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen on loopback: %v", err) + } + server := &http.Server{Handler: handler} + if certFile != "" || keyFile != "" { + if err := configureTLSServer(server, certFile, keyFile); err != nil { + t.Fatalf("configure TLS: %v", err) + } + } + serveErr := make(chan error, 1) + go func() { + if certFile == "" && keyFile == "" { + serveErr <- server.Serve(listener) + return + } + serveErr <- server.ServeTLS(listener, "", "") + }() + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if err := server.Shutdown(ctx); err != nil { + t.Errorf("shutdown origin: %v", err) + } + if err := <-serveErr; err != nil && !errors.Is(err, http.ErrServerClosed) { + t.Errorf("serve origin: %v", err) + } + }) + + address := listener.Addr().String() + return originServer{ + httpURL: "http://" + address, + httpsURL: "https://" + address, + wsURL: "ws://" + address, + wssURL: "wss://" + address, + } +} diff --git a/internal/e2e/fixtures/testserver/http.go b/internal/e2e/fixtures/testserver/http.go index 5eaf72806d..982d150cc2 100644 --- a/internal/e2e/fixtures/testserver/http.go +++ b/internal/e2e/fixtures/testserver/http.go @@ -15,6 +15,7 @@ package main import ( + "io" "log" "net/http" "os" @@ -23,36 +24,54 @@ import ( "github.com/spf13/cobra" ) -// newHTTPCmd is a plain HTTP/1.1 origin an Actor's egress lands on. It exists so -// a test can assert the destination port is recovered from SO_ORIGINAL_DST. -// It can also verify an injected Authorization header against a mounted token. +// newHTTPHandler is the HTTP/1.1 origin an Actor's egress lands on. It exists so +// a test can assert the destination port is recovered from SO_ORIGINAL_DST +// rather than defaulted from the URL scheme: the actor fetches its /healthz on a +// non-standard port, and the gateway's access log is expected to carry that +// port. The optional body makes the same fixture useful for assertions about a +// response inside the egress tunnel. +func newHTTPHandler(body, authorizationFile string) http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + if body != "" { + _, _ = io.WriteString(w, body) + } + }) + if authorizationFile != "" { + mux.HandleFunc("/credential", credentialHandler(authorizationFile)) + } + return mux +} + func newHTTPCmd() *cobra.Command { - var listenAddress, authorizationFile string + var ( + authorizationFile string + listenAddress string + certFile string + keyFile string + body string + ) cmd := &cobra.Command{ Use: "http", - Short: "Serve a plain HTTP/1.1 origin answering /healthz.", + Short: "Serve an HTTP/1.1 origin answering /healthz.", Args: cobra.NoArgs, RunE: func(_ *cobra.Command, _ []string) error { - mux := http.NewServeMux() - mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) { - w.WriteHeader(http.StatusOK) - }) - if authorizationFile != "" { - mux.HandleFunc("/credential", credentialHandler(authorizationFile)) - } - server := &http.Server{ Addr: listenAddress, - Handler: mux, + Handler: newHTTPHandler(body, authorizationFile), ReadHeaderTimeout: 10 * time.Second, WriteTimeout: 2 * time.Minute, } log.Printf("testserver http: listening on %s", listenAddress) - return server.ListenAndServe() + return serveHTTP(server, certFile, keyFile) }, } cmd.Flags().StringVar(&listenAddress, "listen", ":8080", "Address the HTTP origin listens on.") + cmd.Flags().StringVar(&certFile, "tls-cert", "", "PEM certificate file; enables HTTPS when paired with --tls-key.") + cmd.Flags().StringVar(&keyFile, "tls-key", "", "PEM private key file; enables HTTPS when paired with --tls-cert.") cmd.Flags().StringVar(&authorizationFile, "authorization-file", "", "Enable /credential, requiring a Bearer token matching this file.") + cmd.Flags().StringVar(&body, "body", "", "Response body for /healthz.") return cmd } diff --git a/internal/e2e/fixtures/testserver/http_test.go b/internal/e2e/fixtures/testserver/http_test.go index b42c292677..bf747237a2 100644 --- a/internal/e2e/fixtures/testserver/http_test.go +++ b/internal/e2e/fixtures/testserver/http_test.go @@ -15,13 +15,81 @@ package main import ( + "crypto/tls" + "crypto/x509" + "io" "net/http" "net/http/httptest" "os" "path/filepath" "testing" + "time" ) +func TestHTTPHandlerDefaultsToEmptyHealthz(t *testing.T) { + server := startOriginServer(t, newHTTPHandler("", ""), "", "") + + client := &http.Client{Timeout: 2 * time.Second} + response, err := client.Get(server.httpURL + "/healthz") + if err != nil { + t.Fatalf("GET /healthz: %v", err) + } + defer response.Body.Close() + + if response.StatusCode != http.StatusOK { + t.Fatalf("GET /healthz status = %d, want %d", response.StatusCode, http.StatusOK) + } + body, err := io.ReadAll(response.Body) + if err != nil { + t.Fatalf("reading /healthz: %v", err) + } + if len(body) != 0 { + t.Errorf("GET /healthz body = %q, want empty", body) + } +} + +func TestHTTPSTestResponseUsesHTTP11AndVerifiedIdentity(t *testing.T) { + cert := writeOriginCertificate(t) + server := startOriginServer(t, newHTTPHandler("fixture response", ""), cert.certFile, cert.keyFile) + + roots := x509.NewCertPool() + if !roots.AppendCertsFromPEM(cert.caPEM) { + t.Fatal("adding fixture CA to trust pool") + } + protocols := new(http.Protocols) + protocols.SetHTTP1(true) + protocols.SetHTTP2(true) + client := &http.Client{ + Timeout: 2 * time.Second, + Transport: &http.Transport{ + Protocols: protocols, + TLSClientConfig: &tls.Config{ + RootCAs: roots, + ServerName: "127.0.0.1", + }, + }, + } + response, err := client.Get(server.httpsURL + "/healthz") + if err != nil { + t.Fatalf("GET /healthz over verified TLS: %v", err) + } + defer response.Body.Close() + + if response.Proto != "HTTP/1.1" { + t.Errorf("HTTPS response protocol with HTTP/2 offered = %q, want HTTP/1.1", response.Proto) + } + if response.StatusCode != http.StatusOK { + t.Errorf("HTTPS response status = %d, want %d", response.StatusCode, http.StatusOK) + } + body, err := io.ReadAll(response.Body) + if err != nil { + t.Fatalf("reading HTTPS response: %v", err) + } + if string(body) != "fixture response" { + t.Errorf("HTTPS response body = %q, want %q", body, "fixture response") + } +} + func TestCredentialHandler(t *testing.T) { path := filepath.Join(t.TempDir(), "token") if err := os.WriteFile(path, []byte("expected-token"), 0600); err != nil { @@ -38,7 +106,7 @@ func TestCredentialHandler(t *testing.T) { req := httptest.NewRequest(http.MethodGet, "/credential", nil) req.Header.Set("Authorization", tc.header) resp := httptest.NewRecorder() - credentialHandler(path)(resp, req) + newHTTPHandler("fixture response", path).ServeHTTP(resp, req) if resp.Code != tc.status || resp.Body.Len() != 0 { t.Errorf("header %q: status=%d body=%q, want status=%d and no body", tc.header, resp.Code, resp.Body.String(), tc.status) } diff --git a/internal/e2e/fixtures/testserver/tls.go b/internal/e2e/fixtures/testserver/tls.go new file mode 100644 index 0000000000..b370552dcc --- /dev/null +++ b/internal/e2e/fixtures/testserver/tls.go @@ -0,0 +1,55 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "crypto/tls" + "fmt" + "net/http" +) + +// configureTLSServer loads the origin certificate and key and offers only +// HTTP/1.1 over TLS. Invalid credentials fail setup before the listener starts. +func configureTLSServer(server *http.Server, certFile, keyFile string) error { + if certFile == "" && keyFile == "" { + return nil + } + if certFile == "" || keyFile == "" { + return fmt.Errorf("--tls-cert and --tls-key must be provided together") + } + + certificate, err := tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + return fmt.Errorf("loading TLS certificate and key: %w", err) + } + server.TLSConfig = &tls.Config{ + Certificates: []tls.Certificate{certificate}, + MinVersion: tls.VersionTLS12, + NextProtos: []string{"http/1.1"}, + } + return nil +} + +func serveHTTP(server *http.Server, certFile, keyFile string) error { + if certFile == "" && keyFile == "" { + return server.ListenAndServe() + } + + if err := configureTLSServer(server, certFile, keyFile); err != nil { + return err + } + + return server.ListenAndServeTLS("", "") +} diff --git a/internal/e2e/fixtures/testserver/tls_test.go b/internal/e2e/fixtures/testserver/tls_test.go new file mode 100644 index 0000000000..f8073cba62 --- /dev/null +++ b/internal/e2e/fixtures/testserver/tls_test.go @@ -0,0 +1,35 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "net/http" + "testing" +) + +func TestTLSConfigurationRejectsIncompleteOrUnreadableCredentials(t *testing.T) { + for name, paths := range map[string][2]string{ + "missing key": {"cert.pem", ""}, + "missing cert": {"", "key.pem"}, + "unreadable pair": {"missing-cert.pem", "missing-key.pem"}, + } { + t.Run(name, func(t *testing.T) { + server := &http.Server{} + if err := configureTLSServer(server, paths[0], paths[1]); err == nil { + t.Fatal("configureTLSServer succeeded, want error") + } + }) + } +} diff --git a/internal/e2e/fixtures/testserver/websocket.go b/internal/e2e/fixtures/testserver/websocket.go index efd189d5a8..011043c1ef 100644 --- a/internal/e2e/fixtures/testserver/websocket.go +++ b/internal/e2e/fixtures/testserver/websocket.go @@ -26,53 +26,70 @@ var upgrader = websocket.Upgrader{ CheckOrigin: func(r *http.Request) bool { return true }, } +func newWebsocketHandler(echo bool) http.Handler { + mux := http.NewServeMux() + + mux.HandleFunc("/readyz", func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("ok")) + }) + + mux.HandleFunc("/ws", func(w http.ResponseWriter, r *http.Request) { + c, err := upgrader.Upgrade(w, r, nil) + if err != nil { + log.Print("upgrade:", err) + return + } + defer c.Close() + for { + mt, message, err := c.ReadMessage() + if err != nil { + log.Println("read:", err) + break + } + log.Printf("recv: %s", message) + + var response []byte + switch { + case echo: + response = message + case string(message) == "PING": + response = []byte("PONG") + default: + continue + } + if err := c.WriteMessage(mt, response); err != nil { + log.Println("write:", err) + break + } + } + }) + return mux +} + func newWebsocketCmd() *cobra.Command { - var listenAddress string + var ( + listenAddress string + certFile string + keyFile string + echo bool + ) cmd := &cobra.Command{ Use: "websocket", Short: "Serve a websocket server for e2e tests.", Args: cobra.NoArgs, RunE: func(_ *cobra.Command, _ []string) error { - mux := http.NewServeMux() - - mux.HandleFunc("/readyz", func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusOK) - w.Write([]byte("ok")) - }) - - mux.HandleFunc("/ws", func(w http.ResponseWriter, r *http.Request) { - c, err := upgrader.Upgrade(w, r, nil) - if err != nil { - log.Print("upgrade:", err) - return - } - defer c.Close() - for { - mt, message, err := c.ReadMessage() - if err != nil { - log.Println("read:", err) - break - } - log.Printf("recv: %s", message) - - if string(message) == "PING" { - err = c.WriteMessage(mt, []byte("PONG")) - if err != nil { - log.Println("write:", err) - break - } - } - } - }) - server := &http.Server{ Addr: listenAddress, - Handler: mux, + Handler: newWebsocketHandler(echo), } log.Printf("testserver websocket: listening on %s", listenAddress) - return server.ListenAndServe() + return serveHTTP(server, certFile, keyFile) }, } cmd.Flags().StringVar(&listenAddress, "listen", ":80", "Address the websocket server listens on.") + cmd.Flags().StringVar(&certFile, "tls-cert", "", "PEM certificate file; enables HTTPS when paired with --tls-key.") + cmd.Flags().StringVar(&keyFile, "tls-key", "", "PEM private key file; enables HTTPS when paired with --tls-cert.") + cmd.Flags().BoolVar(&echo, "echo", false, "Echo every WebSocket message with its original type and payload.") return cmd } diff --git a/internal/e2e/fixtures/testserver/websocket_test.go b/internal/e2e/fixtures/testserver/websocket_test.go new file mode 100644 index 0000000000..f5f601a947 --- /dev/null +++ b/internal/e2e/fixtures/testserver/websocket_test.go @@ -0,0 +1,109 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "crypto/tls" + "crypto/x509" + "net/http" + "testing" + "time" + + "github.com/gorilla/websocket" +) + +func TestWebsocketDefaultRespondsToPingWithPong(t *testing.T) { + server := startOriginServer(t, newWebsocketHandler(false), "", "") + dialer := websocket.Dialer{HandshakeTimeout: 2 * time.Second} + conn, response, err := dialer.Dial(server.wsURL+"/ws", nil) + if err != nil { + t.Fatalf("dial websocket: %v", err) + } + defer conn.Close() + if response.StatusCode != http.StatusSwitchingProtocols { + t.Fatalf("websocket handshake status = %d, want %d", response.StatusCode, http.StatusSwitchingProtocols) + } + if err := conn.SetReadDeadline(time.Now().Add(2 * time.Second)); err != nil { + t.Fatalf("set websocket read deadline: %v", err) + } + if err := conn.SetWriteDeadline(time.Now().Add(2 * time.Second)); err != nil { + t.Fatalf("set websocket write deadline: %v", err) + } + + if err := conn.WriteMessage(websocket.TextMessage, []byte("PING")); err != nil { + t.Fatalf("write PING: %v", err) + } + messageType, message, err := conn.ReadMessage() + if err != nil { + t.Fatalf("read PONG: %v", err) + } + if messageType != websocket.TextMessage || string(message) != "PONG" { + t.Errorf("PING response = type %d, %q; want text PONG", messageType, message) + } +} + +func TestWebsocketEchoPreservesMessageTypeAndPayload(t *testing.T) { + cert := writeOriginCertificate(t) + server := startOriginServer(t, newWebsocketHandler(true), cert.certFile, cert.keyFile) + roots := x509.NewCertPool() + if !roots.AppendCertsFromPEM(cert.caPEM) { + t.Fatal("adding fixture CA to trust pool") + } + dialer := websocket.Dialer{ + HandshakeTimeout: 2 * time.Second, + TLSClientConfig: &tls.Config{ + RootCAs: roots, + ServerName: "127.0.0.1", + }, + } + conn, response, err := dialer.Dial(server.wssURL+"/ws", nil) + if err != nil { + t.Fatalf("dial secure websocket: %v", err) + } + defer conn.Close() + if response.StatusCode != http.StatusSwitchingProtocols { + t.Fatalf("secure websocket handshake status = %d, want %d", response.StatusCode, http.StatusSwitchingProtocols) + } + if err := conn.SetReadDeadline(time.Now().Add(2 * time.Second)); err != nil { + t.Fatalf("set secure websocket read deadline: %v", err) + } + if err := conn.SetWriteDeadline(time.Now().Add(2 * time.Second)); err != nil { + t.Fatalf("set secure websocket write deadline: %v", err) + } + + messages := []struct { + messageType int + payload []byte + }{ + {websocket.TextMessage, []byte("first message")}, + {websocket.BinaryMessage, []byte{0, 1, 2, 255}}, + {websocket.TextMessage, []byte("PING")}, + } + for i, want := range messages { + if err := conn.WriteMessage(want.messageType, want.payload); err != nil { + t.Fatalf("write message %d: %v", i, err) + } + gotType, gotPayload, err := conn.ReadMessage() + if err != nil { + t.Fatalf("read echo %d: %v", i, err) + } + if gotType != want.messageType { + t.Errorf("echo %d message type = %d, want %d", i, gotType, want.messageType) + } + if string(gotPayload) != string(want.payload) { + t.Errorf("echo %d payload = %q, want %q", i, gotPayload, want.payload) + } + } +} diff --git a/internal/e2e/router_client.go b/internal/e2e/router_client.go index 300f88474e..9b8c68f10a 100644 --- a/internal/e2e/router_client.go +++ b/internal/e2e/router_client.go @@ -137,6 +137,14 @@ func (c *RouterClient) request(ctx context.Context, method string, actorRef reso // egress side. The caller owns the returned connection and must Close it; // the underlying port-forward is torn down by RouterClient.Close. func (c *RouterClient) Connect(ctx context.Context, actorRef resources.ActorRef, port int) (net.Conn, error) { + return c.ConnectWithHeaders(ctx, actorRef, port, http.Header{ + atenet.TargetActorHeader: []string{actorRef.String()}, + }) +} + +// ConnectWithHeaders is like Connect but allows specifying custom headers on +// the outer CONNECT request (e.g. testing repeated or missing headers). +func (c *RouterClient) ConnectWithHeaders(ctx context.Context, actorRef resources.ActorRef, port int, header http.Header) (net.Conn, error) { if err := c.ensureConnectPortForward(ctx); err != nil { return nil, err } @@ -151,9 +159,7 @@ func (c *RouterClient) Connect(ctx context.Context, actorRef resources.ActorRef, Method: http.MethodConnect, URL: &url.URL{Host: destination}, Host: destination, - Header: http.Header{ - atenet.TargetActorHeader: []string{actorRef.String()}, - }, + Header: header, } if err := req.Write(rawConn); err != nil { _ = rawConn.Close() diff --git a/internal/e2e/sandbox.go b/internal/e2e/sandbox.go index 68e4835cb6..db5eb8b89d 100644 --- a/internal/e2e/sandbox.go +++ b/internal/e2e/sandbox.go @@ -21,7 +21,7 @@ import ( ) // SandboxClassMicroVM is the kata + cloud-hypervisor runtime, spelled as the -// WorkerPool/ActorTemplate spec.sandboxClass field spells it. +// WorkerPool spec.sandboxClasses[].name and ActorTemplate sandboxClass spell it. const SandboxClassMicroVM = "microvm" // sandboxClassEnv selects which runtime's fixtures the suites build actors @@ -184,7 +184,7 @@ func fixtureSubstitutions(bucket, name string) (inline, blocks map[string]string "${FIXTURE_SUFFIX}": "-" + name, } blocks = map[string]string{ - "${WORKERPOOL_RUNTIME}": "", + "${WORKERPOOL_RUNTIME}": " sandboxClasses:\n - name: gvisor", "${TEMPLATE_SANDBOX_CLASS}": "", "${TEMPLATE_RESOURCES}": "", // Off unless the caller opts in; see WithTrustBundle. @@ -199,7 +199,7 @@ func fixtureSubstitutions(bucket, name string) (inline, blocks map[string]string // The micro-VM ActorTemplates name the cluster-wide SandboxConfig // hack/install-microvm-deps.sh installs (configName: microvm), so a // missing or stale one fails loudly. The pool only selects the class. - blocks["${WORKERPOOL_RUNTIME}"] = " sandboxClass: microvm" + blocks["${WORKERPOOL_RUNTIME}"] = " sandboxClasses:\n - name: microvm" // Must match the WorkerPool's: a snapshot is not portable across sandbox // classes, so only same-class pools are eligible to run these actors. blocks["${TEMPLATE_SANDBOX_CLASS}"] = " sandboxClass: microvm" diff --git a/internal/e2e/sandbox_test.go b/internal/e2e/sandbox_test.go index f0510b9309..1d63f07355 100644 --- a/internal/e2e/sandbox_test.go +++ b/internal/e2e/sandbox_test.go @@ -40,6 +40,10 @@ var substrateFixtures = []struct { Pool: "internal/e2e/fixtures/probe/probe-sized.yaml.tmpl", Template: "internal/e2e/fixtures/probe/probe-sized-template.yaml.tmpl", }, 1}, + {SubstrateFixtureManifests{ + Pool: "internal/e2e/fixtures/security/atespace-a-pool.yaml.tmpl", + Template: "internal/e2e/fixtures/security/atespace-template.yaml.tmpl", + }, 1}, {SubstrateFixtureManifests{ Pool: "internal/e2e/fixtures/capabilities/capabilities.yaml.tmpl", Template: "internal/e2e/fixtures/capabilities/capabilities-templates.yaml.tmpl", @@ -128,8 +132,8 @@ func TestRenderSubstrateFixtures_GVisor(t *testing.T) { if !strings.HasSuffix(pool.Spec.WorkerImage, "/cmd/ateom-gvisor") { t.Errorf("WorkerPool workerImage = %q, want the gVisor ateom", pool.Spec.WorkerImage) } - if pool.Spec.SandboxClass != "" { - t.Errorf("WorkerPool carries micro-VM runtime fields: class=%q", pool.Spec.SandboxClass) + if pool.Spec.DefaultSandboxClass() != v1alpha1.SandboxClassGvisor { + t.Errorf("WorkerPool runtime = class %q, want gvisor", pool.Spec.DefaultSandboxClass()) } templates := renderTemplates(t, fixture.manifests.Template) @@ -177,8 +181,8 @@ func TestRenderSubstrateFixtures_MicroVM(t *testing.T) { if !strings.HasSuffix(pool.Spec.WorkerImage, "/cmd/ateom-microvm") { t.Errorf("WorkerPool workerImage = %q, want the micro-VM ateom", pool.Spec.WorkerImage) } - if pool.Spec.SandboxClass != SandboxClassMicroVM { - t.Errorf("WorkerPool runtime = class %q, want microvm", pool.Spec.SandboxClass) + if pool.Spec.DefaultSandboxClass() != SandboxClassMicroVM { + t.Errorf("WorkerPool runtime = class %q, want microvm", pool.Spec.DefaultSandboxClass()) } templates := renderTemplates(t, fixture.manifests.Template) diff --git a/internal/e2e/serverpod.go b/internal/e2e/serverpod.go index ea4d186337..b94850b67b 100644 --- a/internal/e2e/serverpod.go +++ b/internal/e2e/serverpod.go @@ -18,13 +18,17 @@ import ( "context" "fmt" "net" + "os" + "slices" "strconv" "strings" "testing" "time" corev1 "k8s.io/api/core/v1" + discoveryv1 "k8s.io/api/discovery/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/yaml" ) // serverPodTemplate is the one manifest every ServerPod is rendered from. @@ -71,6 +75,8 @@ type ServerPod struct { // HealthPath is the HTTP readiness path, defaulting to /healthz. Ignored // when GRPCProbe is set. HealthPath string + // HealthScheme selects HTTP or HTTPS for the readiness probe. Empty means HTTP. + HealthScheme corev1.URIScheme // Volumes and VolumeMounts carry whatever credentials the server needs. // Typed, rather than more YAML in the template, so the Secret names here // sit beside the code that creates them instead of drifting from it. @@ -179,5 +185,107 @@ func serverReadinessProbe(spec ServerPod, targetPort string) string { if path == "" { path = "/healthz" } - return fmt.Sprintf(" httpGet:\n path: %s\n port: %s", path, targetPort) + scheme := spec.HealthScheme + if scheme == "" { + scheme = corev1.URISchemeHTTP + } + return fmt.Sprintf(" httpGet:\n path: %s\n port: %s\n scheme: %s", path, targetPort, scheme) +} + +func serverEndpointReady(endpointSlices []discoveryv1.EndpointSlice, pod *corev1.Pod, port int32) bool { + for _, slice := range endpointSlices { + hasPort := false + for _, p := range slice.Ports { + if p.Port != nil && *p.Port == port && p.Protocol != nil && *p.Protocol == corev1.ProtocolTCP { + hasPort = true + } + } + if !hasPort { + continue + } + for _, endpoint := range slice.Endpoints { + ref := endpoint.TargetRef + if ref == nil || ref.Kind != "Pod" || ref.Namespace != pod.Namespace || ref.Name != pod.Name || ref.UID != pod.UID { + continue + } + if endpoint.Conditions.Ready == nil || !*endpoint.Conditions.Ready { + continue + } + if endpoint.Conditions.Terminating != nil && *endpoint.Conditions.Terminating { + continue + } + if slices.Contains(endpoint.Addresses, pod.Status.PodIP) { + return true + } + } + } + return false +} + +// CreateServerService reserves the origin address before its Pod starts, so +// callers can issue a certificate for that IP. Namespace must already exist. +// DeployServerPod subsequently applies the same Service without replacing it. +func CreateServerService(t *testing.T, ctx context.Context, spec ServerPod) Server { + t.Helper() + if spec.Namespace == "" { + t.Fatal("CreateServerService requires a test namespace") + } + raw, err := os.ReadFile(renderServerPod(t, spec, spec.Namespace)) + if err != nil { + t.Fatalf("reading origin manifest: %v", err) + } + for doc := range strings.SplitSeq(string(raw), "\n---\n") { + var kind metav1.TypeMeta + if err := yaml.Unmarshal([]byte(doc), &kind); err != nil { + t.Fatalf("decoding origin kind: %v", err) + } + if kind.Kind != "Service" { + continue + } + var service corev1.Service + if err := yaml.UnmarshalStrict([]byte(doc), &service); err != nil { + t.Fatalf("decoding origin Service: %v", err) + } + created, err := GetClients().K8s.CoreV1().Services(spec.Namespace).Create(ctx, &service, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("creating origin Service: %v", err) + } + if net.ParseIP(created.Spec.ClusterIP) == nil { + t.Fatalf("origin Service has invalid IP %q", created.Spec.ClusterIP) + } + return Server{Namespace: spec.Namespace, ClusterIP: created.Spec.ClusterIP, Port: spec.Port} + } + t.Fatal("origin manifest has no Service") + return Server{} +} + +// WaitForServerEndpoint waits for publication of the current origin Pod as a +// ready Service backend. It does not perform the protocol exchange under test. +func WaitForServerEndpoint(t *testing.T, ctx context.Context, spec ServerPod) { + t.Helper() + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + clients := GetClients() + pod, err := clients.K8s.CoreV1().Pods(spec.Namespace).Get(ctx, spec.Name, metav1.GetOptions{}) + if err != nil { + t.Fatalf("reading origin Pod for endpoint readiness: %v", err) + } + port := spec.TargetPort + if port == 0 { + port = spec.Port + } + for { + endpoints, err := clients.K8s.DiscoveryV1().EndpointSlices(spec.Namespace).List(ctx, metav1.ListOptions{LabelSelector: discoveryv1.LabelServiceName + "=" + spec.Name}) + if err != nil { + t.Fatalf("reading origin EndpointSlices: %v", err) + } + if serverEndpointReady(endpoints.Items, pod, int32(port)) { + return + } + select { + case <-ctx.Done(): + t.Fatalf("origin %s/%s has no ready endpoint for Pod %s on port %d: %v", spec.Namespace, spec.Name, pod.UID, port, ctx.Err()) + case <-time.After(200 * time.Millisecond): + } + } } diff --git a/internal/e2e/serverpod_test.go b/internal/e2e/serverpod_test.go index 6acc1398a6..f081f48fe3 100644 --- a/internal/e2e/serverpod_test.go +++ b/internal/e2e/serverpod_test.go @@ -22,6 +22,7 @@ import ( "testing" corev1 "k8s.io/api/core/v1" + discoveryv1 "k8s.io/api/discovery/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" "sigs.k8s.io/yaml" @@ -286,3 +287,41 @@ func TestRenderServerPod_Volumes(t *testing.T) { } } } + +func TestRenderServerPod_HTTPSProbe(t *testing.T) { + pod, _ := renderServerPodDocs(t, ServerPod{Name: "tls", Port: 8443, HealthScheme: corev1.URISchemeHTTPS}) + probe := pod.Spec.Containers[0].ReadinessProbe.HTTPGet + if probe.Scheme != corev1.URISchemeHTTPS || probe.Port.IntValue() != 8443 { + t.Fatalf("probe = %+v, want HTTPS on 8443", probe) + } +} + +func TestServerEndpointReady(t *testing.T) { + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "origin", Namespace: "test", UID: "current"}, Status: corev1.PodStatus{PodIP: "10.0.0.2"}} + base := discoveryv1.EndpointSlice{ + Ports: []discoveryv1.EndpointPort{{Port: ptr.To(int32(8443)), Protocol: ptr.To(corev1.ProtocolTCP)}}, + Endpoints: []discoveryv1.Endpoint{{Addresses: []string{"10.0.0.2"}, TargetRef: &corev1.ObjectReference{Kind: "Pod", Namespace: "test", Name: "origin", UID: "current"}, Conditions: discoveryv1.EndpointConditions{Ready: ptr.To(true)}}}, + } + cases := []struct { + name string + change func(*discoveryv1.EndpointSlice) + want bool + }{ + {"ready origin", func(*discoveryv1.EndpointSlice) {}, true}, + {"not ready", func(s *discoveryv1.EndpointSlice) { s.Endpoints[0].Conditions.Ready = ptr.To(false) }, false}, + {"unknown readiness", func(s *discoveryv1.EndpointSlice) { s.Endpoints[0].Conditions.Ready = nil }, false}, + {"previous pod", func(s *discoveryv1.EndpointSlice) { s.Endpoints[0].TargetRef.UID = "previous" }, false}, + {"wrong address", func(s *discoveryv1.EndpointSlice) { s.Endpoints[0].Addresses = []string{"10.0.0.3"} }, false}, + {"wrong port", func(s *discoveryv1.EndpointSlice) { s.Ports[0].Port = ptr.To(int32(443)) }, false}, + {"terminating", func(s *discoveryv1.EndpointSlice) { s.Endpoints[0].Conditions.Terminating = ptr.To(true) }, false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + s := base.DeepCopy() + tc.change(s) + if got := serverEndpointReady([]discoveryv1.EndpointSlice{*s}, pod, 8443); got != tc.want { + t.Fatalf("ready = %v, want %v", got, tc.want) + } + }) + } +} diff --git a/internal/e2e/suites/demo/demo_test.go b/internal/e2e/suites/demo/demo_test.go index 1224145b97..61fceea190 100644 --- a/internal/e2e/suites/demo/demo_test.go +++ b/internal/e2e/suites/demo/demo_test.go @@ -50,7 +50,7 @@ func TestActorLifecycle(t *testing.T) { clients := e2e.GetClients() // Create actor template. - at, err := createActorTemplate(ctx, t, clients, nsObj, ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL) + at, err := createActorTemplate(ctx, t, clients, nsObj, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY) if err != nil { t.Fatalf("failed to initialize ActorTemplate: %v", err) } @@ -101,7 +101,7 @@ func TestActorSnapshotLifecycle(t *testing.T) { clients := e2e.GetClients() nsObj := e2e.CreateNamespace(t) - at, err := createActorTemplate(ctx, t, clients, nsObj, ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL) + at, err := createActorTemplate(ctx, t, clients, nsObj, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY) if err != nil { t.Fatalf("failed to initialize ActorTemplate: %v", err) } @@ -143,7 +143,7 @@ func TestActorSnapshotLifecycle(t *testing.T) { if err != nil { t.Fatalf("failed to suspend source Actor: %v", err) } - snapshotURI := suspended.GetActor().GetStatus().GetExternalSnapshot().GetSnapshotUri() + snapshotURI := durableSnapshotURI(suspended.GetActor().GetStatus()) if snapshotURI == "" { t.Fatal("suspended Actor has no external snapshot") } @@ -159,15 +159,11 @@ func TestActorSnapshotLifecycle(t *testing.T) { if err != nil { t.Fatalf("failed to create the Tag: %v", err) } - if got := tagToUpdate.GetStatus().GetSnapshot().GetSnapshotUri(); got == "" || got == snapshotURI { + if got := tagSnapshotURI(tagToUpdate); got == "" || got == snapshotURI { t.Fatalf("Tag %s snapshot uri = %q, want a copy of its own", tagRef.GetName(), got) } - wantTagURI, err := resources.NewTagSnapshotURI(tagToUpdate.GetStatus().GetStorageLocation(), tagToUpdate.GetMetadata().GetAtespace(), tagToUpdate.GetMetadata().GetUid()) - if err != nil { - t.Fatalf("NewTagSnapshotURI: %v", err) - } - if got := tagToUpdate.GetStatus().GetSnapshot().GetSnapshotUri(); got != wantTagURI.String() { - t.Errorf("tag snapshot URI = %q, want UID-based URI %q", got, wantTagURI) + if _, err := resources.ParseSnapshotURI(tagSnapshotURI(tagToUpdate)); err != nil { + t.Errorf("ParseSnapshotURI(%q): %v", tagSnapshotURI(tagToUpdate), err) } listed, err := clients.SubstrateAPI.ListTags(ctx, &ateapipb.ListTagsRequest{Atespace: demoAtespace}) if err != nil { @@ -222,9 +218,9 @@ func TestDurableDirLifecycle(t *testing.T) { tc actorLifecycleTestCase }{ { - name: "onCommit:Full", + name: "preferredFidelity:MEMORY", tc: actorLifecycleTestCase{ - onCommit: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, + fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, wantMemoryAfterPause: 2, wantFileAfterPause: 2, wantMemoryAfterSuspend: 3, @@ -234,9 +230,9 @@ func TestDurableDirLifecycle(t *testing.T) { { // Pause captures the same Data scope as suspend, so memory // cold-boots on every resume. - name: "onCommit:Data", + name: "preferredFidelity:VOLUMES", tc: actorLifecycleTestCase{ - onCommit: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA, + fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, wantMemoryAfterPause: 1, wantFileAfterPause: 2, wantMemoryAfterSuspend: 1, @@ -245,28 +241,28 @@ func TestDurableDirLifecycle(t *testing.T) { }, { // Suspend from PAUSED uploads the Full local snapshot. - name: "onCommit:Full, suspend from PAUSED", + name: "preferredFidelity:MEMORY, suspend from PAUSED", tc: actorLifecycleTestCase{ - onCommit: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, - wantMemoryAfterPause: 2, - wantFileAfterPause: 2, - wantMemoryAfterSuspend: 3, - wantFileAfterSuspend: 3, - wantSnapshotContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, - suspendWhilePaused: true, + fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + wantMemoryAfterPause: 2, + wantFileAfterPause: 2, + wantMemoryAfterSuspend: 3, + wantFileAfterSuspend: 3, + wantSnapshotFidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, + suspendWhilePaused: true, }, }, { // Suspend from PAUSED uploads the Data local snapshot. - name: "onCommit:Data, suspend from PAUSED", + name: "preferredFidelity:VOLUMES, suspend from PAUSED", tc: actorLifecycleTestCase{ - onCommit: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA, - wantMemoryAfterPause: 1, - wantFileAfterPause: 2, - wantMemoryAfterSuspend: 1, - wantFileAfterSuspend: 3, - wantSnapshotContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA, - suspendWhilePaused: true, + fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, + wantMemoryAfterPause: 1, + wantFileAfterPause: 2, + wantMemoryAfterSuspend: 1, + wantFileAfterSuspend: 3, + wantSnapshotFidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, + suspendWhilePaused: true, }, }, } @@ -293,9 +289,9 @@ func TestMultipleDurableDirLifecycle(t *testing.T) { tc actorLifecycleTestCase }{ { - name: "onCommit:Full", + name: "preferredFidelity:MEMORY", tc: actorLifecycleTestCase{ - onCommit: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, + fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, wantMemoryAfterPause: 2, wantFileAfterPause: 2, wantMemoryAfterSuspend: 3, @@ -304,9 +300,9 @@ func TestMultipleDurableDirLifecycle(t *testing.T) { }, }, { - name: "onCommit:Data", + name: "preferredFidelity:VOLUMES", tc: actorLifecycleTestCase{ - onCommit: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA, + fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, wantMemoryAfterPause: 1, wantFileAfterPause: 2, wantMemoryAfterSuspend: 1, @@ -335,9 +331,9 @@ func TestExternalVolumeLifecycle(t *testing.T) { tc actorLifecycleTestCase }{ { - name: "onCommit:Data", + name: "preferredFidelity:VOLUMES", tc: actorLifecycleTestCase{ - onCommit: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA, + fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, wantMemoryAfterPause: 1, wantFileAfterPause: 2, wantMemoryAfterSuspend: 1, @@ -360,7 +356,7 @@ func TestDeleteActorAnyStateWithExternalVolume(t *testing.T) { clients := e2e.GetClients() nsObj := e2e.CreateNamespace(t) - at, err := createActorTemplateWithExternalVolume(ctx, t, clients, nsObj, ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA) + at, err := createActorTemplateWithExternalVolume(ctx, t, clients, nsObj, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES) if err != nil { t.Fatalf("failed to initialize ActorTemplate: %v", err) } @@ -391,12 +387,12 @@ func TestDeleteActorAnyStateWithExternalVolume(t *testing.T) { if err != nil { t.Fatalf("failed to get actor: %v", err) } - if len(actor.GetStatus().GetActorVolumes()) == 0 { + if len(actor.GetStatus().GetExternalVolumes()) == 0 { t.Fatalf("expected actor to have volumes, got 0") } - for _, vol := range actor.GetStatus().GetActorVolumes() { + for _, vol := range actor.GetStatus().GetExternalVolumes() { if vol.Status != ateapipb.ExternalVolume_STATUS_CREATED { - t.Fatalf("expected volume %q to be CREATED, got %s", vol.VolumeName, vol.Status) + t.Fatalf("expected volume %q to be CREATED, got %s", vol.Name, vol.Status) } } @@ -427,7 +423,7 @@ func TestExternalVolume_NodeMigration(t *testing.T) { clients := e2e.GetClients() nsObj := e2e.CreateNamespace(t) - at, err := createActorTemplateWithExternalVolume(ctx, t, clients, nsObj, ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA) + at, err := createActorTemplateWithExternalVolume(ctx, t, clients, nsObj, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES) if err != nil { t.Fatalf("failed to initialize ActorTemplate: %v", err) } @@ -515,7 +511,7 @@ func TestExternalVolume_NodeMigration(t *testing.T) { // 5. Suspend & Resume actor. // 6. Call to actor and validate memory and file counters. type actorLifecycleTestCase struct { - onCommit ateapipb.SnapshotContentScope + fidelity ateapipb.SnapshotFidelity wantMemoryAfterPause int wantFileAfterPause int wantMemoryAfterSuspend int @@ -527,9 +523,9 @@ type actorLifecycleTestCase struct { // wrong place, they would diverge. checkSecondFileCounter bool - // wantSnapshotContentScope, when set, asserts the content scope recorded + // wantSnapshotFidelity, when set, asserts the content scope recorded // on the ActorSnapshot the suspend produced. - wantSnapshotContentScope ateapipb.SnapshotContentScope + wantSnapshotFidelity ateapipb.SnapshotFidelity // microVMOnly skips the case outside the micro-VM environment. microVMOnly bool @@ -542,7 +538,7 @@ type actorLifecycleTestCase struct { suspendWhilePaused bool } -func runActorLifecycleTestCase(t *testing.T, prefix string, createTemplate func(context.Context, *testing.T, *e2e.Clients, *e2e.Namespace, ateapipb.SnapshotContentScope) (*ateapipb.ActorTemplate, error), tc actorLifecycleTestCase) { +func runActorLifecycleTestCase(t *testing.T, prefix string, createTemplate func(context.Context, *testing.T, *e2e.Clients, *e2e.Namespace, ateapipb.SnapshotFidelity) (*ateapipb.ActorTemplate, error), tc actorLifecycleTestCase) { // Create namespace nsObj := e2e.CreateNamespace(t) @@ -550,7 +546,7 @@ func runActorLifecycleTestCase(t *testing.T, prefix string, createTemplate func( clients := e2e.GetClients() // Create actor template. - at, err := createTemplate(ctx, t, clients, nsObj, tc.onCommit) + at, err := createTemplate(ctx, t, clients, nsObj, tc.fidelity) if err != nil { t.Fatalf("failed to initialize ActorTemplate: %v", err) } @@ -651,13 +647,13 @@ func runActorLifecycleTestCase(t *testing.T, prefix string, createTemplate func( if err != nil { t.Fatalf("failed to get suspended Actor: %v", err) } - if suspendedActor.GetStatus().GetLocalSnapshot() != nil { - t.Errorf("suspended Actor still carries LocalSnapshot: %v", suspendedActor.GetStatus().GetLocalSnapshot()) + if hasLocalSnapshot(suspendedActor.GetStatus()) { + t.Errorf("suspended Actor still carries LocalSnapshot: %v", suspendedActor.GetStatus().GetSnapshots()) } } - if tc.wantSnapshotContentScope != ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_UNSPECIFIED { - validateSnapshotContentScope(ctx, t, clients, actorID, tc.wantSnapshotContentScope) + if tc.wantSnapshotFidelity != ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED { + validateSnapshotFidelity(ctx, t, clients, actorID, tc.wantSnapshotFidelity) } // Resuming the actor @@ -679,9 +675,9 @@ func runActorLifecycleTestCase(t *testing.T, prefix string, createTemplate func( } } -// validateSnapshotContentScope asserts the content scope recorded on the +// validateSnapshotFidelity asserts the content scope recorded on the // suspended actor's external snapshot. -func validateSnapshotContentScope(ctx context.Context, t *testing.T, clients *e2e.Clients, actorID string, want ateapipb.SnapshotContentScope) { +func validateSnapshotFidelity(ctx context.Context, t *testing.T, clients *e2e.Clients, actorID string, want ateapipb.SnapshotFidelity) { t.Helper() actor, err := clients.SubstrateAPI.GetActor(ctx, &ateapipb.GetActorRequest{ Actor: &ateapipb.ObjectRef{Atespace: demoAtespace, Name: actorID}, @@ -689,12 +685,62 @@ func validateSnapshotContentScope(ctx context.Context, t *testing.T, clients *e2 if err != nil { t.Fatalf("failed to get suspended Actor: %v", err) } - if actor.GetStatus().GetExternalSnapshot().GetSnapshotUri() == "" { + if durableSnapshotURI(actor.GetStatus()) == "" { t.Fatal("suspended Actor has no external snapshot") } - if got := actor.GetStatus().GetExternalSnapshot().GetContentScope(); got != want { - t.Errorf("snapshot %q content scope = %v, want %v", actor.GetStatus().GetExternalSnapshot().GetSnapshotUri(), got, want) + if got := durableSnapshotStorage(actor.GetStatus()).GetFidelity(); got != want { + t.Errorf("snapshot %q content scope = %v, want %v", durableSnapshotURI(actor.GetStatus()), got, want) + } +} + +func durableSnapshot(status *ateapipb.ActorStatus) *ateapipb.Snapshot { + var best *ateapipb.Snapshot + for _, snap := range status.GetSnapshots() { + for _, st := range snap.GetStorage() { + if st.GetDurability() == ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE && + st.GetStatus() == ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED && + st.GetObject().GetSnapshotUri() != "" { + if best == nil || snap.GetGeneration() > best.GetGeneration() { + best = snap + } + } + } } + return best +} + +func durableSnapshotStorage(status *ateapipb.ActorStatus) *ateapipb.SnapshotStorage { + snap := durableSnapshot(status) + for _, st := range snap.GetStorage() { + if st.GetDurability() == ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE { + return st + } + } + return nil +} + +func durableSnapshotURI(status *ateapipb.ActorStatus) string { + return durableSnapshotStorage(status).GetObject().GetSnapshotUri() +} + +func hasLocalSnapshot(status *ateapipb.ActorStatus) bool { + for _, snap := range status.GetSnapshots() { + for _, st := range snap.GetStorage() { + if st.GetDurability() == ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL { + return true + } + } + } + return false +} + +func tagSnapshotURI(tag *ateapipb.Tag) string { + for _, st := range tag.GetStatus().GetSnapshot().GetStorage() { + if st.GetDurability() == ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE { + return st.GetObject().GetSnapshotUri() + } + } + return "" } // validateSecondFileCounter checks the counter the workload keeps in its second @@ -953,7 +999,7 @@ func revertActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj if err != nil { t.Fatalf("failed to suspend Actor: %v", err) } - snapshotURI := suspended.GetActor().GetStatus().GetExternalSnapshot().GetSnapshotUri() + snapshotURI := durableSnapshotURI(suspended.GetActor().GetStatus()) if snapshotURI == "" { t.Fatal("suspend wrote no external snapshot") } @@ -978,7 +1024,7 @@ func revertActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj if got := reverted.GetActor().GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_SUSPENDED { t.Fatalf("state after revert = %v, want SUSPENDED", got) } - if got := reverted.GetActor().GetStatus().GetExternalSnapshot().GetSnapshotUri(); got != snapshotURI { + if got := durableSnapshotURI(reverted.GetActor().GetStatus()); got != snapshotURI { t.Fatalf("snapshot URI after revert = %q, want %q", got, snapshotURI) } if reverted.GetActor().GetStatus().GetWorkerAssignment() != nil { @@ -1013,10 +1059,10 @@ func revertActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj if got := reverted.GetActor().GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_SUSPENDED { t.Fatalf("state after revert from PAUSED = %v, want SUSPENDED", got) } - if reverted.GetActor().GetStatus().GetLocalSnapshot() != nil { + if hasLocalSnapshot(reverted.GetActor().GetStatus()) { t.Fatal("local snapshot pointer survived revert from PAUSED") } - if got := reverted.GetActor().GetStatus().GetExternalSnapshot().GetSnapshotUri(); got != snapshotURI { + if got := durableSnapshotURI(reverted.GetActor().GetStatus()); got != snapshotURI { t.Fatalf("snapshot URI after revert from PAUSED = %q, want %q", got, snapshotURI) } @@ -1184,7 +1230,7 @@ func deletePausedActorAnyState(ctx context.Context, t *testing.T, clients *e2e.C return nil } -func createActorTemplateInternal(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj *e2e.Namespace, base string, onCommit ateapipb.SnapshotContentScope, modifyTemplate func(*ateapipb.ActorTemplate)) (*ateapipb.ActorTemplate, error) { +func createActorTemplateInternal(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj *e2e.Namespace, base string, fidelity ateapipb.SnapshotFidelity, modifyTemplate func(*ateapipb.ActorTemplate)) (*ateapipb.ActorTemplate, error) { env, err := e2e.CheckEnv("BUCKET_NAME") if err != nil { t.Fatalf("CheckEnv failed: %v", err) @@ -1201,19 +1247,19 @@ func createActorTemplateInternal(ctx context.Context, t *testing.T, clients *e2e PoolReplicas: 2, Labels: map[string]string{"demo": nsObj.Name}, SnapshotConfig: &ateapipb.SnapshotConfig{ - StorageLocation: "gs://" + env["BUCKET_NAME"] + "/ate-demo-" + name, - OnCommit: onCommit, + StorageLocation: "gs://" + env["BUCKET_NAME"] + "/ate-demo-" + name, + PreferredFidelity: fidelity, }, Modify: modifyTemplate, }) return at, nil } -func createActorTemplate(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj *e2e.Namespace, onCommit ateapipb.SnapshotContentScope) (*ateapipb.ActorTemplate, error) { - return createActorTemplateInternal(ctx, t, clients, nsObj, "counter", onCommit, nil) +func createActorTemplate(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj *e2e.Namespace, fidelity ateapipb.SnapshotFidelity) (*ateapipb.ActorTemplate, error) { + return createActorTemplateInternal(ctx, t, clients, nsObj, "counter", fidelity, nil) } -func createActorTemplateWithExternalVolume(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj *e2e.Namespace, onCommit ateapipb.SnapshotContentScope) (*ateapipb.ActorTemplate, error) { +func createActorTemplateWithExternalVolume(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj *e2e.Namespace, fidelity ateapipb.SnapshotFidelity) (*ateapipb.ActorTemplate, error) { scName := e2e.StorageClass if !hasStorageClass(ctx, clients, scName) { t.Fatalf("StorageClass %q not found in cluster; provide a valid --storage-class or install the CSI driver via --setup-csi", scName) @@ -1258,7 +1304,7 @@ func createActorTemplateWithExternalVolume(ctx context.Context, t *testing.T, cl }) } } - return createActorTemplateInternal(ctx, t, clients, nsObj, "counter-ext-vol", onCommit, modify) + return createActorTemplateInternal(ctx, t, clients, nsObj, "counter-ext-vol", fidelity, modify) } // secondDurableDirVolume is the extra durable-dir volume (and where the counter @@ -1273,7 +1319,7 @@ const ( // counter's second file counter at it, so both volumes are written on every // request. Only the micro-VM runtime accepts this: gVisor templates are still // capped at one durable-dir volume by the ActorTemplate CEL rules. -func createActorTemplateWithTwoDurableDirs(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj *e2e.Namespace, onCommit ateapipb.SnapshotContentScope) (*ateapipb.ActorTemplate, error) { +func createActorTemplateWithTwoDurableDirs(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj *e2e.Namespace, fidelity ateapipb.SnapshotFidelity) (*ateapipb.ActorTemplate, error) { modify := func(at *ateapipb.ActorTemplate) { for _, c := range at.GetContainers() { if c.GetName() != "counter" { @@ -1290,7 +1336,7 @@ func createActorTemplateWithTwoDurableDirs(ctx context.Context, t *testing.T, cl DurableDir: &ateapipb.DurableDirVolumeSource{}, }) } - return createActorTemplateInternal(ctx, t, clients, nsObj, "counter-two-durabledirs", onCommit, modify) + return createActorTemplateInternal(ctx, t, clients, nsObj, "counter-two-durabledirs", fidelity, modify) } func hasStorageClass(ctx context.Context, clients *e2e.Clients, name string) bool { @@ -1492,7 +1538,7 @@ func TestWorkerPodDeletion(t *testing.T) { clients := e2e.GetClients() // Create actor template. - at, err := createActorTemplate(ctx, t, clients, nsObj, ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL) + at, err := createActorTemplate(ctx, t, clients, nsObj, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY) if err != nil { t.Fatalf("failed to initialize ActorTemplate: %v", err) } @@ -1622,7 +1668,7 @@ func TestRevertCrashedActor(t *testing.T) { ctx := context.Background() clients := e2e.GetClients() - at, err := createActorTemplate(ctx, t, clients, nsObj, ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL) + at, err := createActorTemplate(ctx, t, clients, nsObj, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY) if err != nil { t.Fatalf("failed to initialize ActorTemplate: %v", err) } @@ -1658,7 +1704,7 @@ func TestRevertCrashedActor(t *testing.T) { if err != nil { t.Fatalf("failed to suspend Actor: %v", err) } - snapshotURI := suspended.GetActor().GetStatus().GetExternalSnapshot().GetSnapshotUri() + snapshotURI := durableSnapshotURI(suspended.GetActor().GetStatus()) if snapshotURI == "" { t.Fatal("suspend wrote no external snapshot") } @@ -1708,7 +1754,7 @@ func TestRevertCrashedActor(t *testing.T) { if got := reverted.GetActor().GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_SUSPENDED { t.Fatalf("state after revert = %v, want SUSPENDED", got) } - if got := reverted.GetActor().GetStatus().GetExternalSnapshot().GetSnapshotUri(); got != snapshotURI { + if got := durableSnapshotURI(reverted.GetActor().GetStatus()); got != snapshotURI { t.Fatalf("snapshot URI after revert = %q, want %q", got, snapshotURI) } if reverted.GetActor().GetStatus().GetWorkerAssignment() != nil { diff --git a/internal/e2e/suites/demo/termination_test.go b/internal/e2e/suites/demo/termination_test.go index 6f18ef6045..fdb7c72df9 100644 --- a/internal/e2e/suites/demo/termination_test.go +++ b/internal/e2e/suites/demo/termination_test.go @@ -42,7 +42,7 @@ func TestGracefulWorkerTermination(t *testing.T) { ctx := context.Background() clients := e2e.GetClients() - at, err := createActorTemplate(ctx, t, clients, nsObj, ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL) + at, err := createActorTemplate(ctx, t, clients, nsObj, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY) if err != nil { t.Fatalf("failed to initialize ActorTemplate: %v", err) } @@ -150,7 +150,7 @@ func TestGracefulWorkerTerminationSuspend(t *testing.T) { ctx := context.Background() clients := e2e.GetClients() - at, err := createActorTemplate(ctx, t, clients, nsObj, ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL) + at, err := createActorTemplate(ctx, t, clients, nsObj, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY) if err != nil { t.Fatalf("failed to initialize ActorTemplate: %v", err) } diff --git a/internal/e2e/suites/demo/updatetemplate_test.go b/internal/e2e/suites/demo/updatetemplate_test.go index 6da3badecf..b689189043 100644 --- a/internal/e2e/suites/demo/updatetemplate_test.go +++ b/internal/e2e/suites/demo/updatetemplate_test.go @@ -36,26 +36,26 @@ func TestUpdateTemplateLifecycle(t *testing.T) { t.Parallel() tests := []struct { name string - onCommit ateapipb.SnapshotContentScope + fidelity ateapipb.SnapshotFidelity }{ { - name: "onCommit:Data", - onCommit: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA, + name: "preferredFidelity:VOLUMES", + fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, }, { - name: "onCommit:Full", - onCommit: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, + name: "preferredFidelity:MEMORY", + fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { t.Parallel() - runUpdateTemplateTestCase(t, test.onCommit) + runUpdateTemplateTestCase(t, test.fidelity) }) } } -func runUpdateTemplateTestCase(t *testing.T, onCommit ateapipb.SnapshotContentScope) { +func runUpdateTemplateTestCase(t *testing.T, fidelity ateapipb.SnapshotFidelity) { nsObj := e2e.CreateNamespace(t) ctx := context.Background() clients := e2e.GetClients() @@ -70,8 +70,8 @@ func runUpdateTemplateTestCase(t *testing.T, onCommit ateapipb.SnapshotContentSc // in the durable dir is readable — the response's "file content" both // proves B's spec took effect and re-reads the preserved data. nameA, nameB := "update-a-"+nsObj.Name, "update-b-"+nsObj.Name - createdA := createUpdateTestTemplate(ctx, t, clients, nsObj, nameA, "update-a", env["BUCKET_NAME"], onCommit, nil) - createdB := createUpdateTestTemplate(ctx, t, clients, nsObj, nameB, "update-b", env["BUCKET_NAME"], onCommit, func(tmpl *ateapipb.ActorTemplate) { + createdA := createUpdateTestTemplate(ctx, t, clients, nsObj, nameA, "update-a", env["BUCKET_NAME"], fidelity, nil) + createdB := createUpdateTestTemplate(ctx, t, clients, nsObj, nameB, "update-b", env["BUCKET_NAME"], fidelity, func(tmpl *ateapipb.ActorTemplate) { ctr := tmpl.Containers[0] ctr.Command = append(ctr.Command, "--validate-existing-file-path=/home/counter/a.txt") }) @@ -138,10 +138,10 @@ func runUpdateTemplateTestCase(t *testing.T, onCommit ateapipb.SnapshotContentSc if err != nil { t.Fatalf("failed to get suspended Actor: %v", err) } - if got, want := suspended.GetStatus().GetExternalSnapshot().GetActorTemplateUid(), createdA.GetMetadata().GetUid(); got != want { + if got, want := durableSnapshot(suspended.GetStatus()).GetActorTemplateUid(), createdA.GetMetadata().GetUid(); got != want { t.Errorf("suspended Actor external_snapshot.actor_template_uid = %q, want template A's %q", got, want) } - if suspended.GetStatus().GetExternalSnapshot().GetSnapshotUri() == "" { + if durableSnapshotURI(suspended.GetStatus()) == "" { t.Error("suspended Actor has no external snapshot") } if wa := suspended.GetStatus().GetWorkerAssignment(); wa != nil { @@ -188,11 +188,11 @@ func runUpdateTemplateTestCase(t *testing.T, onCommit ateapipb.SnapshotContentSc t.Errorf("[after template update] expected %q (template B validating the preserved file), got response: %s", want, resp) } - // Pause under template B while status.external_snapshot still holds the last + // Pause under template B while status.snapshots still holds the last // committed snapshot from template A. Resuming from PAUSED restores the // local checkpoint (which was captured under template B, since templates // can only be updated while SUSPENDED) and preserves the in-memory counter - // when onCommit is FULL. + // when preferredFidelity is FULL. t.Logf("Pausing Actor %q under template B...", actorID) if _, err := clients.SubstrateAPI.PauseActor(ctx, &ateapipb.PauseActorRequest{ Actor: &ateapipb.ObjectRef{Atespace: demoAtespace, Name: actorID}, @@ -207,7 +207,7 @@ func runUpdateTemplateTestCase(t *testing.T, onCommit ateapipb.SnapshotContentSc if err != nil { t.Fatalf("failed to get paused Actor: %v", err) } - if got, want := paused.GetStatus().GetExternalSnapshot().GetActorTemplateUid(), createdA.GetMetadata().GetUid(); got != want { + if got, want := durableSnapshot(paused.GetStatus()).GetActorTemplateUid(), createdA.GetMetadata().GetUid(); got != want { t.Errorf("paused Actor external_snapshot.actor_template_uid = %q, want template A's %q", got, want) } @@ -224,7 +224,7 @@ func runUpdateTemplateTestCase(t *testing.T, onCommit ateapipb.SnapshotContentSc t.Fatalf("failed to call actor after pause/resume under template B: %v", err) } wantMemAfterPause := 2 - if onCommit == ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA { + if fidelity == ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES { wantMemAfterPause = 1 } validateCounterResponse(t, resp, "after pause/resume under template B", wantMemAfterPause, 4) @@ -248,15 +248,15 @@ func runUpdateTemplateTestCase(t *testing.T, onCommit ateapipb.SnapshotContentSc } // Resume prefers a local checkpoint over the external snapshot, so one // surviving here would hide the repoint the next resume has to detect. - if got := revertedStatus.GetLocalSnapshot(); got != nil { - t.Errorf("reverted Actor local_snapshot = %v, want cleared", got) + if hasLocalSnapshot(revertedStatus) { + t.Errorf("reverted Actor local_snapshot present in %v, want cleared", revertedStatus.GetSnapshots()) } // The two halves of the mismatch the next resume has to spot: the spec // names B, the snapshot it would restore was captured under A. if got := reverted.GetActor().GetActorTemplate().GetName(); got != nameB { t.Errorf("reverted Actor actor_template = %q, want %q", got, nameB) } - if got, want := revertedStatus.GetExternalSnapshot().GetActorTemplateUid(), createdA.GetMetadata().GetUid(); got != want { + if got, want := durableSnapshot(revertedStatus).GetActorTemplateUid(), createdA.GetMetadata().GetUid(); got != want { t.Errorf("reverted Actor external_snapshot.actor_template_uid = %q, want template A's %q", got, want) } @@ -298,7 +298,7 @@ func runUpdateTemplateTestCase(t *testing.T, onCommit ateapipb.SnapshotContentSc if err != nil { t.Fatalf("failed to get re-suspended Actor: %v", err) } - if got, want := suspended.GetStatus().GetExternalSnapshot().GetActorTemplateUid(), createdB.GetMetadata().GetUid(); got != want { + if got, want := durableSnapshot(suspended.GetStatus()).GetActorTemplateUid(), createdB.GetMetadata().GetUid(); got != want { t.Errorf("re-suspended Actor external_snapshot.actor_template_uid = %q, want template B's %q", got, want) } } @@ -306,7 +306,7 @@ func runUpdateTemplateTestCase(t *testing.T, onCommit ateapipb.SnapshotContentSc // createUpdateTestTemplate creates a per-test WorkerPool plus a substrate // ActorTemplate copying the deployed counter fixture's resolved runtime, // capturing at the scope under test on both pause and commit. -func createUpdateTestTemplate(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj *e2e.Namespace, name, poolName, bucket string, onCommit ateapipb.SnapshotContentScope, modify func(*ateapipb.ActorTemplate)) *ateapipb.ActorTemplate { +func createUpdateTestTemplate(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj *e2e.Namespace, name, poolName, bucket string, fidelity ateapipb.SnapshotFidelity, modify func(*ateapipb.ActorTemplate)) *ateapipb.ActorTemplate { t.Helper() return e2e.CreateSubstrateCounterTemplate(ctx, t, clients, nsObj.Name, e2e.SubstrateTemplateOptions{ Atespace: demoAtespace, @@ -315,8 +315,8 @@ func createUpdateTestTemplate(ctx context.Context, t *testing.T, clients *e2e.Cl PoolReplicas: 2, Labels: map[string]string{"demo": nsObj.Name}, SnapshotConfig: &ateapipb.SnapshotConfig{ - StorageLocation: "gs://" + bucket + "/ate-demo-" + nsObj.Name, - OnCommit: onCommit, + StorageLocation: "gs://" + bucket + "/ate-demo-" + nsObj.Name, + PreferredFidelity: fidelity, }, Modify: modify, }) diff --git a/internal/e2e/suites/metrics/metrics_test.go b/internal/e2e/suites/metrics/metrics_test.go index 712148fd4d..7ab059e946 100644 --- a/internal/e2e/suites/metrics/metrics_test.go +++ b/internal/e2e/suites/metrics/metrics_test.go @@ -311,17 +311,17 @@ func validateSnapshotPhaseLabels(scrape string) error { } phase := extractLabelValue(line, "ate_snapshot_phase") kind := extractLabelValue(line, "ate_snapshot_kind") - scope := extractLabelValue(line, "ate_snapshot_scope") + fidelity := extractLabelValue(line, "ate_snapshot_fidelity") class := extractLabelValue(line, "ate_sandbox_class") if phase == "" { return fmt.Errorf("%s line is missing ate_snapshot_phase: %q", m, line) } - if kind != "" && scope != "" && class != "" { + if kind != "" && fidelity != "" && class != "" { labelled = true } } if !labelled { - return fmt.Errorf("no %s line carried all of ate_snapshot_kind, ate_snapshot_scope and ate_sandbox_class", m) + return fmt.Errorf("no %s line carried all of ate_snapshot_kind, ate_snapshot_fidelity and ate_sandbox_class", m) } } return nil diff --git a/internal/e2e/suites/networking/accesslog_test.go b/internal/e2e/suites/networking/accesslog_test.go new file mode 100644 index 0000000000..f9057bc5c8 --- /dev/null +++ b/internal/e2e/suites/networking/accesslog_test.go @@ -0,0 +1,143 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package networking + +import ( + "testing" + + "github.com/agent-substrate/substrate/internal/resources" +) + +func TestAccessLogField(t *testing.T) { + line := `http.status=101 ate.actor.name=ws-1 substrate.connect.authority="10.96.0.7:80" ate.atespace=networking-e2e` + for _, test := range []struct { + key, want string + }{ + {"http.status", "101"}, + {"ate.actor.name", "ws-1"}, + {"substrate.connect.authority", "10.96.0.7:80"}, + } { + if got, ok := accessLogField(line, test.key); !ok || got != test.want { + t.Errorf("accessLogField(%q, %q) = (%q, %t), want (%q, true)", line, test.key, got, ok, test.want) + } + } + if got, ok := accessLogField(`other.http.status=503 http.status=101`, "http.status"); !ok || got != "101" { + t.Errorf("accessLogField must match a complete key, got (%q, %t)", got, ok) + } +} + +func TestProtocolGatewayAccessLogMatches(t *testing.T) { + actorRef := resources.ActorRef{Atespace: "networking-e2e", Name: "ws-1"} + for _, test := range []struct { + name, container, line string + want bool + }{ + { + name: "agentgateway websocket success", + container: "agentgateway", + line: `http.status=101 ate.actor.name=ws-1 substrate.connect.authority="10.96.0.7:80" ate.atespace=networking-e2e`, + want: true, + }, + { + name: "agentgateway opaque tunnel success", + container: "agentgateway", + line: `CONNECT tunnel terminated target=10.96.0.7:80 actor_name=ws-1 actor_uid=uid-1 atespace=networking-e2e`, + want: true, + }, + { + name: "agentgateway opaque tunnel wrong identity", + container: "agentgateway", + line: `CONNECT tunnel terminated target=10.96.0.7:80 actor_name=ws-2 actor_uid=uid-1 atespace=networking-e2e substrate.connect.authority="10.96.0.7:80"`, + want: false, + }, + { + name: "agentgateway opaque tunnel wrong target", + container: "agentgateway", + line: `CONNECT tunnel terminated target=10.96.0.8:80 actor_name=ws-1 actor_uid=uid-1 atespace=networking-e2e substrate.connect.authority="10.96.0.7:80"`, + want: false, + }, + { + name: "agentgateway opaque tunnel wrong atespace", + container: "agentgateway", + line: `CONNECT tunnel terminated target=10.96.0.7:80 actor_name=ws-1 actor_uid=uid-1 atespace=other`, + want: false, + }, + { + name: "agentgateway opaque tunnel error", + container: "agentgateway", + line: `CONNECT tunnel terminated target=10.96.0.7:80 actor_name=ws-1 actor_uid=uid-1 atespace=networking-e2e error=denied`, + want: false, + }, + { + name: "agentgateway wrong destination", + container: "agentgateway", + line: `http.status=101 ate.actor.name=ws-1 substrate.connect.authority="10.96.0.8:80" ate.atespace=networking-e2e`, + want: false, + }, + { + name: "agentgateway wrong actor", + container: "agentgateway", + line: `http.status=101 ate.actor.name=ws-2 substrate.connect.authority="10.96.0.7:80" ate.atespace=networking-e2e`, + want: false, + }, + { + name: "agentgateway wrong atespace", + container: "agentgateway", + line: `http.status=101 ate.actor.name=ws-1 substrate.connect.authority="10.96.0.7:80" ate.atespace=other`, + want: false, + }, + { + name: "agentgateway rejected request", + container: "agentgateway", + line: `http.status=403 ate.actor.name=ws-1 substrate.connect.authority="10.96.0.7:80" ate.atespace=networking-e2e`, + want: false, + }, + { + name: "envoy success", + container: "envoy", + line: `authority=10.96.0.7:80 code=200 peer_san="spiffe://substrate-actor.local/ateom-for-actor/networking-e2e/ws-1"`, + want: true, + }, + { + name: "envoy wrong actor identity", + container: "envoy", + line: `authority=10.96.0.7:80 code=200 peer_san="spiffe://substrate-actor.local/ateom-for-actor/networking-e2e/ws-2"`, + want: false, + }, + { + name: "envoy wrong atespace identity", + container: "envoy", + line: `authority=10.96.0.7:80 code=200 peer_san="spiffe://substrate-actor.local/ateom-for-actor/other/ws-1"`, + want: false, + }, + { + name: "envoy actor certificate identity is not the transport peer", + container: "envoy", + line: `authority=10.96.0.7:80 code=200 peer_san="spiffe://substrate-actor.local/actor/networking-e2e/ws-1"`, + want: false, + }, + } { + t.Run(test.name, func(t *testing.T) { + got := protocolGatewayAccessLogMatches(gatewayAccessLogLine{container: test.container, text: test.line}, actorRef, "10.96.0.7:80", true) + if got != test.want { + t.Fatalf("protocolGatewayAccessLogMatches() = %t, want %t", got, test.want) + } + }) + } + valid := gatewayAccessLogLine{container: "agentgateway", text: `CONNECT tunnel terminated target=10.96.0.7:80 actor_name=ws-1 actor_uid=uid-1 atespace=networking-e2e`} + if protocolGatewayAccessLogMatches(valid, actorRef, "10.96.0.7:80", false) { + t.Fatal("opaque CONNECT evidence must not satisfy the MITM matcher") + } +} diff --git a/internal/e2e/suites/networking/arbitraryport_test.go b/internal/e2e/suites/networking/arbitraryport_test.go index 3042a7b763..0e5325e70e 100644 --- a/internal/e2e/suites/networking/arbitraryport_test.go +++ b/internal/e2e/suites/networking/arbitraryport_test.go @@ -125,6 +125,32 @@ func TestActorArbitraryPortAccess(t *testing.T) { } t.Logf("tunneled request to an unlisted port correctly returned HTTP %d; body: %s", resp.StatusCode, body) }) + + t.Run("duplicate actor header rejected on CONNECT", func(t *testing.T) { + for _, secondActor := range []string{actorRef.String(), "untrusted/actor"} { + t.Run(secondActor, func(t *testing.T) { + headers := http.Header{ + atenet.TargetActorHeader: []string{actorRef.String(), secondActor}, + } + conn, err := router.ConnectWithHeaders(ctx, actorRef, counterExtraPort, headers) + if err != nil { + // A rejected CONNECT handshake is a valid rejection. + t.Logf("CONNECT with duplicate headers rejected: %v", err) + return + } + defer conn.Close() + + resp, body, err := requestTunneled(conn, actorRef.Name) + if err != nil { + t.Logf("tunneled request with duplicate CONNECT headers correctly failed: %v", err) + return + } + if resp.StatusCode != http.StatusNotFound { + t.Fatalf("request through CONNECT with duplicate actor headers returned %d, want 404; body: %s", resp.StatusCode, body) + } + }) + } + }) } // waitForTunneledRouteReady retries an entire CONNECT exchange while the diff --git a/internal/e2e/suites/networking/microvmclock_test.go b/internal/e2e/suites/networking/microvmclock_test.go new file mode 100644 index 0000000000..22b1b25300 --- /dev/null +++ b/internal/e2e/suites/networking/microvmclock_test.go @@ -0,0 +1,140 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package networking + +import ( + "context" + "crypto/x509" + "encoding/json" + "encoding/pem" + "net/http" + "testing" + "time" + + "github.com/agent-substrate/substrate/internal/e2e" + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// TestMicroVMClockAfterDelayedRestore probes delayed-restore TLS behavior by +// issuing the origin trust chain only after a suspended actor has been idle. +// The only wait here is the deliberate interval; ingress readiness is checked +// separately before the one measured HTTPS operation. +func TestMicroVMClockAfterDelayedRestore(t *testing.T) { + if !e2e.IsMicroVM() { + t.Skip("micro-VM clock contract") + } + + ctx := t.Context() + actorName, router, actorRef := prepareProtocolActor(t, ctx, "clock") + clients := e2e.GetClients() + t.Logf("clock evidence host_utc_before_suspend=%s actor=%s/%s", time.Now().UTC().Format(time.RFC3339Nano), actorRef.Atespace, actorName) + beforeSuspend, err := clients.SubstrateAPI.GetActor(ctx, &ateapipb.GetActorRequest{Actor: actorRef.ToObjectRef()}) + if err != nil { + t.Fatalf("GetActor before SuspendActor: %v", err) + } + if assignment := beforeSuspend.GetStatus().GetWorkerAssignment(); assignment == nil || assignment.GetWorker().GetName() == "" { + t.Fatalf("actor had no worker assignment before SuspendActor: %v", assignment) + } + suspendBoundary := time.Now().UTC() + suspended, err := clients.SubstrateAPI.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: actorRef.ToObjectRef()}) + if err != nil { + t.Fatalf("SuspendActor: %v", err) + } + if actor := suspended.GetActor(); actor.GetStatus().GetState() != ateapipb.ActorState_ACTOR_STATE_SUSPENDED { + t.Fatalf("SuspendActor returned state %s, want SUSPENDED", actor.GetStatus().GetState()) + } else if actor.GetStatus().GetWorkerAssignment() != nil { + t.Fatalf("SuspendActor returned worker assignment %v, want it cleared", actor.GetStatus().GetWorkerAssignment()) + } + + // Keep this interval long enough to separate the snapshot boundary from the + // fresh certificate issuance and resumed request. + const delay = 70 * time.Second + t.Logf("clock evidence suspended; waiting %s before issuing a fresh root and leaf", delay) + timer := time.NewTimer(delay) + select { + case <-timer.C: + case <-ctx.Done(): + timer.Stop() + t.Fatal(ctx.Err()) + } + t.Logf("clock evidence host_utc_after_delay=%s", time.Now().UTC().Format(time.RFC3339Nano)) + + target, address, rootCA := prepareProtocolOrigin(t, ctx, e2e.ServerPod{ + Name: "clock-origin", ImportPath: "github.com/agent-substrate/substrate/internal/e2e/fixtures/testserver", + Args: []string{"http", "--body=delayed restore clock"}, Port: 8443, + }, true) + logOriginCertificateValidity(t, ctx, target, rootCA, suspendBoundary) + + if _, err := e2e.ResumeActorAwaitCapacity(t, ctx, clients, &ateapipb.ResumeActorRequest{Actor: actorRef.ToObjectRef()}); err != nil { + t.Fatalf("ResumeActor: %v", err) + } + waitForRouteReady(t, "restored actor ingress readiness", func() (*http.Response, error) { + return router.Get(ctx, actorRef, "/readyz") + }) + + since := metav1.NewTime(time.Now().Add(-time.Minute)) + raw := postEgressOnce(t, ctx, router, actorRef, "/", map[string]any{ + "url": "https://" + address + "/healthz", "http1": true, + }) + var got struct { + StatusCode int `json:"statusCode"` + Body string `json:"body"` + Protocol string `json:"protocol"` + TLS bool `json:"tls"` + Error string `json:"error"` + } + if err := json.Unmarshal(raw, &got); err != nil { + t.Fatalf("decoding delayed-restore HTTPS observation: %v", err) + } + if got.Error != "" || got.StatusCode != 200 || got.Body != "delayed restore clock" || got.Protocol != "HTTP/1.1" || !got.TLS { + t.Errorf("delayed-restore HTTPS observation = %+v; want one verified TLS HTTP/1.1 8443 request", got) + } + assertProtocolGateway(t, ctx, since, actorName, target.Address()) +} + +func logOriginCertificateValidity(t *testing.T, ctx context.Context, target e2e.Server, rootCA string, suspendBoundary time.Time) { + t.Helper() + rootBlock, _ := pem.Decode([]byte(rootCA)) + if rootBlock == nil { + t.Fatalf("origin root CA is not PEM") + } + root, err := x509.ParseCertificate(rootBlock.Bytes) + if err != nil { + t.Fatalf("parsing origin root CA: %v", err) + } + secret, err := e2e.GetClients().K8s.CoreV1().Secrets(target.Namespace).Get(ctx, "origin-tls", metav1.GetOptions{}) + if err != nil { + t.Fatalf("reading origin leaf metadata: %v", err) + } + leaf, err := x509.ParseCertificate(mustPEMCertificate(t, secret.Data[corev1.TLSCertKey])) + if err != nil { + t.Fatalf("parsing origin leaf: %v", err) + } + if !leaf.NotBefore.After(suspendBoundary) { + t.Fatalf("origin leaf NotBefore %s is not after suspend boundary %s", leaf.NotBefore.UTC(), suspendBoundary) + } + t.Logf("clock evidence host_utc_after_origin=%s root_not_before=%s root_not_after=%s leaf_not_before=%s leaf_not_after=%s", time.Now().UTC().Format(time.RFC3339Nano), root.NotBefore.UTC().Format(time.RFC3339Nano), root.NotAfter.UTC().Format(time.RFC3339Nano), leaf.NotBefore.UTC().Format(time.RFC3339Nano), leaf.NotAfter.UTC().Format(time.RFC3339Nano)) +} + +func mustPEMCertificate(t *testing.T, data []byte) []byte { + t.Helper() + block, _ := pem.Decode(data) + if block == nil { + t.Fatalf("origin leaf is not PEM") + } + return block.Bytes +} diff --git a/internal/e2e/suites/networking/networking_test.go b/internal/e2e/suites/networking/networking_test.go index b255f09900..c8d3585ef3 100644 --- a/internal/e2e/suites/networking/networking_test.go +++ b/internal/e2e/suites/networking/networking_test.go @@ -19,21 +19,29 @@ import ( "encoding/json" "fmt" "io" + "net" "net/http" + "slices" "strconv" "strings" "testing" "time" + "github.com/agent-substrate/substrate/internal/atenet" "github.com/agent-substrate/substrate/internal/e2e" + "github.com/agent-substrate/substrate/internal/localca" "github.com/agent-substrate/substrate/internal/resources" + "github.com/agent-substrate/substrate/internal/testcert" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" corev1 "k8s.io/api/core/v1" + discoveryv1 "k8s.io/api/discovery/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) const networkingAtespace = "networking-e2e" +// egressFixture returns the unified egress demo deployed by the E2E workflow. +func egressFixture() e2e.Fixture { return e2e.EgressFixture() } func TestActorDirectAccess(t *testing.T) { ctx := context.Background() _, actorName, actor := createAndResumeSubstrateActor(t, ctx, "direct", e2e.SubstrateCounterFixture()) @@ -50,6 +58,28 @@ func TestActorDirectAccess(t *testing.T) { }) t.Logf("Actor access through ingress succeeded; body: %s", body) }) + t.Run("duplicate actor header rejected on plain HTTP", func(t *testing.T) { + actorRef := resources.ActorRef{Atespace: networkingAtespace, Name: actorName} + for _, secondActor := range []string{actorRef.String(), "untrusted/actor"} { + t.Run(secondActor, func(t *testing.T) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, router.BaseURL()+"/readyz", nil) + if err != nil { + t.Fatalf("creating request: %v", err) + } + req.Header.Add(atenet.TargetActorHeader, actorRef.String()) + req.Header.Add(atenet.TargetActorHeader, secondActor) + resp, err := (&http.Client{Timeout: 10 * time.Second}).Do(req) + if err != nil { + t.Fatalf("sending request with duplicate actor headers: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusNotFound { + body, _ := io.ReadAll(resp.Body) + t.Fatalf("duplicate actor header on plain HTTP returned %d, want 404; body: %s", resp.StatusCode, body) + } + }) + } + }) } // egressHTTPTarget returns a copy of the origin TestActorEgress dials: @@ -345,12 +375,13 @@ func waitForAccessLog(t *testing.T, ctx context.Context, since metav1.Time, want // accessLogField returns the value of the key=value field named key in an Envoy // access log line whose fields are separated by spaces. func accessLogField(line, key string) (string, bool) { - _, rest, ok := strings.Cut(line, key+"=") - if !ok { - return "", false + for field := range strings.FieldsSeq(line) { + fieldKey, value, ok := strings.Cut(field, "=") + if ok && fieldKey == key { + return strings.Trim(value, `"`), true + } } - value, _, _ := strings.Cut(rest, " ") - return value, true + return "", false } // createAndResumeActorWithEgress creates an actor from template, gives it an @@ -466,3 +497,413 @@ func assertDirectActorAccess(t *testing.T, ctx context.Context, clients *e2e.Cli } t.Logf("direct Actor access through %s/%s:80 was blocked as expected: %v", actor.GetStatus().GetWorkerAssignment().GetWorkerNamespace(), actor.GetStatus().GetWorkerAssignment().GetWorkerPod(), err) } + +// TestActorEgressHTTPSNonStandardPort verifies HTTP/1.1 inside TLS while +// preserving the original destination port through the egress tunnel. +func TestActorEgressHTTPSNonStandardPort(t *testing.T) { + ctx := t.Context() + const expectedBody = "https nonstandard origin" + target, address, _ := prepareProtocolOrigin(t, ctx, e2e.ServerPod{ + Name: "https-origin", + ImportPath: "github.com/agent-substrate/substrate/internal/e2e/fixtures/testserver", + Args: []string{"http", "--body=" + expectedBody}, + Port: 8443, + }, true) + actorName, router, actorRef := prepareProtocolActor(t, ctx, "https-port") + since := metav1.NewTime(time.Now().Add(-time.Minute)) + raw := postEgressOnce(t, ctx, router, actorRef, "/", map[string]any{ + "url": "https://" + address + "/healthz", "http1": true, + }) + var got struct { + StatusCode int `json:"statusCode"` + Body string `json:"body"` + Protocol string `json:"protocol"` + TLS bool `json:"tls"` + Error string `json:"error"` + } + if err := json.Unmarshal(raw, &got); err != nil { + t.Fatalf("decoding HTTPS observation: %v", err) + } + if got.Error != "" || got.StatusCode != http.StatusOK || got.Body != expectedBody || got.Protocol != "HTTP/1.1" || !got.TLS { + t.Errorf("HTTPS observation = %+v; want verified TLS, HTTP/1.1, status 200 and body %q", got, expectedBody) + } + assertProtocolGateway(t, ctx, since, actorName, target.Address()) +} + +func prepareProtocolOrigin(t *testing.T, ctx context.Context, spec e2e.ServerPod, encrypted bool) (e2e.Server, string, string) { + t.Helper() + spec.Namespace = e2e.CreateNamespace(t).Name + registerOriginDiagnostics(t, spec.Namespace, spec.Name) + var reserved e2e.Server + var rootCA string + if encrypted { + reserved = e2e.CreateServerService(t, ctx, spec) + // The unified gateway verifies local origins against service-DNS roots. + secret, err := e2e.GetClients().K8s.CoreV1().Secrets("podcertificate-controller-system").Get(ctx, "service-dns-ca-pool", metav1.GetOptions{}) + if err != nil { + t.Fatalf("reading service-DNS CA pool: %v", err) + } + pool, err := localca.Unmarshal(secret.Data["pool"]) + if err != nil { + t.Fatalf("parsing service-DNS CA pool: %v", err) + } + if len(pool.CAs) == 0 { + t.Fatal("service-DNS CA pool is empty") + } + material := testcert.ServerTLSWithCA(t, pool.CAs[0], net.ParseIP(reserved.ClusterIP), spec.Name+"."+spec.Namespace+".svc.cluster.local") + rootCA = string(material.RootCA) + originSecret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "origin-tls", Namespace: spec.Namespace}, Type: corev1.SecretTypeTLS, + Data: map[string][]byte{corev1.TLSCertKey: material.Certificate, corev1.TLSPrivateKeyKey: material.PrivateKey}} + if _, err := e2e.GetClients().K8s.CoreV1().Secrets(spec.Namespace).Create(ctx, originSecret, metav1.CreateOptions{}); err != nil { + t.Fatalf("creating origin credentials: %v", err) + } + spec.Volumes = []corev1.Volume{{Name: "tls", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: originSecret.Name}}}} + spec.VolumeMounts = []corev1.VolumeMount{{Name: "tls", MountPath: "/run/origin-tls", ReadOnly: true}} + spec.Args = append(spec.Args, "--tls-cert=/run/origin-tls/tls.crt", "--tls-key=/run/origin-tls/tls.key") + spec.HealthScheme = corev1.URISchemeHTTPS + } + target := e2e.DeployServerPod(t, ctx, spec) + if encrypted && target.ClusterIP != reserved.ClusterIP { + t.Fatalf("Service IP changed from %s to %s after certificate issuance", reserved.ClusterIP, target.ClusterIP) + } + e2e.WaitForServerEndpoint(t, ctx, spec) + family := "IPv6" + if net.ParseIP(target.ClusterIP).To4() != nil { + family = "IPv4" + } + t.Logf("origin %s/%s at %s (%s), TLS=%v", target.Namespace, spec.Name, target.Address(), family, encrypted) + address := protocolOriginRequest(target, spec.Name) + return target, address, rootCA +} + +const ( + originDiagnosticLogLines = 80 + originDiagnosticLogBytes = 16 << 10 + originDiagnosticFieldBytes = 1024 + originDiagnosticItems = 20 +) + +func registerOriginDiagnostics(t *testing.T, namespace, name string) { + t.Helper() + t.Cleanup(func() { + if !t.Failed() { + return + } + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + collectOriginDiagnostics(t, ctx, namespace, name) + }) +} + +func collectOriginDiagnostics(t *testing.T, ctx context.Context, namespace, name string) { + t.Helper() + clients := e2e.GetClients().K8s + observed := time.Now().UTC().Format(time.RFC3339Nano) + t.Logf("origin diagnostics observed_at=%s namespace=%s name=%s", observed, namespace, name) + + pod, err := clients.CoreV1().Pods(namespace).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + t.Logf("origin diagnostics: get Pod %s/%s: %v", namespace, name, err) + } else { + deletion := "" + if pod.DeletionTimestamp != nil { + deletion = formatOriginTimestamp(*pod.DeletionTimestamp) + } + t.Logf("origin Pod status: uid=%s phase=%s pod_ip=%s host_ip=%s reason=%s message=%q created=%s deletion=%s containers=%s", pod.UID, pod.Status.Phase, pod.Status.PodIP, pod.Status.HostIP, pod.Status.Reason, boundedOriginDiagnosticField(pod.Status.Message), formatOriginTimestamp(pod.CreationTimestamp), deletion, formatOriginContainerStatuses(pod.Status.ContainerStatuses)) + for _, previous := range []bool{false, true} { + logs, logErr := clients.CoreV1().Pods(namespace).GetLogs(name, &corev1.PodLogOptions{ + Previous: previous, + TailLines: ptrInt64(originDiagnosticLogLines), + LimitBytes: ptrInt64(originDiagnosticLogBytes), + Timestamps: true, + }).DoRaw(ctx) + label := "current" + if previous { + label = "previous" + } + if logErr != nil { + t.Logf("origin %s logs unavailable: %v", label, logErr) + continue + } + t.Logf("origin %s logs (bounded):\n%s", label, boundedOriginDiagnosticText(string(logs), originDiagnosticLogLines, originDiagnosticLogBytes)) + } + } + + service, err := clients.CoreV1().Services(namespace).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + t.Logf("origin diagnostics: get Service %s/%s: %v", namespace, name, err) + } else { + t.Logf("origin Service: cluster_ip=%s ports=%s selector=%s", service.Spec.ClusterIP, formatOriginServicePorts(service.Spec.Ports), formatOriginSelector(service.Spec.Selector)) + } + + slices, err := clients.DiscoveryV1().EndpointSlices(namespace).List(ctx, metav1.ListOptions{LabelSelector: discoveryv1.LabelServiceName + "=" + name, Limit: 20}) + if err != nil { + t.Logf("origin diagnostics: list EndpointSlices for %s/%s: %v", namespace, name, err) + } else { + for _, slice := range slices.Items { + t.Logf("origin EndpointSlice: name=%s uid=%s created=%s ports=%s endpoints=%s", slice.Name, slice.UID, formatOriginTimestamp(slice.CreationTimestamp), formatOriginEndpointPorts(slice.Ports), formatOriginEndpoints(slice.Endpoints)) + } + } + + events, err := clients.CoreV1().Events(namespace).List(ctx, metav1.ListOptions{Limit: 50}) + if err != nil { + t.Logf("origin diagnostics: list namespace events %s: %v", namespace, err) + return + } + for _, event := range events.Items { + if event.InvolvedObject.Name != name { + continue + } + t.Logf("origin event: type=%s reason=%s message=%q count=%d last=%s", event.Type, event.Reason, boundedOriginDiagnosticField(event.Message), event.Count, event.LastTimestamp.UTC().Format(time.RFC3339Nano)) + } +} + +func ptrInt64(value int) *int64 { + result := int64(value) + return &result +} + +func boundedOriginDiagnosticText(value string, maxLines, maxBytes int) string { + lines := strings.Split(value, "\n") + if len(lines) > maxLines { + lines = lines[len(lines)-maxLines:] + } + result := strings.Join(lines, "\n") + if len(result) > maxBytes { + result = result[len(result)-maxBytes:] + } + return result +} + +func boundedOriginDiagnosticField(value string) string { + if len(value) <= originDiagnosticFieldBytes { + return value + } + return value[:originDiagnosticFieldBytes-3] + "..." +} + +func formatOriginServicePorts(ports []corev1.ServicePort) string { + values := make([]string, 0, len(ports)) + for _, port := range ports[:min(len(ports), originDiagnosticItems)] { + values = append(values, fmt.Sprintf("%s/%d->%s/%s", port.Name, port.Port, port.TargetPort.String(), port.Protocol)) + } + return boundedOriginDiagnosticField(strings.Join(values, ",")) +} + +func formatOriginSelector(selector map[string]string) string { + values := make([]string, 0, len(selector)) + for key, value := range selector { + values = append(values, boundedOriginDiagnosticField(key)+"="+boundedOriginDiagnosticField(value)) + } + slices.Sort(values) + if len(values) > originDiagnosticItems { + values = values[:originDiagnosticItems] + } + return boundedOriginDiagnosticField(strings.Join(values, ",")) +} + +func formatOriginEndpointPorts(ports []discoveryv1.EndpointPort) string { + values := make([]string, 0, len(ports)) + for _, port := range ports { + protocol := "" + if port.Protocol != nil { + protocol = string(*port.Protocol) + } + values = append(values, fmt.Sprintf("%s/%d/%s", boundedOriginDiagnosticField(valueString(port.Name)), valueInt32(port.Port), protocol)) + } + if len(values) > originDiagnosticItems { + values = values[:originDiagnosticItems] + } + return boundedOriginDiagnosticField(strings.Join(values, ",")) +} + +func formatOriginEndpoints(endpoints []discoveryv1.Endpoint) string { + values := make([]string, 0, len(endpoints)) + for _, endpoint := range endpoints { + ready, terminating := "unknown", "unknown" + if endpoint.Conditions.Ready != nil { + ready = strconv.FormatBool(*endpoint.Conditions.Ready) + } + if endpoint.Conditions.Terminating != nil { + terminating = strconv.FormatBool(*endpoint.Conditions.Terminating) + } + uid := "" + if endpoint.TargetRef != nil { + uid = string(endpoint.TargetRef.UID) + } + addresses := make([]string, 0, len(endpoint.Addresses)) + for _, address := range endpoint.Addresses { + addresses = append(addresses, boundedOriginDiagnosticField(address)) + } + if len(addresses) > originDiagnosticItems { + addresses = addresses[:originDiagnosticItems] + } + values = append(values, boundedOriginDiagnosticField(fmt.Sprintf("addresses=%s uid=%s ready=%s terminating=%s", strings.Join(addresses, ","), uid, ready, terminating))) + } + if len(values) > originDiagnosticItems { + values = values[:originDiagnosticItems] + } + return strings.Join(values, "; ") +} + +func formatOriginTimestamp(value metav1.Time) string { + if value.IsZero() { + return "" + } + return value.UTC().Format(time.RFC3339Nano) +} + +func formatOriginContainerStatuses(statuses []corev1.ContainerStatus) string { + values := make([]string, 0, len(statuses)) + for _, status := range statuses[:min(len(statuses), originDiagnosticItems)] { + values = append(values, boundedOriginDiagnosticField(fmt.Sprintf("%s ready=%t restarts=%d", status.Name, status.Ready, status.RestartCount))) + } + return strings.Join(values, "; ") +} + +func valueInt32(value *int32) int32 { + if value == nil { + return 0 + } + return *value +} + +func valueString(value *string) string { + if value == nil { + return "" + } + return *value +} + +// The unified gateway needs the origin's DNS name for SNI; the outer CONNECT +// log still records target.Address(). +func protocolOriginRequest(target e2e.Server, service string) string { + return net.JoinHostPort(service+"."+target.Namespace+".svc.cluster.local", strconv.Itoa(target.Port)) +} + +func TestProtocolOriginRequest(t *testing.T) { + for _, tc := range []struct { + name, ip string + }{ + {"IPv4", "10.0.0.7"}, + {"IPv6", "fd00::7"}, + } { + t.Run(tc.name, func(t *testing.T) { + target := e2e.Server{Namespace: "test", ClusterIP: tc.ip, Port: 8443} + if got, want := protocolOriginRequest(target, "origin"), "origin.test.svc.cluster.local:8443"; got != want { + t.Fatalf("origin request = %q, want %q", got, want) + } + }) + } +} + +func TestBoundedOriginDiagnosticText(t *testing.T) { + got := boundedOriginDiagnosticText("one\ntwo\nthree\nfour", 2, 16) + if got != "three\nfour" { + t.Fatalf("bounded diagnostic text = %q, want %q", got, "three\\nfour") + } + if got := boundedOriginDiagnosticText("123456789", 10, 4); got != "6789" { + t.Fatalf("byte-bounded diagnostic text = %q, want %q", got, "6789") + } +} + +func prepareProtocolActor(t *testing.T, ctx context.Context, prefix string) (string, *e2e.RouterClient, resources.ActorRef) { + t.Helper() + https8443 := e2e.EgressAllowHTTPS("*") + https8443.Https.Ports = &ateapipb.Ports{Numbers: []int32{8443}} + rules := append(e2e.EgressAllowAll(), https8443) + _, actorName, _ := createAndResumeActorWithEgress(t, ctx, prefix, egressFixture(), rules...) + router := mustRouterClient(t, ctx) + t.Cleanup(func() { router.Close() }) + ref := resources.ActorRef{Atespace: networkingAtespace, Name: actorName} + waitForRouteReady(t, "egress actor ingress readiness", func() (*http.Response, error) { return router.Get(ctx, ref, "/readyz") }) + return actorName, router, ref +} + +// postEgressOnce starts the measured operation only after readiness. Unlike +// postThroughEgressActor, an outbound failure is never retried. +func postEgressOnce(t *testing.T, ctx context.Context, router *e2e.RouterClient, actorRef resources.ActorRef, path string, input any) []byte { + return postEgressWithStatus(t, ctx, router, actorRef, path, input, http.StatusOK) +} + +func postEgressWithStatus(t *testing.T, ctx context.Context, router *e2e.RouterClient, actorRef resources.ActorRef, path string, input any, wantStatus int) []byte { + t.Helper() + payload, err := json.Marshal(input) + if err != nil { + t.Fatalf("encoding actor operation: %v", err) + } + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + response, err := router.PostJSON(ctx, actorRef, path, payload) + if err != nil { + t.Fatalf("actor %s operation %s transport failed: %v", actorRef.Name, path, err) + } + defer response.Body.Close() + raw, err := io.ReadAll(io.LimitReader(response.Body, (1<<20)+1)) + if err != nil { + t.Fatalf("reading actor operation: %v", err) + } + if len(raw) > 1<<20 { + t.Fatal("actor operation response exceeds 1 MiB") + } + if response.StatusCode != wantStatus { + t.Fatalf("actor %s operation %s returned HTTP %d, want %d: %s", actorRef.Name, path, response.StatusCode, wantStatus, raw) + } + return raw +} + +func assertProtocolGateway(t *testing.T, ctx context.Context, since metav1.Time, actorName, authority string) { + t.Helper() + actorRef := resources.ActorRef{Atespace: networkingAtespace, Name: actorName} + identity := resources.AteomForActorSPIFFEID(actorRef).String() + waitForAccessLog(t, ctx, since, "successful gateway traffic for "+identity+" to "+authority, func(lines []gatewayAccessLogLine) bool { + for _, entry := range lines { + if protocolGatewayAccessLogMatches(entry, actorRef, authority, false) { + t.Logf("gateway evidence: %s", entry.text) + return true + } + } + return false + }) +} + +func protocolGatewayAccessLogMatches(entry gatewayAccessLogLine, actorRef resources.ActorRef, authority string, allowOpaque bool) bool { + switch entry.container { + case "envoy": + gotAuthority, _ := accessLogField(entry.text, "authority") + code, _ := accessLogField(entry.text, "code") + peers, _ := accessLogField(entry.text, "peer_san") + if gotAuthority != authority || code != "200" { + return false + } + for peer := range strings.SplitSeq(peers, ",") { + if peer == resources.AteomForActorSPIFFEID(actorRef).String() { + return true + } + } + case "agentgateway": + gotAuthority, _ := accessLogField(entry.text, "substrate.connect.authority") + if allowOpaque && strings.Contains(entry.text, "CONNECT tunnel terminated") { + // Baseline TCP routes carry opaque CONNECT traffic and therefore do + // not emit an HTTP status. The accepted CONNECT record identifies the + // authenticated actor and destination; the operation assertion above + // supplies end-to-end success. + target, targetOK := accessLogField(entry.text, "target") + tunnelActor, actorOK := accessLogField(entry.text, "actor_name") + tunnelSpace, spaceOK := accessLogField(entry.text, "atespace") + _, errorOK := accessLogField(entry.text, "error") + return targetOK && target == authority && actorOK && tunnelActor == actorRef.Name && spaceOK && tunnelSpace == actorRef.Atespace && !errorOK + } + gotActor, _ := accessLogField(entry.text, "ate.actor.name") + gotAtespace, _ := accessLogField(entry.text, "ate.atespace") + status, _ := accessLogField(entry.text, "http.status") + if gotAuthority != authority || gotActor != actorRef.Name || gotAtespace != actorRef.Atespace { + return false + } + if status == "200" || status == "101" { + return true + } + return false + } + return false +} diff --git a/internal/e2e/suites/networking/websocketegress_test.go b/internal/e2e/suites/networking/websocketegress_test.go new file mode 100644 index 0000000000..34b318b194 --- /dev/null +++ b/internal/e2e/suites/networking/websocketegress_test.go @@ -0,0 +1,95 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package networking + +import ( + "encoding/json" + "net/http" + "testing" + + "github.com/agent-substrate/substrate/internal/e2e" +) + +// egressWebSocketResponse mirrors observations returned by the actor. The +// outer HTTP status is already checked before these protocol assertions run. +type egressWebSocketResponse struct { + StatusCode int `json:"statusCode"` + Protocol string `json:"protocol"` + HandshakeBody string `json:"handshakeBody"` + TLS bool `json:"tls"` + Messages []egressWebSocketMessage `json:"messages"` + Error string `json:"error"` +} + +type egressWebSocketMessage struct { + Type int `json:"type"` + Message string `json:"message"` +} + +func TestActorEgressWebSocket(t *testing.T) { + ctx := t.Context() + _, address, _ := prepareProtocolOrigin(t, ctx, e2e.ServerPod{ + Name: "ws-origin", ImportPath: "github.com/agent-substrate/substrate/internal/e2e/fixtures/testserver", + Args: []string{"websocket", "--echo"}, Port: 80, TargetPort: 8080, HealthPath: "/readyz", + }, false) + _, router, actorRef := prepareProtocolActor(t, ctx, "ws") + raw := postEgressWithStatus(t, ctx, router, actorRef, "/websocket", map[string]any{ + "url": "ws://" + address + "/ws", "messages": []string{"ws-message"}, + }, http.StatusBadGateway) + var got egressWebSocketResponse + if err := json.Unmarshal(raw, &got); err != nil { + t.Fatalf("decoding WebSocket observation: %v", err) + } + if got.StatusCode != http.StatusForbidden { + t.Errorf("WebSocket handshake status = %d, want %d", got.StatusCode, http.StatusForbidden) + } + if want := e2e.CurrentAtenetDataplane().EgressUpgradeDeniedBody(); got.HandshakeBody != want { + t.Errorf("WebSocket handshake body = %q, want %q", got.HandshakeBody, want) + } + if got.Error == "" { + t.Error("WebSocket handshake error is empty") + } + if len(got.Messages) != 0 { + t.Errorf("WebSocket messages = %+v, want no echoes", got.Messages) + } +} + +func TestActorEgressSecureWebSocket(t *testing.T) { + ctx := t.Context() + _, address, _ := prepareProtocolOrigin(t, ctx, e2e.ServerPod{ + Name: "wss-origin", ImportPath: "github.com/agent-substrate/substrate/internal/e2e/fixtures/testserver", + Args: []string{"websocket", "--echo"}, Port: 443, TargetPort: 8443, HealthPath: "/readyz", + }, true) + _, router, actorRef := prepareProtocolActor(t, ctx, "wss") + raw := postEgressWithStatus(t, ctx, router, actorRef, "/websocket", map[string]any{ + "url": "wss://" + address + "/ws", "messages": []string{"wss-message"}, + }, http.StatusBadGateway) + var got egressWebSocketResponse + if err := json.Unmarshal(raw, &got); err != nil { + t.Fatalf("decoding secure WebSocket observation: %v", err) + } + if got.StatusCode != http.StatusForbidden { + t.Errorf("secure WebSocket handshake status = %d, want %d", got.StatusCode, http.StatusForbidden) + } + if want := e2e.CurrentAtenetDataplane().EgressUpgradeDeniedBody(); got.HandshakeBody != want { + t.Errorf("secure WebSocket handshake body = %q, want %q", got.HandshakeBody, want) + } + if got.Error == "" { + t.Error("secure WebSocket handshake error is empty") + } + if len(got.Messages) != 0 { + t.Errorf("secure WebSocket messages = %+v, want no echoes", got.Messages) + } +} diff --git a/internal/e2e/suites/networkpolicy/networkpolicy_test.go b/internal/e2e/suites/networkpolicy/networkpolicy_test.go index c2cf26b9d4..ff021982c7 100644 --- a/internal/e2e/suites/networkpolicy/networkpolicy_test.go +++ b/internal/e2e/suites/networkpolicy/networkpolicy_test.go @@ -51,8 +51,9 @@ func TestNetworkPolicyLifecycleAndReconciliation(t *testing.T) { Namespace: nsObj.Name, }, Spec: v1alpha1.WorkerPoolSpec{ - Replicas: 1, - WorkerImage: "ateom:v1", + Replicas: 1, + WorkerImage: "ateom:v1", + SandboxClasses: []v1alpha1.WorkerPoolSandboxClass{{Name: v1alpha1.SandboxClassGvisor}}, }, } diff --git a/internal/e2e/suites/security/README.md b/internal/e2e/suites/security/README.md new file mode 100644 index 0000000000..0c06735843 --- /dev/null +++ b/internal/e2e/suites/security/README.md @@ -0,0 +1,119 @@ +# Security e2e suite + +End-to-end tests for Substrate's isolation boundaries. This is the runbook +for running them on a local kind cluster and for watching what they do while +they run. + +Prereqs: Go, kubectl, a running Docker. kind is auto-managed. + +## Steps to run the security tests + +1. Create the cluster (in the repo root): + +``` +hack/create-kind-cluster.sh +``` + +2. Install the control plane (slow step — pulls ~570 MB): + +``` +hack/install-ate-kind.sh --deploy-ate-system +``` + +Stop here — the NFS/CSI and demo steps CI runs are for other suites; this suite needs none of them. + +3. Run the cross-atespace isolation test: + +``` +hack/run-e2e-kind.sh ./internal/e2e/suites/security -v -run TestCrossAtespaceIsolation +``` + +The wrapper sets KO_DOCKER_REPO, BUCKET_NAME, the kube context, and builds the probe image via ko. To run only the durable-mount and host-state subtests and skip the tag-scope one: + +``` +hack/run-e2e-kind.sh ./internal/e2e/suites/security -v \ + -run 'TestCrossAtespaceIsolation/(VictimDurableMarkerNotVisibleThroughAttackerMount|VictimHostStateNotReachableFromAttackerSandbox)' +``` + +Note the setup (victim run/write/suspend, attacker reuse) runs regardless — the subtest filter only selects which assertions execute. + +4. On failure, a failed suite keeps its namespaces so you can read the worker logs: + + +kubectl get ns -l ate.dev/e2e +kubectl -n logs -c ateom +KIND_CLUSTER_NAME=kind hack/cleanup-e2e.sh # reclaim when done + + + +## Run the GitHub checks locally + +The PR workflow (`.github/workflows/pr-workflow.yaml`) runs three checks: +`run-tests`, `E2E ()`, and `e2e-test`, which only aggregates the +E2E matrix. Every one of them compiles this suite, so a build error here +fails all three. + +Fast, no cluster needed. This is what CI's `run-tests` job does, minus the +JUnit wrapping, and it also type-checks every E2E suite: + +``` +go vet ./... && go test -race ./... && hack/verify-all.sh && (cd tools/apitool && go test -race ./...) +``` + +`hack/verify-all.sh` runs gofmt, boilerplate, licenses, go.mod, metrics +registry, and generated-code checks. +Commit or stash first: the generated-code check refuses a dirty tree. The +metrics check needs Docker or a local `weaver` at the version pinned in +`hack/verify/metrics.sh`. The root-gated tests CI runs under sudo need Linux: + +``` +hack/run-root-tests.sh -race -v +``` + +Full E2E, same as CI's gVisor lane. Do steps 1 and 2 above, then run every +suite with CI's flags instead of the `-run` filter from step 3: + +``` +hack/run-e2e-kind.sh -v -args --no-color +``` + +CI also deploys the NFS CSI driver and the demos before this, and repeats the +run with `E2E_SANDBOX_CLASS=microvm`. Neither matters for this suite. + +## Commands to run while the test is running to observe +Watch the lifecycle (start these first, before/at test launch) + +# The two atespace namespaces appear and vanish — this is the run's heartbeat +kubectl get ns -l ate.dev/e2e -w + +# The one shared worker pod: watch the victim's pod get reused, not replaced +kubectl get pods -n ate-e2e-sec-a -w +The reuse itself (the point of the test) +Actors aren't in k8s, so use the plugin. This is where you see victim B run, suspend, then attacker A bind to the same worker: + + +# Actor state + worker assignment, atespace B (victim) then A (attacker) +watch -n1 'kubectl ate get actors -a ate-e2e-sec-b; echo ---; kubectl ate get actors -a ate-e2e-sec-a' + +# Which worker each actor is on — the name should match across the two atespaces +kubectl ate get workers +kubectl ate top workers # capacity: Actors used/total (you'll see 1/1 then 0/1 then 1/1) +The worker name that B's victim shows, then A's attacker shows after the suspend, is the same string requireReusedWorker asserts on. Seeing them match live is the manual version of that check. + +The shared worker pod on the host + +# The pod hosting both actors in turn +kubectl get pods -n ate-e2e-sec-a -o wide + +# ateom logs — the sandbox lifecycle, checkpoints, resumes +kubectl logs -n ate-e2e-sec-a -l app=ateom -c ateom -f + +# The on-host per-actor layout the host-state probes assert against. +# Two UID-keyed durable dirs (victim's + attacker's) live side by side here: +POD=$(kubectl get pods -n ate-e2e-sec-a -l app=ateom -o name | head -1) +kubectl exec -n ate-e2e-sec-a "$POD" -c ateom -- ls -la /var/lib/ateom-gvisor/actors +Control plane / snapshots (for the tag-scope subtest) + +kubectl ate get tags -a ate-e2e-sec-a # atespace-a-tag appears during the last subtest +kubectl ate get atespaces # ate-e2e-sec-a / -b listed while live +kubectl logs -n ate-system -l app=ateapi -f # Create \ No newline at end of file diff --git a/internal/e2e/suites/security/crossatespace_test.go b/internal/e2e/suites/security/crossatespace_test.go new file mode 100644 index 0000000000..355229d310 --- /dev/null +++ b/internal/e2e/suites/security/crossatespace_test.go @@ -0,0 +1,475 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package security holds the end-to-end tests for Substrate's security +// boundaries. Every assertion here is negative: something must not be +// possible. Each is paired with a positive control showing the probe could +// have seen the violation, because a negative test that stops exercising its +// boundary keeps passing. +// +// This file covers the boundary between two atespaces whose actors are +// identical except for the atespace they run in. The victim actor runs, writes +// state, and vacates its worker; the attacker actor, in the other atespace, +// then reuses that worker and must find none of it. A single-worker pool +// (fixtures/security/atespace-a-pool.yaml.tmpl) forces the reuse, and the test +// checks that it happened. +package security + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/url" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/agent-substrate/substrate/internal/e2e" + "github.com/agent-substrate/substrate/internal/nodepath" + "github.com/agent-substrate/substrate/internal/resources" + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +const ( + // templateName is the ActorTemplate name both atespaces deploy. + templateName = "probe" + + // dataMount is where atespace-template.yaml.tmpl mounts the durable-dir + // volume, and durableVolume is that volume's name, which is also the last + // segment of its mount point on the worker. Keep both in sync with the + // manifest. + dataMount = "/data" + durableVolume = "data" + + // writtenByProbe is the fixed body the probe's /writefile endpoint writes. + // Actors are told apart by the path they write, not by the content. + writtenByProbe = "written by probe" + + // markerA and markerB are named for the atespace that owns them, so a file + // seen across the boundary is attributable. + markerA = dataMount + "/atespace-a-marker" + markerB = dataMount + "/atespace-b-marker" + + // neverWritten is a path no actor creates. Reading it proves /readfile + // reports a missing file as an error, which every negative assertion in + // this file relies on. + neverWritten = dataMount + "/never-written" +) + +// fileResponse mirrors the probe's /readfile and /writefile responses. +type fileResponse struct { + Path string `json:"path"` + Content string `json:"content"` + OK string `json:"ok"` + Error string `json:"error"` +} + +// atespaces is the pair of atespaces under test. +type atespaces struct { + a, b string +} + +// TestCrossAtespaceIsolation checks that the atespace boundary holds across +// worker reuse for an actor's stored data and its snapshots. +// +// The victim, in atespace B, runs, writes its durable marker, and is +// suspended. The attacker, in atespace A, then starts on the worker the victim +// left and must find none of its state. +// +// The fixture is deployed once for the whole test. Deploying waits for each +// atespace's golden snapshot, which on the micro-VM lane is a cold boot plus a +// checkpoint, so per-subtest fixtures would dominate the runtime. +func TestCrossAtespaceIsolation(t *testing.T) { + env, err := e2e.CheckEnv("BUCKET_NAME", "KO_DOCKER_REPO") + if err != nil { + t.Fatalf("CheckEnv failed: %v", err) + } + ctx := context.Background() + clients := e2e.GetClients() + + ns := deployAtespaces(t, ctx, clients, env["BUCKET_NAME"]) + + rc, err := e2e.NewRouterClient(ctx) + if err != nil { + t.Fatalf("NewRouterClient: %v", err) + } + defer rc.Close() + + // Victim: the actor in atespace B runs first. Its UID and worker are recorded while it is + // RUNNING: the attacker probes host paths keyed by that UID and must land + // on that worker. + const victim = "victim" + createAndResumeActor(t, ctx, clients, ns.b, victim) + victimActor := resources.ActorRef{Atespace: ns.b, Name: victim} + victimUID := actorUID(t, ctx, clients, ns.b, victim) + victimWorker := assignedWorker(t, ctx, clients, ns.b, victim) + + writeFile(t, ctx, rc, victimActor, markerB) + // Control: the victim reads back its own marker, so the attacker's failure + // to read it below means the boundary held, not that the marker was never + // written. + if got := readFile(t, ctx, rc, victimActor, markerB); got.Content != writtenByProbe { + t.Fatalf("control: victim %s reading its OWN %s = %q (error %q), want %q — the probe cannot see its own durable data, so nothing below is meaningful", + victimActor, markerB, got.Content, got.Error, writtenByProbe) + } + + // Suspend the victim to free the worker. The durable marker stays on the + // worker; the snapshot goes to object storage. + if _, err := clients.SubstrateAPI.SuspendActor(ctx, &ateapipb.SuspendActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: ns.b, Name: victim}, + }); err != nil { + t.Fatalf("SuspendActor %s/%s: %v", ns.b, victim, err) + } + waitForActorState(t, ctx, clients, ns.b, victim, ateapipb.ActorState_ACTOR_STATE_SUSPENDED) + + // Attacker: the actor in atespace A takes the freed worker. + const attacker = "attacker" + createAndResumeActor(t, ctx, clients, ns.a, attacker) + attackerActor := resources.ActorRef{Atespace: ns.a, Name: attacker} + requireReusedWorker(t, ctx, clients, ns.a, attacker, victimWorker) + + // Controls for every negative assertion below. A failure is fatal: the + // probe cannot tell a denial from a missing file, so the subtests would + // pass for the wrong reason. + if got := readFile(t, ctx, rc, attackerActor, neverWritten); got.Error == "" { + t.Fatalf("control: attacker %s read %s, which nothing ever created, and reported no error (content %q) — /readfile does not report misses as errors, so every negative assertion here would pass vacuously", + attackerActor, neverWritten, got.Content) + } + writeFile(t, ctx, rc, attackerActor, markerA) + if got := readFile(t, ctx, rc, attackerActor, markerA); got.Content != writtenByProbe { + t.Fatalf("control: attacker %s reading its OWN %s = %q (error %q), want %q — the probe cannot see its own durable data", + attackerActor, markerA, got.Content, got.Error, writtenByProbe) + } + + // Durable mount: the attacker's /data must be its own. The victim's + // marker is known to exist, so reading it through the attacker's mount + // means durable directories are shared or mis-keyed across reuse. + t.Run("VictimDurableMarkerNotVisibleThroughAttackerMount", func(t *testing.T) { + if got := readFile(t, ctx, rc, attackerActor, markerB); got.Error == "" { + t.Errorf("attacker %s read %s, the victim's durable marker in atespace %s, through its own mount: content %q — durable-dir volumes are shared across the worker-reuse boundary", + attackerActor, markerB, ns.b, got.Content) + } + }) + + // Host state: the mount is confined (above); this checks the sandbox + // does not expose the host directories that mount is built from. The paths + // come from internal/nodepath so they move with the worker layout; see + // actorHostPath. + t.Run("VictimHostStateNotReachableFromAttackerSandbox", func(t *testing.T) { + for _, probe := range []struct { + what string + path string + }{ + {"the victim's durable marker on the host", actorHostPath(victimUID, "durable-dir", durableVolume, filepath.Base(markerB))}, + {"the victim's projected actor identity", actorHostPath(victimUID, "system-info", "system-info", "actor-id")}, + {"the victim's sandbox asset record", actorHostPath(victimUID, "sandbox-assets.json")}, + } { + if got := readFile(t, ctx, rc, attackerActor, probe.path); got.Error == "" { + t.Errorf("attacker %s read %s (%s): content %q — the sandbox exposes the worker's per-actor host state across the reuse boundary", + attackerActor, probe.path, probe.what, got.Content) + } + } + + // A write is the more serious direction of the same boundary: reaching the + // victim's durable directory means an actor can corrupt another atespace's data. + injected := actorHostPath(victimUID, "durable-dir", durableVolume, "injected") + if got := writeFileAllowingFailure(t, ctx, rc, attackerActor, injected); got.Error == "" { + t.Errorf("attacker %s wrote %s, inside the victim's durable directory (atespace %s) on the worker — an actor can corrupt another atespace's stored data", + attackerActor, injected, ns.b) + } + }) + + // Tag scope. Runs last: it suspends the attacker to snapshot it. + t.Run("TagScopedToAtespaceCannotSeedAnotherAtespace", func(t *testing.T) { + testTagScopeBoundary(t, ctx, clients, rc, ns, attacker) + }) +} + +// actorHostPath is nodepath.ActorsDir// on the worker host. The +// leaves (durable-dir, system-info, sandbox-assets.json) mirror +// cmd/atelet/internal/ateletpath, which this suite cannot import. +func actorHostPath(actorUID string, elem ...string) string { + return filepath.Join(append([]string{nodepath.ActorsDir, actorUID}, elem...)...) +} + +// testTagScopeBoundary checks that an Actor cannot be seeded from a snapshot +// taken in another atespace. A snapshot carries the source's memory, root filesystem +// and durable data, so loading one across the boundary hands over everything. +// +// Substrate enforces this through the Tag's scope in ateapi's CreateActor. The +// negative case matches the error message, not just the code. +// +// source is a running actor in atespace A holding markerA; it is suspended here +// to produce the snapshot the tag captures. +func testTagScopeBoundary(t *testing.T, ctx context.Context, clients *e2e.Clients, rc *e2e.RouterClient, ns atespaces, source string) { + t.Helper() + + suspended, err := clients.SubstrateAPI.SuspendActor(ctx, &ateapipb.SuspendActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: ns.a, Name: source}, + }) + if err != nil { + t.Fatalf("SuspendActor %s/%s: %v", ns.a, source, err) + } + if durableSnapshotURI(suspended.GetActor().GetStatus()) == "" { + t.Fatalf("suspended actor %s/%s has no external snapshot, so there is nothing for a tag to capture", ns.a, source) + } + + tagRef := &ateapipb.ObjectRef{Atespace: ns.a, Name: "atespace-a-tag"} + t.Cleanup(func() { + _, _ = clients.SubstrateAPI.DeleteTag(context.Background(), &ateapipb.DeleteTagRequest{Tag: tagRef}) + }) + // Tags outlive the fixture namespace, so an interrupted run can leave one + // behind and wedge reruns on AlreadyExists. Clear it first, best-effort. + _, _ = clients.SubstrateAPI.DeleteTag(ctx, &ateapipb.DeleteTagRequest{Tag: tagRef}) + tag, err := clients.SubstrateAPI.CreateTag(ctx, &ateapipb.CreateTagRequest{ + Tag: &ateapipb.Tag{ + Metadata: &ateapipb.ResourceMetadata{Atespace: ns.a, Name: tagRef.GetName()}, + Scope: ateapipb.TagScope_TAG_SCOPE_ATESPACE, + SourceActor: &ateapipb.ObjectRef{Atespace: ns.a, Name: source}, + }, + }) + if err != nil { + t.Fatalf("CreateTag %s/%s: %v", ns.a, tagRef.GetName(), err) + } + if snapshotURI(tag.GetStatus().GetSnapshot()) == "" { + t.Fatalf("tag %s/%s has no snapshot uri, so it could seed an Actor in NO atespace and the negative case below would be vacuous", + ns.a, tagRef.GetName()) + } + + // The boundary: the other atespace may not name this tag. + crossRef := &ateapipb.ObjectRef{Atespace: ns.b, Name: "cross-atespace-clone"} + t.Cleanup(func() { + _, _ = clients.SubstrateAPI.DeleteActor(context.Background(), &ateapipb.DeleteActorRequest{Actor: crossRef}) + }) + _, err = clients.SubstrateAPI.CreateActor(ctx, &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: ns.b, Name: crossRef.GetName()}, + ActorTemplate: &ateapipb.ObjectRef{Atespace: ns.b, Name: templateName}, + SourceTag: tagRef, + }}) + switch { + case err == nil: + t.Errorf("CreateActor %s/%s seeded from tag %s/%s (scope ATESPACE) succeeded — an actor can be seeded from another atespace's snapshot", + ns.b, crossRef.GetName(), ns.a, tagRef.GetName()) + case status.Code(err) != codes.FailedPrecondition: + t.Errorf("CreateActor %s/%s seeded from tag %s/%s: got %v, want FailedPrecondition", + ns.b, crossRef.GetName(), ns.a, tagRef.GetName(), err) + case !strings.Contains(status.Convert(err).Message(), "not published outside its Atespace"): + // Rejected by some other precondition, not the scope check. + t.Errorf("CreateActor %s/%s seeded from tag %s/%s was rejected by a different precondition: %q; want the tag-scope rejection", + ns.b, crossRef.GetName(), ns.a, tagRef.GetName(), status.Convert(err).Message()) + } + + // Positive control: the same tag seeds an Actor in its own atespace and the + // clone carries the source's durable data. Otherwise an unusable tag would + // make the rejection above look like enforcement. + const clone = "same-atespace-clone" + cloneRef := &ateapipb.ObjectRef{Atespace: ns.a, Name: clone} + t.Cleanup(func() { + cleanupCtx := context.Background() + _, _ = clients.SubstrateAPI.SuspendActor(cleanupCtx, &ateapipb.SuspendActorRequest{Actor: cloneRef}) + _, _ = clients.SubstrateAPI.DeleteActor(cleanupCtx, &ateapipb.DeleteActorRequest{Actor: cloneRef}) + }) + if _, err := clients.SubstrateAPI.CreateActor(ctx, &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: ns.a, Name: clone}, + ActorTemplate: &ateapipb.ObjectRef{Atespace: ns.a, Name: templateName}, + SourceTag: tagRef, + }}); err != nil { + t.Fatalf("control: CreateActor %s/%s from tag %s in its own atespace: %v — the tag is unusable, so the cross-atespace rejection above proves nothing", + ns.a, clone, tagRef.GetName(), err) + } + if _, err := e2e.ResumeActorAwaitCapacity(t, ctx, clients, &ateapipb.ResumeActorRequest{Actor: cloneRef}); err != nil { + t.Fatalf("control: ResumeActor %s/%s: %v", ns.a, clone, err) + } + waitForActorState(t, ctx, clients, ns.a, clone, ateapipb.ActorState_ACTOR_STATE_RUNNING) + if got := readFile(t, ctx, rc, resources.ActorRef{Atespace: ns.a, Name: clone}, markerA); got.Content != writtenByProbe { + t.Fatalf("control: actor %s/%s restored from tag %s reads %s = %q (error %q), want %q — the tag carries no durable data, so the cross-atespace rejection above proves nothing", + ns.a, clone, tagRef.GetName(), markerA, got.Content, got.Error, writtenByProbe) + } +} + +// deployAtespaces installs both atespaces and returns their names. A goes +// first: it brings the only WorkerPool, and B's golden snapshot needs a worker +// to boot on. +func deployAtespaces(t *testing.T, ctx context.Context, clients *e2e.Clients, bucket string) atespaces { + t.Helper() + a, _ := e2e.DeploySubstrateFixture(t, ctx, clients, e2e.SubstrateFixtureManifests{ + Pool: "internal/e2e/fixtures/security/atespace-a-pool.yaml.tmpl", + Template: "internal/e2e/fixtures/security/atespace-template.yaml.tmpl", + }, bucket, "sec-a", false) + b, _ := e2e.DeploySubstrateFixture(t, ctx, clients, e2e.SubstrateFixtureManifests{ + Pool: "internal/e2e/fixtures/security/atespace-b-namespace.yaml.tmpl", + Template: "internal/e2e/fixtures/security/atespace-template.yaml.tmpl", + }, bucket, "sec-b", false) + if a == b { + t.Fatalf("both fixtures rendered to atespace %q — the fixture no longer isolates anything", a) + } + return atespaces{a: a, b: b} +} + +// requireReusedWorker fails unless the actor landed on priorWorker. The fixture +// runs one worker for both atespaces, so any other worker means the attacker never +// touched the host the victim left its state on and the test proves nothing. +func requireReusedWorker(t *testing.T, ctx context.Context, clients *e2e.Clients, atespace, name, priorWorker string) { + t.Helper() + got := assignedWorker(t, ctx, clients, atespace, name) + if got != priorWorker { + t.Fatalf("actor %s/%s landed on worker %q, not the worker %q the victim vacated; the fixture deploys one worker for both atespaces, so the reuse boundary under test is not being exercised", + atespace, name, got, priorWorker) + } + t.Logf("actor %s/%s reused worker %s", atespace, name, priorWorker) +} + +func assignedWorker(t *testing.T, ctx context.Context, clients *e2e.Clients, atespace, name string) string { + t.Helper() + actor, err := clients.SubstrateAPI.GetActor(ctx, &ateapipb.GetActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: atespace, Name: name}, + }) + if err != nil { + t.Fatalf("GetActor %s/%s: %v", atespace, name, err) + } + worker := actor.GetStatus().GetWorkerAssignment().GetWorker().GetName() + if worker == "" { + t.Fatalf("actor %s/%s has no worker assignment while RUNNING", atespace, name) + } + return worker +} + +func actorUID(t *testing.T, ctx context.Context, clients *e2e.Clients, atespace, name string) string { + t.Helper() + actor, err := clients.SubstrateAPI.GetActor(ctx, &ateapipb.GetActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: atespace, Name: name}, + }) + if err != nil { + t.Fatalf("GetActor %s/%s: %v", atespace, name, err) + } + uid := actor.GetMetadata().GetUid() + if uid == "" { + t.Fatalf("actor %s/%s has no uid", atespace, name) + } + return uid +} + +// createAndResumeActor creates an actor from its atespace's template and resumes +// it from the golden snapshot, removing it when the test ends. +func createAndResumeActor(t *testing.T, ctx context.Context, clients *e2e.Clients, atespace, name string) { + t.Helper() + ref := &ateapipb.ObjectRef{Atespace: atespace, Name: name} + // Actor records outlive the fixture namespace, so an interrupted run can + // leave one behind and wedge reruns on AlreadyExists. DeleteActor needs + // SUSPENDED or CRASHED, hence the suspend; both are best-effort. + _, _ = clients.SubstrateAPI.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: ref}) + _, _ = clients.SubstrateAPI.DeleteActor(ctx, &ateapipb.DeleteActorRequest{Actor: ref}) + if _, err := clients.SubstrateAPI.CreateActor(ctx, &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: atespace, Name: name}, + ActorTemplate: &ateapipb.ObjectRef{Atespace: atespace, Name: templateName}, + }}); err != nil { + t.Fatalf("CreateActor %s/%s: %v", atespace, name, err) + } + t.Cleanup(func() { + cleanupCtx := context.Background() + _, _ = clients.SubstrateAPI.SuspendActor(cleanupCtx, &ateapipb.SuspendActorRequest{Actor: ref}) + if _, err := clients.SubstrateAPI.DeleteActor(cleanupCtx, &ateapipb.DeleteActorRequest{Actor: ref}); err != nil { + t.Logf("cleanup: DeleteActor %s/%s failed, actor leaked (remove with: kubectl ate delete actor %s -a %s): %v", atespace, name, name, atespace, err) + } + }) + + if _, err := e2e.ResumeActorAwaitCapacity(t, ctx, clients, &ateapipb.ResumeActorRequest{Actor: ref}); err != nil { + t.Fatalf("ResumeActor %s/%s: %v", atespace, name, err) + } + waitForActorState(t, ctx, clients, atespace, name, ateapipb.ActorState_ACTOR_STATE_RUNNING) +} + +func waitForActorState(t *testing.T, ctx context.Context, clients *e2e.Clients, atespace, name string, want ateapipb.ActorState) { + t.Helper() + deadline := time.Now().Add(60 * time.Second) + for time.Now().Before(deadline) { + resp, err := clients.SubstrateAPI.GetActor(ctx, &ateapipb.GetActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: atespace, Name: name}, + }) + if err == nil && resp.GetStatus().GetState() == want { + return + } + time.Sleep(time.Second) + } + t.Fatalf("timed out waiting for actor %s/%s to reach state %v", atespace, name, want) +} + +// writeFile writes the probe's fixed body at path and fails the test if the +// write did not land, so a later negative assertion cannot pass on missing +// state. +func writeFile(t *testing.T, ctx context.Context, rc *e2e.RouterClient, actor resources.ActorRef, path string) { + t.Helper() + if got := writeFileAllowingFailure(t, ctx, rc, actor, path); got.OK != "true" { + t.Fatalf("actor %s writing %s: %q", actor, path, got.Error) + } +} + +// writeFileAllowingFailure is writeFile for assertions that expect the write +// to be refused; it returns the probe's report instead of failing. +func writeFileAllowingFailure(t *testing.T, ctx context.Context, rc *e2e.RouterClient, actor resources.ActorRef, path string) fileResponse { + t.Helper() + return probeFileOp(t, ctx, rc, actor, "/writefile", path) +} + +func readFile(t *testing.T, ctx context.Context, rc *e2e.RouterClient, actor resources.ActorRef, path string) fileResponse { + t.Helper() + return probeFileOp(t, ctx, rc, actor, "/readfile", path) +} + +// probeFileOp calls one of the probe's file endpoints. A transport or HTTP +// failure is fatal rather than returned: the probe never ran the operation, +// which is not the same as the actor being denied. +func probeFileOp(t *testing.T, ctx context.Context, rc *e2e.RouterClient, actor resources.ActorRef, endpoint, path string) fileResponse { + t.Helper() + resp, err := rc.Get(ctx, actor, endpoint+"?path="+url.QueryEscape(path)) + if err != nil { + t.Fatalf("GET %s for %s: %v", endpoint, actor, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + t.Fatalf("GET %s for %s: status %d, body %q", endpoint, actor, resp.StatusCode, body) + } + var out fileResponse + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + t.Fatalf("decoding %s for %s: %v", endpoint, actor, err) + } + return out +} + +func durableSnapshotURI(status *ateapipb.ActorStatus) string { + var best *ateapipb.Snapshot + for _, snap := range status.GetSnapshots() { + if uri := snapshotURI(snap); uri != "" { + if best == nil || snap.GetGeneration() > best.GetGeneration() { + best = snap + } + } + } + return snapshotURI(best) +} + +func snapshotURI(snap *ateapipb.Snapshot) string { + for _, st := range snap.GetStorage() { + if st.GetDurability() == ateapipb.SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE && + st.GetStatus() == ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED { + return st.GetObject().GetSnapshotUri() + } + } + return "" +} diff --git a/internal/e2e/suites/security/testmain_test.go b/internal/e2e/suites/security/testmain_test.go new file mode 100644 index 0000000000..3f30d513f4 --- /dev/null +++ b/internal/e2e/suites/security/testmain_test.go @@ -0,0 +1,24 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package security + +import ( + "os" + "testing" + + "github.com/agent-substrate/substrate/internal/e2e" +) + +func TestMain(m *testing.M) { os.Exit(e2e.RunTestMain(m)) } diff --git a/internal/e2e/template.go b/internal/e2e/template.go index 7a31a614bc..9f90201a3a 100644 --- a/internal/e2e/template.go +++ b/internal/e2e/template.go @@ -85,9 +85,9 @@ func CreateSubstrateTemplateFrom(ctx context.Context, t *testing.T, clients *Cli Labels: opts.Labels, }, Spec: v1alpha1.WorkerPoolSpec{ - Replicas: opts.PoolReplicas, - WorkerImage: existingWp.Spec.WorkerImage, - SandboxClass: existingWp.Spec.SandboxClass, + Replicas: opts.PoolReplicas, + WorkerImage: existingWp.Spec.WorkerImage, + SandboxClasses: existingWp.Spec.SandboxClasses, }, } if _, err := clients.SubstrateK8s.ApiV1alpha1().WorkerPools(namespace).Create(ctx, wp, metav1.CreateOptions{}); err != nil { diff --git a/internal/hardware/hardware.go b/internal/hardware/hardware.go index 56f1e6c2e3..4fab18123c 100644 --- a/internal/hardware/hardware.go +++ b/internal/hardware/hardware.go @@ -12,45 +12,86 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package hardware probes host hardware identity and evaluates snapshot -// hardware compatibility between workers and snapshots. +// Package hardware probes host hardware compatibility attributes and evaluates +// sandbox runtime compatibility between workers and snapshots. package hardware import ( "runtime" + "github.com/agent-substrate/substrate/internal/proto/ateletpb" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" ) const ( + // SchemaVersionV1 is the initial compatibility schema version. + SchemaVersionV1 = "v1" + // AttrArchitecture is the CPU architecture ("amd64", "arm64", etc.). AttrArchitecture = "architecture" - // TODO: Add AttrCPUVendor ("cpu_vendor"), AttrCPUModel ("cpu_model"), and - // other hardware attributes as snapshot compatibility expands. + // TODO: Add AttrCPUFeatures ("cpu_features"), AttrGVisorAssetHash + // ("gvisor_asset_hash"), and other compatibility attributes as snapshot + // compatibility expands. ) -// ProbeHost inspects the current host and returns its HardwareIdentity. +// ProbeHost inspects the current host and returns its v1 VersionedSandboxCompat +// in atelet wire format. // -// TODO: Probe and populate cpu_vendor, cpu_model, and other host hardware -// attributes (e.g. via CPUID on amd64 and MIDR_EL1 on arm64). -func ProbeHost() *ateapipb.HardwareIdentity { - return &ateapipb.HardwareIdentity{ - Attributes: map[string]string{ - AttrArchitecture: runtime.GOARCH, +// TODO: Probe and populate cpu_features and other host hardware attributes +// (e.g. via CPUID on amd64 and MIDR_EL1 on arm64). +func ProbeHost() *ateletpb.VersionedSandboxCompat { + return &ateletpb.VersionedSandboxCompat{ + SchemaVersion: SchemaVersionV1, + Attributes: []*ateletpb.AttributeEntry{ + { + Key: AttrArchitecture, + Value: runtime.GOARCH, + }, }, } } -// Matches reports whether worker satisfies the hardware identity recorded on -// snap. A nil or empty snapshot HardwareIdentity imposes no constraint. -// -// TODO: Distinguish memory-restore matching (full CPU vendor/model) from -// cold-boot fallback matching (architecture only) once additional hardware -// attributes are populated. -func Matches(worker, snap *ateapipb.HardwareIdentity) bool { - for k, v := range snap.GetAttributes() { - if worker.GetAttributes()[k] != v { +// Matches reports whether a worker's SandboxRuntime can restore a snapshot +// stamped with snap. A nil snapshot SandboxRuntime imposes no constraint. +// Otherwise the sandbox classes must be equal and MatchesCompat must hold. +func Matches(worker, snap *ateapipb.SandboxRuntime) bool { + if snap == nil { + return true + } + if worker == nil || worker.GetSandboxClass() != snap.GetSandboxClass() { + return false + } + return MatchesCompat(worker.GetVersion(), snap.GetVersion()) +} + +// MatchesCompat reports whether worker can restore a snapshot stamped with +// snap: same non-empty schema version, and every attribute on snap present on +// worker with an equal value. Keys only the worker has are ignored; a new +// schema version is how older snapshots are excluded. +func MatchesCompat(worker, snap *ateapipb.VersionedSandboxCompat) bool { + if worker == nil || snap == nil { + return false + } + if worker.GetSchemaVersion() == "" || worker.GetSchemaVersion() != snap.GetSchemaVersion() { + return false + } + wAttrs := worker.GetAttributes() + sAttrs := snap.GetAttributes() + // API validation bounds attributes to at most 32 entries with unique keys, + // so the nested scan is small and avoids map allocation on the hot path. + for _, sa := range sAttrs { + found := false + for _, wa := range wAttrs { + if wa.GetKey() == sa.GetKey() { + if wa.GetValue() != sa.GetValue() { + return false + } + found = true + break + } + } + if !found { return false } } diff --git a/internal/hardware/hardware_test.go b/internal/hardware/hardware_test.go index 3fb46757cb..5f0da993b3 100644 --- a/internal/hardware/hardware_test.go +++ b/internal/hardware/hardware_test.go @@ -22,27 +22,68 @@ import ( ) func TestProbeHost(t *testing.T) { - hw := ProbeHost() - if got := hw.GetAttributes()[AttrArchitecture]; got != runtime.GOARCH { - t.Errorf("ProbeHost() architecture = %q, want %q", got, runtime.GOARCH) + compat := ProbeHost() + if got := compat.GetSchemaVersion(); got != SchemaVersionV1 { + t.Errorf("ProbeHost() schema_version = %q, want %q", got, SchemaVersionV1) + } + attrs := compat.GetAttributes() + if len(attrs) != 1 || attrs[0].GetKey() != AttrArchitecture || attrs[0].GetValue() != runtime.GOARCH { + t.Errorf("ProbeHost() attributes = %v, want [{%s: %s}]", attrs, AttrArchitecture, runtime.GOARCH) } } func TestMatches(t *testing.T) { - amd64Worker := &ateapipb.HardwareIdentity{Attributes: map[string]string{AttrArchitecture: "amd64"}} - arm64Worker := &ateapipb.HardwareIdentity{Attributes: map[string]string{AttrArchitecture: "arm64"}} + runtimeWith := func(class, ver string, attrs ...*ateapipb.AttributeEntry) *ateapipb.SandboxRuntime { + return &ateapipb.SandboxRuntime{ + SandboxClass: class, + Version: &ateapipb.VersionedSandboxCompat{ + SchemaVersion: ver, + Attributes: attrs, + }, + } + } + amd64GVisor := runtimeWith("gvisor", "v1", + &ateapipb.AttributeEntry{Key: AttrArchitecture, Value: "amd64"}, + &ateapipb.AttributeEntry{Key: "cpu_features", Value: "GenuineIntel"}, + ) + amd64GVisorReordered := runtimeWith("gvisor", "v1", + &ateapipb.AttributeEntry{Key: "cpu_features", Value: "GenuineIntel"}, + &ateapipb.AttributeEntry{Key: AttrArchitecture, Value: "amd64"}, + ) + arm64GVisor := runtimeWith("gvisor", "v1", + &ateapipb.AttributeEntry{Key: AttrArchitecture, Value: "arm64"}, + &ateapipb.AttributeEntry{Key: "cpu_features", Value: "GenuineIntel"}, + ) + amd64MicroVM := runtimeWith("microvm", "v1", + &ateapipb.AttributeEntry{Key: AttrArchitecture, Value: "amd64"}, + &ateapipb.AttributeEntry{Key: "cpu_features", Value: "GenuineIntel"}, + ) + amd64GVisorV2 := runtimeWith("gvisor", "v2", + &ateapipb.AttributeEntry{Key: AttrArchitecture, Value: "amd64"}, + &ateapipb.AttributeEntry{Key: "cpu_features", Value: "GenuineIntel"}, + ) + amd64GVisorExtraAttr := runtimeWith("gvisor", "v1", + &ateapipb.AttributeEntry{Key: AttrArchitecture, Value: "amd64"}, + &ateapipb.AttributeEntry{Key: "cpu_features", Value: "GenuineIntel"}, + &ateapipb.AttributeEntry{Key: "gvisor_asset_hash", Value: "abc"}, + ) tests := []struct { name string - worker *ateapipb.HardwareIdentity - snap *ateapipb.HardwareIdentity + worker *ateapipb.SandboxRuntime + snap *ateapipb.SandboxRuntime want bool }{ - {"nil snapshot imposes no constraint", amd64Worker, nil, true}, - {"empty snapshot imposes no constraint", amd64Worker, &ateapipb.HardwareIdentity{}, true}, - {"same architecture matches", amd64Worker, amd64Worker, true}, - {"different architecture fails", arm64Worker, amd64Worker, false}, - {"nil worker fails when snapshot is stamped", nil, amd64Worker, false}, + {"nil snapshot imposes no constraint", amd64GVisor, nil, true}, + {"exact match", amd64GVisor, amd64GVisor, true}, + {"attribute order ignored", amd64GVisor, amd64GVisorReordered, true}, + {"different attribute value fails", arm64GVisor, amd64GVisor, false}, + {"different sandbox class fails", amd64MicroVM, amd64GVisor, false}, + {"different schema version fails", amd64GVisorV2, amd64GVisor, false}, + {"attribute the snapshot predates is not checked", amd64GVisorExtraAttr, amd64GVisor, true}, + {"attribute missing on worker fails", amd64GVisor, amd64GVisorExtraAttr, false}, + {"nil worker fails when snapshot is stamped", nil, amd64GVisor, false}, + {"nil version fails", &ateapipb.SandboxRuntime{SandboxClass: "gvisor"}, amd64GVisor, false}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { @@ -52,3 +93,38 @@ func TestMatches(t *testing.T) { }) } } + +func TestMatchesCompat(t *testing.T) { + compat := func(ver string, attrs ...*ateapipb.AttributeEntry) *ateapipb.VersionedSandboxCompat { + return &ateapipb.VersionedSandboxCompat{SchemaVersion: ver, Attributes: attrs} + } + amd64 := &ateapipb.AttributeEntry{Key: AttrArchitecture, Value: "amd64"} + arm64 := &ateapipb.AttributeEntry{Key: AttrArchitecture, Value: "arm64"} + cpu := &ateapipb.AttributeEntry{Key: "cpu_features", Value: "GenuineIntel"} + + tests := []struct { + name string + worker, snap *ateapipb.VersionedSandboxCompat + want bool + }{ + {"equal", compat("v1", amd64), compat("v1", amd64), true}, + {"key only the worker has is ignored", compat("v1", amd64, cpu), compat("v1", amd64), true}, + {"key only the snapshot has fails", compat("v1", amd64), compat("v1", amd64, cpu), false}, + {"value differs", compat("v1", arm64), compat("v1", amd64), false}, + {"schema version differs", compat("v2", amd64), compat("v1", amd64), false}, + {"both schema versions empty", compat("", amd64), compat("", amd64), false}, + {"no attributes on either", compat("v1"), compat("v1"), true}, + // Unlike Matches, a nil snapshot identity here is a stamped snapshot + // missing its identity, not the absence of a constraint. + {"nil snapshot identity", compat("v1", amd64), nil, false}, + {"nil worker identity", nil, compat("v1", amd64), false}, + {"both nil", nil, nil, false}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := MatchesCompat(tc.worker, tc.snap); got != tc.want { + t.Errorf("MatchesCompat() = %v, want %v", got, tc.want) + } + }) + } +} diff --git a/internal/imagecache/README.md b/internal/imagecache/README.md index 63492fabc5..1b81f4c272 100644 --- a/internal/imagecache/README.md +++ b/internal/imagecache/README.md @@ -107,7 +107,7 @@ layer diffIDs in order — layers shared by N images exist once. requested digest (and the per-platform child digest for multi-arch refs). `prepareOCIDirectory` in atelet then writes `rootfs-overlay.json` -(`OverlaySpec`) into the bundle next to `config.json`, listing the layer +(`OverlaySpec`) into the bundle, listing the layer directories bottom-first plus any `ExtraDirs` (in-rootfs bind-mount targets, e.g. the actor identity mount at `/run/ate`), and creates the empty bundle-local `rootfs/`, `upper/`, and `work/` directories. diff --git a/internal/imagecache/imagecache.go b/internal/imagecache/imagecache.go index 977c077135..16793503ca 100644 --- a/internal/imagecache/imagecache.go +++ b/internal/imagecache/imagecache.go @@ -21,7 +21,7 @@ // // - atelet (plain root, all capabilities dropped) pulls layers and unpacks // them into the pool (Store.EnsureImage), and writes a rootfs-overlay.json -// next to each bundle's config.json (WriteSpec). Whiteout entries are +// into each bundle (WriteSpec). Whiteout entries are // recorded in per-layer metadata rather than materialized, because // overlayfs whiteouts are char devices (CAP_MKNOD) with trusted.* xattrs // for opaque dirs (CAP_SYS_ADMIN). diff --git a/internal/imagecache/spec.go b/internal/imagecache/spec.go index 47c2f5db13..e1e0873440 100644 --- a/internal/imagecache/spec.go +++ b/internal/imagecache/spec.go @@ -25,16 +25,16 @@ import ( ) // ImageVolumeMountPath is where one image volume is composed inside a -// bundle. atelet writes it into config.json as the bind source; the ateom -// mounts the volume there (see setupImageVolumes). +// bundle. The ateom mounts the volume there (see setupImageVolumes) and binds +// it into the container. func ImageVolumeMountPath(bundlePath, volumeName string) string { return filepath.Join(bundlePath, "volumes", volumeName) } -// OverlaySpecFileName is the file atelet writes into each container bundle, -// next to config.json, describing how to compose the bundle's rootfs from -// cached layers. Its absence means the bundle's rootfs is a plain directory -// (e.g. one prepared by a pre-imagecache atelet) and needs no mount. +// OverlaySpecFileName is the file atelet writes into each container bundle +// describing how to compose the bundle's rootfs from cached layers. Its +// absence means the bundle's rootfs is a plain directory (e.g. one prepared by +// a pre-imagecache atelet) and needs no mount. const OverlaySpecFileName = "rootfs-overlay.json" // OverlaySpec is the contract between atelet (which cannot mount) and the diff --git a/internal/localjwtauthority/localjwtauthority.go b/internal/localjwtauthority/localjwtauthority.go index 8db9683fbd..6e3fc5adc2 100644 --- a/internal/localjwtauthority/localjwtauthority.go +++ b/internal/localjwtauthority/localjwtauthority.go @@ -27,6 +27,7 @@ import ( "fmt" "hash" "os" + "slices" "sync" "time" @@ -189,6 +190,56 @@ func (p *ConcretePool) VerificationKeys() ([]*VerificationKey, error) { return keys, nil } +// AddAuthority adds authority to the pool without activating it, so relying +// parties can learn its key before it signs anything. +func (p *ConcretePool) AddAuthority(authority *Authority) error { + if authority.ID == "" { + return fmt.Errorf("authority has no ID") + } + if p.index(authority.ID) >= 0 { + return fmt.Errorf("authority %q already present", authority.ID) + } + p.Authorities = append(p.Authorities, authority) + return nil +} + +// Activate makes the authority with id the one that signs. +func (p *ConcretePool) Activate(id string) error { + if p.index(id) < 0 { + return fmt.Errorf("authority %q not present", id) + } + p.ActiveForSigning = id + return nil +} + +// RemoveAuthority removes the authority with id, which must not be the one +// that signs. +func (p *ConcretePool) RemoveAuthority(id string) error { + i := p.index(id) + if i < 0 { + return fmt.Errorf("authority %q not present", id) + } + if id == p.ActiveID() { + return fmt.Errorf("authority %q is active for signing", id) + } + p.Authorities = slices.Delete(p.Authorities, i, i+1) + return nil +} + +// ActiveID returns the ID of the authority that signs: ActiveForSigning, or +// the first authority's when none is designated. +func (p *ConcretePool) ActiveID() string { + if p.ActiveForSigning == "" && len(p.Authorities) > 0 { + return p.Authorities[0].ID + } + return p.ActiveForSigning +} + +// index returns the position of the authority with id, or -1. +func (p *ConcretePool) index(id string) int { + return slices.IndexFunc(p.Authorities, func(a *Authority) bool { return a.ID == id }) +} + type wireHeader struct { Type string `json:"typ,omitempty"` Algorithm string `json:"alg,omitempty"` @@ -338,6 +389,10 @@ func Unmarshal(wireBytes []byte) (*ConcretePool, error) { return pool, nil } +// rsaKeyBits is the size of the RSA keys GenerateAuthority makes. RS256 does +// not fix one, and smaller keys are too weak. +const rsaKeyBits = 4096 + // GenerateAuthority generates a JWT signing key for algorithm, which must be // RS256 or ES256. An empty id defaults to the Thumbprint of the public key. func GenerateAuthority(algorithm, id string) (*Authority, error) { @@ -345,7 +400,7 @@ func GenerateAuthority(algorithm, id string) (*Authority, error) { var err error switch algorithm { case "RS256": - key, err = rsa.GenerateKey(rand.Reader, 2048) + key, err = rsa.GenerateKey(rand.Reader, rsaKeyBits) case "ES256": key, err = ecdsa.GenerateKey(elliptic.P256(), rand.Reader) default: diff --git a/internal/localjwtauthority/localjwtauthority_test.go b/internal/localjwtauthority/localjwtauthority_test.go index ee5cd4fc2c..4b3e8e5243 100644 --- a/internal/localjwtauthority/localjwtauthority_test.go +++ b/internal/localjwtauthority/localjwtauthority_test.go @@ -190,6 +190,122 @@ func TestSignJWTHeader(t *testing.T) { } } +func authorityIDs(p *ConcretePool) []string { + var ids []string + for _, a := range p.Authorities { + ids = append(ids, a.ID) + } + return ids +} + +func testPool(t *testing.T, ids ...string) *ConcretePool { + t.Helper() + pool := &ConcretePool{ActiveForSigning: ids[0]} + for _, id := range ids { + authority, err := GenerateAuthority("ES256", id) + if err != nil { + t.Fatal(err) + } + pool.Authorities = append(pool.Authorities, authority) + } + return pool +} + +func TestPoolRotation(t *testing.T) { + pool := testPool(t, "old") + next, err := GenerateAuthority("RS256", "next") + if err != nil { + t.Fatal(err) + } + + if err := pool.AddAuthority(next); err != nil { + t.Fatalf("AddAuthority: %v", err) + } + if pool.ActiveForSigning != "old" { + t.Errorf("after AddAuthority, active = %q, want old", pool.ActiveForSigning) + } + if err := pool.Activate("next"); err != nil { + t.Fatalf("Activate: %v", err) + } + if err := pool.RemoveAuthority("old"); err != nil { + t.Fatalf("RemoveAuthority: %v", err) + } + + if diff := cmp.Diff([]string{"next"}, authorityIDs(pool)); diff != "" { + t.Errorf("authorities (-want +got):\n%s", diff) + } + jwt, err := pool.SignJWT(&actoridjwt.Claims{Subject: "actor/a/b", Audiences: []string{"aud"}}) + if err != nil { + t.Fatalf("SignJWT: %v", err) + } + header, err := base64.RawURLEncoding.DecodeString(strings.Split(jwt, ".")[0]) + if err != nil { + t.Fatal(err) + } + if want := `{"typ":"JWT","alg":"RS256","kid":"next"}`; string(header) != want { + t.Errorf("header = %s, want %s", header, want) + } +} + +func TestPoolRotationRejects(t *testing.T) { + dup, err := GenerateAuthority("ES256", "1") + if err != nil { + t.Fatal(err) + } + noID := &Authority{Algorithm: "ES256", SigningKey: dup.SigningKey} + + for _, tc := range []struct { + name string + pool *ConcretePool + change func(*ConcretePool) error + }{ + {name: "add duplicate ID", pool: testPool(t, "1", "2"), change: func(p *ConcretePool) error { return p.AddAuthority(dup) }}, + {name: "add without ID", pool: testPool(t, "1"), change: func(p *ConcretePool) error { return p.AddAuthority(noID) }}, + {name: "activate absent", pool: testPool(t, "1"), change: func(p *ConcretePool) error { return p.Activate("2") }}, + {name: "remove absent", pool: testPool(t, "1"), change: func(p *ConcretePool) error { return p.RemoveAuthority("2") }}, + {name: "remove active", pool: testPool(t, "1", "2"), change: func(p *ConcretePool) error { return p.RemoveAuthority("1") }}, + {name: "remove first with none designated", pool: func() *ConcretePool { + p := testPool(t, "1", "2") + p.ActiveForSigning = "" + return p + }(), change: func(p *ConcretePool) error { return p.RemoveAuthority("1") }}, + } { + t.Run(tc.name, func(t *testing.T) { + wantIDs, wantActive := authorityIDs(tc.pool), tc.pool.ActiveForSigning + if err := tc.change(tc.pool); err == nil { + t.Error("got nil error") + } + if diff := cmp.Diff(wantIDs, authorityIDs(tc.pool)); diff != "" { + t.Errorf("authorities changed (-want +got):\n%s", diff) + } + if tc.pool.ActiveForSigning != wantActive { + t.Errorf("active = %q, want %q", tc.pool.ActiveForSigning, wantActive) + } + }) + } +} + +func TestPoolActiveID(t *testing.T) { + designated := testPool(t, "1", "2") + designated.ActiveForSigning = "2" + undesignated := testPool(t, "1", "2") + undesignated.ActiveForSigning = "" + + for _, tc := range []struct { + name string + pool *ConcretePool + want string + }{ + {name: "designated", pool: designated, want: "2"}, + {name: "none designated", pool: undesignated, want: "1"}, + {name: "empty", pool: &ConcretePool{}, want: ""}, + } { + if got := tc.pool.ActiveID(); got != tc.want { + t.Errorf("%s: ActiveID() = %q, want %q", tc.name, got, tc.want) + } + } +} + func TestGenerateAuthority(t *testing.T) { for _, alg := range []string{"RS256", "ES256"} { t.Run(alg, func(t *testing.T) { @@ -209,8 +325,8 @@ func TestGenerateAuthority(t *testing.T) { } switch key := authority.SigningKey.(type) { case *rsa.PrivateKey: - if alg != "RS256" || key.N.BitLen() != 2048 { - t.Errorf("got a %d-bit RSA key for %s, want 2048-bit for RS256", key.N.BitLen(), alg) + if alg != "RS256" || key.N.BitLen() != 4096 { + t.Errorf("got a %d-bit RSA key for %s, want 4096-bit for RS256", key.N.BitLen(), alg) } case *ecdsa.PrivateKey: if alg != "ES256" || key.Curve != elliptic.P256() { diff --git a/internal/objectstore/objectstore.go b/internal/objectstore/objectstore.go index 671e580647..59675e157b 100644 --- a/internal/objectstore/objectstore.go +++ b/internal/objectstore/objectstore.go @@ -41,6 +41,9 @@ import ( // their round trips rather than about throughput. const prefixConcurrency = 8 +// ErrEmptySource reports that CopyPrefix found no objects to copy. +var ErrEmptySource = errors.New("no objects to copy") + // Store is the object-storage surface the control plane needs. Every method // addresses objects by name only. type Store interface { @@ -120,7 +123,7 @@ func CopyPrefix(ctx context.Context, s Store, src, dst resources.StoragePrefix) // leave the destination naming an external snapshot that cannot be // restored from. if len(objects) == 0 { - return fmt.Errorf("external snapshot %s has no objects to copy", src) + return fmt.Errorf("external snapshot %s: %w", src, ErrEmptySource) } group, ctx := errgroup.WithContext(ctx) group.SetLimit(prefixConcurrency) diff --git a/internal/objectstoreplugin/calls_test.go b/internal/objectstoreplugin/calls_test.go new file mode 100644 index 0000000000..dd272f9919 --- /dev/null +++ b/internal/objectstoreplugin/calls_test.go @@ -0,0 +1,41 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package objectstoreplugin + +import ( + "errors" + "testing" + + "github.com/agent-substrate/substrate/internal/apierror" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func TestCallError(t *testing.T) { + if err := CallError(nil); err != nil { + t.Errorf("CallError(nil) = %v, want nil", err) + } + unavailable := status.Error(codes.Unavailable, "connection refused") + if got := apierror.Code(CallError(unavailable)); got != codes.Unavailable { + t.Errorf("apierror.Code(CallError(Unavailable)) = %s, want %s", got, codes.Unavailable) + } + for _, code := range []codes.Code{codes.NotFound, codes.Internal, codes.FailedPrecondition} { + in := status.Error(code, "plugin error") + got := CallError(in) + if !errors.Is(got, in) || status.Code(got) != code { + t.Errorf("CallError(%s) = %v, want the plugin's status unchanged", code, got) + } + } +} diff --git a/internal/objectstoreplugin/control.go b/internal/objectstoreplugin/control.go new file mode 100644 index 0000000000..69b46787f4 --- /dev/null +++ b/internal/objectstoreplugin/control.go @@ -0,0 +1,65 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package objectstoreplugin + +import ( + "context" + + "github.com/agent-substrate/substrate/internal/objectstore" + "github.com/agent-substrate/substrate/internal/resources" + objectstorev1 "github.com/agent-substrate/substrate/pkg/proto/objectstorepb/v1" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// ControlPlugin serves ControlProvider on an object store. +type ControlPlugin struct { + objectstorev1.UnimplementedControlProviderServer + + store objectstore.Store +} + +// NewControlPlugin returns a ControlPlugin backed by store. +func NewControlPlugin(store objectstore.Store) *ControlPlugin { + return &ControlPlugin{store: store} +} + +// CleanupSnapshot deletes every object under the given prefix. +func (p *ControlPlugin) CleanupSnapshot(ctx context.Context, req *objectstorev1.CleanupSnapshotRequest) (*objectstorev1.CleanupSnapshotResponse, error) { + prefix, err := resources.ParseStoragePrefix(req.GetSnapshotUri()) + if err != nil { + return nil, status.Error(codes.InvalidArgument, err.Error()) + } + if err := objectstore.DeletePrefix(ctx, p.store, prefix); err != nil { + return nil, toStatus(err) + } + return &objectstorev1.CleanupSnapshotResponse{}, nil +} + +// CopySnapshot copies every object of src_uri to dst_uri. +func (p *ControlPlugin) CopySnapshot(ctx context.Context, req *objectstorev1.CopySnapshotRequest) (*objectstorev1.CopySnapshotResponse, error) { + src, err := resources.ParseStoragePrefix(req.GetSrcUri()) + if err != nil { + return nil, status.Error(codes.InvalidArgument, err.Error()) + } + dst, err := resources.ParseStoragePrefix(req.GetDstUri()) + if err != nil { + return nil, status.Error(codes.InvalidArgument, err.Error()) + } + if err := objectstore.CopyPrefix(ctx, p.store, src, dst); err != nil { + return nil, toStatus(err) + } + return &objectstorev1.CopySnapshotResponse{}, nil +} diff --git a/internal/objectstoreplugin/node.go b/internal/objectstoreplugin/node.go new file mode 100644 index 0000000000..6c7f8fa35d --- /dev/null +++ b/internal/objectstoreplugin/node.go @@ -0,0 +1,216 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package objectstoreplugin implements the snapshot plugin API on GCS or S3 +// object storage, using the plugin's own credentials. +// +// Snapshot files are stored as zstd-compressed objects named .zstd +// under the snapshot URI, except the manifest, which is stored as is. This is +// the layout atelet has always written, so snapshots taken before the plugin +// existed stay readable. +package objectstoreplugin + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + + "github.com/agent-substrate/substrate/internal/resources" + "github.com/agent-substrate/substrate/pkg/objectstorage" + objectstorev1 "github.com/agent-substrate/substrate/pkg/proto/objectstorepb/v1" + "golang.org/x/sync/errgroup" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// This plugin stores each file as .zstd, except manifestFile, which it +// stores as-is so that snapshots written before atelet used this plugin stay +// readable. +const manifestFile = "manifest.json" + +// NodePlugin serves NodeProvider on an object storage client. +type NodePlugin struct { + objectstorev1.UnimplementedNodeProviderServer + + gcsClient objectstorage.ObjectStorage + // root confines every local path a caller names. + root string +} + +// NewNodePlugin returns a NodePlugin that reads and writes local files only +// below root. +func NewNodePlugin(gcsClient objectstorage.ObjectStorage, root string) (*NodePlugin, error) { + if !filepath.IsAbs(root) { + return nil, fmt.Errorf("root %q is not an absolute path", root) + } + return &NodePlugin{gcsClient: gcsClient, root: filepath.Clean(root)}, nil +} + +// FetchSnapshot downloads the requested snapshot files into write_path. +func (p *NodePlugin) FetchSnapshot(ctx context.Context, req *objectstorev1.FetchSnapshotRequest) (*objectstorev1.FetchSnapshotResponse, error) { + uri, dstDir, err := p.validate(req.GetSnapshotUri(), req.GetWritePath(), req.GetFiles()) + if err != nil { + return nil, err + } + root, err := p.openDir(dstDir) + if err != nil { + return nil, toStatus(fmt.Errorf("while opening restore directory: %w", err)) + } + defer root.Close() + + g, gCtx := errgroup.WithContext(ctx) + for _, fileName := range req.GetFiles() { + fileName := fileName + g.Go(func() error { + if fileName == manifestFile { + return p.fetchManifest(gCtx, uri, root) + } + objectURI, err := uri.ObjectURI(fileName + ".zstd") + if err != nil { + return fmt.Errorf("while addressing %s in GCS: %w", fileName, err) + } + local, err := root.OpenFile(fileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600) + if err != nil { + return fmt.Errorf("while opening %s in restore directory: %w", fileName, err) + } + fetchErr := objectstorage.FetchFileFromGCSWithZstd(gCtx, p.gcsClient, objectURI, local) + closeErr := local.Close() + if err := errors.Join(fetchErr, closeErr); err != nil { + return fmt.Errorf("while downloading %s from GCS: %w", fileName, err) + } + return nil + }) + } + if err := g.Wait(); err != nil { + return nil, toStatus(err) + } + return &objectstorev1.FetchSnapshotResponse{}, nil +} + +// UploadSnapshot uploads the requested files from local_path into the +// snapshot. +func (p *NodePlugin) UploadSnapshot(ctx context.Context, req *objectstorev1.UploadSnapshotRequest) (*objectstorev1.UploadSnapshotResponse, error) { + uri, srcDir, err := p.validate(req.GetSnapshotUri(), req.GetLocalPath(), req.GetFiles()) + if err != nil { + return nil, err + } + root, err := p.openDir(srcDir) + if err != nil { + return nil, toStatus(fmt.Errorf("while opening snapshot directory: %w", err)) + } + defer root.Close() + + g, gCtx := errgroup.WithContext(ctx) + for _, fileName := range req.GetFiles() { + g.Go(func() error { + local, err := root.Open(fileName) + if err != nil { + return fmt.Errorf("while opening %s in snapshot directory: %w", fileName, err) + } + defer local.Close() + info, err := local.Stat() + if err != nil { + return fmt.Errorf("while inspecting %s in snapshot directory: %w", fileName, err) + } + if !info.Mode().IsRegular() { + return fmt.Errorf("snapshot file %s is not a regular file", fileName) + } + if fileName == manifestFile { + return p.uploadManifest(gCtx, uri, local) + } + + objectURI, err := uri.ObjectURI(fileName + ".zstd") + if err != nil { + return fmt.Errorf("while addressing %s in GCS: %w", fileName, err) + } + if err := objectstorage.SendFileToGCSWithZstd(gCtx, p.gcsClient, objectURI, local); err != nil { + return fmt.Errorf("while uploading %s to GCS: %w", fileName, err) + } + return nil + }) + } + if err := g.Wait(); err != nil { + return nil, toStatus(err) + } + return &objectstorev1.UploadSnapshotResponse{}, nil +} + +// fetchManifest downloads the uncompressed manifest into root. +func (p *NodePlugin) fetchManifest(ctx context.Context, uri resources.SnapshotURI, root *os.Root) error { + manifestURI, err := uri.ObjectURI(manifestFile) + if err != nil { + return err + } + manifest, err := objectstorage.FetchFromGCS(ctx, p.gcsClient, manifestURI) + if err != nil { + return fmt.Errorf("while fetching snapshot manifest: %w", err) + } + if err := root.WriteFile(manifestFile, manifest, 0o600); err != nil { + return fmt.Errorf("while writing %s in restore directory: %w", manifestFile, err) + } + return nil +} + +// uploadManifest uploads the manifest from local uncompressed. +func (p *NodePlugin) uploadManifest(ctx context.Context, uri resources.SnapshotURI, local *os.File) error { + manifest, err := io.ReadAll(local) + if err != nil { + return fmt.Errorf("while reading %s in snapshot directory: %w", manifestFile, err) + } + manifestURI, err := uri.ObjectURI(manifestFile) + if err != nil { + return fmt.Errorf("while addressing snapshot manifest in GCS: %w", err) + } + if err := objectstorage.SendBytesToGCS(ctx, p.gcsClient, manifestURI, manifest); err != nil { + return fmt.Errorf("while uploading snapshot manifest: %w", err) + } + return nil +} + +// validate checks a request's snapshot URI, local directory and file names, +// returning the parsed URI and the local directory relative to p.root. +func (p *NodePlugin) validate(snapshotURI, dir string, files []string) (resources.SnapshotURI, string, error) { + uri, err := resources.ParseSnapshotURI(snapshotURI) + if err != nil { + return resources.SnapshotURI{}, "", status.Error(codes.InvalidArgument, err.Error()) + } + if !filepath.IsAbs(dir) { + return resources.SnapshotURI{}, "", status.Errorf(codes.InvalidArgument, "local path %q is not absolute", dir) + } + rel, err := filepath.Rel(p.root, dir) + if err != nil || !filepath.IsLocal(rel) { + return resources.SnapshotURI{}, "", status.Errorf(codes.InvalidArgument, "local path %q is outside %q", dir, p.root) + } + if len(files) == 0 { + return resources.SnapshotURI{}, "", status.Error(codes.InvalidArgument, "no files requested") + } + if err := resources.ValidateSnapshotFileNames(files); err != nil { + return resources.SnapshotURI{}, "", status.Error(codes.InvalidArgument, err.Error()) + } + return uri, rel, nil +} + +// openDir opens rel, a directory below p.root, through an os.Root at p.root, +// so a symlink anywhere on the path cannot lead outside p.root. +func (p *NodePlugin) openDir(rel string) (*os.Root, error) { + base, err := os.OpenRoot(p.root) + if err != nil { + return nil, err + } + defer base.Close() + return base.OpenRoot(rel) +} diff --git a/internal/objectstoreplugin/objectstoreplugintest/objectstoreplugintest.go b/internal/objectstoreplugin/objectstoreplugintest/objectstoreplugintest.go new file mode 100644 index 0000000000..1e4b592d2e --- /dev/null +++ b/internal/objectstoreplugin/objectstoreplugintest/objectstoreplugintest.go @@ -0,0 +1,61 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package objectstoreplugintest provides snapshot plugin clients that call the +// object-store plugins directly, without a socket, for tests that drive atelet +// or ate-api-server through the real plugin. +package objectstoreplugintest + +import ( + "context" + + "github.com/agent-substrate/substrate/internal/objectstore" + "github.com/agent-substrate/substrate/internal/objectstoreplugin" + objectstorev1 "github.com/agent-substrate/substrate/pkg/proto/objectstorepb/v1" + "google.golang.org/grpc" +) + +// NodeClient returns a client that calls p directly. +func NodeClient(p *objectstoreplugin.NodePlugin) objectstorev1.NodeProviderClient { + return nodeClient{p} +} + +type nodeClient struct { + p *objectstoreplugin.NodePlugin +} + +func (c nodeClient) FetchSnapshot(ctx context.Context, in *objectstorev1.FetchSnapshotRequest, _ ...grpc.CallOption) (*objectstorev1.FetchSnapshotResponse, error) { + return c.p.FetchSnapshot(ctx, in) +} + +func (c nodeClient) UploadSnapshot(ctx context.Context, in *objectstorev1.UploadSnapshotRequest, _ ...grpc.CallOption) (*objectstorev1.UploadSnapshotResponse, error) { + return c.p.UploadSnapshot(ctx, in) +} + +// ControlClient returns a client that calls a ControlPlugin on store directly. +func ControlClient(store objectstore.Store) objectstorev1.ControlProviderClient { + return controlClient{objectstoreplugin.NewControlPlugin(store)} +} + +type controlClient struct { + p *objectstoreplugin.ControlPlugin +} + +func (c controlClient) CleanupSnapshot(ctx context.Context, in *objectstorev1.CleanupSnapshotRequest, _ ...grpc.CallOption) (*objectstorev1.CleanupSnapshotResponse, error) { + return c.p.CleanupSnapshot(ctx, in) +} + +func (c controlClient) CopySnapshot(ctx context.Context, in *objectstorev1.CopySnapshotRequest, _ ...grpc.CallOption) (*objectstorev1.CopySnapshotResponse, error) { + return c.p.CopySnapshot(ctx, in) +} diff --git a/internal/objectstoreplugin/plugin_test.go b/internal/objectstoreplugin/plugin_test.go new file mode 100644 index 0000000000..eafac6f033 --- /dev/null +++ b/internal/objectstoreplugin/plugin_test.go @@ -0,0 +1,459 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package objectstoreplugin + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "testing" + "time" + + "github.com/agent-substrate/substrate/pkg/objectstorage" + objectstorev1 "github.com/agent-substrate/substrate/pkg/proto/objectstorepb/v1" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/health" + healthpb "google.golang.org/grpc/health/grpc_health_v1" + "google.golang.org/grpc/status" +) + +const ( + testBucket = "bucket" + testPrefix = "root/atespaces/team-a/actors/uid1/snapshots/snap1" + testURI = "gs://" + testBucket + "/" + testPrefix +) + +// memObjects is an in-memory backend implementing both objectstorage.ObjectStorage +// and objectstore.Store, keyed by "bucket/object". +type memObjects struct { + mu sync.Mutex + m map[string][]byte +} + +func newMemObjects() *memObjects { return &memObjects{m: map[string][]byte{}} } + +func (s *memObjects) PutObject(_ context.Context, bucket, object string, r io.Reader) error { + b, err := io.ReadAll(r) + if err != nil { + return err + } + s.mu.Lock() + defer s.mu.Unlock() + s.m[bucket+"/"+object] = b + return nil +} + +func (s *memObjects) GetObject(_ context.Context, bucket, object string) (io.ReadCloser, error) { + s.mu.Lock() + defer s.mu.Unlock() + b, ok := s.m[bucket+"/"+object] + if !ok { + return nil, fmt.Errorf("%w: %s/%s", objectstorage.ErrObjectNotFound, bucket, object) + } + return io.NopCloser(bytes.NewReader(b)), nil +} + +func (s *memObjects) List(_ context.Context, bucket, prefix string) ([]string, error) { + s.mu.Lock() + defer s.mu.Unlock() + var out []string + for k := range s.m { + if name, ok := strings.CutPrefix(k, bucket+"/"); ok && strings.HasPrefix(name, prefix) { + out = append(out, name) + } + } + return out, nil +} + +func (s *memObjects) Delete(_ context.Context, bucket, object string) error { + s.mu.Lock() + defer s.mu.Unlock() + delete(s.m, bucket+"/"+object) + return nil +} + +func (s *memObjects) Copy(_ context.Context, srcBucket, srcObject, dstBucket, dstObject string) error { + s.mu.Lock() + defer s.mu.Unlock() + s.m[dstBucket+"/"+dstObject] = s.m[srcBucket+"/"+srcObject] + return nil +} + +func (s *memObjects) keys() []string { + s.mu.Lock() + defer s.mu.Unlock() + var out []string + for k := range s.m { + out = append(out, k) + } + sort.Strings(out) + return out +} + +// serve starts the plugins on a real Unix socket, as the sidecar does, and +// returns a connection to it. +func serve(t *testing.T, backend *memObjects, root string) *grpc.ClientConn { + t.Helper() + // Unix socket paths are length-limited; t.TempDir can exceed that on macOS. + sockDir, err := os.MkdirTemp("", "snapplug") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { os.RemoveAll(sockDir) }) + lis, err := Listen(filepath.Join(sockDir, "plugin.sock")) + if err != nil { + t.Fatal(err) + } + node, err := NewNodePlugin(backend, root) + if err != nil { + t.Fatal(err) + } + srv := grpc.NewServer() + objectstorev1.RegisterNodeProviderServer(srv, node) + objectstorev1.RegisterControlProviderServer(srv, NewControlPlugin(backend)) + healthpb.RegisterHealthServer(srv, health.NewServer()) + go srv.Serve(lis) + t.Cleanup(srv.Stop) + + conn, err := Dial(filepath.Join(sockDir, "plugin.sock"), ReadyWait) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { conn.Close() }) + return conn +} + +func TestWaitReady(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if err := WaitReady(ctx, serve(t, newMemObjects(), t.TempDir())); err != nil { + t.Errorf("WaitReady against a serving plugin = %v", err) + } + + // A socket nobody listens on must fail when ctx ends, not hang: WaitReady + // waits for the plugin as long as ctx allows. + conn, err := Dial(filepath.Join(t.TempDir(), "missing.sock"), ReadyWait) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + short, cancelShort := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancelShort() + err = WaitReady(short, conn) + if status.Code(err) != codes.DeadlineExceeded { + t.Errorf("WaitReady against a missing socket = %v, want %s", err, codes.DeadlineExceeded) + } +} + +func TestUploadFetchRoundTrip(t *testing.T) { + ctx := context.Background() + backend := newMemObjects() + root := t.TempDir() + client := objectstorev1.NewNodeProviderClient(serve(t, backend, root)) + + src := filepath.Join(root, "src") + dst := filepath.Join(root, "dst") + for _, d := range []string{src, dst} { + if err := os.Mkdir(d, 0o755); err != nil { + t.Fatal(err) + } + } + files := map[string][]byte{ + "memory.img": bytes.Repeat([]byte("abcdefgh"), 1<<16), + "state.bin": []byte("vm state"), + manifestFile: []byte(`{"snapshotFiles":["memory.img","state.bin"]}`), + } + for name, content := range files { + if err := os.WriteFile(filepath.Join(src, name), content, 0o644); err != nil { + t.Fatal(err) + } + } + + if _, err := client.UploadSnapshot(ctx, &objectstorev1.UploadSnapshotRequest{ + SnapshotUri: testURI, LocalPath: src, Files: []string{"memory.img", "state.bin"}, + }); err != nil { + t.Fatalf("UploadSnapshot(data) = %v", err) + } + if _, err := client.UploadSnapshot(ctx, &objectstorev1.UploadSnapshotRequest{ + SnapshotUri: testURI, LocalPath: src, Files: []string{manifestFile}, + }); err != nil { + t.Fatalf("UploadSnapshot(manifest) = %v", err) + } + + // The stored layout is the one atelet has always written. + want := []string{ + testBucket + "/" + testPrefix + "/manifest.json", + testBucket + "/" + testPrefix + "/memory.img.zstd", + testBucket + "/" + testPrefix + "/state.bin.zstd", + } + if got := backend.keys(); !equal(got, want) { + t.Fatalf("stored objects = %v, want %v", got, want) + } + if got := backend.m[want[0]]; !bytes.Equal(got, files[manifestFile]) { + t.Errorf("stored manifest = %q, want it uncompressed", got) + } + + if _, err := client.FetchSnapshot(ctx, &objectstorev1.FetchSnapshotRequest{ + SnapshotUri: testURI, WritePath: dst, Files: []string{manifestFile, "memory.img", "state.bin"}, + }); err != nil { + t.Fatalf("FetchSnapshot = %v", err) + } + for name, content := range files { + got, err := os.ReadFile(filepath.Join(dst, name)) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, content) { + t.Errorf("fetched %s differs from the uploaded file", name) + } + } +} + +func TestFetchMissingIsNotFound(t *testing.T) { + root := t.TempDir() + client := objectstorev1.NewNodeProviderClient(serve(t, newMemObjects(), root)) + for _, file := range []string{manifestFile, "memory.img"} { + _, err := client.FetchSnapshot(context.Background(), &objectstorev1.FetchSnapshotRequest{ + SnapshotUri: testURI, WritePath: root, Files: []string{file}, + }) + if status.Code(err) != codes.NotFound { + t.Errorf("FetchSnapshot(%s) = %v, want NotFound", file, err) + } + } +} + +func TestNodeRequestValidation(t *testing.T) { + root := t.TempDir() + client := objectstorev1.NewNodeProviderClient(serve(t, newMemObjects(), root)) + for _, tc := range []struct { + name string + uri string + dir string + files []string + }{ + {"bad uri", "gs://bucket/not-a-snapshot", root, []string{"a"}}, + {"relative dir", testURI, "relative/dir", []string{"a"}}, + {"dir outside root", testURI, filepath.Dir(root), []string{"a"}}, + {"dir escapes root", testURI, root + "/../elsewhere", []string{"a"}}, + {"sibling sharing root's prefix", testURI, root + "-other", []string{"a"}}, + {"no files", testURI, root, nil}, + {"empty file", testURI, root, []string{""}}, + {"nested file", testURI, root, []string{"sub/a"}}, + {"parent file", testURI, root, []string{".."}}, + {"duplicate file", testURI, root, []string{"a", "b", "a"}}, + } { + t.Run(tc.name, func(t *testing.T) { + _, fetchErr := client.FetchSnapshot(context.Background(), &objectstorev1.FetchSnapshotRequest{ + SnapshotUri: tc.uri, WritePath: tc.dir, Files: tc.files, + }) + _, uploadErr := client.UploadSnapshot(context.Background(), &objectstorev1.UploadSnapshotRequest{ + SnapshotUri: tc.uri, LocalPath: tc.dir, Files: tc.files, + }) + for _, err := range []error{fetchErr, uploadErr} { + if status.Code(err) != codes.InvalidArgument { + t.Errorf("got %v, want InvalidArgument", err) + } + } + }) + } +} + +func TestUploadRejectsSymlinkOutsideDir(t *testing.T) { + for _, file := range []string{manifestFile, "checkpoint.img"} { + t.Run(file, func(t *testing.T) { + parent := t.TempDir() + backend := newMemObjects() + client := objectstorev1.NewNodeProviderClient(serve(t, backend, parent)) + checkpointDir := filepath.Join(parent, "checkpoint-state") + if err := os.Mkdir(checkpointDir, 0o700); err != nil { + t.Fatal(err) + } + outside := filepath.Join(parent, "outside") + if err := os.WriteFile(outside, []byte("secret"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outside, filepath.Join(checkpointDir, file)); err != nil { + t.Fatal(err) + } + + _, err := client.UploadSnapshot(context.Background(), &objectstorev1.UploadSnapshotRequest{ + SnapshotUri: testURI, LocalPath: checkpointDir, Files: []string{file}, + }) + if err == nil { + t.Fatal("UploadSnapshot followed a symlink outside the snapshot directory") + } + if got := backend.keys(); len(got) != 0 { + t.Fatalf("uploaded objects = %v, want none", got) + } + }) + } +} + +func TestFetchRejectsSymlinkOutsideDir(t *testing.T) { + for _, file := range []string{manifestFile, "checkpoint.img"} { + t.Run(file, func(t *testing.T) { + ctx := context.Background() + parent := t.TempDir() + backend := newMemObjects() + client := objectstorev1.NewNodeProviderClient(serve(t, backend, parent)) + src := filepath.Join(parent, "src") + restoreDir := filepath.Join(parent, "restore-state") + for _, d := range []string{src, restoreDir} { + if err := os.Mkdir(d, 0o700); err != nil { + t.Fatal(err) + } + } + if err := os.WriteFile(filepath.Join(src, file), []byte("replacement"), 0o600); err != nil { + t.Fatal(err) + } + if _, err := client.UploadSnapshot(ctx, &objectstorev1.UploadSnapshotRequest{ + SnapshotUri: testURI, LocalPath: src, Files: []string{file}, + }); err != nil { + t.Fatal(err) + } + outside := filepath.Join(parent, "outside") + if err := os.WriteFile(outside, []byte("keep me"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outside, filepath.Join(restoreDir, file)); err != nil { + t.Fatal(err) + } + + _, err := client.FetchSnapshot(ctx, &objectstorev1.FetchSnapshotRequest{ + SnapshotUri: testURI, WritePath: restoreDir, Files: []string{file}, + }) + if err == nil { + t.Fatal("FetchSnapshot followed a symlink outside the restore directory") + } + if got, err := os.ReadFile(outside); err != nil || string(got) != "keep me" { + t.Fatalf("outside file = %q, %v; want unchanged", got, err) + } + }) + } +} + +// A directory below --root that is a symlink to elsewhere passes the lexical +// check, so only opening it through os.Root keeps the transfer inside --root. +func TestRejectsDirSymlinkOutsideRoot(t *testing.T) { + ctx := context.Background() + parent := t.TempDir() + root := filepath.Join(parent, "root") + outside := filepath.Join(parent, "outside") + for _, d := range []string{root, outside} { + if err := os.Mkdir(d, 0o700); err != nil { + t.Fatal(err) + } + } + if err := os.WriteFile(filepath.Join(outside, "checkpoint.img"), []byte("secret"), 0o600); err != nil { + t.Fatal(err) + } + link := filepath.Join(root, "link") + if err := os.Symlink(outside, link); err != nil { + t.Fatal(err) + } + backend := newMemObjects() + client := objectstorev1.NewNodeProviderClient(serve(t, backend, root)) + + if _, err := client.UploadSnapshot(ctx, &objectstorev1.UploadSnapshotRequest{ + SnapshotUri: testURI, LocalPath: link, Files: []string{"checkpoint.img"}, + }); err == nil { + t.Error("UploadSnapshot read through a directory symlink outside --root") + } + if got := backend.keys(); len(got) != 0 { + t.Errorf("uploaded objects = %v, want none", got) + } + + backend.m[testBucket+"/"+testPrefix+"/"+manifestFile] = []byte("m") + if _, err := client.FetchSnapshot(ctx, &objectstorev1.FetchSnapshotRequest{ + SnapshotUri: testURI, WritePath: link, Files: []string{manifestFile}, + }); err == nil { + t.Error("FetchSnapshot succeeded through a directory symlink outside --root") + } + if _, err := os.Stat(filepath.Join(outside, manifestFile)); !os.IsNotExist(err) { + t.Errorf("FetchSnapshot created a file outside --root: %v", err) + } +} + +func TestCleanupSnapshot(t *testing.T) { + ctx := context.Background() + backend := newMemObjects() + client := objectstorev1.NewControlProviderClient(serve(t, backend, t.TempDir())) + // uid10 shares a string prefix with uid1, so it survives only if cleanup + // stops at a path-segment boundary. + keep := testBucket + "/root/atespaces/team-a/actors/uid10/snapshots/snap1/manifest.json" + backend.m[testBucket+"/"+testPrefix+"/manifest.json"] = []byte("m") + backend.m[testBucket+"/"+testPrefix+"/memory.img.zstd"] = []byte("d") + backend.m[keep] = []byte("other actor") + + // An owner prefix collects every snapshot below it, and nothing beside it. + owner := "gs://" + testBucket + "/root/atespaces/team-a/actors/uid1" + if _, err := client.CleanupSnapshot(ctx, &objectstorev1.CleanupSnapshotRequest{SnapshotUri: owner}); err != nil { + t.Fatalf("CleanupSnapshot = %v", err) + } + if got := backend.keys(); !equal(got, []string{keep}) { + t.Errorf("objects left = %v, want only %s", got, keep) + } + // Cleaning up again is a no-op. + if _, err := client.CleanupSnapshot(ctx, &objectstorev1.CleanupSnapshotRequest{SnapshotUri: owner}); err != nil { + t.Errorf("repeated CleanupSnapshot = %v", err) + } + // A bare bucket is never a valid prefix. + for _, uri := range []string{"gs://" + testBucket, "gs://" + testBucket + "/", "not a uri?x=1"} { + _, err := client.CleanupSnapshot(ctx, &objectstorev1.CleanupSnapshotRequest{SnapshotUri: uri}) + if status.Code(err) != codes.InvalidArgument { + t.Errorf("CleanupSnapshot(%q) = %v, want InvalidArgument", uri, err) + } + } +} + +func TestCopySnapshot(t *testing.T) { + ctx := context.Background() + backend := newMemObjects() + client := objectstorev1.NewControlProviderClient(serve(t, backend, t.TempDir())) + backend.m[testBucket+"/"+testPrefix+"/manifest.json"] = []byte("m") + tag := "gs://" + testBucket + "/root/atespaces/team-a/tags/tag1" + + if _, err := client.CopySnapshot(ctx, &objectstorev1.CopySnapshotRequest{SrcUri: testURI, DstUri: tag}); err != nil { + t.Fatalf("CopySnapshot = %v", err) + } + if got := backend.m[testBucket+"/root/atespaces/team-a/tags/tag1/manifest.json"]; string(got) != "m" { + t.Errorf("copied manifest = %q, want %q", got, "m") + } + + empty := "gs://" + testBucket + "/root/atespaces/team-a/actors/uid9/snapshots/none" + _, err := client.CopySnapshot(ctx, &objectstorev1.CopySnapshotRequest{SrcUri: empty, DstUri: tag}) + if status.Code(err) != codes.FailedPrecondition { + t.Errorf("CopySnapshot(empty source) = %v, want FailedPrecondition", err) + } +} + +func equal(a, b []string) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} diff --git a/internal/objectstoreplugin/socket.go b/internal/objectstoreplugin/socket.go new file mode 100644 index 0000000000..a5881aea04 --- /dev/null +++ b/internal/objectstoreplugin/socket.go @@ -0,0 +1,168 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package objectstoreplugin + +import ( + "context" + "errors" + "fmt" + "io/fs" + "net" + "os" + "path/filepath" + "time" + + "github.com/agent-substrate/substrate/internal/apierror" + "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" + "google.golang.org/grpc" + "google.golang.org/grpc/backoff" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials/insecure" + healthpb "google.golang.org/grpc/health/grpc_health_v1" + "google.golang.org/grpc/status" +) + +// ReadyWait is the ready wait servers pass to Dial. It rides through a sidecar +// restart, including the first 10s step of kubelet's crash-loop back-off, and +// fails a plugin that stays down well before a caller's deadline would. +const ReadyWait = 30 * time.Second + +// reconnectParams keeps reconnect attempts to the local socket frequent, so a +// plugin that comes back is reached within the ready wait; gRPC's default back-off +// grows to two minutes. +var reconnectParams = grpc.ConnectParams{ + Backoff: backoff.Config{ + BaseDelay: 100 * time.Millisecond, + Multiplier: 1.6, + Jitter: 0.2, + MaxDelay: time.Second, + }, + MinConnectTimeout: 5 * time.Second, +} + +// Listen removes any stale socket at path and listens on a fresh one that +// only the socket's owner can connect to. +func Listen(path string) (net.Listener, error) { + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return nil, fmt.Errorf("while creating socket directory: %w", err) + } + if err := os.Remove(path); err != nil && !errors.Is(err, fs.ErrNotExist) { + return nil, fmt.Errorf("while removing stale socket %s: %w", path, err) + } + lis, err := net.Listen("unix", path) + if err != nil { + return nil, fmt.Errorf("while listening on %s: %w", path, err) + } + if err := os.Chmod(path, 0o600); err != nil { + lis.Close() + return nil, fmt.Errorf("while restricting socket %s: %w", path, err) + } + return lis, nil +} + +// Dial returns a long-lived client connection to a plugin socket. A unary +// call waits up to readyWait for the connection to become ready, then fails +// with codes.Unavailable, so a plugin that is lost after startup fails calls +// promptly while a brief restart does not. A call that starts on a ready +// connection is not bounded by readyWait, and one in flight when the plugin +// goes fails with codes.Unavailable at once. A caller that passes +// grpc.WaitForReady itself opts out of the bound and waits as long as its +// context allows; WaitReady does so at startup. +// +// The bound covers unary calls only. A streaming call gets gRPC's plain +// fail-fast and fails at once while the plugin restarts, so the first +// streaming RPC on this connection needs a grpc.WithStreamInterceptor that +// applies the same wait. +// +// The connection is unauthenticated: the socket is reachable only from +// inside the pod, and only by its owner. +func Dial(path string, readyWait time.Duration) (*grpc.ClientConn, error) { + conn, err := grpc.NewClient("unix://"+path, + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithStatsHandler(otelgrpc.NewClientHandler()), + grpc.WithConnectParams(reconnectParams), + grpc.WithUnaryInterceptor(boundedWaitForReady(readyWait)), + ) + if err != nil { + return nil, fmt.Errorf("while dialing snapshot plugin at %s: %w", path, err) + } + return conn, nil +} + +// boundedWaitForReady waits up to wait for cc to become ready before issuing +// a call that sets no grpc.WaitForReady option, then issues it fail-fast. +func boundedWaitForReady(wait time.Duration) grpc.UnaryClientInterceptor { + return func(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { + for _, opt := range opts { + if _, ok := opt.(grpc.FailFastCallOption); ok { + return invoker(ctx, method, req, reply, cc, opts...) + } + } + if err := awaitReady(ctx, cc, wait); err != nil { + return err + } + return invoker(ctx, method, req, reply, cc, opts...) + } +} + +// awaitReady returns nil once cc is ready or shut down, the latter left for +// the call itself to report. It returns codes.Unavailable if cc is not ready +// within wait, and ctx's error as a status if ctx ends first. +func awaitReady(ctx context.Context, cc *grpc.ClientConn, wait time.Duration) error { + waitCtx, cancel := context.WithTimeout(ctx, wait) + defer cancel() + for { + state := cc.GetState() + switch state { + case connectivity.Ready, connectivity.Shutdown: + return nil + case connectivity.Idle: + cc.Connect() + } + if !cc.WaitForStateChange(waitCtx, state) { + if ctx.Err() != nil { + return status.FromContextError(ctx.Err()).Err() + } + return status.Errorf(codes.Unavailable, "snapshot plugin at %s is not ready after %s (connection %s)", cc.Target(), wait, state) + } + } +} + +// WaitReady blocks until the plugin behind conn reports that it is serving, +// or ctx ends, however long the plugin takes to come up. Bounding ctx is what +// turns a plugin that never comes up (a wrong socket path, a crash loop) into +// an error at startup. +func WaitReady(ctx context.Context, conn *grpc.ClientConn) error { + resp, err := healthpb.NewHealthClient(conn).Check(ctx, &healthpb.HealthCheckRequest{}, grpc.WaitForReady(true)) + if err != nil { + return fmt.Errorf("while waiting for the snapshot plugin at %s: %w", conn.Target(), err) + } + if resp.GetStatus() != healthpb.HealthCheckResponse_SERVING { + return fmt.Errorf("snapshot plugin at %s is %s", conn.Target(), resp.GetStatus()) + } + return nil +} + +// CallError returns the error a server reports to its own caller for a failed +// plugin call. codes.Unavailable, which a call reports when the plugin cannot +// be reached, becomes an apierror.Unavailable so the server's caller retries +// rather than treating it as Internal. Any other error is returned unchanged. +func CallError(err error) error { + if status.Code(err) == codes.Unavailable { + return apierror.Unavailable("snapshot plugin: %w", err) + } + return err +} diff --git a/internal/objectstoreplugin/socket_test.go b/internal/objectstoreplugin/socket_test.go new file mode 100644 index 0000000000..18a8f6f659 --- /dev/null +++ b/internal/objectstoreplugin/socket_test.go @@ -0,0 +1,268 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package objectstoreplugin + +import ( + "context" + "os" + "path/filepath" + "sync" + "testing" + "time" + + objectstorev1 "github.com/agent-substrate/substrate/pkg/proto/objectstorepb/v1" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/health" + healthpb "google.golang.org/grpc/health/grpc_health_v1" + "google.golang.org/grpc/status" +) + +// slowControl is a ControlProvider whose CleanupSnapshot takes delay, and +// closes started when a call arrives. +type slowControl struct { + objectstorev1.UnimplementedControlProviderServer + + delay time.Duration + once sync.Once + started chan struct{} +} + +func newSlowControl(delay time.Duration) *slowControl { + return &slowControl{delay: delay, started: make(chan struct{})} +} + +func (c *slowControl) CleanupSnapshot(ctx context.Context, _ *objectstorev1.CleanupSnapshotRequest) (*objectstorev1.CleanupSnapshotResponse, error) { + c.once.Do(func() { close(c.started) }) + select { + case <-time.After(c.delay): + return &objectstorev1.CleanupSnapshotResponse{}, nil + case <-ctx.Done(): + return nil, status.FromContextError(ctx.Err()).Err() + } +} + +// sidecar serves a ControlProvider on a Unix socket and can be stopped and +// started again on the same path, as kubelet restarts a sidecar container. +type sidecar struct { + t *testing.T + path string + control objectstorev1.ControlProviderServer + + mu sync.Mutex + srv *grpc.Server +} + +func newSidecar(t *testing.T, control objectstorev1.ControlProviderServer) *sidecar { + t.Helper() + // Unix socket paths are length-limited; t.TempDir can exceed that on macOS. + dir, err := os.MkdirTemp("", "snapplug") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { os.RemoveAll(dir) }) + s := &sidecar{t: t, path: filepath.Join(dir, "plugin.sock"), control: control} + t.Cleanup(s.stop) + return s +} + +// start may run on any goroutine, so it reports failures with Error. +func (s *sidecar) start() { + lis, err := Listen(s.path) + if err != nil { + s.t.Errorf("Listen: %v", err) + return + } + srv := grpc.NewServer() + objectstorev1.RegisterControlProviderServer(srv, s.control) + healthpb.RegisterHealthServer(srv, health.NewServer()) + go srv.Serve(lis) + s.mu.Lock() + s.srv = srv + s.mu.Unlock() +} + +// stop ends the server and every connection to it at once, as a crashed +// sidecar does. +func (s *sidecar) stop() { + s.mu.Lock() + srv := s.srv + s.srv = nil + s.mu.Unlock() + if srv != nil { + srv.Stop() + } +} + +// dialReady connects to s with the given wait and passes the startup +// readiness check. +func (s *sidecar) dialReady(wait time.Duration) (*grpc.ClientConn, objectstorev1.ControlProviderClient) { + s.t.Helper() + conn, err := Dial(s.path, wait) + if err != nil { + s.t.Fatal(err) + } + s.t.Cleanup(func() { conn.Close() }) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if err := WaitReady(ctx, conn); err != nil { + s.t.Fatalf("WaitReady = %v", err) + } + return conn, objectstorev1.NewControlProviderClient(conn) +} + +// lose stops s and waits until conn has seen the plugin go, so the next call +// starts on a lost connection rather than racing the loss. +func (s *sidecar) lose(conn *grpc.ClientConn) { + s.t.Helper() + s.stop() + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + for conn.GetState() == connectivity.Ready { + if !conn.WaitForStateChange(ctx, connectivity.Ready) { + s.t.Fatal("connection still ready 10s after the plugin stopped") + } + } +} + +var cleanupReq = &objectstorev1.CleanupSnapshotRequest{SnapshotUri: testURI} + +// A plugin lost after startup fails a call with Unavailable once the ready +// wait runs out, not when the caller's deadline does. +func TestCallFailsPromptlyAfterPluginLoss(t *testing.T) { + const wait = 300 * time.Millisecond + s := newSidecar(t, newSlowControl(0)) + s.start() + conn, client := s.dialReady(wait) + s.lose(conn) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + start := time.Now() + _, err := client.CleanupSnapshot(ctx, cleanupReq) + elapsed := time.Since(start) + if status.Code(err) != codes.Unavailable { + t.Fatalf("CleanupSnapshot after plugin loss = %v, want %s", err, codes.Unavailable) + } + if elapsed > 5*time.Second { + t.Errorf("CleanupSnapshot failed after %s, want about %s, well before the caller's 10s deadline", elapsed, wait) + } +} + +// A call that ends with the caller's context while waiting reports the +// context's error, not Unavailable. +func TestCallWaitEndsWithCallerContext(t *testing.T) { + s := newSidecar(t, newSlowControl(0)) + s.start() + conn, client := s.dialReady(time.Minute) + s.lose(conn) + + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + _, err := client.CleanupSnapshot(ctx, cleanupReq) + if status.Code(err) != codes.DeadlineExceeded { + t.Errorf("CleanupSnapshot = %v, want %s", err, codes.DeadlineExceeded) + } +} + +// A call that starts on a ready connection runs as long as the plugin takes, +// however short the ready wait. +func TestSlowCallOutlastsReadyWait(t *testing.T) { + const wait = 100 * time.Millisecond + s := newSidecar(t, newSlowControl(10*wait)) + s.start() + _, client := s.dialReady(wait) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if _, err := client.CleanupSnapshot(ctx, cleanupReq); err != nil { + t.Errorf("CleanupSnapshot on a slow, healthy plugin = %v, want success", err) + } +} + +// A plugin that restarts within the ready wait serves the call. +func TestCallRidesThroughPluginRestart(t *testing.T) { + s := newSidecar(t, newSlowControl(0)) + s.start() + conn, client := s.dialReady(10 * time.Second) + s.lose(conn) + + restarted := make(chan struct{}) + go func() { + defer close(restarted) + time.Sleep(time.Second) + s.start() + }() + defer func() { <-restarted }() + + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + if _, err := client.CleanupSnapshot(ctx, cleanupReq); err != nil { + t.Errorf("CleanupSnapshot across a plugin restart = %v, want success", err) + } +} + +// A call in flight when the plugin dies fails with Unavailable right away. +func TestInFlightCallFailsWhenPluginDies(t *testing.T) { + control := newSlowControl(time.Minute) + s := newSidecar(t, control) + s.start() + _, client := s.dialReady(time.Minute) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + errc := make(chan error, 1) + go func() { + _, err := client.CleanupSnapshot(ctx, cleanupReq) + errc <- err + }() + <-control.started + start := time.Now() + s.stop() + err := <-errc + if status.Code(err) != codes.Unavailable { + t.Fatalf("in-flight CleanupSnapshot = %v, want %s", err, codes.Unavailable) + } + if elapsed := time.Since(start); elapsed > 5*time.Second { + t.Errorf("in-flight CleanupSnapshot failed %s after the plugin died, want promptly", elapsed) + } +} + +// WaitReady at startup waits for the plugin as long as its context allows, +// not only for the per-call ready wait. +func TestWaitReadyOutlastsReadyWait(t *testing.T) { + const wait = 100 * time.Millisecond + s := newSidecar(t, newSlowControl(0)) + conn, err := Dial(s.path, wait) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + + started := make(chan struct{}) + go func() { + defer close(started) + time.Sleep(10 * wait) + s.start() + }() + defer func() { <-started }() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if err := WaitReady(ctx, conn); err != nil { + t.Errorf("WaitReady for a plugin that starts after the ready wait = %v, want success", err) + } +} diff --git a/internal/objectstoreplugin/status.go b/internal/objectstoreplugin/status.go new file mode 100644 index 0000000000..aa9008eb3b --- /dev/null +++ b/internal/objectstoreplugin/status.go @@ -0,0 +1,41 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package objectstoreplugin + +import ( + "context" + "errors" + + "github.com/agent-substrate/substrate/internal/objectstore" + "github.com/agent-substrate/substrate/pkg/objectstorage" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// toStatus maps a plugin error to a gRPC status, keeping NotFound +// distinguishable so callers can probe for a snapshot. +func toStatus(err error) error { + switch { + case errors.Is(err, objectstorage.ErrObjectNotFound): + return status.Error(codes.NotFound, err.Error()) + case errors.Is(err, objectstore.ErrEmptySource): + return status.Error(codes.FailedPrecondition, err.Error()) + case errors.Is(err, context.Canceled): + return status.Error(codes.Canceled, err.Error()) + case errors.Is(err, context.DeadlineExceeded): + return status.Error(codes.DeadlineExceeded, err.Error()) + } + return status.Error(codes.Internal, err.Error()) +} diff --git a/internal/ocispec/microvm_test.go b/internal/ocispec/microvm_test.go index fb3f808fa5..bfde5f6595 100644 --- a/internal/ocispec/microvm_test.go +++ b/internal/ocispec/microvm_test.go @@ -17,7 +17,7 @@ package ocispec import ( "testing" - "github.com/agent-substrate/substrate/internal/proto/ateletpb" + "github.com/agent-substrate/substrate/internal/proto/ateompb" "github.com/opencontainers/runtime-spec/specs-go" ) @@ -121,10 +121,10 @@ func eqInt64Ptr(a, b *int64) bool { // unbound the container. func TestShapeMicroVM_KeepsDeclaredContainerLimits(t *testing.T) { const declared = 64 * 1024 * 1024 - spec := Build(Options{ + spec := Build(Options{Container: &ateompb.Container{ContainerSpec: &ateompb.ContainerSpec{ Args: []string{"/app"}, - Resources: &ateletpb.ResourceLimits{MemoryBytes: declared}, - }) + Resources: &ateompb.ResourceLimits{MemoryBytes: declared}, + }}}) if err := ShapeMicroVM(spec, MicroVMOptions{ActorDirs: parityActorDirs, ContainerID: "app"}); err != nil { t.Fatalf("ShapeMicroVM() = %v", err) } @@ -140,7 +140,7 @@ func TestShapeMicroVM_KeepsDeclaredContainerLimits(t *testing.T) { // A container that declares nothing must stay unbounded inside the guest: guest // RAM is the real ceiling, and a cap equal to the whole guest can never bind. func TestShapeMicroVM_LeavesUndeclaredContainerUnlimited(t *testing.T) { - spec := Build(Options{Args: []string{"/app"}}) + spec := Build(Options{Container: &ateompb.Container{ContainerSpec: &ateompb.ContainerSpec{Args: []string{"/app"}}}}) if err := ShapeMicroVM(spec, MicroVMOptions{ActorDirs: parityActorDirs, ContainerID: "app"}); err != nil { t.Fatalf("ShapeMicroVM() = %v", err) } diff --git a/internal/ocispec/ocispec.go b/internal/ocispec/ocispec.go index 7b672f2453..dbaec1cf85 100644 --- a/internal/ocispec/ocispec.go +++ b/internal/ocispec/ocispec.go @@ -24,7 +24,8 @@ import ( "path/filepath" "github.com/agent-substrate/substrate/internal/imagecache" - "github.com/agent-substrate/substrate/internal/proto/ateletpb" + "github.com/agent-substrate/substrate/internal/nodepath" + "github.com/agent-substrate/substrate/internal/proto/ateompb" "github.com/opencontainers/runtime-spec/specs-go" ) @@ -34,27 +35,15 @@ const specFile = "config.json" // hostname is the UTS hostname for actor containers. const hostname = "actor" -// Options describes one actor container. Args, Env and Capabilities arrive -// already resolved. +// Options describes one actor container. type Options struct { - Args []string - Env []string - // NetNSPath is the network namespace the ateom runs the actor in. - NetNSPath string - Volumes []*ateletpb.Volume - VolumeMounts []*ateletpb.VolumeMount - Capabilities []string - // Resources are the container's own declared limits, or nil for none. - Resources *ateletpb.ResourceLimits - - // The actor's directories the bind mount sources are joined from, one - // subdirectory per volume name. - DurableDirVolumeMountsDir string - VolumesDir string - SystemInfoVolumeRootsDir string - // BundlePath is this container's bundle, where its image volumes are - // composed. - BundlePath string + // ActorUID names the actor's network namespace. + ActorUID string + // ActorDirs are the actor's directories the bind mount sources are joined + // from, one subdirectory per volume name. + ActorDirs *ateompb.ActorDirs + // Container carries the process and its volume mounts. + Container *ateompb.Container } const ( @@ -71,7 +60,7 @@ const ( // ociResources maps the resolved limits onto the OCI spec's linux.resources. // Returns nil when nothing is set, so the spec carries no linux.resources at // all for a container that declares no limits. -func ociResources(r *ateletpb.ResourceLimits) *specs.LinuxResources { +func ociResources(r *ateompb.ResourceLimits) *specs.LinuxResources { if r == nil || (r.GetMemoryBytes() <= 0 && r.GetCpuMillis() <= 0) { return nil } @@ -89,19 +78,21 @@ func ociResources(r *ateletpb.ResourceLimits) *specs.LinuxResources { // Build returns a runtime-neutral OCI spec for an actor container. func Build(o Options) *specs.Spec { + container := o.Container + containerSpec := container.GetContainerSpec() spec := &specs.Spec{ Process: &specs.Process{ User: specs.User{ UID: 0, GID: 0, }, - Args: o.Args, - Env: o.Env, + Args: containerSpec.GetArgs(), + Env: containerSpec.GetEnv(), Cwd: "/", Capabilities: &specs.LinuxCapabilities{ - Bounding: o.Capabilities, - Effective: o.Capabilities, - Permitted: o.Capabilities, + Bounding: containerSpec.GetCapabilities(), + Effective: containerSpec.GetCapabilities(), + Permitted: containerSpec.GetCapabilities(), // Inheritable and Ambient stay empty; capabilities inherit via // Bounding. // @@ -152,7 +143,7 @@ func Build(o Options) *specs.Spec { }, { Type: "network", - Path: o.NetNSPath, // Will be created by ateom + Path: nodepath.ActorNetNSPath(o.ActorUID), }, { Type: "ipc", @@ -164,54 +155,36 @@ func Build(o Options) *specs.Spec { Type: "mount", }, }, - Resources: ociResources(o.Resources), + Resources: ociResources(containerSpec.GetResources()), }, } - volumesByName := make(map[string]*ateletpb.Volume, len(o.Volumes)) - for _, vol := range o.Volumes { - volumesByName[vol.GetName()] = vol - } - for _, vm := range o.VolumeMounts { - var srcPath string - options := []string{"bind", "rw"} - switch volumesByName[vm.GetName()].GetSource().(type) { - case *ateletpb.Volume_DurableDir: - srcPath = filepath.Join(o.DurableDirVolumeMountsDir, vm.GetName()) - case *ateletpb.Volume_External: - srcPath = filepath.Join(o.VolumesDir, vm.GetName()) - case *ateletpb.Volume_SystemInfo: - srcPath = filepath.Join(o.SystemInfoVolumeRootsDir, vm.GetName()) - options = []string{"bind", "ro"} - case *ateletpb.Volume_Image: - srcPath = imagecache.ImageVolumeMountPath(o.BundlePath, vm.GetName()) - options = []string{"bind", "ro"} - default: - continue - } + // Mount order across volume kinds does not matter: the API rejects mounts + // that nest under one another. + bind := func(dest, src, mode string) { spec.Mounts = append(spec.Mounts, specs.Mount{ - Destination: vm.GetMountPath(), + Destination: dest, Type: "bind", - Source: srcPath, - Options: options, + Source: src, + Options: []string{"bind", mode}, }) } - - return spec -} - -// Load reads the OCI spec of the bundle at bundlePath. -func Load(bundlePath string) (*specs.Spec, error) { - specPath := path.Join(bundlePath, specFile) - b, err := os.ReadFile(specPath) - if err != nil { - return nil, fmt.Errorf("reading %q: %w", specPath, err) + actorDirs := o.ActorDirs + for _, m := range container.GetDurableDirVolumeMounts() { + bind(m.GetMountPath(), filepath.Join(actorDirs.GetDurableDirVolumeMountsDir(), m.GetVolumeName()), "rw") + } + for _, m := range container.GetCsiVolumeMounts() { + bind(m.GetMountPath(), filepath.Join(actorDirs.GetVolumesDir(), m.GetVolumeName()), "rw") } - var spec specs.Spec - if err := json.Unmarshal(b, &spec); err != nil { - return nil, fmt.Errorf("parsing %q: %w", specPath, err) + for _, m := range container.GetSystemInfoVolumeMounts() { + bind(m.GetMountPath(), filepath.Join(actorDirs.GetSystemInfoVolumeRootsDir(), m.GetVolumeName()), "ro") } - return &spec, nil + bundlePath := filepath.Join(actorDirs.GetOciBundleDir(), container.GetName()) + for _, m := range container.GetImageVolumeMounts() { + bind(m.GetMountPath(), imagecache.ImageVolumeMountPath(bundlePath, m.GetVolumeName()), "ro") + } + + return spec } // Save writes spec as the OCI spec of the bundle at bundlePath. diff --git a/internal/ocispec/ocispec_test.go b/internal/ocispec/ocispec_test.go index d19d4ae80a..2d06ed5c3d 100644 --- a/internal/ocispec/ocispec_test.go +++ b/internal/ocispec/ocispec_test.go @@ -15,11 +15,15 @@ package ocispec import ( + "encoding/json" + "os" + "path/filepath" "slices" "testing" "github.com/agent-substrate/substrate/internal/imagecache" - "github.com/agent-substrate/substrate/internal/proto/ateletpb" + "github.com/agent-substrate/substrate/internal/nodepath" + "github.com/agent-substrate/substrate/internal/proto/ateompb" "github.com/opencontainers/runtime-spec/specs-go" ) @@ -35,18 +39,8 @@ func mountFor(t *testing.T, spec *specs.Spec, dest string) specs.Mount { return specs.Mount{} } -func durableVolume(name string) *ateletpb.Volume { - return &ateletpb.Volume{Name: name, Source: &ateletpb.Volume_DurableDir{DurableDir: &ateletpb.DurableDirVolume{}}} -} - // Each volume mount becomes a bind of its host directory, rw or ro by kind. func TestBuild_VolumeMounts(t *testing.T) { - volumes := []*ateletpb.Volume{ - durableVolume("data"), - {Name: "sysinfo", Source: &ateletpb.Volume_SystemInfo{SystemInfo: &ateletpb.SystemInfoVolume{}}}, - {Name: "csi", Source: &ateletpb.Volume_External{External: &ateletpb.ExternalVolumeSource{}}}, - {Name: "agent", Source: &ateletpb.Volume_Image{Image: &ateletpb.ImageVolumeSource{}}}, - } const ( durableDir = "/node/actors/a/durable-dir" volumesDir = "/node/actors/a/volumes" @@ -54,19 +48,23 @@ func TestBuild_VolumeMounts(t *testing.T) { bundle = "/node/actors/a/bundles/app" ) spec := Build(Options{ - Args: []string{"/app"}, - Volumes: volumes, - VolumeMounts: []*ateletpb.VolumeMount{ - {Name: "data", MountPath: "/var/data"}, - {Name: "data", MountPath: "/home/counter"}, - {Name: "sysinfo", MountPath: "/run/ate"}, - {Name: "csi", MountPath: "/mnt/csi"}, - {Name: "agent", MountPath: "/ate"}, + ActorDirs: &ateompb.ActorDirs{ + OciBundleDir: "/node/actors/a/bundles", + DurableDirVolumeMountsDir: durableDir, + VolumesDir: volumesDir, + SystemInfoVolumeRootsDir: sysInfoDir, + }, + Container: &ateompb.Container{ + Name: "app", + ContainerSpec: &ateompb.ContainerSpec{Args: []string{"/app"}}, + DurableDirVolumeMounts: []*ateompb.DurableDirVolumeMount{ + {VolumeName: "data", MountPath: "/var/data"}, + {VolumeName: "data", MountPath: "/home/counter"}, + }, + SystemInfoVolumeMounts: []*ateompb.SystemInfoVolumeMount{{VolumeName: "sysinfo", MountPath: "/run/ate"}}, + CsiVolumeMounts: []*ateompb.VolumeMount{{VolumeName: "csi", MountPath: "/mnt/csi"}}, + ImageVolumeMounts: []*ateompb.ImageVolumeMount{{VolumeName: "agent", MountPath: "/ate"}}, }, - DurableDirVolumeMountsDir: durableDir, - VolumesDir: volumesDir, - SystemInfoVolumeRootsDir: sysInfoDir, - BundlePath: bundle, }) for _, tc := range []struct { @@ -93,22 +91,22 @@ func TestBuild_VolumeMounts(t *testing.T) { } } -// A mount naming an undeclared volume is skipped. -func TestBuild_UnknownVolumeMountSkipped(t *testing.T) { - spec := Build(Options{ - VolumeMounts: []*ateletpb.VolumeMount{{Name: "missing", MountPath: "/mnt/missing"}}, - }) - for _, m := range spec.Mounts { - if m.Destination == "/mnt/missing" { - t.Fatalf("mount for an undeclared volume was emitted: %v", m) - } +// The container joins the network namespace ateom creates for the actor. +func TestBuild_JoinsTheActorNetNS(t *testing.T) { + spec := Build(Options{ActorUID: "uid-a", Container: &ateompb.Container{ContainerSpec: &ateompb.ContainerSpec{Args: []string{"/app"}}}}) + i := slices.IndexFunc(spec.Linux.Namespaces, func(ns specs.LinuxNamespace) bool { return ns.Type == specs.NetworkNamespace }) + if i < 0 { + t.Fatal("spec has no network namespace") + } + if got, want := spec.Linux.Namespaces[i].Path, nodepath.ActorNetNSPath("uid-a"); got != want { + t.Errorf("network namespace path = %q, want %q", got, want) } } // The resolved set lands in bounding, effective and permitted only. func TestBuild_Capabilities(t *testing.T) { want := []string{"CAP_CHOWN", "CAP_KILL"} - spec := Build(Options{Args: []string{"/app"}, Capabilities: want}) + spec := Build(Options{Container: &ateompb.Container{ContainerSpec: &ateompb.ContainerSpec{Args: []string{"/app"}, Capabilities: want}}}) caps := spec.Process.Capabilities if caps == nil { @@ -141,7 +139,7 @@ func TestBuild_Capabilities(t *testing.T) { // The pause container gets no capabilities. func TestBuild_NoCapabilitiesForPause(t *testing.T) { - spec := Build(Options{Args: []string{"/pause"}}) + spec := Build(Options{Container: &ateompb.Container{ContainerSpec: &ateompb.ContainerSpec{Args: []string{"/pause"}}}}) caps := spec.Process.Capabilities if caps == nil { @@ -163,15 +161,19 @@ func TestBuild_NoCapabilitiesForPause(t *testing.T) { } } -func TestSaveLoadRoundTrip(t *testing.T) { +func TestSave(t *testing.T) { bundle := t.TempDir() - want := Build(Options{Args: []string{"/app"}, NetNSPath: "/run/netns/x"}) + want := Build(Options{ActorUID: "uid-a", Container: &ateompb.Container{ContainerSpec: &ateompb.ContainerSpec{Args: []string{"/app"}}}}) if err := Save(bundle, want); err != nil { t.Fatalf("Save() = %v", err) } - got, err := Load(bundle) + b, err := os.ReadFile(filepath.Join(bundle, "config.json")) if err != nil { - t.Fatalf("Load() = %v", err) + t.Fatal(err) + } + var got specs.Spec + if err := json.Unmarshal(b, &got); err != nil { + t.Fatal(err) } if got.Hostname != want.Hostname || !slices.Equal(got.Process.Args, want.Process.Args) || len(got.Mounts) != len(want.Mounts) { t.Errorf("round trip changed the spec:\n got %+v\nwant %+v", got, want) @@ -182,11 +184,11 @@ func TestOCIResources(t *testing.T) { if got := ociResources(nil); got != nil { t.Errorf("ociResources(nil) = %v, want nil", got) } - if got := ociResources(&ateletpb.ResourceLimits{}); got != nil { + if got := ociResources(&ateompb.ResourceLimits{}); got != nil { t.Errorf("ociResources(zero) = %v, want nil so the spec is unchanged", got) } - got := ociResources(&ateletpb.ResourceLimits{MemoryBytes: 268435456, CpuMillis: 200}) + got := ociResources(&ateompb.ResourceLimits{MemoryBytes: 268435456, CpuMillis: 200}) if got == nil { t.Fatal("ociResources() = nil, want limits") } @@ -213,7 +215,7 @@ func TestOCIResources(t *testing.T) { // EINVAL at container create. func TestOCIResources_ClampsQuotaToKernelMinimum(t *testing.T) { for _, millis := range []int64{1, 5, 9} { - got := ociResources(&ateletpb.ResourceLimits{CpuMillis: millis}) + got := ociResources(&ateompb.ResourceLimits{CpuMillis: millis}) if got == nil || got.CPU == nil || got.CPU.Quota == nil { t.Fatalf("cpu=%dm: ociResources() = %v, want a quota", millis, got) } @@ -223,7 +225,7 @@ func TestOCIResources_ClampsQuotaToKernelMinimum(t *testing.T) { } } // At the floor the quota is exact, not clamped. - got := ociResources(&ateletpb.ResourceLimits{CpuMillis: 10}) + got := ociResources(&ateompb.ResourceLimits{CpuMillis: 10}) if *got.CPU.Quota != cpuQuotaMinUS { t.Errorf("cpu=10m: quota = %d, want exactly %d", *got.CPU.Quota, cpuQuotaMinUS) } @@ -232,24 +234,24 @@ func TestOCIResources_ClampsQuotaToKernelMinimum(t *testing.T) { // A negative limit must not produce a non-nil but empty LinuxResources, which // would put a bare "resources": {} into the spec. func TestOCIResources_NegativeIsUnset(t *testing.T) { - if got := ociResources(&ateletpb.ResourceLimits{MemoryBytes: -1, CpuMillis: -1}); got != nil { + if got := ociResources(&ateompb.ResourceLimits{MemoryBytes: -1, CpuMillis: -1}); got != nil { t.Errorf("ociResources(negative) = %+v, want nil", got) } } // A container without limits carries no linux.resources at all. func TestBuild_NoResourcesLeavesLinuxUntouched(t *testing.T) { - spec := Build(Options{Args: []string{"/pause"}}) + spec := Build(Options{Container: &ateompb.Container{ContainerSpec: &ateompb.ContainerSpec{Args: []string{"/app"}}}}) if spec.Linux.Resources != nil { t.Errorf("Linux.Resources = %v, want nil when no limits are declared", spec.Linux.Resources) } } func TestBuild_ResourcesApplied(t *testing.T) { - spec := Build(Options{ + spec := Build(Options{Container: &ateompb.Container{ContainerSpec: &ateompb.ContainerSpec{ Args: []string{"/app"}, - Resources: &ateletpb.ResourceLimits{MemoryBytes: 67108864}, - }) + Resources: &ateompb.ResourceLimits{MemoryBytes: 67108864}, + }}}) if spec.Linux.Resources == nil || spec.Linux.Resources.Memory == nil { t.Fatalf("Linux.Resources = %v, want a memory limit", spec.Linux.Resources) } diff --git a/internal/ocispec/parity_test.go b/internal/ocispec/parity_test.go index 4fbd80fe31..468c511721 100644 --- a/internal/ocispec/parity_test.go +++ b/internal/ocispec/parity_test.go @@ -16,12 +16,10 @@ package ocispec import ( "encoding/json" - "path/filepath" "slices" "strings" "testing" - "github.com/agent-substrate/substrate/internal/proto/ateletpb" "github.com/agent-substrate/substrate/internal/proto/ateompb" "github.com/agent-substrate/substrate/internal/sizing" "github.com/opencontainers/runtime-spec/specs-go" @@ -39,26 +37,21 @@ var parityActorDirs = &ateompb.ActorDirs{ // parityOptions mounts one volume of every kind. var parityOptions = Options{ - Args: []string{"/app"}, - DurableDirVolumeMountsDir: parityActorDirs.GetDurableDirVolumeMountsDir(), - VolumesDir: parityActorDirs.GetVolumesDir(), - SystemInfoVolumeRootsDir: parityActorDirs.GetSystemInfoVolumeRootsDir(), - BundlePath: filepath.Join(parityActorDirs.GetOciBundleDir(), "app"), - Volumes: []*ateletpb.Volume{ - durableVolume("data"), - {Name: "sysinfo", Source: &ateletpb.Volume_SystemInfo{SystemInfo: &ateletpb.SystemInfoVolume{}}}, - {Name: "csi", Source: &ateletpb.Volume_External{External: &ateletpb.ExternalVolumeSource{}}}, - {Name: "agent", Source: &ateletpb.Volume_Image{Image: &ateletpb.ImageVolumeSource{}}}, - }, - VolumeMounts: []*ateletpb.VolumeMount{ - {Name: "data", MountPath: "/var/data"}, - // System-info volume mount. - {Name: "sysinfo", MountPath: "/run/ate"}, - {Name: "csi", MountPath: "/mnt/csi"}, - {Name: "agent", MountPath: "/ate"}, + ActorUID: testActorUID, + ActorDirs: parityActorDirs, + Container: &ateompb.Container{ + Name: "app", + ContainerSpec: &ateompb.ContainerSpec{Args: []string{"/app"}}, + DurableDirVolumeMounts: []*ateompb.DurableDirVolumeMount{{VolumeName: "data", MountPath: "/var/data"}}, + SystemInfoVolumeMounts: []*ateompb.SystemInfoVolumeMount{{VolumeName: "sysinfo", MountPath: "/run/ate"}}, + CsiVolumeMounts: []*ateompb.VolumeMount{{VolumeName: "csi", MountPath: "/mnt/csi"}}, + ImageVolumeMounts: []*ateompb.ImageVolumeMount{{VolumeName: "agent", MountPath: "/ate"}}, }, } +// parityMountPaths are where parityOptions mounts its volumes. +var parityMountPaths = []string{"/var/data", "/run/ate", "/mnt/csi", "/ate"} + var paritySize = sizing.SandboxSize{MilliCPU: 500, MemoryBytes: 512 << 20} // Build's output contains no runtime-specific literals. @@ -96,10 +89,9 @@ func TestShapers_PreserveEveryVolumeMount(t *testing.T) { if err := tc.shape(spec); err != nil { t.Fatalf("shaping the spec: %v", err) } - for _, vm := range parityOptions.VolumeMounts { - m := mountFor(t, spec, vm.GetMountPath()) - if m.Source == "" { - t.Errorf("mount %q has no source after shaping", vm.GetMountPath()) + for _, dest := range parityMountPaths { + if m := mountFor(t, spec, dest); m.Source == "" { + t.Errorf("mount %q has no source after shaping", dest) } } // Neither shaper overrides the hostname. @@ -139,7 +131,7 @@ func TestShapeMicroVM_TranslatesSourcesIntoTheShare(t *testing.T) { // ShapeMicroVM errors on a bind that is not staged into the share. func TestShapeMicroVM_UnstagedSourceIsAnError(t *testing.T) { - spec := Build(Options{Args: []string{"/app"}}) + spec := Build(Options{Container: &ateompb.Container{ContainerSpec: &ateompb.ContainerSpec{Args: []string{"/app"}}}}) spec.Mounts = append(spec.Mounts, specs.Mount{Destination: "/mnt/new", Type: "bind", Source: "/var/lib/ate/new-kind/x"}) if err := ShapeMicroVM(spec, MicroVMOptions{ActorDirs: parityActorDirs, ContainerID: "app"}); err == nil { t.Fatal("ShapeMicroVM() = nil, want an error for a bind that is not staged into the share") diff --git a/internal/proto/ateletpb/atelet.pb.go b/internal/proto/ateletpb/atelet.pb.go index ea6f25b7bd..e11f9cc298 100644 --- a/internal/proto/ateletpb/atelet.pb.go +++ b/internal/proto/ateletpb/atelet.pb.go @@ -140,58 +140,65 @@ func (CheckpointType) EnumDescriptor() ([]byte, []int) { return file_atelet_proto_rawDescGZIP(), []int{1} } -type SnapshotScope int32 +// SnapshotFidelity mirrors the public ateapi enum: the state layers a +// snapshot holds, lowest to highest, each including the ones below it. +type SnapshotFidelity int32 const ( // Not valid option; should never happen. - SnapshotScope_SNAPSHOT_SCOPE_UNSPECIFIED SnapshotScope = 0 - // Capture process memory plus the full filesystem delta on top of the OCI - // image (including any attached DurableDir volumes). - SnapshotScope_SNAPSHOT_SCOPE_FULL SnapshotScope = 1 + SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED SnapshotFidelity = 0 // Capture only the contents of attached volumes that support snapshots // (currently DurableDir-typed volumes). Memory and the rest of rootfs are // excluded. - SnapshotScope_SNAPSHOT_SCOPE_DATA SnapshotScope = 2 + SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES SnapshotFidelity = 1 + // Volumes plus the root filesystem changes made since boot. No runtime + // captures this yet; requests carrying it are rejected. + SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS SnapshotFidelity = 2 + // Capture process memory plus the full filesystem delta on top of the OCI + // image (including any attached DurableDir volumes). + SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY SnapshotFidelity = 3 ) -// Enum value maps for SnapshotScope. +// Enum value maps for SnapshotFidelity. var ( - SnapshotScope_name = map[int32]string{ - 0: "SNAPSHOT_SCOPE_UNSPECIFIED", - 1: "SNAPSHOT_SCOPE_FULL", - 2: "SNAPSHOT_SCOPE_DATA", - } - SnapshotScope_value = map[string]int32{ - "SNAPSHOT_SCOPE_UNSPECIFIED": 0, - "SNAPSHOT_SCOPE_FULL": 1, - "SNAPSHOT_SCOPE_DATA": 2, + SnapshotFidelity_name = map[int32]string{ + 0: "SNAPSHOT_FIDELITY_UNSPECIFIED", + 1: "SNAPSHOT_FIDELITY_VOLUMES", + 2: "SNAPSHOT_FIDELITY_ROOTFS", + 3: "SNAPSHOT_FIDELITY_MEMORY", + } + SnapshotFidelity_value = map[string]int32{ + "SNAPSHOT_FIDELITY_UNSPECIFIED": 0, + "SNAPSHOT_FIDELITY_VOLUMES": 1, + "SNAPSHOT_FIDELITY_ROOTFS": 2, + "SNAPSHOT_FIDELITY_MEMORY": 3, } ) -func (x SnapshotScope) Enum() *SnapshotScope { - p := new(SnapshotScope) +func (x SnapshotFidelity) Enum() *SnapshotFidelity { + p := new(SnapshotFidelity) *p = x return p } -func (x SnapshotScope) String() string { +func (x SnapshotFidelity) String() string { return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) } -func (SnapshotScope) Descriptor() protoreflect.EnumDescriptor { +func (SnapshotFidelity) Descriptor() protoreflect.EnumDescriptor { return file_atelet_proto_enumTypes[2].Descriptor() } -func (SnapshotScope) Type() protoreflect.EnumType { +func (SnapshotFidelity) Type() protoreflect.EnumType { return &file_atelet_proto_enumTypes[2] } -func (x SnapshotScope) Number() protoreflect.EnumNumber { +func (x SnapshotFidelity) Number() protoreflect.EnumNumber { return protoreflect.EnumNumber(x) } -// Deprecated: Use SnapshotScope.Descriptor instead. -func (SnapshotScope) EnumDescriptor() ([]byte, []int) { +// Deprecated: Use SnapshotFidelity.Descriptor instead. +func (SnapshotFidelity) EnumDescriptor() ([]byte, []int) { return file_atelet_proto_rawDescGZIP(), []int{2} } @@ -202,13 +209,21 @@ type RegisterWorkerRequest struct { // // +k8s:required Capacity *WorkerResources `protobuf:"bytes,1,opt,name=capacity,proto3" json:"capacity,omitempty"` - // Hardware identity of the worker. atelet converts it to the control - // plane's HardwareIdentity when forwarding. + // Runtime that new and cold-booted actors start under. + // Snapshots matching default_runtime can also warm-boot on this worker. + // atelet converts it to the control plane's SandboxRuntime when forwarding. // // +k8s:required - Hardware *HardwareIdentity `protobuf:"bytes,2,opt,name=hardware,proto3" json:"hardware,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + DefaultRuntime *SandboxRuntime `protobuf:"bytes,2,opt,name=default_runtime,json=defaultRuntime,proto3" json:"default_runtime,omitempty"` + // Additional runtimes (excluding default_runtime) that this worker can + // restore snapshots from right now. + // + // +k8s:optional + // +k8s:maxItems=32 + // +k8s:listType=atomic + RestorableRuntimes []*SandboxRuntime `protobuf:"bytes,3,rep,name=restorable_runtimes,json=restorableRuntimes,proto3" json:"restorable_runtimes,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *RegisterWorkerRequest) Reset() { @@ -248,43 +263,55 @@ func (x *RegisterWorkerRequest) GetCapacity() *WorkerResources { return nil } -func (x *RegisterWorkerRequest) GetHardware() *HardwareIdentity { +func (x *RegisterWorkerRequest) GetDefaultRuntime() *SandboxRuntime { if x != nil { - return x.Hardware + return x.DefaultRuntime } return nil } -// HardwareIdentity identifies the hardware properties of a worker. It mirrors -// the control plane's HardwareIdentity and is validated by the same rules. -type HardwareIdentity struct { +func (x *RegisterWorkerRequest) GetRestorableRuntimes() []*SandboxRuntime { + if x != nil { + return x.RestorableRuntimes + } + return nil +} + +// SandboxRuntime describes a sandbox runtime and its compatibility identity as +// reported by a worker. It mirrors the control plane's SandboxRuntime and is +// validated by the same rules. +type SandboxRuntime struct { state protoimpl.MessageState `protogen:"open.v1"` - // Key-value pairs describing hardware properties (currently "architecture"; - // TODO: add "cpu_vendor", "cpu_model", etc.). + // e.g. "gvisor", "microvm" // - // +k8s:optional - // +k8s:maxProperties=32 - // +k8s:eachKey=+k8s:maxLength=128 - // +k8s:eachVal=+k8s:maxLength=256 - Attributes map[string]string `protobuf:"bytes,1,rep,name=attributes,proto3" json:"attributes,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + // +k8s:required + // +k8s:maxLength=63 + SandboxClass string `protobuf:"bytes,1,opt,name=sandbox_class,json=sandboxClass,proto3" json:"sandbox_class,omitempty"` + // Primary compatibility version stamped onto an actor's snapshot when a + // checkpoint is taken on this runtime. Snapshots matching `version` + // can warm-boot on this runtime. + // TODO: Add support for additional compatibility versions. + // + // +k8s:required + Version *VersionedSandboxCompat `protobuf:"bytes,2,opt,name=version,proto3" json:"version,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } -func (x *HardwareIdentity) Reset() { - *x = HardwareIdentity{} +func (x *SandboxRuntime) Reset() { + *x = SandboxRuntime{} mi := &file_atelet_proto_msgTypes[1] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } -func (x *HardwareIdentity) String() string { +func (x *SandboxRuntime) String() string { return protoimpl.X.MessageStringOf(x) } -func (*HardwareIdentity) ProtoMessage() {} +func (*SandboxRuntime) ProtoMessage() {} -func (x *HardwareIdentity) ProtoReflect() protoreflect.Message { +func (x *SandboxRuntime) ProtoReflect() protoreflect.Message { mi := &file_atelet_proto_msgTypes[1] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -296,18 +323,159 @@ func (x *HardwareIdentity) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use HardwareIdentity.ProtoReflect.Descriptor instead. -func (*HardwareIdentity) Descriptor() ([]byte, []int) { +// Deprecated: Use SandboxRuntime.ProtoReflect.Descriptor instead. +func (*SandboxRuntime) Descriptor() ([]byte, []int) { return file_atelet_proto_rawDescGZIP(), []int{1} } -func (x *HardwareIdentity) GetAttributes() map[string]string { +func (x *SandboxRuntime) GetSandboxClass() string { + if x != nil { + return x.SandboxClass + } + return "" +} + +func (x *SandboxRuntime) GetVersion() *VersionedSandboxCompat { + if x != nil { + return x.Version + } + return nil +} + +// VersionedSandboxCompat holds the compatibility schema version and the +// key-value compatibility attributes for a SandboxRuntime. +type VersionedSandboxCompat struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Compatibility schema version (e.g. "v1") defining which runtime and + // hardware attributes are recorded and how they are computed. + // + // +k8s:required + // +k8s:maxLength=64 + SchemaVersion string `protobuf:"bytes,1,opt,name=schema_version,json=schemaVersion,proto3" json:"schema_version,omitempty"` + // Key-value compatibility attributes defined by this schema version, covering + // both runtime assets (e.g. "gvisor_asset_hash") and effective guest hardware + // features after masking (e.g. "architecture", "cpu_features"). + // Matched against a snapshot under the same schema_version: every attribute + // on the snapshot must be present on the worker with an equal value; keys + // only the worker has are ignored. Bump schema_version when a key's meaning + // changes or older snapshots must no longer match. + // Order as sent does not matter: ate-api-server sorts attributes by key + // before recording them. + // + // +k8s:required + // +k8s:maxItems=32 + // +k8s:listType=map + // +k8s:listMapKey=key + Attributes []*AttributeEntry `protobuf:"bytes,2,rep,name=attributes,proto3" json:"attributes,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *VersionedSandboxCompat) Reset() { + *x = VersionedSandboxCompat{} + mi := &file_atelet_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *VersionedSandboxCompat) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*VersionedSandboxCompat) ProtoMessage() {} + +func (x *VersionedSandboxCompat) ProtoReflect() protoreflect.Message { + mi := &file_atelet_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use VersionedSandboxCompat.ProtoReflect.Descriptor instead. +func (*VersionedSandboxCompat) Descriptor() ([]byte, []int) { + return file_atelet_proto_rawDescGZIP(), []int{2} +} + +func (x *VersionedSandboxCompat) GetSchemaVersion() string { + if x != nil { + return x.SchemaVersion + } + return "" +} + +func (x *VersionedSandboxCompat) GetAttributes() []*AttributeEntry { if x != nil { return x.Attributes } return nil } +// AttributeEntry is a single key-value compatibility attribute. +type AttributeEntry struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Attribute name (e.g. "architecture", "cpu_features", "gvisor_asset_hash"). + // + // +k8s:required + // +k8s:maxLength=128 + Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` + // Attribute value corresponding to key. + // + // +k8s:required + // +k8s:maxLength=256 + Value string `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *AttributeEntry) Reset() { + *x = AttributeEntry{} + mi := &file_atelet_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *AttributeEntry) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AttributeEntry) ProtoMessage() {} + +func (x *AttributeEntry) ProtoReflect() protoreflect.Message { + mi := &file_atelet_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AttributeEntry.ProtoReflect.Descriptor instead. +func (*AttributeEntry) Descriptor() ([]byte, []int) { + return file_atelet_proto_rawDescGZIP(), []int{3} +} + +func (x *AttributeEntry) GetKey() string { + if x != nil { + return x.Key + } + return "" +} + +func (x *AttributeEntry) GetValue() string { + if x != nil { + return x.Value + } + return "" +} + // WorkerResources is what a worker can supply. It mirrors the control plane's // WorkerResources and is validated by the same rules. type WorkerResources struct { @@ -329,7 +497,7 @@ type WorkerResources struct { func (x *WorkerResources) Reset() { *x = WorkerResources{} - mi := &file_atelet_proto_msgTypes[2] + mi := &file_atelet_proto_msgTypes[4] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -341,7 +509,7 @@ func (x *WorkerResources) String() string { func (*WorkerResources) ProtoMessage() {} func (x *WorkerResources) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[2] + mi := &file_atelet_proto_msgTypes[4] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -354,7 +522,7 @@ func (x *WorkerResources) ProtoReflect() protoreflect.Message { // Deprecated: Use WorkerResources.ProtoReflect.Descriptor instead. func (*WorkerResources) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{2} + return file_atelet_proto_rawDescGZIP(), []int{4} } func (x *WorkerResources) GetResources() *Resources { @@ -388,7 +556,7 @@ type Resources struct { func (x *Resources) Reset() { *x = Resources{} - mi := &file_atelet_proto_msgTypes[3] + mi := &file_atelet_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -400,7 +568,7 @@ func (x *Resources) String() string { func (*Resources) ProtoMessage() {} func (x *Resources) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[3] + mi := &file_atelet_proto_msgTypes[5] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -413,7 +581,7 @@ func (x *Resources) ProtoReflect() protoreflect.Message { // Deprecated: Use Resources.ProtoReflect.Descriptor instead. func (*Resources) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{3} + return file_atelet_proto_rawDescGZIP(), []int{5} } func (x *Resources) GetLimits() []*Limits { @@ -441,7 +609,7 @@ type Limits struct { func (x *Limits) Reset() { *x = Limits{} - mi := &file_atelet_proto_msgTypes[4] + mi := &file_atelet_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -453,7 +621,7 @@ func (x *Limits) String() string { func (*Limits) ProtoMessage() {} func (x *Limits) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[4] + mi := &file_atelet_proto_msgTypes[6] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -466,7 +634,7 @@ func (x *Limits) ProtoReflect() protoreflect.Message { // Deprecated: Use Limits.ProtoReflect.Descriptor instead. func (*Limits) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{4} + return file_atelet_proto_rawDescGZIP(), []int{6} } func (x *Limits) GetName() string { @@ -491,7 +659,7 @@ type RegisterWorkerResponse struct { func (x *RegisterWorkerResponse) Reset() { *x = RegisterWorkerResponse{} - mi := &file_atelet_proto_msgTypes[5] + mi := &file_atelet_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -503,7 +671,7 @@ func (x *RegisterWorkerResponse) String() string { func (*RegisterWorkerResponse) ProtoMessage() {} func (x *RegisterWorkerResponse) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[5] + mi := &file_atelet_proto_msgTypes[7] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -516,7 +684,7 @@ func (x *RegisterWorkerResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RegisterWorkerResponse.ProtoReflect.Descriptor instead. func (*RegisterWorkerResponse) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{5} + return file_atelet_proto_rawDescGZIP(), []int{7} } type RequestActorSuspendRequest struct { @@ -543,7 +711,7 @@ type RequestActorSuspendRequest struct { func (x *RequestActorSuspendRequest) Reset() { *x = RequestActorSuspendRequest{} - mi := &file_atelet_proto_msgTypes[6] + mi := &file_atelet_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -555,7 +723,7 @@ func (x *RequestActorSuspendRequest) String() string { func (*RequestActorSuspendRequest) ProtoMessage() {} func (x *RequestActorSuspendRequest) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[6] + mi := &file_atelet_proto_msgTypes[8] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -568,7 +736,7 @@ func (x *RequestActorSuspendRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use RequestActorSuspendRequest.ProtoReflect.Descriptor instead. func (*RequestActorSuspendRequest) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{6} + return file_atelet_proto_rawDescGZIP(), []int{8} } func (x *RequestActorSuspendRequest) GetActorAtespace() string { @@ -600,7 +768,7 @@ type RequestActorSuspendResponse struct { func (x *RequestActorSuspendResponse) Reset() { *x = RequestActorSuspendResponse{} - mi := &file_atelet_proto_msgTypes[7] + mi := &file_atelet_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -612,7 +780,7 @@ func (x *RequestActorSuspendResponse) String() string { func (*RequestActorSuspendResponse) ProtoMessage() {} func (x *RequestActorSuspendResponse) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[7] + mi := &file_atelet_proto_msgTypes[9] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -625,7 +793,7 @@ func (x *RequestActorSuspendResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RequestActorSuspendResponse.ProtoReflect.Descriptor instead. func (*RequestActorSuspendResponse) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{7} + return file_atelet_proto_rawDescGZIP(), []int{9} } type MintActorCertificateRequest struct { @@ -656,7 +824,7 @@ type MintActorCertificateRequest struct { func (x *MintActorCertificateRequest) Reset() { *x = MintActorCertificateRequest{} - mi := &file_atelet_proto_msgTypes[8] + mi := &file_atelet_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -668,7 +836,7 @@ func (x *MintActorCertificateRequest) String() string { func (*MintActorCertificateRequest) ProtoMessage() {} func (x *MintActorCertificateRequest) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[8] + mi := &file_atelet_proto_msgTypes[10] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -681,7 +849,7 @@ func (x *MintActorCertificateRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use MintActorCertificateRequest.ProtoReflect.Descriptor instead. func (*MintActorCertificateRequest) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{8} + return file_atelet_proto_rawDescGZIP(), []int{10} } func (x *MintActorCertificateRequest) GetActorAtespace() string { @@ -722,7 +890,7 @@ type MintActorCertificateResponse struct { func (x *MintActorCertificateResponse) Reset() { *x = MintActorCertificateResponse{} - mi := &file_atelet_proto_msgTypes[9] + mi := &file_atelet_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -734,7 +902,7 @@ func (x *MintActorCertificateResponse) String() string { func (*MintActorCertificateResponse) ProtoMessage() {} func (x *MintActorCertificateResponse) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[9] + mi := &file_atelet_proto_msgTypes[11] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -747,7 +915,7 @@ func (x *MintActorCertificateResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use MintActorCertificateResponse.ProtoReflect.Descriptor instead. func (*MintActorCertificateResponse) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{9} + return file_atelet_proto_rawDescGZIP(), []int{11} } func (x *MintActorCertificateResponse) GetActorCertificates() [][]byte { @@ -760,7 +928,7 @@ func (x *MintActorCertificateResponse) GetActorCertificates() [][]byte { type TerminateRequest struct { state protoimpl.MessageState `protogen:"open.v1"` // If unset, the atelet will just cleanup node resources for the actor. - TargetAteomUid string `protobuf:"bytes,1,opt,name=target_ateom_uid,json=targetAteomUid,proto3" json:"target_ateom_uid,omitempty"` + WorkerPodUid string `protobuf:"bytes,1,opt,name=worker_pod_uid,json=workerPodUid,proto3" json:"worker_pod_uid,omitempty"` Atespace string `protobuf:"bytes,2,opt,name=atespace,proto3" json:"atespace,omitempty"` ActorName string `protobuf:"bytes,3,opt,name=actor_name,json=actorName,proto3" json:"actor_name,omitempty"` ActorUid string `protobuf:"bytes,4,opt,name=actor_uid,json=actorUid,proto3" json:"actor_uid,omitempty"` @@ -773,7 +941,7 @@ type TerminateRequest struct { func (x *TerminateRequest) Reset() { *x = TerminateRequest{} - mi := &file_atelet_proto_msgTypes[10] + mi := &file_atelet_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -785,7 +953,7 @@ func (x *TerminateRequest) String() string { func (*TerminateRequest) ProtoMessage() {} func (x *TerminateRequest) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[10] + mi := &file_atelet_proto_msgTypes[12] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -798,12 +966,12 @@ func (x *TerminateRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use TerminateRequest.ProtoReflect.Descriptor instead. func (*TerminateRequest) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{10} + return file_atelet_proto_rawDescGZIP(), []int{12} } -func (x *TerminateRequest) GetTargetAteomUid() string { +func (x *TerminateRequest) GetWorkerPodUid() string { if x != nil { - return x.TargetAteomUid + return x.WorkerPodUid } return "" } @@ -858,7 +1026,7 @@ type TerminateResponse struct { func (x *TerminateResponse) Reset() { *x = TerminateResponse{} - mi := &file_atelet_proto_msgTypes[11] + mi := &file_atelet_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -870,7 +1038,7 @@ func (x *TerminateResponse) String() string { func (*TerminateResponse) ProtoMessage() {} func (x *TerminateResponse) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[11] + mi := &file_atelet_proto_msgTypes[13] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -883,12 +1051,12 @@ func (x *TerminateResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use TerminateResponse.ProtoReflect.Descriptor instead. func (*TerminateResponse) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{11} + return file_atelet_proto_rawDescGZIP(), []int{13} } type RunRequest struct { state protoimpl.MessageState `protogen:"open.v1"` - TargetAteomUid string `protobuf:"bytes,1,opt,name=target_ateom_uid,json=targetAteomUid,proto3" json:"target_ateom_uid,omitempty"` + WorkerPodUid string `protobuf:"bytes,1,opt,name=worker_pod_uid,json=workerPodUid,proto3" json:"worker_pod_uid,omitempty"` Atespace string `protobuf:"bytes,2,opt,name=atespace,proto3" json:"atespace,omitempty"` ActorName string `protobuf:"bytes,3,opt,name=actor_name,json=actorName,proto3" json:"actor_name,omitempty"` ActorUid string `protobuf:"bytes,4,opt,name=actor_uid,json=actorUid,proto3" json:"actor_uid,omitempty"` @@ -912,7 +1080,7 @@ type RunRequest struct { func (x *RunRequest) Reset() { *x = RunRequest{} - mi := &file_atelet_proto_msgTypes[12] + mi := &file_atelet_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -924,7 +1092,7 @@ func (x *RunRequest) String() string { func (*RunRequest) ProtoMessage() {} func (x *RunRequest) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[12] + mi := &file_atelet_proto_msgTypes[14] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -937,12 +1105,12 @@ func (x *RunRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use RunRequest.ProtoReflect.Descriptor instead. func (*RunRequest) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{12} + return file_atelet_proto_rawDescGZIP(), []int{14} } -func (x *RunRequest) GetTargetAteomUid() string { +func (x *RunRequest) GetWorkerPodUid() string { if x != nil { - return x.TargetAteomUid + return x.WorkerPodUid } return "" } @@ -1029,7 +1197,7 @@ type EgressGateway struct { func (x *EgressGateway) Reset() { *x = EgressGateway{} - mi := &file_atelet_proto_msgTypes[13] + mi := &file_atelet_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1041,7 +1209,7 @@ func (x *EgressGateway) String() string { func (*EgressGateway) ProtoMessage() {} func (x *EgressGateway) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[13] + mi := &file_atelet_proto_msgTypes[15] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1054,7 +1222,7 @@ func (x *EgressGateway) ProtoReflect() protoreflect.Message { // Deprecated: Use EgressGateway.ProtoReflect.Descriptor instead. func (*EgressGateway) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{13} + return file_atelet_proto_rawDescGZIP(), []int{15} } func (x *EgressGateway) GetAddress() string { @@ -1078,7 +1246,7 @@ type AssetFile struct { func (x *AssetFile) Reset() { *x = AssetFile{} - mi := &file_atelet_proto_msgTypes[14] + mi := &file_atelet_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1090,7 +1258,7 @@ func (x *AssetFile) String() string { func (*AssetFile) ProtoMessage() {} func (x *AssetFile) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[14] + mi := &file_atelet_proto_msgTypes[16] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1103,7 +1271,7 @@ func (x *AssetFile) ProtoReflect() protoreflect.Message { // Deprecated: Use AssetFile.ProtoReflect.Descriptor instead. func (*AssetFile) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{14} + return file_atelet_proto_rawDescGZIP(), []int{16} } func (x *AssetFile) GetUrl() string { @@ -1131,7 +1299,7 @@ type ArchAssets struct { func (x *ArchAssets) Reset() { *x = ArchAssets{} - mi := &file_atelet_proto_msgTypes[15] + mi := &file_atelet_proto_msgTypes[17] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1143,7 +1311,7 @@ func (x *ArchAssets) String() string { func (*ArchAssets) ProtoMessage() {} func (x *ArchAssets) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[15] + mi := &file_atelet_proto_msgTypes[17] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1156,7 +1324,7 @@ func (x *ArchAssets) ProtoReflect() protoreflect.Message { // Deprecated: Use ArchAssets.ProtoReflect.Descriptor instead. func (*ArchAssets) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{15} + return file_atelet_proto_rawDescGZIP(), []int{17} } func (x *ArchAssets) GetFiles() map[string]*AssetFile { @@ -1187,7 +1355,7 @@ type SandboxAssets struct { func (x *SandboxAssets) Reset() { *x = SandboxAssets{} - mi := &file_atelet_proto_msgTypes[16] + mi := &file_atelet_proto_msgTypes[18] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1199,7 +1367,7 @@ func (x *SandboxAssets) String() string { func (*SandboxAssets) ProtoMessage() {} func (x *SandboxAssets) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[16] + mi := &file_atelet_proto_msgTypes[18] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1212,7 +1380,7 @@ func (x *SandboxAssets) ProtoReflect() protoreflect.Message { // Deprecated: Use SandboxAssets.ProtoReflect.Descriptor instead. func (*SandboxAssets) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{16} + return file_atelet_proto_rawDescGZIP(), []int{18} } func (x *SandboxAssets) GetSandboxClass() string { @@ -1247,7 +1415,7 @@ type WorkloadSpec struct { func (x *WorkloadSpec) Reset() { *x = WorkloadSpec{} - mi := &file_atelet_proto_msgTypes[17] + mi := &file_atelet_proto_msgTypes[19] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1259,7 +1427,7 @@ func (x *WorkloadSpec) String() string { func (*WorkloadSpec) ProtoMessage() {} func (x *WorkloadSpec) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[17] + mi := &file_atelet_proto_msgTypes[19] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1272,7 +1440,7 @@ func (x *WorkloadSpec) ProtoReflect() protoreflect.Message { // Deprecated: Use WorkloadSpec.ProtoReflect.Descriptor instead. func (*WorkloadSpec) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{17} + return file_atelet_proto_rawDescGZIP(), []int{19} } func (x *WorkloadSpec) GetContainers() []*Container { @@ -1297,7 +1465,7 @@ type DurableDirVolume struct { func (x *DurableDirVolume) Reset() { *x = DurableDirVolume{} - mi := &file_atelet_proto_msgTypes[18] + mi := &file_atelet_proto_msgTypes[20] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1309,7 +1477,7 @@ func (x *DurableDirVolume) String() string { func (*DurableDirVolume) ProtoMessage() {} func (x *DurableDirVolume) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[18] + mi := &file_atelet_proto_msgTypes[20] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1322,7 +1490,7 @@ func (x *DurableDirVolume) ProtoReflect() protoreflect.Message { // Deprecated: Use DurableDirVolume.ProtoReflect.Descriptor instead. func (*DurableDirVolume) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{18} + return file_atelet_proto_rawDescGZIP(), []int{20} } type ExternalVolumeSource struct { @@ -1339,7 +1507,7 @@ type ExternalVolumeSource struct { func (x *ExternalVolumeSource) Reset() { *x = ExternalVolumeSource{} - mi := &file_atelet_proto_msgTypes[19] + mi := &file_atelet_proto_msgTypes[21] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1351,7 +1519,7 @@ func (x *ExternalVolumeSource) String() string { func (*ExternalVolumeSource) ProtoMessage() {} func (x *ExternalVolumeSource) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[19] + mi := &file_atelet_proto_msgTypes[21] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1364,7 +1532,7 @@ func (x *ExternalVolumeSource) ProtoReflect() protoreflect.Message { // Deprecated: Use ExternalVolumeSource.ProtoReflect.Descriptor instead. func (*ExternalVolumeSource) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{19} + return file_atelet_proto_rawDescGZIP(), []int{21} } func (x *ExternalVolumeSource) GetStorageVolumeId() string { @@ -1404,7 +1572,7 @@ type ImageVolumeSource struct { func (x *ImageVolumeSource) Reset() { *x = ImageVolumeSource{} - mi := &file_atelet_proto_msgTypes[20] + mi := &file_atelet_proto_msgTypes[22] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1416,7 +1584,7 @@ func (x *ImageVolumeSource) String() string { func (*ImageVolumeSource) ProtoMessage() {} func (x *ImageVolumeSource) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[20] + mi := &file_atelet_proto_msgTypes[22] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1429,7 +1597,7 @@ func (x *ImageVolumeSource) ProtoReflect() protoreflect.Message { // Deprecated: Use ImageVolumeSource.ProtoReflect.Descriptor instead. func (*ImageVolumeSource) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{20} + return file_atelet_proto_rawDescGZIP(), []int{22} } func (x *ImageVolumeSource) GetReference() string { @@ -1451,7 +1619,7 @@ type ActorMetadataItem struct { func (x *ActorMetadataItem) Reset() { *x = ActorMetadataItem{} - mi := &file_atelet_proto_msgTypes[21] + mi := &file_atelet_proto_msgTypes[23] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1463,7 +1631,7 @@ func (x *ActorMetadataItem) String() string { func (*ActorMetadataItem) ProtoMessage() {} func (x *ActorMetadataItem) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[21] + mi := &file_atelet_proto_msgTypes[23] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1476,7 +1644,7 @@ func (x *ActorMetadataItem) ProtoReflect() protoreflect.Message { // Deprecated: Use ActorMetadataItem.ProtoReflect.Descriptor instead. func (*ActorMetadataItem) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{21} + return file_atelet_proto_rawDescGZIP(), []int{23} } func (x *ActorMetadataItem) GetField() ActorMetadataField { @@ -1504,7 +1672,7 @@ type ActorMetadataDataSource struct { func (x *ActorMetadataDataSource) Reset() { *x = ActorMetadataDataSource{} - mi := &file_atelet_proto_msgTypes[22] + mi := &file_atelet_proto_msgTypes[24] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1516,7 +1684,7 @@ func (x *ActorMetadataDataSource) String() string { func (*ActorMetadataDataSource) ProtoMessage() {} func (x *ActorMetadataDataSource) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[22] + mi := &file_atelet_proto_msgTypes[24] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1529,7 +1697,7 @@ func (x *ActorMetadataDataSource) ProtoReflect() protoreflect.Message { // Deprecated: Use ActorMetadataDataSource.ProtoReflect.Descriptor instead. func (*ActorMetadataDataSource) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{22} + return file_atelet_proto_rawDescGZIP(), []int{24} } func (x *ActorMetadataDataSource) GetItems() []*ActorMetadataItem { @@ -1550,7 +1718,7 @@ type TrustBundleDataSource struct { func (x *TrustBundleDataSource) Reset() { *x = TrustBundleDataSource{} - mi := &file_atelet_proto_msgTypes[23] + mi := &file_atelet_proto_msgTypes[25] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1562,7 +1730,7 @@ func (x *TrustBundleDataSource) String() string { func (*TrustBundleDataSource) ProtoMessage() {} func (x *TrustBundleDataSource) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[23] + mi := &file_atelet_proto_msgTypes[25] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1575,7 +1743,7 @@ func (x *TrustBundleDataSource) ProtoReflect() protoreflect.Message { // Deprecated: Use TrustBundleDataSource.ProtoReflect.Descriptor instead. func (*TrustBundleDataSource) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{23} + return file_atelet_proto_rawDescGZIP(), []int{25} } func (x *TrustBundleDataSource) GetPath() string { @@ -1605,7 +1773,7 @@ type SystemInfoDataSource struct { func (x *SystemInfoDataSource) Reset() { *x = SystemInfoDataSource{} - mi := &file_atelet_proto_msgTypes[24] + mi := &file_atelet_proto_msgTypes[26] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1617,7 +1785,7 @@ func (x *SystemInfoDataSource) String() string { func (*SystemInfoDataSource) ProtoMessage() {} func (x *SystemInfoDataSource) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[24] + mi := &file_atelet_proto_msgTypes[26] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1630,7 +1798,7 @@ func (x *SystemInfoDataSource) ProtoReflect() protoreflect.Message { // Deprecated: Use SystemInfoDataSource.ProtoReflect.Descriptor instead. func (*SystemInfoDataSource) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{24} + return file_atelet_proto_rawDescGZIP(), []int{26} } func (x *SystemInfoDataSource) GetDataSource() isSystemInfoDataSource_DataSource { @@ -1686,7 +1854,7 @@ type SystemInfoVolume struct { func (x *SystemInfoVolume) Reset() { *x = SystemInfoVolume{} - mi := &file_atelet_proto_msgTypes[25] + mi := &file_atelet_proto_msgTypes[27] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1698,7 +1866,7 @@ func (x *SystemInfoVolume) String() string { func (*SystemInfoVolume) ProtoMessage() {} func (x *SystemInfoVolume) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[25] + mi := &file_atelet_proto_msgTypes[27] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1711,7 +1879,7 @@ func (x *SystemInfoVolume) ProtoReflect() protoreflect.Message { // Deprecated: Use SystemInfoVolume.ProtoReflect.Descriptor instead. func (*SystemInfoVolume) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{25} + return file_atelet_proto_rawDescGZIP(), []int{27} } func (x *SystemInfoVolume) GetDataSources() []*SystemInfoDataSource { @@ -1737,7 +1905,7 @@ type Volume struct { func (x *Volume) Reset() { *x = Volume{} - mi := &file_atelet_proto_msgTypes[26] + mi := &file_atelet_proto_msgTypes[28] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1749,7 +1917,7 @@ func (x *Volume) String() string { func (*Volume) ProtoMessage() {} func (x *Volume) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[26] + mi := &file_atelet_proto_msgTypes[28] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1762,7 +1930,7 @@ func (x *Volume) ProtoReflect() protoreflect.Message { // Deprecated: Use Volume.ProtoReflect.Descriptor instead. func (*Volume) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{26} + return file_atelet_proto_rawDescGZIP(), []int{28} } func (x *Volume) GetName() string { @@ -1853,7 +2021,7 @@ type VolumeMount struct { func (x *VolumeMount) Reset() { *x = VolumeMount{} - mi := &file_atelet_proto_msgTypes[27] + mi := &file_atelet_proto_msgTypes[29] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1865,7 +2033,7 @@ func (x *VolumeMount) String() string { func (*VolumeMount) ProtoMessage() {} func (x *VolumeMount) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[27] + mi := &file_atelet_proto_msgTypes[29] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1878,7 +2046,7 @@ func (x *VolumeMount) ProtoReflect() protoreflect.Message { // Deprecated: Use VolumeMount.ProtoReflect.Descriptor instead. func (*VolumeMount) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{27} + return file_atelet_proto_rawDescGZIP(), []int{29} } func (x *VolumeMount) GetName() string { @@ -1914,7 +2082,7 @@ type Container struct { func (x *Container) Reset() { *x = Container{} - mi := &file_atelet_proto_msgTypes[28] + mi := &file_atelet_proto_msgTypes[30] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1926,7 +2094,7 @@ func (x *Container) String() string { func (*Container) ProtoMessage() {} func (x *Container) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[28] + mi := &file_atelet_proto_msgTypes[30] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1939,7 +2107,7 @@ func (x *Container) ProtoReflect() protoreflect.Message { // Deprecated: Use Container.ProtoReflect.Descriptor instead. func (*Container) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{28} + return file_atelet_proto_rawDescGZIP(), []int{30} } func (x *Container) GetName() string { @@ -2015,7 +2183,7 @@ type SecurityContext struct { func (x *SecurityContext) Reset() { *x = SecurityContext{} - mi := &file_atelet_proto_msgTypes[29] + mi := &file_atelet_proto_msgTypes[31] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2027,7 +2195,7 @@ func (x *SecurityContext) String() string { func (*SecurityContext) ProtoMessage() {} func (x *SecurityContext) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[29] + mi := &file_atelet_proto_msgTypes[31] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2040,7 +2208,7 @@ func (x *SecurityContext) ProtoReflect() protoreflect.Message { // Deprecated: Use SecurityContext.ProtoReflect.Descriptor instead. func (*SecurityContext) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{29} + return file_atelet_proto_rawDescGZIP(), []int{31} } func (x *SecurityContext) GetCapabilities() *Capabilities { @@ -2062,7 +2230,7 @@ type Capabilities struct { func (x *Capabilities) Reset() { *x = Capabilities{} - mi := &file_atelet_proto_msgTypes[30] + mi := &file_atelet_proto_msgTypes[32] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2074,7 +2242,7 @@ func (x *Capabilities) String() string { func (*Capabilities) ProtoMessage() {} func (x *Capabilities) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[30] + mi := &file_atelet_proto_msgTypes[32] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2087,7 +2255,7 @@ func (x *Capabilities) ProtoReflect() protoreflect.Message { // Deprecated: Use Capabilities.ProtoReflect.Descriptor instead. func (*Capabilities) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{30} + return file_atelet_proto_rawDescGZIP(), []int{32} } func (x *Capabilities) GetAdd() []string { @@ -2118,7 +2286,7 @@ type ResourceLimits struct { func (x *ResourceLimits) Reset() { *x = ResourceLimits{} - mi := &file_atelet_proto_msgTypes[31] + mi := &file_atelet_proto_msgTypes[33] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2130,7 +2298,7 @@ func (x *ResourceLimits) String() string { func (*ResourceLimits) ProtoMessage() {} func (x *ResourceLimits) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[31] + mi := &file_atelet_proto_msgTypes[33] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2143,7 +2311,7 @@ func (x *ResourceLimits) ProtoReflect() protoreflect.Message { // Deprecated: Use ResourceLimits.ProtoReflect.Descriptor instead. func (*ResourceLimits) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{31} + return file_atelet_proto_rawDescGZIP(), []int{33} } func (x *ResourceLimits) GetMemoryBytes() int64 { @@ -2170,7 +2338,7 @@ type EnvEntry struct { func (x *EnvEntry) Reset() { *x = EnvEntry{} - mi := &file_atelet_proto_msgTypes[32] + mi := &file_atelet_proto_msgTypes[34] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2182,7 +2350,7 @@ func (x *EnvEntry) String() string { func (*EnvEntry) ProtoMessage() {} func (x *EnvEntry) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[32] + mi := &file_atelet_proto_msgTypes[34] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2195,7 +2363,7 @@ func (x *EnvEntry) ProtoReflect() protoreflect.Message { // Deprecated: Use EnvEntry.ProtoReflect.Descriptor instead. func (*EnvEntry) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{32} + return file_atelet_proto_rawDescGZIP(), []int{34} } func (x *EnvEntry) GetName() string { @@ -2225,7 +2393,7 @@ type WakeupProbe struct { func (x *WakeupProbe) Reset() { *x = WakeupProbe{} - mi := &file_atelet_proto_msgTypes[33] + mi := &file_atelet_proto_msgTypes[35] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2237,7 +2405,7 @@ func (x *WakeupProbe) String() string { func (*WakeupProbe) ProtoMessage() {} func (x *WakeupProbe) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[33] + mi := &file_atelet_proto_msgTypes[35] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2250,7 +2418,7 @@ func (x *WakeupProbe) ProtoReflect() protoreflect.Message { // Deprecated: Use WakeupProbe.ProtoReflect.Descriptor instead. func (*WakeupProbe) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{33} + return file_atelet_proto_rawDescGZIP(), []int{35} } func (x *WakeupProbe) GetHttpGet() *HTTPGetAction { @@ -2280,7 +2448,7 @@ type HTTPGetAction struct { func (x *HTTPGetAction) Reset() { *x = HTTPGetAction{} - mi := &file_atelet_proto_msgTypes[34] + mi := &file_atelet_proto_msgTypes[36] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2292,7 +2460,7 @@ func (x *HTTPGetAction) String() string { func (*HTTPGetAction) ProtoMessage() {} func (x *HTTPGetAction) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[34] + mi := &file_atelet_proto_msgTypes[36] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2305,7 +2473,7 @@ func (x *HTTPGetAction) ProtoReflect() protoreflect.Message { // Deprecated: Use HTTPGetAction.ProtoReflect.Descriptor instead. func (*HTTPGetAction) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{34} + return file_atelet_proto_rawDescGZIP(), []int{36} } func (x *HTTPGetAction) GetPath() string { @@ -2330,7 +2498,7 @@ type RunResponse struct { func (x *RunResponse) Reset() { *x = RunResponse{} - mi := &file_atelet_proto_msgTypes[35] + mi := &file_atelet_proto_msgTypes[37] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2342,7 +2510,7 @@ func (x *RunResponse) String() string { func (*RunResponse) ProtoMessage() {} func (x *RunResponse) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[35] + mi := &file_atelet_proto_msgTypes[37] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2355,7 +2523,7 @@ func (x *RunResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RunResponse.ProtoReflect.Descriptor instead. func (*RunResponse) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{35} + return file_atelet_proto_rawDescGZIP(), []int{37} } type LocalCheckpointConfiguration struct { @@ -2371,7 +2539,7 @@ type LocalCheckpointConfiguration struct { func (x *LocalCheckpointConfiguration) Reset() { *x = LocalCheckpointConfiguration{} - mi := &file_atelet_proto_msgTypes[36] + mi := &file_atelet_proto_msgTypes[38] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2383,7 +2551,7 @@ func (x *LocalCheckpointConfiguration) String() string { func (*LocalCheckpointConfiguration) ProtoMessage() {} func (x *LocalCheckpointConfiguration) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[36] + mi := &file_atelet_proto_msgTypes[38] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2396,7 +2564,7 @@ func (x *LocalCheckpointConfiguration) ProtoReflect() protoreflect.Message { // Deprecated: Use LocalCheckpointConfiguration.ProtoReflect.Descriptor instead. func (*LocalCheckpointConfiguration) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{36} + return file_atelet_proto_rawDescGZIP(), []int{38} } func (x *LocalCheckpointConfiguration) GetSnapshotName() string { @@ -2417,7 +2585,7 @@ type ExternalCheckpointConfiguration struct { func (x *ExternalCheckpointConfiguration) Reset() { *x = ExternalCheckpointConfiguration{} - mi := &file_atelet_proto_msgTypes[37] + mi := &file_atelet_proto_msgTypes[39] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2429,7 +2597,7 @@ func (x *ExternalCheckpointConfiguration) String() string { func (*ExternalCheckpointConfiguration) ProtoMessage() {} func (x *ExternalCheckpointConfiguration) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[37] + mi := &file_atelet_proto_msgTypes[39] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2442,7 +2610,7 @@ func (x *ExternalCheckpointConfiguration) ProtoReflect() protoreflect.Message { // Deprecated: Use ExternalCheckpointConfiguration.ProtoReflect.Descriptor instead. func (*ExternalCheckpointConfiguration) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{37} + return file_atelet_proto_rawDescGZIP(), []int{39} } func (x *ExternalCheckpointConfiguration) GetSnapshotUri() string { @@ -2467,7 +2635,7 @@ type ExternalRestoreConfiguration struct { func (x *ExternalRestoreConfiguration) Reset() { *x = ExternalRestoreConfiguration{} - mi := &file_atelet_proto_msgTypes[38] + mi := &file_atelet_proto_msgTypes[40] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2479,7 +2647,7 @@ func (x *ExternalRestoreConfiguration) String() string { func (*ExternalRestoreConfiguration) ProtoMessage() {} func (x *ExternalRestoreConfiguration) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[38] + mi := &file_atelet_proto_msgTypes[40] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2492,7 +2660,7 @@ func (x *ExternalRestoreConfiguration) ProtoReflect() protoreflect.Message { // Deprecated: Use ExternalRestoreConfiguration.ProtoReflect.Descriptor instead. func (*ExternalRestoreConfiguration) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{38} + return file_atelet_proto_rawDescGZIP(), []int{40} } func (x *ExternalRestoreConfiguration) GetSnapshotUri() string { @@ -2504,7 +2672,7 @@ func (x *ExternalRestoreConfiguration) GetSnapshotUri() string { type CheckpointRequest struct { state protoimpl.MessageState `protogen:"open.v1"` - TargetAteomUid string `protobuf:"bytes,1,opt,name=target_ateom_uid,json=targetAteomUid,proto3" json:"target_ateom_uid,omitempty"` + WorkerPodUid string `protobuf:"bytes,1,opt,name=worker_pod_uid,json=workerPodUid,proto3" json:"worker_pod_uid,omitempty"` Atespace string `protobuf:"bytes,2,opt,name=atespace,proto3" json:"atespace,omitempty"` ActorName string `protobuf:"bytes,3,opt,name=actor_name,json=actorName,proto3" json:"actor_name,omitempty"` ActorUid string `protobuf:"bytes,4,opt,name=actor_uid,json=actorUid,proto3" json:"actor_uid,omitempty"` @@ -2522,15 +2690,15 @@ type CheckpointRequest struct { // *CheckpointRequest_LocalConfig // *CheckpointRequest_ExternalConfig Config isCheckpointRequest_Config `protobuf_oneof:"config"` - // What should be included in the checkpoint. - Scope SnapshotScope `protobuf:"varint,11,opt,name=scope,proto3,enum=atelet.SnapshotScope" json:"scope,omitempty"` + // Fidelity the checkpoint captures. + Fidelity SnapshotFidelity `protobuf:"varint,11,opt,name=fidelity,proto3,enum=atelet.SnapshotFidelity" json:"fidelity,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } func (x *CheckpointRequest) Reset() { *x = CheckpointRequest{} - mi := &file_atelet_proto_msgTypes[39] + mi := &file_atelet_proto_msgTypes[41] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2542,7 +2710,7 @@ func (x *CheckpointRequest) String() string { func (*CheckpointRequest) ProtoMessage() {} func (x *CheckpointRequest) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[39] + mi := &file_atelet_proto_msgTypes[41] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2555,12 +2723,12 @@ func (x *CheckpointRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use CheckpointRequest.ProtoReflect.Descriptor instead. func (*CheckpointRequest) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{39} + return file_atelet_proto_rawDescGZIP(), []int{41} } -func (x *CheckpointRequest) GetTargetAteomUid() string { +func (x *CheckpointRequest) GetWorkerPodUid() string { if x != nil { - return x.TargetAteomUid + return x.WorkerPodUid } return "" } @@ -2639,11 +2807,11 @@ func (x *CheckpointRequest) GetExternalConfig() *ExternalCheckpointConfiguration return nil } -func (x *CheckpointRequest) GetScope() SnapshotScope { +func (x *CheckpointRequest) GetFidelity() SnapshotFidelity { if x != nil { - return x.Scope + return x.Fidelity } - return SnapshotScope_SNAPSHOT_SCOPE_UNSPECIFIED + return SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED } type isCheckpointRequest_Config interface { @@ -2670,7 +2838,7 @@ type CheckpointResponse struct { func (x *CheckpointResponse) Reset() { *x = CheckpointResponse{} - mi := &file_atelet_proto_msgTypes[40] + mi := &file_atelet_proto_msgTypes[42] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2682,7 +2850,7 @@ func (x *CheckpointResponse) String() string { func (*CheckpointResponse) ProtoMessage() {} func (x *CheckpointResponse) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[40] + mi := &file_atelet_proto_msgTypes[42] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2695,7 +2863,7 @@ func (x *CheckpointResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use CheckpointResponse.ProtoReflect.Descriptor instead. func (*CheckpointResponse) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{40} + return file_atelet_proto_rawDescGZIP(), []int{42} } type UploadPausedCheckpointRequest struct { @@ -2711,19 +2879,19 @@ type UploadPausedCheckpointRequest struct { LocalSnapshotName string `protobuf:"bytes,6,opt,name=local_snapshot_name,json=localSnapshotName,proto3" json:"local_snapshot_name,omitempty"` // Destination object-storage URI (the actor's in-progress snapshot URI). DestinationSnapshotUri string `protobuf:"bytes,7,opt,name=destination_snapshot_uri,json=destinationSnapshotUri,proto3" json:"destination_snapshot_uri,omitempty"` - // Scope the uploaded snapshot must have (the commit scope; FULL or DATA). - // The scope the pause checkpoint captured is not sent: atelet reads it from + // Fidelity the uploaded snapshot must have (MEMORY or VOLUMES). The + // fidelity the pause checkpoint captured is not sent: atelet reads it from // the local snapshot's own manifest, which is authoritative. When they - // differ, atelet converts where possible (a FULL capture to a DATA upload - // of the data-scope files ateom reported) and rejects otherwise. - DesiredScope SnapshotScope `protobuf:"varint,8,opt,name=desired_scope,json=desiredScope,proto3,enum=atelet.SnapshotScope" json:"desired_scope,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // differ, atelet converts where possible (a MEMORY capture to a VOLUMES + // upload of the volume files ateom reported) and rejects otherwise. + DesiredFidelity SnapshotFidelity `protobuf:"varint,8,opt,name=desired_fidelity,json=desiredFidelity,proto3,enum=atelet.SnapshotFidelity" json:"desired_fidelity,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *UploadPausedCheckpointRequest) Reset() { *x = UploadPausedCheckpointRequest{} - mi := &file_atelet_proto_msgTypes[41] + mi := &file_atelet_proto_msgTypes[43] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2735,7 +2903,7 @@ func (x *UploadPausedCheckpointRequest) String() string { func (*UploadPausedCheckpointRequest) ProtoMessage() {} func (x *UploadPausedCheckpointRequest) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[41] + mi := &file_atelet_proto_msgTypes[43] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2748,7 +2916,7 @@ func (x *UploadPausedCheckpointRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use UploadPausedCheckpointRequest.ProtoReflect.Descriptor instead. func (*UploadPausedCheckpointRequest) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{41} + return file_atelet_proto_rawDescGZIP(), []int{43} } func (x *UploadPausedCheckpointRequest) GetAtespace() string { @@ -2800,11 +2968,11 @@ func (x *UploadPausedCheckpointRequest) GetDestinationSnapshotUri() string { return "" } -func (x *UploadPausedCheckpointRequest) GetDesiredScope() SnapshotScope { +func (x *UploadPausedCheckpointRequest) GetDesiredFidelity() SnapshotFidelity { if x != nil { - return x.DesiredScope + return x.DesiredFidelity } - return SnapshotScope_SNAPSHOT_SCOPE_UNSPECIFIED + return SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED } type UploadPausedCheckpointResponse struct { @@ -2815,7 +2983,7 @@ type UploadPausedCheckpointResponse struct { func (x *UploadPausedCheckpointResponse) Reset() { *x = UploadPausedCheckpointResponse{} - mi := &file_atelet_proto_msgTypes[42] + mi := &file_atelet_proto_msgTypes[44] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2827,7 +2995,7 @@ func (x *UploadPausedCheckpointResponse) String() string { func (*UploadPausedCheckpointResponse) ProtoMessage() {} func (x *UploadPausedCheckpointResponse) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[42] + mi := &file_atelet_proto_msgTypes[44] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2840,12 +3008,12 @@ func (x *UploadPausedCheckpointResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use UploadPausedCheckpointResponse.ProtoReflect.Descriptor instead. func (*UploadPausedCheckpointResponse) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{42} + return file_atelet_proto_rawDescGZIP(), []int{44} } type RestoreRequest struct { state protoimpl.MessageState `protogen:"open.v1"` - TargetAteomUid string `protobuf:"bytes,1,opt,name=target_ateom_uid,json=targetAteomUid,proto3" json:"target_ateom_uid,omitempty"` + WorkerPodUid string `protobuf:"bytes,1,opt,name=worker_pod_uid,json=workerPodUid,proto3" json:"worker_pod_uid,omitempty"` Atespace string `protobuf:"bytes,2,opt,name=atespace,proto3" json:"atespace,omitempty"` ActorName string `protobuf:"bytes,3,opt,name=actor_name,json=actorName,proto3" json:"actor_name,omitempty"` ActorUid string `protobuf:"bytes,4,opt,name=actor_uid,json=actorUid,proto3" json:"actor_uid,omitempty"` @@ -2860,13 +3028,13 @@ type RestoreRequest struct { // *RestoreRequest_LocalConfig // *RestoreRequest_ExternalConfig Config isRestoreRequest_Config `protobuf_oneof:"config"` - // What content to restore from the checkpoint. - Scope SnapshotScope `protobuf:"varint,11,opt,name=scope,proto3,enum=atelet.SnapshotScope" json:"scope,omitempty"` + // Fidelity to restore from the checkpoint. + Fidelity SnapshotFidelity `protobuf:"varint,11,opt,name=fidelity,proto3,enum=atelet.SnapshotFidelity" json:"fidelity,omitempty"` // When absent the actor has no egress: its TCP is captured and refused. EgressGateway *EgressGateway `protobuf:"bytes,13,opt,name=egress_gateway,json=egressGateway,proto3,oneof" json:"egress_gateway,omitempty"` // The actor's declared size, from the ActorTemplate's resource limits. For - // gVisor and micro-VM DATA-scope restores the sandbox is (re)sized to these; - // for a FULL micro-VM restore the size baked into the snapshot wins. Zero + // gVisor and micro-VM VOLUMES restores the sandbox is (re)sized to these; + // for a MEMORY micro-VM restore the size baked into the snapshot wins. Zero // means "unset": keep the runtime default. CpuMilli int64 `protobuf:"varint,14,opt,name=cpu_milli,json=cpuMilli,proto3" json:"cpu_milli,omitempty"` // CPU limit in millicores (1000 = one core). MemoryBytes int64 `protobuf:"varint,15,opt,name=memory_bytes,json=memoryBytes,proto3" json:"memory_bytes,omitempty"` // Memory limit in bytes. @@ -2879,7 +3047,7 @@ type RestoreRequest struct { func (x *RestoreRequest) Reset() { *x = RestoreRequest{} - mi := &file_atelet_proto_msgTypes[43] + mi := &file_atelet_proto_msgTypes[45] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2891,7 +3059,7 @@ func (x *RestoreRequest) String() string { func (*RestoreRequest) ProtoMessage() {} func (x *RestoreRequest) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[43] + mi := &file_atelet_proto_msgTypes[45] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2904,12 +3072,12 @@ func (x *RestoreRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use RestoreRequest.ProtoReflect.Descriptor instead. func (*RestoreRequest) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{43} + return file_atelet_proto_rawDescGZIP(), []int{45} } -func (x *RestoreRequest) GetTargetAteomUid() string { +func (x *RestoreRequest) GetWorkerPodUid() string { if x != nil { - return x.TargetAteomUid + return x.WorkerPodUid } return "" } @@ -2988,11 +3156,11 @@ func (x *RestoreRequest) GetExternalConfig() *ExternalRestoreConfiguration { return nil } -func (x *RestoreRequest) GetScope() SnapshotScope { +func (x *RestoreRequest) GetFidelity() SnapshotFidelity { if x != nil { - return x.Scope + return x.Fidelity } - return SnapshotScope_SNAPSHOT_SCOPE_UNSPECIFIED + return SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED } func (x *RestoreRequest) GetEgressGateway() *EgressGateway { @@ -3047,7 +3215,7 @@ type RestoreResponse struct { func (x *RestoreResponse) Reset() { *x = RestoreResponse{} - mi := &file_atelet_proto_msgTypes[44] + mi := &file_atelet_proto_msgTypes[46] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3059,7 +3227,7 @@ func (x *RestoreResponse) String() string { func (*RestoreResponse) ProtoMessage() {} func (x *RestoreResponse) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[44] + mi := &file_atelet_proto_msgTypes[46] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3072,24 +3240,29 @@ func (x *RestoreResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RestoreResponse.ProtoReflect.Descriptor instead. func (*RestoreResponse) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{44} + return file_atelet_proto_rawDescGZIP(), []int{46} } var File_atelet_proto protoreflect.FileDescriptor const file_atelet_proto_rawDesc = "" + "\n" + - "\fatelet.proto\x12\x06atelet\"\x82\x01\n" + + "\fatelet.proto\x12\x06atelet\"\xd6\x01\n" + "\x15RegisterWorkerRequest\x123\n" + - "\bcapacity\x18\x01 \x01(\v2\x17.atelet.WorkerResourcesR\bcapacity\x124\n" + - "\bhardware\x18\x02 \x01(\v2\x18.atelet.HardwareIdentityR\bhardware\"\x9b\x01\n" + - "\x10HardwareIdentity\x12H\n" + + "\bcapacity\x18\x01 \x01(\v2\x17.atelet.WorkerResourcesR\bcapacity\x12?\n" + + "\x0fdefault_runtime\x18\x02 \x01(\v2\x16.atelet.SandboxRuntimeR\x0edefaultRuntime\x12G\n" + + "\x13restorable_runtimes\x18\x03 \x03(\v2\x16.atelet.SandboxRuntimeR\x12restorableRuntimes\"o\n" + + "\x0eSandboxRuntime\x12#\n" + + "\rsandbox_class\x18\x01 \x01(\tR\fsandboxClass\x128\n" + + "\aversion\x18\x02 \x01(\v2\x1e.atelet.VersionedSandboxCompatR\aversion\"w\n" + + "\x16VersionedSandboxCompat\x12%\n" + + "\x0eschema_version\x18\x01 \x01(\tR\rschemaVersion\x126\n" + "\n" + - "attributes\x18\x01 \x03(\v2(.atelet.HardwareIdentity.AttributesEntryR\n" + - "attributes\x1a=\n" + - "\x0fAttributesEntry\x12\x10\n" + + "attributes\x18\x02 \x03(\v2\x16.atelet.AttributeEntryR\n" + + "attributes\"8\n" + + "\x0eAttributeEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + - "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"Z\n" + + "\x05value\x18\x02 \x01(\tR\x05value\"Z\n" + "\x0fWorkerResources\x12/\n" + "\tresources\x18\x01 \x01(\v2\x11.atelet.ResourcesR\tresources\x12\x16\n" + "\x06actors\x18\x02 \x01(\x05R\x06actors\"3\n" + @@ -3112,9 +3285,9 @@ const file_atelet_proto_rawDesc = "" + "\tactor_uid\x18\x05 \x01(\tR\bactorUid\x12>\n" + "\x1bcertificate_signing_request\x18\x01 \x01(\fR\x19certificateSigningRequest\"M\n" + "\x1cMintActorCertificateResponse\x12-\n" + - "\x12actor_certificates\x18\x01 \x03(\fR\x11actorCertificates\"\xa6\x02\n" + - "\x10TerminateRequest\x12(\n" + - "\x10target_ateom_uid\x18\x01 \x01(\tR\x0etargetAteomUid\x12\x1a\n" + + "\x12actor_certificates\x18\x01 \x03(\fR\x11actorCertificates\"\xa2\x02\n" + + "\x10TerminateRequest\x12$\n" + + "\x0eworker_pod_uid\x18\x01 \x01(\tR\fworkerPodUid\x12\x1a\n" + "\batespace\x18\x02 \x01(\tR\batespace\x12\x1d\n" + "\n" + "actor_name\x18\x03 \x01(\tR\tactorName\x12\x1b\n" + @@ -3122,10 +3295,10 @@ const file_atelet_proto_rawDesc = "" + "\x17actor_template_atespace\x18\x05 \x01(\tR\x15actorTemplateAtespace\x12.\n" + "\x13actor_template_name\x18\x06 \x01(\tR\x11actorTemplateName\x12(\n" + "\x04spec\x18\a \x01(\v2\x14.atelet.WorkloadSpecR\x04spec\"\x13\n" + - "\x11TerminateResponse\"\xf4\x03\n" + + "\x11TerminateResponse\"\xf0\x03\n" + "\n" + - "RunRequest\x12(\n" + - "\x10target_ateom_uid\x18\x01 \x01(\tR\x0etargetAteomUid\x12\x1a\n" + + "RunRequest\x12$\n" + + "\x0eworker_pod_uid\x18\x01 \x01(\tR\fworkerPodUid\x12\x1a\n" + "\batespace\x18\x02 \x01(\tR\batespace\x12\x1d\n" + "\n" + "actor_name\x18\x03 \x01(\tR\tactorName\x12\x1b\n" + @@ -3240,9 +3413,9 @@ const file_atelet_proto_rawDesc = "" + "\x1fExternalCheckpointConfiguration\x12!\n" + "\fsnapshot_uri\x18\x01 \x01(\tR\vsnapshotUri\"A\n" + "\x1cExternalRestoreConfiguration\x12!\n" + - "\fsnapshot_uri\x18\x01 \x01(\tR\vsnapshotUri\"\xa9\x04\n" + - "\x11CheckpointRequest\x12(\n" + - "\x10target_ateom_uid\x18\x01 \x01(\tR\x0etargetAteomUid\x12\x1a\n" + + "\fsnapshot_uri\x18\x01 \x01(\tR\vsnapshotUri\"\xae\x04\n" + + "\x11CheckpointRequest\x12$\n" + + "\x0eworker_pod_uid\x18\x01 \x01(\tR\fworkerPodUid\x12\x1a\n" + "\batespace\x18\x02 \x01(\tR\batespace\x12\x1d\n" + "\n" + "actor_name\x18\x03 \x01(\tR\tactorName\x12\x1b\n" + @@ -3253,10 +3426,10 @@ const file_atelet_proto_rawDesc = "" + "\x04type\x18\b \x01(\x0e2\x16.atelet.CheckpointTypeR\x04type\x12I\n" + "\flocal_config\x18\t \x01(\v2$.atelet.LocalCheckpointConfigurationH\x00R\vlocalConfig\x12R\n" + "\x0fexternal_config\x18\n" + - " \x01(\v2'.atelet.ExternalCheckpointConfigurationH\x00R\x0eexternalConfig\x12+\n" + - "\x05scope\x18\v \x01(\x0e2\x15.atelet.SnapshotScopeR\x05scopeB\b\n" + + " \x01(\v2'.atelet.ExternalCheckpointConfigurationH\x00R\x0eexternalConfig\x124\n" + + "\bfidelity\x18\v \x01(\x0e2\x18.atelet.SnapshotFidelityR\bfidelityB\b\n" + "\x06config\"\x14\n" + - "\x12CheckpointResponse\"\x85\x03\n" + + "\x12CheckpointResponse\"\x8e\x03\n" + "\x1dUploadPausedCheckpointRequest\x12\x1a\n" + "\batespace\x18\x01 \x01(\tR\batespace\x12\x1d\n" + "\n" + @@ -3265,11 +3438,11 @@ const file_atelet_proto_rawDesc = "" + "\x17actor_template_atespace\x18\x04 \x01(\tR\x15actorTemplateAtespace\x12.\n" + "\x13actor_template_name\x18\x05 \x01(\tR\x11actorTemplateName\x12.\n" + "\x13local_snapshot_name\x18\x06 \x01(\tR\x11localSnapshotName\x128\n" + - "\x18destination_snapshot_uri\x18\a \x01(\tR\x16destinationSnapshotUri\x12:\n" + - "\rdesired_scope\x18\b \x01(\x0e2\x15.atelet.SnapshotScopeR\fdesiredScope\" \n" + - "\x1eUploadPausedCheckpointResponse\"\xf7\x05\n" + - "\x0eRestoreRequest\x12(\n" + - "\x10target_ateom_uid\x18\x01 \x01(\tR\x0etargetAteomUid\x12\x1a\n" + + "\x18destination_snapshot_uri\x18\a \x01(\tR\x16destinationSnapshotUri\x12C\n" + + "\x10desired_fidelity\x18\b \x01(\x0e2\x18.atelet.SnapshotFidelityR\x0fdesiredFidelity\" \n" + + "\x1eUploadPausedCheckpointResponse\"\xfc\x05\n" + + "\x0eRestoreRequest\x12$\n" + + "\x0eworker_pod_uid\x18\x01 \x01(\tR\fworkerPodUid\x12\x1a\n" + "\batespace\x18\x02 \x01(\tR\batespace\x12\x1d\n" + "\n" + "actor_name\x18\x03 \x01(\tR\tactorName\x12\x1b\n" + @@ -3280,8 +3453,8 @@ const file_atelet_proto_rawDesc = "" + "\x04type\x18\b \x01(\x0e2\x16.atelet.CheckpointTypeR\x04type\x12I\n" + "\flocal_config\x18\t \x01(\v2$.atelet.LocalCheckpointConfigurationH\x00R\vlocalConfig\x12O\n" + "\x0fexternal_config\x18\n" + - " \x01(\v2$.atelet.ExternalRestoreConfigurationH\x00R\x0eexternalConfig\x12+\n" + - "\x05scope\x18\v \x01(\x0e2\x15.atelet.SnapshotScopeR\x05scope\x12A\n" + + " \x01(\v2$.atelet.ExternalRestoreConfigurationH\x00R\x0eexternalConfig\x124\n" + + "\bfidelity\x18\v \x01(\x0e2\x18.atelet.SnapshotFidelityR\bfidelity\x12A\n" + "\x0eegress_gateway\x18\r \x01(\v2\x15.atelet.EgressGatewayH\x01R\regressGateway\x88\x01\x01\x12\x1b\n" + "\tcpu_milli\x18\x0e \x01(\x03R\bcpuMilli\x12!\n" + "\fmemory_bytes\x18\x0f \x01(\x03R\vmemoryBytes\x12<\n" + @@ -3297,11 +3470,12 @@ const file_atelet_proto_rawDesc = "" + "\x0eCheckpointType\x12\x1f\n" + "\x1bCHECKPOINT_TYPE_UNSPECIFIED\x10\x00\x12\x19\n" + "\x15CHECKPOINT_TYPE_LOCAL\x10\x01\x12\x1c\n" + - "\x18CHECKPOINT_TYPE_EXTERNAL\x10\x02*a\n" + - "\rSnapshotScope\x12\x1e\n" + - "\x1aSNAPSHOT_SCOPE_UNSPECIFIED\x10\x00\x12\x17\n" + - "\x13SNAPSHOT_SCOPE_FULL\x10\x01\x12\x17\n" + - "\x13SNAPSHOT_SCOPE_DATA\x10\x022\xa8\x02\n" + + "\x18CHECKPOINT_TYPE_EXTERNAL\x10\x02*\x90\x01\n" + + "\x10SnapshotFidelity\x12!\n" + + "\x1dSNAPSHOT_FIDELITY_UNSPECIFIED\x10\x00\x12\x1d\n" + + "\x19SNAPSHOT_FIDELITY_VOLUMES\x10\x01\x12\x1c\n" + + "\x18SNAPSHOT_FIDELITY_ROOTFS\x10\x02\x12\x1c\n" + + "\x18SNAPSHOT_FIDELITY_MEMORY\x10\x032\xa8\x02\n" + "\fAteomSupport\x12c\n" + "\x14MintActorCertificate\x12#.atelet.MintActorCertificateRequest\x1a$.atelet.MintActorCertificateResponse\"\x00\x12Q\n" + "\x0eRegisterWorker\x12\x1d.atelet.RegisterWorkerRequest\x1a\x1e.atelet.RegisterWorkerResponse\"\x00\x12`\n" + @@ -3327,130 +3501,133 @@ func file_atelet_proto_rawDescGZIP() []byte { } var file_atelet_proto_enumTypes = make([]protoimpl.EnumInfo, 3) -var file_atelet_proto_msgTypes = make([]protoimpl.MessageInfo, 50) +var file_atelet_proto_msgTypes = make([]protoimpl.MessageInfo, 51) var file_atelet_proto_goTypes = []any{ (ActorMetadataField)(0), // 0: atelet.ActorMetadataField (CheckpointType)(0), // 1: atelet.CheckpointType - (SnapshotScope)(0), // 2: atelet.SnapshotScope + (SnapshotFidelity)(0), // 2: atelet.SnapshotFidelity (*RegisterWorkerRequest)(nil), // 3: atelet.RegisterWorkerRequest - (*HardwareIdentity)(nil), // 4: atelet.HardwareIdentity - (*WorkerResources)(nil), // 5: atelet.WorkerResources - (*Resources)(nil), // 6: atelet.Resources - (*Limits)(nil), // 7: atelet.Limits - (*RegisterWorkerResponse)(nil), // 8: atelet.RegisterWorkerResponse - (*RequestActorSuspendRequest)(nil), // 9: atelet.RequestActorSuspendRequest - (*RequestActorSuspendResponse)(nil), // 10: atelet.RequestActorSuspendResponse - (*MintActorCertificateRequest)(nil), // 11: atelet.MintActorCertificateRequest - (*MintActorCertificateResponse)(nil), // 12: atelet.MintActorCertificateResponse - (*TerminateRequest)(nil), // 13: atelet.TerminateRequest - (*TerminateResponse)(nil), // 14: atelet.TerminateResponse - (*RunRequest)(nil), // 15: atelet.RunRequest - (*EgressGateway)(nil), // 16: atelet.EgressGateway - (*AssetFile)(nil), // 17: atelet.AssetFile - (*ArchAssets)(nil), // 18: atelet.ArchAssets - (*SandboxAssets)(nil), // 19: atelet.SandboxAssets - (*WorkloadSpec)(nil), // 20: atelet.WorkloadSpec - (*DurableDirVolume)(nil), // 21: atelet.DurableDirVolume - (*ExternalVolumeSource)(nil), // 22: atelet.ExternalVolumeSource - (*ImageVolumeSource)(nil), // 23: atelet.ImageVolumeSource - (*ActorMetadataItem)(nil), // 24: atelet.ActorMetadataItem - (*ActorMetadataDataSource)(nil), // 25: atelet.ActorMetadataDataSource - (*TrustBundleDataSource)(nil), // 26: atelet.TrustBundleDataSource - (*SystemInfoDataSource)(nil), // 27: atelet.SystemInfoDataSource - (*SystemInfoVolume)(nil), // 28: atelet.SystemInfoVolume - (*Volume)(nil), // 29: atelet.Volume - (*VolumeMount)(nil), // 30: atelet.VolumeMount - (*Container)(nil), // 31: atelet.Container - (*SecurityContext)(nil), // 32: atelet.SecurityContext - (*Capabilities)(nil), // 33: atelet.Capabilities - (*ResourceLimits)(nil), // 34: atelet.ResourceLimits - (*EnvEntry)(nil), // 35: atelet.EnvEntry - (*WakeupProbe)(nil), // 36: atelet.WakeupProbe - (*HTTPGetAction)(nil), // 37: atelet.HTTPGetAction - (*RunResponse)(nil), // 38: atelet.RunResponse - (*LocalCheckpointConfiguration)(nil), // 39: atelet.LocalCheckpointConfiguration - (*ExternalCheckpointConfiguration)(nil), // 40: atelet.ExternalCheckpointConfiguration - (*ExternalRestoreConfiguration)(nil), // 41: atelet.ExternalRestoreConfiguration - (*CheckpointRequest)(nil), // 42: atelet.CheckpointRequest - (*CheckpointResponse)(nil), // 43: atelet.CheckpointResponse - (*UploadPausedCheckpointRequest)(nil), // 44: atelet.UploadPausedCheckpointRequest - (*UploadPausedCheckpointResponse)(nil), // 45: atelet.UploadPausedCheckpointResponse - (*RestoreRequest)(nil), // 46: atelet.RestoreRequest - (*RestoreResponse)(nil), // 47: atelet.RestoreResponse - nil, // 48: atelet.HardwareIdentity.AttributesEntry - nil, // 49: atelet.ArchAssets.FilesEntry - nil, // 50: atelet.SandboxAssets.AssetsEntry - nil, // 51: atelet.ExternalVolumeSource.VolumeContextEntry - nil, // 52: atelet.ExternalVolumeSource.PublishContextEntry + (*SandboxRuntime)(nil), // 4: atelet.SandboxRuntime + (*VersionedSandboxCompat)(nil), // 5: atelet.VersionedSandboxCompat + (*AttributeEntry)(nil), // 6: atelet.AttributeEntry + (*WorkerResources)(nil), // 7: atelet.WorkerResources + (*Resources)(nil), // 8: atelet.Resources + (*Limits)(nil), // 9: atelet.Limits + (*RegisterWorkerResponse)(nil), // 10: atelet.RegisterWorkerResponse + (*RequestActorSuspendRequest)(nil), // 11: atelet.RequestActorSuspendRequest + (*RequestActorSuspendResponse)(nil), // 12: atelet.RequestActorSuspendResponse + (*MintActorCertificateRequest)(nil), // 13: atelet.MintActorCertificateRequest + (*MintActorCertificateResponse)(nil), // 14: atelet.MintActorCertificateResponse + (*TerminateRequest)(nil), // 15: atelet.TerminateRequest + (*TerminateResponse)(nil), // 16: atelet.TerminateResponse + (*RunRequest)(nil), // 17: atelet.RunRequest + (*EgressGateway)(nil), // 18: atelet.EgressGateway + (*AssetFile)(nil), // 19: atelet.AssetFile + (*ArchAssets)(nil), // 20: atelet.ArchAssets + (*SandboxAssets)(nil), // 21: atelet.SandboxAssets + (*WorkloadSpec)(nil), // 22: atelet.WorkloadSpec + (*DurableDirVolume)(nil), // 23: atelet.DurableDirVolume + (*ExternalVolumeSource)(nil), // 24: atelet.ExternalVolumeSource + (*ImageVolumeSource)(nil), // 25: atelet.ImageVolumeSource + (*ActorMetadataItem)(nil), // 26: atelet.ActorMetadataItem + (*ActorMetadataDataSource)(nil), // 27: atelet.ActorMetadataDataSource + (*TrustBundleDataSource)(nil), // 28: atelet.TrustBundleDataSource + (*SystemInfoDataSource)(nil), // 29: atelet.SystemInfoDataSource + (*SystemInfoVolume)(nil), // 30: atelet.SystemInfoVolume + (*Volume)(nil), // 31: atelet.Volume + (*VolumeMount)(nil), // 32: atelet.VolumeMount + (*Container)(nil), // 33: atelet.Container + (*SecurityContext)(nil), // 34: atelet.SecurityContext + (*Capabilities)(nil), // 35: atelet.Capabilities + (*ResourceLimits)(nil), // 36: atelet.ResourceLimits + (*EnvEntry)(nil), // 37: atelet.EnvEntry + (*WakeupProbe)(nil), // 38: atelet.WakeupProbe + (*HTTPGetAction)(nil), // 39: atelet.HTTPGetAction + (*RunResponse)(nil), // 40: atelet.RunResponse + (*LocalCheckpointConfiguration)(nil), // 41: atelet.LocalCheckpointConfiguration + (*ExternalCheckpointConfiguration)(nil), // 42: atelet.ExternalCheckpointConfiguration + (*ExternalRestoreConfiguration)(nil), // 43: atelet.ExternalRestoreConfiguration + (*CheckpointRequest)(nil), // 44: atelet.CheckpointRequest + (*CheckpointResponse)(nil), // 45: atelet.CheckpointResponse + (*UploadPausedCheckpointRequest)(nil), // 46: atelet.UploadPausedCheckpointRequest + (*UploadPausedCheckpointResponse)(nil), // 47: atelet.UploadPausedCheckpointResponse + (*RestoreRequest)(nil), // 48: atelet.RestoreRequest + (*RestoreResponse)(nil), // 49: atelet.RestoreResponse + nil, // 50: atelet.ArchAssets.FilesEntry + nil, // 51: atelet.SandboxAssets.AssetsEntry + nil, // 52: atelet.ExternalVolumeSource.VolumeContextEntry + nil, // 53: atelet.ExternalVolumeSource.PublishContextEntry } var file_atelet_proto_depIdxs = []int32{ - 5, // 0: atelet.RegisterWorkerRequest.capacity:type_name -> atelet.WorkerResources - 4, // 1: atelet.RegisterWorkerRequest.hardware:type_name -> atelet.HardwareIdentity - 48, // 2: atelet.HardwareIdentity.attributes:type_name -> atelet.HardwareIdentity.AttributesEntry - 6, // 3: atelet.WorkerResources.resources:type_name -> atelet.Resources - 7, // 4: atelet.Resources.limits:type_name -> atelet.Limits - 20, // 5: atelet.TerminateRequest.spec:type_name -> atelet.WorkloadSpec - 20, // 6: atelet.RunRequest.spec:type_name -> atelet.WorkloadSpec - 19, // 7: atelet.RunRequest.sandbox_assets:type_name -> atelet.SandboxAssets - 16, // 8: atelet.RunRequest.egress_gateway:type_name -> atelet.EgressGateway - 49, // 9: atelet.ArchAssets.files:type_name -> atelet.ArchAssets.FilesEntry - 50, // 10: atelet.SandboxAssets.assets:type_name -> atelet.SandboxAssets.AssetsEntry - 31, // 11: atelet.WorkloadSpec.containers:type_name -> atelet.Container - 29, // 12: atelet.WorkloadSpec.volumes:type_name -> atelet.Volume - 51, // 13: atelet.ExternalVolumeSource.volume_context:type_name -> atelet.ExternalVolumeSource.VolumeContextEntry - 52, // 14: atelet.ExternalVolumeSource.publish_context:type_name -> atelet.ExternalVolumeSource.PublishContextEntry - 0, // 15: atelet.ActorMetadataItem.field:type_name -> atelet.ActorMetadataField - 24, // 16: atelet.ActorMetadataDataSource.items:type_name -> atelet.ActorMetadataItem - 25, // 17: atelet.SystemInfoDataSource.actor_metadata:type_name -> atelet.ActorMetadataDataSource - 26, // 18: atelet.SystemInfoDataSource.trust_bundle:type_name -> atelet.TrustBundleDataSource - 27, // 19: atelet.SystemInfoVolume.data_sources:type_name -> atelet.SystemInfoDataSource - 21, // 20: atelet.Volume.durable_dir:type_name -> atelet.DurableDirVolume - 22, // 21: atelet.Volume.external:type_name -> atelet.ExternalVolumeSource - 28, // 22: atelet.Volume.system_info:type_name -> atelet.SystemInfoVolume - 23, // 23: atelet.Volume.image:type_name -> atelet.ImageVolumeSource - 35, // 24: atelet.Container.env:type_name -> atelet.EnvEntry - 36, // 25: atelet.Container.wakeup_probe:type_name -> atelet.WakeupProbe - 30, // 26: atelet.Container.volume_mounts:type_name -> atelet.VolumeMount - 32, // 27: atelet.Container.security_context:type_name -> atelet.SecurityContext - 34, // 28: atelet.Container.resources:type_name -> atelet.ResourceLimits - 33, // 29: atelet.SecurityContext.capabilities:type_name -> atelet.Capabilities - 37, // 30: atelet.WakeupProbe.http_get:type_name -> atelet.HTTPGetAction - 20, // 31: atelet.CheckpointRequest.spec:type_name -> atelet.WorkloadSpec - 1, // 32: atelet.CheckpointRequest.type:type_name -> atelet.CheckpointType - 39, // 33: atelet.CheckpointRequest.local_config:type_name -> atelet.LocalCheckpointConfiguration - 40, // 34: atelet.CheckpointRequest.external_config:type_name -> atelet.ExternalCheckpointConfiguration - 2, // 35: atelet.CheckpointRequest.scope:type_name -> atelet.SnapshotScope - 2, // 36: atelet.UploadPausedCheckpointRequest.desired_scope:type_name -> atelet.SnapshotScope - 20, // 37: atelet.RestoreRequest.spec:type_name -> atelet.WorkloadSpec - 1, // 38: atelet.RestoreRequest.type:type_name -> atelet.CheckpointType - 39, // 39: atelet.RestoreRequest.local_config:type_name -> atelet.LocalCheckpointConfiguration - 41, // 40: atelet.RestoreRequest.external_config:type_name -> atelet.ExternalRestoreConfiguration - 2, // 41: atelet.RestoreRequest.scope:type_name -> atelet.SnapshotScope - 16, // 42: atelet.RestoreRequest.egress_gateway:type_name -> atelet.EgressGateway - 19, // 43: atelet.RestoreRequest.sandbox_assets:type_name -> atelet.SandboxAssets - 17, // 44: atelet.ArchAssets.FilesEntry.value:type_name -> atelet.AssetFile - 18, // 45: atelet.SandboxAssets.AssetsEntry.value:type_name -> atelet.ArchAssets - 11, // 46: atelet.AteomSupport.MintActorCertificate:input_type -> atelet.MintActorCertificateRequest - 3, // 47: atelet.AteomSupport.RegisterWorker:input_type -> atelet.RegisterWorkerRequest - 9, // 48: atelet.AteomSupport.RequestActorSuspend:input_type -> atelet.RequestActorSuspendRequest - 15, // 49: atelet.AteomHerder.Run:input_type -> atelet.RunRequest - 42, // 50: atelet.AteomHerder.Checkpoint:input_type -> atelet.CheckpointRequest - 46, // 51: atelet.AteomHerder.Restore:input_type -> atelet.RestoreRequest - 44, // 52: atelet.AteomHerder.UploadPausedCheckpoint:input_type -> atelet.UploadPausedCheckpointRequest - 13, // 53: atelet.AteomHerder.Terminate:input_type -> atelet.TerminateRequest - 12, // 54: atelet.AteomSupport.MintActorCertificate:output_type -> atelet.MintActorCertificateResponse - 8, // 55: atelet.AteomSupport.RegisterWorker:output_type -> atelet.RegisterWorkerResponse - 10, // 56: atelet.AteomSupport.RequestActorSuspend:output_type -> atelet.RequestActorSuspendResponse - 38, // 57: atelet.AteomHerder.Run:output_type -> atelet.RunResponse - 43, // 58: atelet.AteomHerder.Checkpoint:output_type -> atelet.CheckpointResponse - 47, // 59: atelet.AteomHerder.Restore:output_type -> atelet.RestoreResponse - 45, // 60: atelet.AteomHerder.UploadPausedCheckpoint:output_type -> atelet.UploadPausedCheckpointResponse - 14, // 61: atelet.AteomHerder.Terminate:output_type -> atelet.TerminateResponse - 54, // [54:62] is the sub-list for method output_type - 46, // [46:54] is the sub-list for method input_type - 46, // [46:46] is the sub-list for extension type_name - 46, // [46:46] is the sub-list for extension extendee - 0, // [0:46] is the sub-list for field type_name + 7, // 0: atelet.RegisterWorkerRequest.capacity:type_name -> atelet.WorkerResources + 4, // 1: atelet.RegisterWorkerRequest.default_runtime:type_name -> atelet.SandboxRuntime + 4, // 2: atelet.RegisterWorkerRequest.restorable_runtimes:type_name -> atelet.SandboxRuntime + 5, // 3: atelet.SandboxRuntime.version:type_name -> atelet.VersionedSandboxCompat + 6, // 4: atelet.VersionedSandboxCompat.attributes:type_name -> atelet.AttributeEntry + 8, // 5: atelet.WorkerResources.resources:type_name -> atelet.Resources + 9, // 6: atelet.Resources.limits:type_name -> atelet.Limits + 22, // 7: atelet.TerminateRequest.spec:type_name -> atelet.WorkloadSpec + 22, // 8: atelet.RunRequest.spec:type_name -> atelet.WorkloadSpec + 21, // 9: atelet.RunRequest.sandbox_assets:type_name -> atelet.SandboxAssets + 18, // 10: atelet.RunRequest.egress_gateway:type_name -> atelet.EgressGateway + 50, // 11: atelet.ArchAssets.files:type_name -> atelet.ArchAssets.FilesEntry + 51, // 12: atelet.SandboxAssets.assets:type_name -> atelet.SandboxAssets.AssetsEntry + 33, // 13: atelet.WorkloadSpec.containers:type_name -> atelet.Container + 31, // 14: atelet.WorkloadSpec.volumes:type_name -> atelet.Volume + 52, // 15: atelet.ExternalVolumeSource.volume_context:type_name -> atelet.ExternalVolumeSource.VolumeContextEntry + 53, // 16: atelet.ExternalVolumeSource.publish_context:type_name -> atelet.ExternalVolumeSource.PublishContextEntry + 0, // 17: atelet.ActorMetadataItem.field:type_name -> atelet.ActorMetadataField + 26, // 18: atelet.ActorMetadataDataSource.items:type_name -> atelet.ActorMetadataItem + 27, // 19: atelet.SystemInfoDataSource.actor_metadata:type_name -> atelet.ActorMetadataDataSource + 28, // 20: atelet.SystemInfoDataSource.trust_bundle:type_name -> atelet.TrustBundleDataSource + 29, // 21: atelet.SystemInfoVolume.data_sources:type_name -> atelet.SystemInfoDataSource + 23, // 22: atelet.Volume.durable_dir:type_name -> atelet.DurableDirVolume + 24, // 23: atelet.Volume.external:type_name -> atelet.ExternalVolumeSource + 30, // 24: atelet.Volume.system_info:type_name -> atelet.SystemInfoVolume + 25, // 25: atelet.Volume.image:type_name -> atelet.ImageVolumeSource + 37, // 26: atelet.Container.env:type_name -> atelet.EnvEntry + 38, // 27: atelet.Container.wakeup_probe:type_name -> atelet.WakeupProbe + 32, // 28: atelet.Container.volume_mounts:type_name -> atelet.VolumeMount + 34, // 29: atelet.Container.security_context:type_name -> atelet.SecurityContext + 36, // 30: atelet.Container.resources:type_name -> atelet.ResourceLimits + 35, // 31: atelet.SecurityContext.capabilities:type_name -> atelet.Capabilities + 39, // 32: atelet.WakeupProbe.http_get:type_name -> atelet.HTTPGetAction + 22, // 33: atelet.CheckpointRequest.spec:type_name -> atelet.WorkloadSpec + 1, // 34: atelet.CheckpointRequest.type:type_name -> atelet.CheckpointType + 41, // 35: atelet.CheckpointRequest.local_config:type_name -> atelet.LocalCheckpointConfiguration + 42, // 36: atelet.CheckpointRequest.external_config:type_name -> atelet.ExternalCheckpointConfiguration + 2, // 37: atelet.CheckpointRequest.fidelity:type_name -> atelet.SnapshotFidelity + 2, // 38: atelet.UploadPausedCheckpointRequest.desired_fidelity:type_name -> atelet.SnapshotFidelity + 22, // 39: atelet.RestoreRequest.spec:type_name -> atelet.WorkloadSpec + 1, // 40: atelet.RestoreRequest.type:type_name -> atelet.CheckpointType + 41, // 41: atelet.RestoreRequest.local_config:type_name -> atelet.LocalCheckpointConfiguration + 43, // 42: atelet.RestoreRequest.external_config:type_name -> atelet.ExternalRestoreConfiguration + 2, // 43: atelet.RestoreRequest.fidelity:type_name -> atelet.SnapshotFidelity + 18, // 44: atelet.RestoreRequest.egress_gateway:type_name -> atelet.EgressGateway + 21, // 45: atelet.RestoreRequest.sandbox_assets:type_name -> atelet.SandboxAssets + 19, // 46: atelet.ArchAssets.FilesEntry.value:type_name -> atelet.AssetFile + 20, // 47: atelet.SandboxAssets.AssetsEntry.value:type_name -> atelet.ArchAssets + 13, // 48: atelet.AteomSupport.MintActorCertificate:input_type -> atelet.MintActorCertificateRequest + 3, // 49: atelet.AteomSupport.RegisterWorker:input_type -> atelet.RegisterWorkerRequest + 11, // 50: atelet.AteomSupport.RequestActorSuspend:input_type -> atelet.RequestActorSuspendRequest + 17, // 51: atelet.AteomHerder.Run:input_type -> atelet.RunRequest + 44, // 52: atelet.AteomHerder.Checkpoint:input_type -> atelet.CheckpointRequest + 48, // 53: atelet.AteomHerder.Restore:input_type -> atelet.RestoreRequest + 46, // 54: atelet.AteomHerder.UploadPausedCheckpoint:input_type -> atelet.UploadPausedCheckpointRequest + 15, // 55: atelet.AteomHerder.Terminate:input_type -> atelet.TerminateRequest + 14, // 56: atelet.AteomSupport.MintActorCertificate:output_type -> atelet.MintActorCertificateResponse + 10, // 57: atelet.AteomSupport.RegisterWorker:output_type -> atelet.RegisterWorkerResponse + 12, // 58: atelet.AteomSupport.RequestActorSuspend:output_type -> atelet.RequestActorSuspendResponse + 40, // 59: atelet.AteomHerder.Run:output_type -> atelet.RunResponse + 45, // 60: atelet.AteomHerder.Checkpoint:output_type -> atelet.CheckpointResponse + 49, // 61: atelet.AteomHerder.Restore:output_type -> atelet.RestoreResponse + 47, // 62: atelet.AteomHerder.UploadPausedCheckpoint:output_type -> atelet.UploadPausedCheckpointResponse + 16, // 63: atelet.AteomHerder.Terminate:output_type -> atelet.TerminateResponse + 56, // [56:64] is the sub-list for method output_type + 48, // [48:56] is the sub-list for method input_type + 48, // [48:48] is the sub-list for extension type_name + 48, // [48:48] is the sub-list for extension extendee + 0, // [0:48] is the sub-list for field type_name } func init() { file_atelet_proto_init() } @@ -3458,22 +3635,22 @@ func file_atelet_proto_init() { if File_atelet_proto != nil { return } - file_atelet_proto_msgTypes[12].OneofWrappers = []any{} - file_atelet_proto_msgTypes[24].OneofWrappers = []any{ + file_atelet_proto_msgTypes[14].OneofWrappers = []any{} + file_atelet_proto_msgTypes[26].OneofWrappers = []any{ (*SystemInfoDataSource_ActorMetadata)(nil), (*SystemInfoDataSource_TrustBundle)(nil), } - file_atelet_proto_msgTypes[26].OneofWrappers = []any{ + file_atelet_proto_msgTypes[28].OneofWrappers = []any{ (*Volume_DurableDir)(nil), (*Volume_External)(nil), (*Volume_SystemInfo)(nil), (*Volume_Image)(nil), } - file_atelet_proto_msgTypes[39].OneofWrappers = []any{ + file_atelet_proto_msgTypes[41].OneofWrappers = []any{ (*CheckpointRequest_LocalConfig)(nil), (*CheckpointRequest_ExternalConfig)(nil), } - file_atelet_proto_msgTypes[43].OneofWrappers = []any{ + file_atelet_proto_msgTypes[45].OneofWrappers = []any{ (*RestoreRequest_LocalConfig)(nil), (*RestoreRequest_ExternalConfig)(nil), } @@ -3483,7 +3660,7 @@ func file_atelet_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_atelet_proto_rawDesc), len(file_atelet_proto_rawDesc)), NumEnums: 3, - NumMessages: 50, + NumMessages: 51, NumExtensions: 0, NumServices: 2, }, diff --git a/internal/proto/ateletpb/atelet.proto b/internal/proto/ateletpb/atelet.proto index 8164bc63d3..034e7e03dc 100644 --- a/internal/proto/ateletpb/atelet.proto +++ b/internal/proto/ateletpb/atelet.proto @@ -30,7 +30,7 @@ service AteomSupport { // the actor filesystem, when/if we support those). rpc MintActorCertificate(MintActorCertificateRequest) returns (MintActorCertificateResponse) {} - // Register capacity and hardware identity for this worker with atelet. + // Register capacity and supported sandbox runtimes for this worker with atelet. rpc RegisterWorker(RegisterWorkerRequest) returns (RegisterWorkerResponse) {} // RequestActorSuspend asks that the actor this worker hosts be suspended, @@ -52,24 +52,84 @@ message RegisterWorkerRequest { // +k8s:required WorkerResources capacity = 1; - // Hardware identity of the worker. atelet converts it to the control - // plane's HardwareIdentity when forwarding. + // Runtime that new and cold-booted actors start under. + // Snapshots matching default_runtime can also warm-boot on this worker. + // atelet converts it to the control plane's SandboxRuntime when forwarding. // // +k8s:required - HardwareIdentity hardware = 2; -} + SandboxRuntime default_runtime = 2; -// HardwareIdentity identifies the hardware properties of a worker. It mirrors -// the control plane's HardwareIdentity and is validated by the same rules. -message HardwareIdentity { - // Key-value pairs describing hardware properties (currently "architecture"; - // TODO: add "cpu_vendor", "cpu_model", etc.). + // Additional runtimes (excluding default_runtime) that this worker can + // restore snapshots from right now. // // +k8s:optional - // +k8s:maxProperties=32 - // +k8s:eachKey=+k8s:maxLength=128 - // +k8s:eachVal=+k8s:maxLength=256 - map attributes = 1; + // +k8s:maxItems=32 + // +k8s:listType=atomic + repeated SandboxRuntime restorable_runtimes = 3; +} + +// SandboxRuntime describes a sandbox runtime and its compatibility identity as +// reported by a worker. It mirrors the control plane's SandboxRuntime and is +// validated by the same rules. +message SandboxRuntime { + // e.g. "gvisor", "microvm" + // + // +k8s:required + // +k8s:maxLength=63 + string sandbox_class = 1; + + // Primary compatibility version stamped onto an actor's snapshot when a + // checkpoint is taken on this runtime. Snapshots matching `version` + // can warm-boot on this runtime. + // TODO: Add support for additional compatibility versions. + // + // +k8s:required + VersionedSandboxCompat version = 2; + + // TODO: Add an informational SandboxConfig version name (e.g. "gvisor-2") + // once workers report one. +} + +// VersionedSandboxCompat holds the compatibility schema version and the +// key-value compatibility attributes for a SandboxRuntime. +message VersionedSandboxCompat { + // Compatibility schema version (e.g. "v1") defining which runtime and + // hardware attributes are recorded and how they are computed. + // + // +k8s:required + // +k8s:maxLength=64 + string schema_version = 1; + + // Key-value compatibility attributes defined by this schema version, covering + // both runtime assets (e.g. "gvisor_asset_hash") and effective guest hardware + // features after masking (e.g. "architecture", "cpu_features"). + // Matched against a snapshot under the same schema_version: every attribute + // on the snapshot must be present on the worker with an equal value; keys + // only the worker has are ignored. Bump schema_version when a key's meaning + // changes or older snapshots must no longer match. + // Order as sent does not matter: ate-api-server sorts attributes by key + // before recording them. + // + // +k8s:required + // +k8s:maxItems=32 + // +k8s:listType=map + // +k8s:listMapKey=key + repeated AttributeEntry attributes = 2; +} + +// AttributeEntry is a single key-value compatibility attribute. +message AttributeEntry { + // Attribute name (e.g. "architecture", "cpu_features", "gvisor_asset_hash"). + // + // +k8s:required + // +k8s:maxLength=128 + string key = 1; + + // Attribute value corresponding to key. + // + // +k8s:required + // +k8s:maxLength=256 + string value = 2; } // WorkerResources is what a worker can supply. It mirrors the control plane's @@ -203,7 +263,7 @@ service AteomHerder { message TerminateRequest { // If unset, the atelet will just cleanup node resources for the actor. - string target_ateom_uid = 1; + string worker_pod_uid = 1; string atespace = 2; string actor_name = 3; @@ -219,7 +279,7 @@ message TerminateResponse { } message RunRequest { - string target_ateom_uid = 1; + string worker_pod_uid = 1; string atespace = 2; string actor_name = 3; @@ -449,16 +509,21 @@ enum CheckpointType { CHECKPOINT_TYPE_EXTERNAL = 2; } -enum SnapshotScope { +// SnapshotFidelity mirrors the public ateapi enum: the state layers a +// snapshot holds, lowest to highest, each including the ones below it. +enum SnapshotFidelity { // Not valid option; should never happen. - SNAPSHOT_SCOPE_UNSPECIFIED = 0; - // Capture process memory plus the full filesystem delta on top of the OCI - // image (including any attached DurableDir volumes). - SNAPSHOT_SCOPE_FULL = 1; + SNAPSHOT_FIDELITY_UNSPECIFIED = 0; // Capture only the contents of attached volumes that support snapshots // (currently DurableDir-typed volumes). Memory and the rest of rootfs are // excluded. - SNAPSHOT_SCOPE_DATA = 2; + SNAPSHOT_FIDELITY_VOLUMES = 1; + // Volumes plus the root filesystem changes made since boot. No runtime + // captures this yet; requests carrying it are rejected. + SNAPSHOT_FIDELITY_ROOTFS = 2; + // Capture process memory plus the full filesystem delta on top of the OCI + // image (including any attached DurableDir volumes). + SNAPSHOT_FIDELITY_MEMORY = 3; } // ExternalRestoreConfiguration is an external snapshot a restore reads. @@ -472,7 +537,7 @@ message ExternalRestoreConfiguration { } message CheckpointRequest { - string target_ateom_uid = 1; + string worker_pod_uid = 1; string atespace = 2; string actor_name = 3; @@ -494,8 +559,8 @@ message CheckpointRequest { ExternalCheckpointConfiguration external_config = 10; } - // What should be included in the checkpoint. - SnapshotScope scope = 11; + // Fidelity the checkpoint captures. + SnapshotFidelity fidelity = 11; } message CheckpointResponse { @@ -517,19 +582,19 @@ message UploadPausedCheckpointRequest { // Destination object-storage URI (the actor's in-progress snapshot URI). string destination_snapshot_uri = 7; - // Scope the uploaded snapshot must have (the commit scope; FULL or DATA). - // The scope the pause checkpoint captured is not sent: atelet reads it from + // Fidelity the uploaded snapshot must have (MEMORY or VOLUMES). The + // fidelity the pause checkpoint captured is not sent: atelet reads it from // the local snapshot's own manifest, which is authoritative. When they - // differ, atelet converts where possible (a FULL capture to a DATA upload - // of the data-scope files ateom reported) and rejects otherwise. - SnapshotScope desired_scope = 8; + // differ, atelet converts where possible (a MEMORY capture to a VOLUMES + // upload of the volume files ateom reported) and rejects otherwise. + SnapshotFidelity desired_fidelity = 8; } message UploadPausedCheckpointResponse { } message RestoreRequest { - string target_ateom_uid = 1; + string worker_pod_uid = 1; string atespace = 2; string actor_name = 3; @@ -548,15 +613,15 @@ message RestoreRequest { ExternalRestoreConfiguration external_config = 10; } - // What content to restore from the checkpoint. - SnapshotScope scope = 11; + // Fidelity to restore from the checkpoint. + SnapshotFidelity fidelity = 11; // When absent the actor has no egress: its TCP is captured and refused. optional EgressGateway egress_gateway = 13; // The actor's declared size, from the ActorTemplate's resource limits. For - // gVisor and micro-VM DATA-scope restores the sandbox is (re)sized to these; - // for a FULL micro-VM restore the size baked into the snapshot wins. Zero + // gVisor and micro-VM VOLUMES restores the sandbox is (re)sized to these; + // for a MEMORY micro-VM restore the size baked into the snapshot wins. Zero // means "unset": keep the runtime default. int64 cpu_milli = 14; // CPU limit in millicores (1000 = one core). int64 memory_bytes = 15; // Memory limit in bytes. diff --git a/internal/proto/ateletpb/atelet_grpc.pb.go b/internal/proto/ateletpb/atelet_grpc.pb.go index 3eaecf4ea6..bec22fb739 100644 --- a/internal/proto/ateletpb/atelet_grpc.pb.go +++ b/internal/proto/ateletpb/atelet_grpc.pb.go @@ -53,7 +53,7 @@ type AteomSupportClient interface { // MintActorCertificate (which would be used for certificates projected into // the actor filesystem, when/if we support those). MintActorCertificate(ctx context.Context, in *MintActorCertificateRequest, opts ...grpc.CallOption) (*MintActorCertificateResponse, error) - // Register capacity and hardware identity for this worker with atelet. + // Register capacity and supported sandbox runtimes for this worker with atelet. RegisterWorker(ctx context.Context, in *RegisterWorkerRequest, opts ...grpc.CallOption) (*RegisterWorkerResponse, error) // RequestActorSuspend asks that the actor this worker hosts be suspended, // for atelet to forward to the control plane's WorkerService, which this @@ -120,7 +120,7 @@ type AteomSupportServer interface { // MintActorCertificate (which would be used for certificates projected into // the actor filesystem, when/if we support those). MintActorCertificate(context.Context, *MintActorCertificateRequest) (*MintActorCertificateResponse, error) - // Register capacity and hardware identity for this worker with atelet. + // Register capacity and supported sandbox runtimes for this worker with atelet. RegisterWorker(context.Context, *RegisterWorkerRequest) (*RegisterWorkerResponse, error) // RequestActorSuspend asks that the actor this worker hosts be suspended, // for atelet to forward to the control plane's WorkerService, which this diff --git a/internal/proto/ateompb/ateom.pb.go b/internal/proto/ateompb/ateom.pb.go index 5ed1349db9..23f5a590c2 100644 --- a/internal/proto/ateompb/ateom.pb.go +++ b/internal/proto/ateompb/ateom.pb.go @@ -35,58 +35,65 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -type SnapshotScope int32 +// SnapshotFidelity mirrors the public ateapi enum: the state layers a +// snapshot holds, lowest to highest, each including the ones below it. +type SnapshotFidelity int32 const ( // Not valid option; should never happen. - SnapshotScope_SNAPSHOT_SCOPE_UNSPECIFIED SnapshotScope = 0 - // Capture process memory plus the full filesystem delta on top of the OCI - // image (including any attached DurableDir volumes). - SnapshotScope_SNAPSHOT_SCOPE_FULL SnapshotScope = 1 + SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED SnapshotFidelity = 0 // Capture only the contents of attached volumes that support snapshots // (currently DurableDir-typed volumes). Memory and the rest of rootfs are // excluded. - SnapshotScope_SNAPSHOT_SCOPE_DATA SnapshotScope = 2 + SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES SnapshotFidelity = 1 + // Volumes plus the root filesystem changes made since boot. No runtime + // captures this yet; requests carrying it are rejected. + SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS SnapshotFidelity = 2 + // Capture process memory plus the full filesystem delta on top of the OCI + // image (including any attached DurableDir volumes). + SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY SnapshotFidelity = 3 ) -// Enum value maps for SnapshotScope. +// Enum value maps for SnapshotFidelity. var ( - SnapshotScope_name = map[int32]string{ - 0: "SNAPSHOT_SCOPE_UNSPECIFIED", - 1: "SNAPSHOT_SCOPE_FULL", - 2: "SNAPSHOT_SCOPE_DATA", - } - SnapshotScope_value = map[string]int32{ - "SNAPSHOT_SCOPE_UNSPECIFIED": 0, - "SNAPSHOT_SCOPE_FULL": 1, - "SNAPSHOT_SCOPE_DATA": 2, + SnapshotFidelity_name = map[int32]string{ + 0: "SNAPSHOT_FIDELITY_UNSPECIFIED", + 1: "SNAPSHOT_FIDELITY_VOLUMES", + 2: "SNAPSHOT_FIDELITY_ROOTFS", + 3: "SNAPSHOT_FIDELITY_MEMORY", + } + SnapshotFidelity_value = map[string]int32{ + "SNAPSHOT_FIDELITY_UNSPECIFIED": 0, + "SNAPSHOT_FIDELITY_VOLUMES": 1, + "SNAPSHOT_FIDELITY_ROOTFS": 2, + "SNAPSHOT_FIDELITY_MEMORY": 3, } ) -func (x SnapshotScope) Enum() *SnapshotScope { - p := new(SnapshotScope) +func (x SnapshotFidelity) Enum() *SnapshotFidelity { + p := new(SnapshotFidelity) *p = x return p } -func (x SnapshotScope) String() string { +func (x SnapshotFidelity) String() string { return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) } -func (SnapshotScope) Descriptor() protoreflect.EnumDescriptor { +func (SnapshotFidelity) Descriptor() protoreflect.EnumDescriptor { return file_ateom_proto_enumTypes[0].Descriptor() } -func (SnapshotScope) Type() protoreflect.EnumType { +func (SnapshotFidelity) Type() protoreflect.EnumType { return &file_ateom_proto_enumTypes[0] } -func (x SnapshotScope) Number() protoreflect.EnumNumber { +func (x SnapshotFidelity) Number() protoreflect.EnumNumber { return protoreflect.EnumNumber(x) } -// Deprecated: Use SnapshotScope.Descriptor instead. -func (SnapshotScope) EnumDescriptor() ([]byte, []int) { +// Deprecated: Use SnapshotFidelity.Descriptor instead. +func (SnapshotFidelity) EnumDescriptor() ([]byte, []int) { return file_ateom_proto_rawDescGZIP(), []int{0} } @@ -691,8 +698,11 @@ type Container struct { SystemInfoVolumeMounts []*SystemInfoVolumeMount `protobuf:"bytes,5,rep,name=system_info_volume_mounts,json=systemInfoVolumeMounts,proto3" json:"system_info_volume_mounts,omitempty"` // image_volume_mounts are the image volumes this container mounts, if any. ImageVolumeMounts []*ImageVolumeMount `protobuf:"bytes,6,rep,name=image_volume_mounts,json=imageVolumeMounts,proto3" json:"image_volume_mounts,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // container_spec is what ateom builds the container's OCI spec from, + // besides its volume mounts. Unset on RPCs that start no process. + ContainerSpec *ContainerSpec `protobuf:"bytes,7,opt,name=container_spec,json=containerSpec,proto3" json:"container_spec,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *Container) Reset() { @@ -767,6 +777,143 @@ func (x *Container) GetImageVolumeMounts() []*ImageVolumeMount { return nil } +func (x *Container) GetContainerSpec() *ContainerSpec { + if x != nil { + return x.ContainerSpec + } + return nil +} + +// ContainerSpec is a container's process and limits, resolved by atelet. +type ContainerSpec struct { + state protoimpl.MessageState `protogen:"open.v1"` + // args is the process argv, already resolved against the image's + // ENTRYPOINT and CMD. + Args []string `protobuf:"bytes,1,rep,name=args,proto3" json:"args,omitempty"` + // env is the process environment as KEY=VALUE, already merged with the + // image's ENV. + Env []string `protobuf:"bytes,2,rep,name=env,proto3" json:"env,omitempty"` + // capabilities is the container's capability set, CAP_-prefixed. + Capabilities []string `protobuf:"bytes,3,rep,name=capabilities,proto3" json:"capabilities,omitempty"` + // resources are the container's own declared limits, if any. + Resources *ResourceLimits `protobuf:"bytes,4,opt,name=resources,proto3" json:"resources,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ContainerSpec) Reset() { + *x = ContainerSpec{} + mi := &file_ateom_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ContainerSpec) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ContainerSpec) ProtoMessage() {} + +func (x *ContainerSpec) ProtoReflect() protoreflect.Message { + mi := &file_ateom_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ContainerSpec.ProtoReflect.Descriptor instead. +func (*ContainerSpec) Descriptor() ([]byte, []int) { + return file_ateom_proto_rawDescGZIP(), []int{7} +} + +func (x *ContainerSpec) GetArgs() []string { + if x != nil { + return x.Args + } + return nil +} + +func (x *ContainerSpec) GetEnv() []string { + if x != nil { + return x.Env + } + return nil +} + +func (x *ContainerSpec) GetCapabilities() []string { + if x != nil { + return x.Capabilities + } + return nil +} + +func (x *ContainerSpec) GetResources() *ResourceLimits { + if x != nil { + return x.Resources + } + return nil +} + +// ResourceLimits are a container's declared limits. +type ResourceLimits struct { + state protoimpl.MessageState `protogen:"open.v1"` + // memory_bytes is the memory limit in bytes. 0 means unset. + MemoryBytes int64 `protobuf:"varint,1,opt,name=memory_bytes,json=memoryBytes,proto3" json:"memory_bytes,omitempty"` + // cpu_millis is the CPU limit in milli-cores (1000 = one core). 0 means unset. + CpuMillis int64 `protobuf:"varint,2,opt,name=cpu_millis,json=cpuMillis,proto3" json:"cpu_millis,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ResourceLimits) Reset() { + *x = ResourceLimits{} + mi := &file_ateom_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ResourceLimits) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ResourceLimits) ProtoMessage() {} + +func (x *ResourceLimits) ProtoReflect() protoreflect.Message { + mi := &file_ateom_proto_msgTypes[8] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ResourceLimits.ProtoReflect.Descriptor instead. +func (*ResourceLimits) Descriptor() ([]byte, []int) { + return file_ateom_proto_rawDescGZIP(), []int{8} +} + +func (x *ResourceLimits) GetMemoryBytes() int64 { + if x != nil { + return x.MemoryBytes + } + return 0 +} + +func (x *ResourceLimits) GetCpuMillis() int64 { + if x != nil { + return x.CpuMillis + } + return 0 +} + // VolumeMount is one volume mounted into a container. type VolumeMount struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -778,7 +925,7 @@ type VolumeMount struct { func (x *VolumeMount) Reset() { *x = VolumeMount{} - mi := &file_ateom_proto_msgTypes[7] + mi := &file_ateom_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -790,7 +937,7 @@ func (x *VolumeMount) String() string { func (*VolumeMount) ProtoMessage() {} func (x *VolumeMount) ProtoReflect() protoreflect.Message { - mi := &file_ateom_proto_msgTypes[7] + mi := &file_ateom_proto_msgTypes[9] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -803,7 +950,7 @@ func (x *VolumeMount) ProtoReflect() protoreflect.Message { // Deprecated: Use VolumeMount.ProtoReflect.Descriptor instead. func (*VolumeMount) Descriptor() ([]byte, []int) { - return file_ateom_proto_rawDescGZIP(), []int{7} + return file_ateom_proto_rawDescGZIP(), []int{9} } func (x *VolumeMount) GetVolumeName() string { @@ -834,7 +981,7 @@ type DurableDirVolumeMount struct { func (x *DurableDirVolumeMount) Reset() { *x = DurableDirVolumeMount{} - mi := &file_ateom_proto_msgTypes[8] + mi := &file_ateom_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -846,7 +993,7 @@ func (x *DurableDirVolumeMount) String() string { func (*DurableDirVolumeMount) ProtoMessage() {} func (x *DurableDirVolumeMount) ProtoReflect() protoreflect.Message { - mi := &file_ateom_proto_msgTypes[8] + mi := &file_ateom_proto_msgTypes[10] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -859,7 +1006,7 @@ func (x *DurableDirVolumeMount) ProtoReflect() protoreflect.Message { // Deprecated: Use DurableDirVolumeMount.ProtoReflect.Descriptor instead. func (*DurableDirVolumeMount) Descriptor() ([]byte, []int) { - return file_ateom_proto_rawDescGZIP(), []int{8} + return file_ateom_proto_rawDescGZIP(), []int{10} } func (x *DurableDirVolumeMount) GetVolumeName() string { @@ -892,7 +1039,7 @@ type SystemInfoVolumeMount struct { func (x *SystemInfoVolumeMount) Reset() { *x = SystemInfoVolumeMount{} - mi := &file_ateom_proto_msgTypes[9] + mi := &file_ateom_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -904,7 +1051,7 @@ func (x *SystemInfoVolumeMount) String() string { func (*SystemInfoVolumeMount) ProtoMessage() {} func (x *SystemInfoVolumeMount) ProtoReflect() protoreflect.Message { - mi := &file_ateom_proto_msgTypes[9] + mi := &file_ateom_proto_msgTypes[11] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -917,7 +1064,7 @@ func (x *SystemInfoVolumeMount) ProtoReflect() protoreflect.Message { // Deprecated: Use SystemInfoVolumeMount.ProtoReflect.Descriptor instead. func (*SystemInfoVolumeMount) Descriptor() ([]byte, []int) { - return file_ateom_proto_rawDescGZIP(), []int{9} + return file_ateom_proto_rawDescGZIP(), []int{11} } func (x *SystemInfoVolumeMount) GetVolumeName() string { @@ -949,7 +1096,7 @@ type ImageVolumeMount struct { func (x *ImageVolumeMount) Reset() { *x = ImageVolumeMount{} - mi := &file_ateom_proto_msgTypes[10] + mi := &file_ateom_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -961,7 +1108,7 @@ func (x *ImageVolumeMount) String() string { func (*ImageVolumeMount) ProtoMessage() {} func (x *ImageVolumeMount) ProtoReflect() protoreflect.Message { - mi := &file_ateom_proto_msgTypes[10] + mi := &file_ateom_proto_msgTypes[12] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -974,7 +1121,7 @@ func (x *ImageVolumeMount) ProtoReflect() protoreflect.Message { // Deprecated: Use ImageVolumeMount.ProtoReflect.Descriptor instead. func (*ImageVolumeMount) Descriptor() ([]byte, []int) { - return file_ateom_proto_rawDescGZIP(), []int{10} + return file_ateom_proto_rawDescGZIP(), []int{12} } func (x *ImageVolumeMount) GetVolumeName() string { @@ -1004,7 +1151,7 @@ type WakeupProbe struct { func (x *WakeupProbe) Reset() { *x = WakeupProbe{} - mi := &file_ateom_proto_msgTypes[11] + mi := &file_ateom_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1016,7 +1163,7 @@ func (x *WakeupProbe) String() string { func (*WakeupProbe) ProtoMessage() {} func (x *WakeupProbe) ProtoReflect() protoreflect.Message { - mi := &file_ateom_proto_msgTypes[11] + mi := &file_ateom_proto_msgTypes[13] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1029,7 +1176,7 @@ func (x *WakeupProbe) ProtoReflect() protoreflect.Message { // Deprecated: Use WakeupProbe.ProtoReflect.Descriptor instead. func (*WakeupProbe) Descriptor() ([]byte, []int) { - return file_ateom_proto_rawDescGZIP(), []int{11} + return file_ateom_proto_rawDescGZIP(), []int{13} } func (x *WakeupProbe) GetHttpGet() *HTTPGetAction { @@ -1059,7 +1206,7 @@ type HTTPGetAction struct { func (x *HTTPGetAction) Reset() { *x = HTTPGetAction{} - mi := &file_ateom_proto_msgTypes[12] + mi := &file_ateom_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1071,7 +1218,7 @@ func (x *HTTPGetAction) String() string { func (*HTTPGetAction) ProtoMessage() {} func (x *HTTPGetAction) ProtoReflect() protoreflect.Message { - mi := &file_ateom_proto_msgTypes[12] + mi := &file_ateom_proto_msgTypes[14] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1084,7 +1231,7 @@ func (x *HTTPGetAction) ProtoReflect() protoreflect.Message { // Deprecated: Use HTTPGetAction.ProtoReflect.Descriptor instead. func (*HTTPGetAction) Descriptor() ([]byte, []int) { - return file_ateom_proto_rawDescGZIP(), []int{12} + return file_ateom_proto_rawDescGZIP(), []int{14} } func (x *HTTPGetAction) GetPath() string { @@ -1109,7 +1256,7 @@ type RunWorkloadResponse struct { func (x *RunWorkloadResponse) Reset() { *x = RunWorkloadResponse{} - mi := &file_ateom_proto_msgTypes[13] + mi := &file_ateom_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1121,7 +1268,7 @@ func (x *RunWorkloadResponse) String() string { func (*RunWorkloadResponse) ProtoMessage() {} func (x *RunWorkloadResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateom_proto_msgTypes[13] + mi := &file_ateom_proto_msgTypes[15] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1134,7 +1281,7 @@ func (x *RunWorkloadResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RunWorkloadResponse.ProtoReflect.Descriptor instead. func (*RunWorkloadResponse) Descriptor() ([]byte, []int) { - return file_ateom_proto_rawDescGZIP(), []int{13} + return file_ateom_proto_rawDescGZIP(), []int{15} } type CheckpointWorkloadRequest struct { @@ -1160,16 +1307,16 @@ type CheckpointWorkloadRequest struct { // runtime_asset_paths maps a runtime asset name to the local on-disk path // atelet fetched it to (see RunWorkloadRequest). Empty for gVisor. RuntimeAssetPaths map[string]string `protobuf:"bytes,9,rep,name=runtime_asset_paths,json=runtimeAssetPaths,proto3" json:"runtime_asset_paths,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` - // What content to include in the checkpoint. - Scope SnapshotScope `protobuf:"varint,10,opt,name=scope,proto3,enum=ateom.SnapshotScope" json:"scope,omitempty"` - ActorDirs *ActorDirs `protobuf:"bytes,11,opt,name=actor_dirs,json=actorDirs,proto3" json:"actor_dirs,omitempty"` + // Fidelity the checkpoint captures. + Fidelity SnapshotFidelity `protobuf:"varint,10,opt,name=fidelity,proto3,enum=ateom.SnapshotFidelity" json:"fidelity,omitempty"` + ActorDirs *ActorDirs `protobuf:"bytes,11,opt,name=actor_dirs,json=actorDirs,proto3" json:"actor_dirs,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } func (x *CheckpointWorkloadRequest) Reset() { *x = CheckpointWorkloadRequest{} - mi := &file_ateom_proto_msgTypes[14] + mi := &file_ateom_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1181,7 +1328,7 @@ func (x *CheckpointWorkloadRequest) String() string { func (*CheckpointWorkloadRequest) ProtoMessage() {} func (x *CheckpointWorkloadRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateom_proto_msgTypes[14] + mi := &file_ateom_proto_msgTypes[16] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1194,7 +1341,7 @@ func (x *CheckpointWorkloadRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use CheckpointWorkloadRequest.ProtoReflect.Descriptor instead. func (*CheckpointWorkloadRequest) Descriptor() ([]byte, []int) { - return file_ateom_proto_rawDescGZIP(), []int{14} + return file_ateom_proto_rawDescGZIP(), []int{16} } func (x *CheckpointWorkloadRequest) GetAtespace() string { @@ -1260,11 +1407,11 @@ func (x *CheckpointWorkloadRequest) GetRuntimeAssetPaths() map[string]string { return nil } -func (x *CheckpointWorkloadRequest) GetScope() SnapshotScope { +func (x *CheckpointWorkloadRequest) GetFidelity() SnapshotFidelity { if x != nil { - return x.Scope + return x.Fidelity } - return SnapshotScope_SNAPSHOT_SCOPE_UNSPECIFIED + return SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED } func (x *CheckpointWorkloadRequest) GetActorDirs() *ActorDirs { @@ -1281,7 +1428,7 @@ type CheckpointWorkloadResponse struct { // its own set (gVisor's image files, cloud-hypervisor's snapshot set, ...). SnapshotFiles []string `protobuf:"bytes,1,rep,name=snapshot_files,json=snapshotFiles,proto3" json:"snapshot_files,omitempty"` // data_snapshot_files is the subset of snapshot_files that restores the - // actor at DATA scope on its own. Empty when the capture holds no durable data. + // actor at VOLUMES fidelity on its own. Empty when the capture holds no durable data. DataSnapshotFiles []string `protobuf:"bytes,2,rep,name=data_snapshot_files,json=dataSnapshotFiles,proto3" json:"data_snapshot_files,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache @@ -1289,7 +1436,7 @@ type CheckpointWorkloadResponse struct { func (x *CheckpointWorkloadResponse) Reset() { *x = CheckpointWorkloadResponse{} - mi := &file_ateom_proto_msgTypes[15] + mi := &file_ateom_proto_msgTypes[17] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1301,7 +1448,7 @@ func (x *CheckpointWorkloadResponse) String() string { func (*CheckpointWorkloadResponse) ProtoMessage() {} func (x *CheckpointWorkloadResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateom_proto_msgTypes[15] + mi := &file_ateom_proto_msgTypes[17] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1314,7 +1461,7 @@ func (x *CheckpointWorkloadResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use CheckpointWorkloadResponse.ProtoReflect.Descriptor instead. func (*CheckpointWorkloadResponse) Descriptor() ([]byte, []int) { - return file_ateom_proto_rawDescGZIP(), []int{15} + return file_ateom_proto_rawDescGZIP(), []int{17} } func (x *CheckpointWorkloadResponse) GetSnapshotFiles() []string { @@ -1346,13 +1493,13 @@ type RestoreWorkloadRequest struct { // runtime_asset_paths maps a runtime asset name to the local on-disk path // atelet fetched it to (see RunWorkloadRequest). Empty for gVisor. RuntimeAssetPaths map[string]string `protobuf:"bytes,9,rep,name=runtime_asset_paths,json=runtimeAssetPaths,proto3" json:"runtime_asset_paths,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` - // What content to restore from the snapshot. - Scope SnapshotScope `protobuf:"varint,10,opt,name=scope,proto3,enum=ateom.SnapshotScope" json:"scope,omitempty"` + // Fidelity to restore from the snapshot. + Fidelity SnapshotFidelity `protobuf:"varint,10,opt,name=fidelity,proto3,enum=ateom.SnapshotFidelity" json:"fidelity,omitempty"` // When absent the actor has no egress: its TCP is captured and refused. EgressGateway *EgressGateway `protobuf:"bytes,12,opt,name=egress_gateway,json=egressGateway,proto3,oneof" json:"egress_gateway,omitempty"` // The actor's declared size, from the ActorTemplate's resource limits. Used to - // (re)size the sandbox on a DATA-scope restore (fresh guest container). On a - // FULL micro-VM restore the size baked into the snapshot is authoritative and + // (re)size the sandbox on a VOLUMES restore (fresh guest container). On a + // MEMORY micro-VM restore the size baked into the snapshot is authoritative and // these are ignored. Zero means "unset": keep the runtime default. CpuMilli int64 `protobuf:"varint,14,opt,name=cpu_milli,json=cpuMilli,proto3" json:"cpu_milli,omitempty"` // CPU limit in millicores (1000 = one core). MemoryBytes int64 `protobuf:"varint,15,opt,name=memory_bytes,json=memoryBytes,proto3" json:"memory_bytes,omitempty"` // Memory limit in bytes. @@ -1366,7 +1513,7 @@ type RestoreWorkloadRequest struct { func (x *RestoreWorkloadRequest) Reset() { *x = RestoreWorkloadRequest{} - mi := &file_ateom_proto_msgTypes[16] + mi := &file_ateom_proto_msgTypes[18] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1378,7 +1525,7 @@ func (x *RestoreWorkloadRequest) String() string { func (*RestoreWorkloadRequest) ProtoMessage() {} func (x *RestoreWorkloadRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateom_proto_msgTypes[16] + mi := &file_ateom_proto_msgTypes[18] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1391,7 +1538,7 @@ func (x *RestoreWorkloadRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use RestoreWorkloadRequest.ProtoReflect.Descriptor instead. func (*RestoreWorkloadRequest) Descriptor() ([]byte, []int) { - return file_ateom_proto_rawDescGZIP(), []int{16} + return file_ateom_proto_rawDescGZIP(), []int{18} } func (x *RestoreWorkloadRequest) GetAtespace() string { @@ -1457,11 +1604,11 @@ func (x *RestoreWorkloadRequest) GetRuntimeAssetPaths() map[string]string { return nil } -func (x *RestoreWorkloadRequest) GetScope() SnapshotScope { +func (x *RestoreWorkloadRequest) GetFidelity() SnapshotFidelity { if x != nil { - return x.Scope + return x.Fidelity } - return SnapshotScope_SNAPSHOT_SCOPE_UNSPECIFIED + return SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED } func (x *RestoreWorkloadRequest) GetEgressGateway() *EgressGateway { @@ -1507,7 +1654,7 @@ type RestoreWorkloadResponse struct { func (x *RestoreWorkloadResponse) Reset() { *x = RestoreWorkloadResponse{} - mi := &file_ateom_proto_msgTypes[17] + mi := &file_ateom_proto_msgTypes[19] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1519,7 +1666,7 @@ func (x *RestoreWorkloadResponse) String() string { func (*RestoreWorkloadResponse) ProtoMessage() {} func (x *RestoreWorkloadResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateom_proto_msgTypes[17] + mi := &file_ateom_proto_msgTypes[19] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1532,7 +1679,7 @@ func (x *RestoreWorkloadResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RestoreWorkloadResponse.ProtoReflect.Descriptor instead. func (*RestoreWorkloadResponse) Descriptor() ([]byte, []int) { - return file_ateom_proto_rawDescGZIP(), []int{17} + return file_ateom_proto_rawDescGZIP(), []int{19} } type GetWorkloadStatsRequest struct { @@ -1548,7 +1695,7 @@ type GetWorkloadStatsRequest struct { func (x *GetWorkloadStatsRequest) Reset() { *x = GetWorkloadStatsRequest{} - mi := &file_ateom_proto_msgTypes[18] + mi := &file_ateom_proto_msgTypes[20] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1560,7 +1707,7 @@ func (x *GetWorkloadStatsRequest) String() string { func (*GetWorkloadStatsRequest) ProtoMessage() {} func (x *GetWorkloadStatsRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateom_proto_msgTypes[18] + mi := &file_ateom_proto_msgTypes[20] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1573,7 +1720,7 @@ func (x *GetWorkloadStatsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetWorkloadStatsRequest.ProtoReflect.Descriptor instead. func (*GetWorkloadStatsRequest) Descriptor() ([]byte, []int) { - return file_ateom_proto_rawDescGZIP(), []int{18} + return file_ateom_proto_rawDescGZIP(), []int{20} } func (x *GetWorkloadStatsRequest) GetActorUid() string { @@ -1621,15 +1768,14 @@ type WorkloadStatsSample struct { // drifts upward with cache the kernel would drop for free under pressure. MemoryWorkingSetBytes uint64 `protobuf:"varint,10,opt,name=memory_working_set_bytes,json=memoryWorkingSetBytes,proto3" json:"memory_working_set_bytes,omitempty"` // Cumulative CPU time since the activation began (epoch_unix_nano), for - // every source. The cgroup source restarts on its own; the guest-agent - // source is rebased on its first read after a restore, since the guest's - // counters survive in guest RAM. An ateom that leaves epoch_unix_nano at zero - // also sends the raw guest counter. A lifetime figure is the sum over epochs - // of each epoch's highest value. + // every source, and never decreasing within it. The cgroup source restarts on + // its own; the guest-agent source counts from its first read after a restore + // that resumes the guest, since the guest's counters survive in guest RAM. A + // lifetime figure is the sum over epochs of each epoch's highest value. CpuUsageUsec uint64 `protobuf:"varint,11,opt,name=cpu_usage_usec,json=cpuUsageUsec,proto3" json:"cpu_usage_usec,omitempty"` ObservedAtUnixNano int64 `protobuf:"varint,12,opt,name=observed_at_unix_nano,json=observedAtUnixNano,proto3" json:"observed_at_unix_nano,omitempty"` // The activation this sample belongs to, as the unix-nano time it began. A - // Run or Restore starts one. Zero from an ateom that does not set it. + // Run or Restore starts one. EpochUnixNano int64 `protobuf:"varint,13,opt,name=epoch_unix_nano,json=epochUnixNano,proto3" json:"epoch_unix_nano,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache @@ -1637,7 +1783,7 @@ type WorkloadStatsSample struct { func (x *WorkloadStatsSample) Reset() { *x = WorkloadStatsSample{} - mi := &file_ateom_proto_msgTypes[19] + mi := &file_ateom_proto_msgTypes[21] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1649,7 +1795,7 @@ func (x *WorkloadStatsSample) String() string { func (*WorkloadStatsSample) ProtoMessage() {} func (x *WorkloadStatsSample) ProtoReflect() protoreflect.Message { - mi := &file_ateom_proto_msgTypes[19] + mi := &file_ateom_proto_msgTypes[21] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1662,7 +1808,7 @@ func (x *WorkloadStatsSample) ProtoReflect() protoreflect.Message { // Deprecated: Use WorkloadStatsSample.ProtoReflect.Descriptor instead. func (*WorkloadStatsSample) Descriptor() ([]byte, []int) { - return file_ateom_proto_rawDescGZIP(), []int{19} + return file_ateom_proto_rawDescGZIP(), []int{21} } func (x *WorkloadStatsSample) GetAtespace() string { @@ -1765,7 +1911,7 @@ type GetWorkloadStatsResponse struct { func (x *GetWorkloadStatsResponse) Reset() { *x = GetWorkloadStatsResponse{} - mi := &file_ateom_proto_msgTypes[20] + mi := &file_ateom_proto_msgTypes[22] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1777,7 +1923,7 @@ func (x *GetWorkloadStatsResponse) String() string { func (*GetWorkloadStatsResponse) ProtoMessage() {} func (x *GetWorkloadStatsResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateom_proto_msgTypes[20] + mi := &file_ateom_proto_msgTypes[22] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1790,7 +1936,7 @@ func (x *GetWorkloadStatsResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use GetWorkloadStatsResponse.ProtoReflect.Descriptor instead. func (*GetWorkloadStatsResponse) Descriptor() ([]byte, []int) { - return file_ateom_proto_rawDescGZIP(), []int{20} + return file_ateom_proto_rawDescGZIP(), []int{22} } func (x *GetWorkloadStatsResponse) GetSample() *WorkloadStatsSample { @@ -1808,7 +1954,7 @@ type GetActiveWorkloadStatsRequest struct { func (x *GetActiveWorkloadStatsRequest) Reset() { *x = GetActiveWorkloadStatsRequest{} - mi := &file_ateom_proto_msgTypes[21] + mi := &file_ateom_proto_msgTypes[23] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1820,7 +1966,7 @@ func (x *GetActiveWorkloadStatsRequest) String() string { func (*GetActiveWorkloadStatsRequest) ProtoMessage() {} func (x *GetActiveWorkloadStatsRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateom_proto_msgTypes[21] + mi := &file_ateom_proto_msgTypes[23] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1833,7 +1979,7 @@ func (x *GetActiveWorkloadStatsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetActiveWorkloadStatsRequest.ProtoReflect.Descriptor instead. func (*GetActiveWorkloadStatsRequest) Descriptor() ([]byte, []int) { - return file_ateom_proto_rawDescGZIP(), []int{21} + return file_ateom_proto_rawDescGZIP(), []int{23} } type GetActiveWorkloadStatsResponse struct { @@ -1853,7 +1999,7 @@ type GetActiveWorkloadStatsResponse struct { func (x *GetActiveWorkloadStatsResponse) Reset() { *x = GetActiveWorkloadStatsResponse{} - mi := &file_ateom_proto_msgTypes[22] + mi := &file_ateom_proto_msgTypes[24] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1865,7 +2011,7 @@ func (x *GetActiveWorkloadStatsResponse) String() string { func (*GetActiveWorkloadStatsResponse) ProtoMessage() {} func (x *GetActiveWorkloadStatsResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateom_proto_msgTypes[22] + mi := &file_ateom_proto_msgTypes[24] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1878,7 +2024,7 @@ func (x *GetActiveWorkloadStatsResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use GetActiveWorkloadStatsResponse.ProtoReflect.Descriptor instead. func (*GetActiveWorkloadStatsResponse) Descriptor() ([]byte, []int) { - return file_ateom_proto_rawDescGZIP(), []int{22} + return file_ateom_proto_rawDescGZIP(), []int{24} } func (x *GetActiveWorkloadStatsResponse) GetSamples() []*WorkloadStatsSample { @@ -1942,14 +2088,24 @@ const file_ateom_proto_rawDesc = "" + "\fWorkloadSpec\x120\n" + "\n" + "containers\x18\x01 \x03(\v2\x10.ateom.ContainerR\n" + - "containers\"\x91\x03\n" + + "containers\"\xce\x03\n" + "\tContainer\x12\x12\n" + "\x04name\x18\x01 \x01(\tR\x04name\x125\n" + "\fwakeup_probe\x18\x02 \x01(\v2\x12.ateom.WakeupProbeR\vwakeupProbe\x12W\n" + "\x19durable_dir_volume_mounts\x18\x03 \x03(\v2\x1c.ateom.DurableDirVolumeMountR\x16durableDirVolumeMounts\x12>\n" + "\x11csi_volume_mounts\x18\x04 \x03(\v2\x12.ateom.VolumeMountR\x0fcsiVolumeMounts\x12W\n" + "\x19system_info_volume_mounts\x18\x05 \x03(\v2\x1c.ateom.SystemInfoVolumeMountR\x16systemInfoVolumeMounts\x12G\n" + - "\x13image_volume_mounts\x18\x06 \x03(\v2\x17.ateom.ImageVolumeMountR\x11imageVolumeMounts\"M\n" + + "\x13image_volume_mounts\x18\x06 \x03(\v2\x17.ateom.ImageVolumeMountR\x11imageVolumeMounts\x12;\n" + + "\x0econtainer_spec\x18\a \x01(\v2\x14.ateom.ContainerSpecR\rcontainerSpec\"\x93\x01\n" + + "\rContainerSpec\x12\x12\n" + + "\x04args\x18\x01 \x03(\tR\x04args\x12\x15\n" + + "\x03env\x18\x02 \x03(\tB\x03\x80\x01\x01R\x03env\x12\"\n" + + "\fcapabilities\x18\x03 \x03(\tR\fcapabilities\x123\n" + + "\tresources\x18\x04 \x01(\v2\x15.ateom.ResourceLimitsR\tresources\"R\n" + + "\x0eResourceLimits\x12!\n" + + "\fmemory_bytes\x18\x01 \x01(\x03R\vmemoryBytes\x12\x1d\n" + + "\n" + + "cpu_millis\x18\x02 \x01(\x03R\tcpuMillis\"M\n" + "\vVolumeMount\x12\x1f\n" + "\vvolume_name\x18\x01 \x01(\tR\n" + "volumeName\x12\x1d\n" + @@ -1976,7 +2132,7 @@ const file_ateom_proto_rawDesc = "" + "\rHTTPGetAction\x12\x12\n" + "\x04path\x18\x01 \x01(\tR\x04path\x12\x12\n" + "\x04port\x18\x02 \x01(\x05R\x04port\"\x15\n" + - "\x13RunWorkloadResponse\"\xd2\x04\n" + + "\x13RunWorkloadResponse\"\xdb\x04\n" + "\x19CheckpointWorkloadRequest\x12\x1a\n" + "\batespace\x18\x01 \x01(\tR\batespace\x12\x1d\n" + "\n" + @@ -1988,9 +2144,9 @@ const file_ateom_proto_rawDesc = "" + "runsc_path\x18\x06 \x01(\tR\trunscPath\x12'\n" + "\x04spec\x18\a \x01(\v2\x13.ateom.WorkloadSpecR\x04spec\x12!\n" + "\fsnapshot_uri\x18\b \x01(\tR\vsnapshotUri\x12g\n" + - "\x13runtime_asset_paths\x18\t \x03(\v27.ateom.CheckpointWorkloadRequest.RuntimeAssetPathsEntryR\x11runtimeAssetPaths\x12*\n" + - "\x05scope\x18\n" + - " \x01(\x0e2\x14.ateom.SnapshotScopeR\x05scope\x12/\n" + + "\x13runtime_asset_paths\x18\t \x03(\v27.ateom.CheckpointWorkloadRequest.RuntimeAssetPathsEntryR\x11runtimeAssetPaths\x123\n" + + "\bfidelity\x18\n" + + " \x01(\x0e2\x17.ateom.SnapshotFidelityR\bfidelity\x12/\n" + "\n" + "actor_dirs\x18\v \x01(\v2\x10.ateom.ActorDirsR\tactorDirs\x1aD\n" + "\x16RuntimeAssetPathsEntry\x12\x10\n" + @@ -1998,7 +2154,7 @@ const file_ateom_proto_rawDesc = "" + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"s\n" + "\x1aCheckpointWorkloadResponse\x12%\n" + "\x0esnapshot_files\x18\x01 \x03(\tR\rsnapshotFiles\x12.\n" + - "\x13data_snapshot_files\x18\x02 \x03(\tR\x11dataSnapshotFiles\"\x93\x06\n" + + "\x13data_snapshot_files\x18\x02 \x03(\tR\x11dataSnapshotFiles\"\x9c\x06\n" + "\x16RestoreWorkloadRequest\x12\x1a\n" + "\batespace\x18\x01 \x01(\tR\batespace\x12\x1d\n" + "\n" + @@ -2010,9 +2166,9 @@ const file_ateom_proto_rawDesc = "" + "runsc_path\x18\x06 \x01(\tR\trunscPath\x12'\n" + "\x04spec\x18\a \x01(\v2\x13.ateom.WorkloadSpecR\x04spec\x12!\n" + "\fsnapshot_uri\x18\b \x01(\tR\vsnapshotUri\x12d\n" + - "\x13runtime_asset_paths\x18\t \x03(\v24.ateom.RestoreWorkloadRequest.RuntimeAssetPathsEntryR\x11runtimeAssetPaths\x12*\n" + - "\x05scope\x18\n" + - " \x01(\x0e2\x14.ateom.SnapshotScopeR\x05scope\x12@\n" + + "\x13runtime_asset_paths\x18\t \x03(\v24.ateom.RestoreWorkloadRequest.RuntimeAssetPathsEntryR\x11runtimeAssetPaths\x123\n" + + "\bfidelity\x18\n" + + " \x01(\x0e2\x17.ateom.SnapshotFidelityR\bfidelity\x12@\n" + "\x0eegress_gateway\x18\f \x01(\v2\x14.ateom.EgressGatewayH\x00R\regressGateway\x88\x01\x01\x12\x1b\n" + "\tcpu_milli\x18\x0e \x01(\x03R\bcpuMilli\x12!\n" + "\fmemory_bytes\x18\x0f \x01(\x03R\vmemoryBytes\x12/\n" + @@ -2046,11 +2202,12 @@ const file_ateom_proto_rawDesc = "" + "\x06sample\x18\x01 \x01(\v2\x1a.ateom.WorkloadStatsSampleR\x06sample\"\x1f\n" + "\x1dGetActiveWorkloadStatsRequest\"V\n" + "\x1eGetActiveWorkloadStatsResponse\x124\n" + - "\asamples\x18\x01 \x03(\v2\x1a.ateom.WorkloadStatsSampleR\asamples*a\n" + - "\rSnapshotScope\x12\x1e\n" + - "\x1aSNAPSHOT_SCOPE_UNSPECIFIED\x10\x00\x12\x17\n" + - "\x13SNAPSHOT_SCOPE_FULL\x10\x01\x12\x17\n" + - "\x13SNAPSHOT_SCOPE_DATA\x10\x02*b\n" + + "\asamples\x18\x01 \x03(\v2\x1a.ateom.WorkloadStatsSampleR\asamples*\x90\x01\n" + + "\x10SnapshotFidelity\x12!\n" + + "\x1dSNAPSHOT_FIDELITY_UNSPECIFIED\x10\x00\x12\x1d\n" + + "\x19SNAPSHOT_FIDELITY_VOLUMES\x10\x01\x12\x1c\n" + + "\x18SNAPSHOT_FIDELITY_ROOTFS\x10\x02\x12\x1c\n" + + "\x18SNAPSHOT_FIDELITY_MEMORY\x10\x03*b\n" + "\fSandboxClass\x12\x1d\n" + "\x19SANDBOX_CLASS_UNSPECIFIED\x10\x00\x12\x18\n" + "\x14SANDBOX_CLASS_GVISOR\x10\x01\x12\x19\n" + @@ -2080,9 +2237,9 @@ func file_ateom_proto_rawDescGZIP() []byte { } var file_ateom_proto_enumTypes = make([]protoimpl.EnumInfo, 3) -var file_ateom_proto_msgTypes = make([]protoimpl.MessageInfo, 26) +var file_ateom_proto_msgTypes = make([]protoimpl.MessageInfo, 28) var file_ateom_proto_goTypes = []any{ - (SnapshotScope)(0), // 0: ateom.SnapshotScope + (SnapshotFidelity)(0), // 0: ateom.SnapshotFidelity (SandboxClass)(0), // 1: ateom.SandboxClass (StatsSource)(0), // 2: ateom.StatsSource (*ActorDirs)(nil), // 3: ateom.ActorDirs @@ -2092,70 +2249,74 @@ var file_ateom_proto_goTypes = []any{ (*EgressGateway)(nil), // 7: ateom.EgressGateway (*WorkloadSpec)(nil), // 8: ateom.WorkloadSpec (*Container)(nil), // 9: ateom.Container - (*VolumeMount)(nil), // 10: ateom.VolumeMount - (*DurableDirVolumeMount)(nil), // 11: ateom.DurableDirVolumeMount - (*SystemInfoVolumeMount)(nil), // 12: ateom.SystemInfoVolumeMount - (*ImageVolumeMount)(nil), // 13: ateom.ImageVolumeMount - (*WakeupProbe)(nil), // 14: ateom.WakeupProbe - (*HTTPGetAction)(nil), // 15: ateom.HTTPGetAction - (*RunWorkloadResponse)(nil), // 16: ateom.RunWorkloadResponse - (*CheckpointWorkloadRequest)(nil), // 17: ateom.CheckpointWorkloadRequest - (*CheckpointWorkloadResponse)(nil), // 18: ateom.CheckpointWorkloadResponse - (*RestoreWorkloadRequest)(nil), // 19: ateom.RestoreWorkloadRequest - (*RestoreWorkloadResponse)(nil), // 20: ateom.RestoreWorkloadResponse - (*GetWorkloadStatsRequest)(nil), // 21: ateom.GetWorkloadStatsRequest - (*WorkloadStatsSample)(nil), // 22: ateom.WorkloadStatsSample - (*GetWorkloadStatsResponse)(nil), // 23: ateom.GetWorkloadStatsResponse - (*GetActiveWorkloadStatsRequest)(nil), // 24: ateom.GetActiveWorkloadStatsRequest - (*GetActiveWorkloadStatsResponse)(nil), // 25: ateom.GetActiveWorkloadStatsResponse - nil, // 26: ateom.RunWorkloadRequest.RuntimeAssetPathsEntry - nil, // 27: ateom.CheckpointWorkloadRequest.RuntimeAssetPathsEntry - nil, // 28: ateom.RestoreWorkloadRequest.RuntimeAssetPathsEntry + (*ContainerSpec)(nil), // 10: ateom.ContainerSpec + (*ResourceLimits)(nil), // 11: ateom.ResourceLimits + (*VolumeMount)(nil), // 12: ateom.VolumeMount + (*DurableDirVolumeMount)(nil), // 13: ateom.DurableDirVolumeMount + (*SystemInfoVolumeMount)(nil), // 14: ateom.SystemInfoVolumeMount + (*ImageVolumeMount)(nil), // 15: ateom.ImageVolumeMount + (*WakeupProbe)(nil), // 16: ateom.WakeupProbe + (*HTTPGetAction)(nil), // 17: ateom.HTTPGetAction + (*RunWorkloadResponse)(nil), // 18: ateom.RunWorkloadResponse + (*CheckpointWorkloadRequest)(nil), // 19: ateom.CheckpointWorkloadRequest + (*CheckpointWorkloadResponse)(nil), // 20: ateom.CheckpointWorkloadResponse + (*RestoreWorkloadRequest)(nil), // 21: ateom.RestoreWorkloadRequest + (*RestoreWorkloadResponse)(nil), // 22: ateom.RestoreWorkloadResponse + (*GetWorkloadStatsRequest)(nil), // 23: ateom.GetWorkloadStatsRequest + (*WorkloadStatsSample)(nil), // 24: ateom.WorkloadStatsSample + (*GetWorkloadStatsResponse)(nil), // 25: ateom.GetWorkloadStatsResponse + (*GetActiveWorkloadStatsRequest)(nil), // 26: ateom.GetActiveWorkloadStatsRequest + (*GetActiveWorkloadStatsResponse)(nil), // 27: ateom.GetActiveWorkloadStatsResponse + nil, // 28: ateom.RunWorkloadRequest.RuntimeAssetPathsEntry + nil, // 29: ateom.CheckpointWorkloadRequest.RuntimeAssetPathsEntry + nil, // 30: ateom.RestoreWorkloadRequest.RuntimeAssetPathsEntry } var file_ateom_proto_depIdxs = []int32{ 8, // 0: ateom.TerminateWorkloadRequest.spec:type_name -> ateom.WorkloadSpec 3, // 1: ateom.TerminateWorkloadRequest.actor_dirs:type_name -> ateom.ActorDirs 8, // 2: ateom.RunWorkloadRequest.spec:type_name -> ateom.WorkloadSpec - 26, // 3: ateom.RunWorkloadRequest.runtime_asset_paths:type_name -> ateom.RunWorkloadRequest.RuntimeAssetPathsEntry + 28, // 3: ateom.RunWorkloadRequest.runtime_asset_paths:type_name -> ateom.RunWorkloadRequest.RuntimeAssetPathsEntry 7, // 4: ateom.RunWorkloadRequest.egress_gateway:type_name -> ateom.EgressGateway 3, // 5: ateom.RunWorkloadRequest.actor_dirs:type_name -> ateom.ActorDirs 9, // 6: ateom.WorkloadSpec.containers:type_name -> ateom.Container - 14, // 7: ateom.Container.wakeup_probe:type_name -> ateom.WakeupProbe - 11, // 8: ateom.Container.durable_dir_volume_mounts:type_name -> ateom.DurableDirVolumeMount - 10, // 9: ateom.Container.csi_volume_mounts:type_name -> ateom.VolumeMount - 12, // 10: ateom.Container.system_info_volume_mounts:type_name -> ateom.SystemInfoVolumeMount - 13, // 11: ateom.Container.image_volume_mounts:type_name -> ateom.ImageVolumeMount - 15, // 12: ateom.WakeupProbe.http_get:type_name -> ateom.HTTPGetAction - 8, // 13: ateom.CheckpointWorkloadRequest.spec:type_name -> ateom.WorkloadSpec - 27, // 14: ateom.CheckpointWorkloadRequest.runtime_asset_paths:type_name -> ateom.CheckpointWorkloadRequest.RuntimeAssetPathsEntry - 0, // 15: ateom.CheckpointWorkloadRequest.scope:type_name -> ateom.SnapshotScope - 3, // 16: ateom.CheckpointWorkloadRequest.actor_dirs:type_name -> ateom.ActorDirs - 8, // 17: ateom.RestoreWorkloadRequest.spec:type_name -> ateom.WorkloadSpec - 28, // 18: ateom.RestoreWorkloadRequest.runtime_asset_paths:type_name -> ateom.RestoreWorkloadRequest.RuntimeAssetPathsEntry - 0, // 19: ateom.RestoreWorkloadRequest.scope:type_name -> ateom.SnapshotScope - 7, // 20: ateom.RestoreWorkloadRequest.egress_gateway:type_name -> ateom.EgressGateway - 3, // 21: ateom.RestoreWorkloadRequest.actor_dirs:type_name -> ateom.ActorDirs - 1, // 22: ateom.WorkloadStatsSample.sandbox_class:type_name -> ateom.SandboxClass - 2, // 23: ateom.WorkloadStatsSample.source:type_name -> ateom.StatsSource - 22, // 24: ateom.GetWorkloadStatsResponse.sample:type_name -> ateom.WorkloadStatsSample - 22, // 25: ateom.GetActiveWorkloadStatsResponse.samples:type_name -> ateom.WorkloadStatsSample - 6, // 26: ateom.Ateom.RunWorkload:input_type -> ateom.RunWorkloadRequest - 17, // 27: ateom.Ateom.CheckpointWorkload:input_type -> ateom.CheckpointWorkloadRequest - 19, // 28: ateom.Ateom.RestoreWorkload:input_type -> ateom.RestoreWorkloadRequest - 21, // 29: ateom.Ateom.GetWorkloadStats:input_type -> ateom.GetWorkloadStatsRequest - 24, // 30: ateom.Ateom.GetActiveWorkloadStats:input_type -> ateom.GetActiveWorkloadStatsRequest - 4, // 31: ateom.Ateom.TerminateWorkload:input_type -> ateom.TerminateWorkloadRequest - 16, // 32: ateom.Ateom.RunWorkload:output_type -> ateom.RunWorkloadResponse - 18, // 33: ateom.Ateom.CheckpointWorkload:output_type -> ateom.CheckpointWorkloadResponse - 20, // 34: ateom.Ateom.RestoreWorkload:output_type -> ateom.RestoreWorkloadResponse - 23, // 35: ateom.Ateom.GetWorkloadStats:output_type -> ateom.GetWorkloadStatsResponse - 25, // 36: ateom.Ateom.GetActiveWorkloadStats:output_type -> ateom.GetActiveWorkloadStatsResponse - 5, // 37: ateom.Ateom.TerminateWorkload:output_type -> ateom.TerminateWorkloadResponse - 32, // [32:38] is the sub-list for method output_type - 26, // [26:32] is the sub-list for method input_type - 26, // [26:26] is the sub-list for extension type_name - 26, // [26:26] is the sub-list for extension extendee - 0, // [0:26] is the sub-list for field type_name + 16, // 7: ateom.Container.wakeup_probe:type_name -> ateom.WakeupProbe + 13, // 8: ateom.Container.durable_dir_volume_mounts:type_name -> ateom.DurableDirVolumeMount + 12, // 9: ateom.Container.csi_volume_mounts:type_name -> ateom.VolumeMount + 14, // 10: ateom.Container.system_info_volume_mounts:type_name -> ateom.SystemInfoVolumeMount + 15, // 11: ateom.Container.image_volume_mounts:type_name -> ateom.ImageVolumeMount + 10, // 12: ateom.Container.container_spec:type_name -> ateom.ContainerSpec + 11, // 13: ateom.ContainerSpec.resources:type_name -> ateom.ResourceLimits + 17, // 14: ateom.WakeupProbe.http_get:type_name -> ateom.HTTPGetAction + 8, // 15: ateom.CheckpointWorkloadRequest.spec:type_name -> ateom.WorkloadSpec + 29, // 16: ateom.CheckpointWorkloadRequest.runtime_asset_paths:type_name -> ateom.CheckpointWorkloadRequest.RuntimeAssetPathsEntry + 0, // 17: ateom.CheckpointWorkloadRequest.fidelity:type_name -> ateom.SnapshotFidelity + 3, // 18: ateom.CheckpointWorkloadRequest.actor_dirs:type_name -> ateom.ActorDirs + 8, // 19: ateom.RestoreWorkloadRequest.spec:type_name -> ateom.WorkloadSpec + 30, // 20: ateom.RestoreWorkloadRequest.runtime_asset_paths:type_name -> ateom.RestoreWorkloadRequest.RuntimeAssetPathsEntry + 0, // 21: ateom.RestoreWorkloadRequest.fidelity:type_name -> ateom.SnapshotFidelity + 7, // 22: ateom.RestoreWorkloadRequest.egress_gateway:type_name -> ateom.EgressGateway + 3, // 23: ateom.RestoreWorkloadRequest.actor_dirs:type_name -> ateom.ActorDirs + 1, // 24: ateom.WorkloadStatsSample.sandbox_class:type_name -> ateom.SandboxClass + 2, // 25: ateom.WorkloadStatsSample.source:type_name -> ateom.StatsSource + 24, // 26: ateom.GetWorkloadStatsResponse.sample:type_name -> ateom.WorkloadStatsSample + 24, // 27: ateom.GetActiveWorkloadStatsResponse.samples:type_name -> ateom.WorkloadStatsSample + 6, // 28: ateom.Ateom.RunWorkload:input_type -> ateom.RunWorkloadRequest + 19, // 29: ateom.Ateom.CheckpointWorkload:input_type -> ateom.CheckpointWorkloadRequest + 21, // 30: ateom.Ateom.RestoreWorkload:input_type -> ateom.RestoreWorkloadRequest + 23, // 31: ateom.Ateom.GetWorkloadStats:input_type -> ateom.GetWorkloadStatsRequest + 26, // 32: ateom.Ateom.GetActiveWorkloadStats:input_type -> ateom.GetActiveWorkloadStatsRequest + 4, // 33: ateom.Ateom.TerminateWorkload:input_type -> ateom.TerminateWorkloadRequest + 18, // 34: ateom.Ateom.RunWorkload:output_type -> ateom.RunWorkloadResponse + 20, // 35: ateom.Ateom.CheckpointWorkload:output_type -> ateom.CheckpointWorkloadResponse + 22, // 36: ateom.Ateom.RestoreWorkload:output_type -> ateom.RestoreWorkloadResponse + 25, // 37: ateom.Ateom.GetWorkloadStats:output_type -> ateom.GetWorkloadStatsResponse + 27, // 38: ateom.Ateom.GetActiveWorkloadStats:output_type -> ateom.GetActiveWorkloadStatsResponse + 5, // 39: ateom.Ateom.TerminateWorkload:output_type -> ateom.TerminateWorkloadResponse + 34, // [34:40] is the sub-list for method output_type + 28, // [28:34] is the sub-list for method input_type + 28, // [28:28] is the sub-list for extension type_name + 28, // [28:28] is the sub-list for extension extendee + 0, // [0:28] is the sub-list for field type_name } func init() { file_ateom_proto_init() } @@ -2164,14 +2325,14 @@ func file_ateom_proto_init() { return } file_ateom_proto_msgTypes[3].OneofWrappers = []any{} - file_ateom_proto_msgTypes[16].OneofWrappers = []any{} + file_ateom_proto_msgTypes[18].OneofWrappers = []any{} type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_ateom_proto_rawDesc), len(file_ateom_proto_rawDesc)), NumEnums: 3, - NumMessages: 26, + NumMessages: 28, NumExtensions: 0, NumServices: 1, }, diff --git a/internal/proto/ateompb/ateom.proto b/internal/proto/ateompb/ateom.proto index 7f234c0be5..89755de037 100644 --- a/internal/proto/ateompb/ateom.proto +++ b/internal/proto/ateompb/ateom.proto @@ -78,27 +78,26 @@ service Ateom { // that dies during boot is attributable rather than anonymous. rpc GetWorkloadStats(GetWorkloadStatsRequest) returns (GetWorkloadStatsResponse) {} - // GetActiveWorkloadStats samples whatever this ateom is currently - // executing, without asserting an identity. It is the discovery read for a + // GetActiveWorkloadStats reports every actor this ateom hosts, without + // asserting an identity. It is the discovery read for a // scraper that enumerates ateoms and holds no worker-to-actor mapping; // GetWorkloadStats above is the verified read for a caller that must be // answered about a specific actor. // - // Every state a blind caller can find is a normal answer here, never an - // error: an "available" ateom answers an empty samples list, and a workload - // with nothing to measure YET (a poll landing in a boot or a restore) - // appears as a sample carrying its attribution with - // source = STATS_SOURCE_UNSPECIFIED -- "not measured", per the sample's own - // contract -- so even a workload that dies during boot is attributable, not - // anonymous. Error codes are reserved for real failures reading a sandbox - // that should be measurable. This is deliberately unlike GetWorkloadStats, - // whose caller asserts knowledge the codes then answer. + // It serves each actor's latest sample from the ateom's own sampling timer, + // so a poll reads no sandbox and has no error path: an "available" ateom + // answers an empty samples list, and a workload with nothing measured yet + // (a boot, a restore, or a guest the last sweep did not reach) appears as a + // sample carrying its attribution with source = STATS_SOURCE_UNSPECIFIED -- + // "not measured", per the sample's own contract -- so even a workload that + // dies during boot is attributable, not anonymous. A sample is up to one + // sampling interval old. This is deliberately unlike GetWorkloadStats, whose + // caller asserts knowledge the codes then answer. // // Consumers MUST attribute each sample solely from the identity echoed // inside it, never from a mapping they hold: without an asserted uid, the // response is the only statement of who was measured. Like GetWorkloadStats - // it is a pure read, safe on a timer, and does not touch the lifecycle - // mutex. + // it is safe on a timer and does not touch the lifecycle mutex. rpc GetActiveWorkloadStats(GetActiveWorkloadStatsRequest) returns (GetActiveWorkloadStatsResponse) {} // TerminateWorkload stops and deletes container workloads and cleans up @@ -200,6 +199,35 @@ message Container { // image_volume_mounts are the image volumes this container mounts, if any. repeated ImageVolumeMount image_volume_mounts = 6; + + // container_spec is what ateom builds the container's OCI spec from, + // besides its volume mounts. Unset on RPCs that start no process. + ContainerSpec container_spec = 7; +} + +// ContainerSpec is a container's process and limits, resolved by atelet. +message ContainerSpec { + // args is the process argv, already resolved against the image's + // ENTRYPOINT and CMD. + repeated string args = 1; + + // env is the process environment as KEY=VALUE, already merged with the + // image's ENV. + repeated string env = 2 [ debug_redact = true ]; + + // capabilities is the container's capability set, CAP_-prefixed. + repeated string capabilities = 3; + + // resources are the container's own declared limits, if any. + ResourceLimits resources = 4; +} + +// ResourceLimits are a container's declared limits. +message ResourceLimits { + // memory_bytes is the memory limit in bytes. 0 means unset. + int64 memory_bytes = 1; + // cpu_millis is the CPU limit in milli-cores (1000 = one core). 0 means unset. + int64 cpu_millis = 2; } // VolumeMount is one volume mounted into a container. @@ -257,16 +285,21 @@ message HTTPGetAction { message RunWorkloadResponse { } -enum SnapshotScope { +// SnapshotFidelity mirrors the public ateapi enum: the state layers a +// snapshot holds, lowest to highest, each including the ones below it. +enum SnapshotFidelity { // Not valid option; should never happen. - SNAPSHOT_SCOPE_UNSPECIFIED = 0; - // Capture process memory plus the full filesystem delta on top of the OCI - // image (including any attached DurableDir volumes). - SNAPSHOT_SCOPE_FULL = 1; + SNAPSHOT_FIDELITY_UNSPECIFIED = 0; // Capture only the contents of attached volumes that support snapshots // (currently DurableDir-typed volumes). Memory and the rest of rootfs are // excluded. - SNAPSHOT_SCOPE_DATA = 2; + SNAPSHOT_FIDELITY_VOLUMES = 1; + // Volumes plus the root filesystem changes made since boot. No runtime + // captures this yet; requests carrying it are rejected. + SNAPSHOT_FIDELITY_ROOTFS = 2; + // Capture process memory plus the full filesystem delta on top of the OCI + // image (including any attached DurableDir volumes). + SNAPSHOT_FIDELITY_MEMORY = 3; } message CheckpointWorkloadRequest { @@ -296,8 +329,8 @@ message CheckpointWorkloadRequest { // atelet fetched it to (see RunWorkloadRequest). Empty for gVisor. map runtime_asset_paths = 9; - // What content to include in the checkpoint. - SnapshotScope scope = 10; + // Fidelity the checkpoint captures. + SnapshotFidelity fidelity = 10; ActorDirs actor_dirs = 11; } @@ -308,7 +341,7 @@ message CheckpointWorkloadResponse { repeated string snapshot_files = 1; // data_snapshot_files is the subset of snapshot_files that restores the - // actor at DATA scope on its own. Empty when the capture holds no durable data. + // actor at VOLUMES fidelity on its own. Empty when the capture holds no durable data. repeated string data_snapshot_files = 2; } @@ -331,15 +364,15 @@ message RestoreWorkloadRequest { // atelet fetched it to (see RunWorkloadRequest). Empty for gVisor. map runtime_asset_paths = 9; - // What content to restore from the snapshot. - SnapshotScope scope = 10; + // Fidelity to restore from the snapshot. + SnapshotFidelity fidelity = 10; // When absent the actor has no egress: its TCP is captured and refused. optional EgressGateway egress_gateway = 12; // The actor's declared size, from the ActorTemplate's resource limits. Used to - // (re)size the sandbox on a DATA-scope restore (fresh guest container). On a - // FULL micro-VM restore the size baked into the snapshot is authoritative and + // (re)size the sandbox on a VOLUMES restore (fresh guest container). On a + // MEMORY micro-VM restore the size baked into the snapshot is authoritative and // these are ignored. Zero means "unset": keep the runtime default. int64 cpu_milli = 14; // CPU limit in millicores (1000 = one core). int64 memory_bytes = 15; // Memory limit in bytes. @@ -429,17 +462,16 @@ message WorkloadStatsSample { // drifts upward with cache the kernel would drop for free under pressure. uint64 memory_working_set_bytes = 10; // Cumulative CPU time since the activation began (epoch_unix_nano), for - // every source. The cgroup source restarts on its own; the guest-agent - // source is rebased on its first read after a restore, since the guest's - // counters survive in guest RAM. An ateom that leaves epoch_unix_nano at zero - // also sends the raw guest counter. A lifetime figure is the sum over epochs - // of each epoch's highest value. + // every source, and never decreasing within it. The cgroup source restarts on + // its own; the guest-agent source counts from its first read after a restore + // that resumes the guest, since the guest's counters survive in guest RAM. A + // lifetime figure is the sum over epochs of each epoch's highest value. uint64 cpu_usage_usec = 11; int64 observed_at_unix_nano = 12; // The activation this sample belongs to, as the unix-nano time it began. A - // Run or Restore starts one. Zero from an ateom that does not set it. + // Run or Restore starts one. int64 epoch_unix_nano = 13; } diff --git a/internal/proto/ateompb/ateom_grpc.pb.go b/internal/proto/ateompb/ateom_grpc.pb.go index 5e05361a83..572fee2b79 100644 --- a/internal/proto/ateompb/ateom_grpc.pb.go +++ b/internal/proto/ateompb/ateom_grpc.pb.go @@ -101,27 +101,26 @@ type AteomClient interface { // "booting" distinguishable from "not here" at all, and it means a workload // that dies during boot is attributable rather than anonymous. GetWorkloadStats(ctx context.Context, in *GetWorkloadStatsRequest, opts ...grpc.CallOption) (*GetWorkloadStatsResponse, error) - // GetActiveWorkloadStats samples whatever this ateom is currently - // executing, without asserting an identity. It is the discovery read for a + // GetActiveWorkloadStats reports every actor this ateom hosts, without + // asserting an identity. It is the discovery read for a // scraper that enumerates ateoms and holds no worker-to-actor mapping; // GetWorkloadStats above is the verified read for a caller that must be // answered about a specific actor. // - // Every state a blind caller can find is a normal answer here, never an - // error: an "available" ateom answers an empty samples list, and a workload - // with nothing to measure YET (a poll landing in a boot or a restore) - // appears as a sample carrying its attribution with - // source = STATS_SOURCE_UNSPECIFIED -- "not measured", per the sample's own - // contract -- so even a workload that dies during boot is attributable, not - // anonymous. Error codes are reserved for real failures reading a sandbox - // that should be measurable. This is deliberately unlike GetWorkloadStats, - // whose caller asserts knowledge the codes then answer. + // It serves each actor's latest sample from the ateom's own sampling timer, + // so a poll reads no sandbox and has no error path: an "available" ateom + // answers an empty samples list, and a workload with nothing measured yet + // (a boot, a restore, or a guest the last sweep did not reach) appears as a + // sample carrying its attribution with source = STATS_SOURCE_UNSPECIFIED -- + // "not measured", per the sample's own contract -- so even a workload that + // dies during boot is attributable, not anonymous. A sample is up to one + // sampling interval old. This is deliberately unlike GetWorkloadStats, whose + // caller asserts knowledge the codes then answer. // // Consumers MUST attribute each sample solely from the identity echoed // inside it, never from a mapping they hold: without an asserted uid, the // response is the only statement of who was measured. Like GetWorkloadStats - // it is a pure read, safe on a timer, and does not touch the lifecycle - // mutex. + // it is safe on a timer and does not touch the lifecycle mutex. GetActiveWorkloadStats(ctx context.Context, in *GetActiveWorkloadStatsRequest, opts ...grpc.CallOption) (*GetActiveWorkloadStatsResponse, error) // TerminateWorkload stops and deletes container workloads and cleans up // network and bundle overlays on ateom. @@ -256,27 +255,26 @@ type AteomServer interface { // "booting" distinguishable from "not here" at all, and it means a workload // that dies during boot is attributable rather than anonymous. GetWorkloadStats(context.Context, *GetWorkloadStatsRequest) (*GetWorkloadStatsResponse, error) - // GetActiveWorkloadStats samples whatever this ateom is currently - // executing, without asserting an identity. It is the discovery read for a + // GetActiveWorkloadStats reports every actor this ateom hosts, without + // asserting an identity. It is the discovery read for a // scraper that enumerates ateoms and holds no worker-to-actor mapping; // GetWorkloadStats above is the verified read for a caller that must be // answered about a specific actor. // - // Every state a blind caller can find is a normal answer here, never an - // error: an "available" ateom answers an empty samples list, and a workload - // with nothing to measure YET (a poll landing in a boot or a restore) - // appears as a sample carrying its attribution with - // source = STATS_SOURCE_UNSPECIFIED -- "not measured", per the sample's own - // contract -- so even a workload that dies during boot is attributable, not - // anonymous. Error codes are reserved for real failures reading a sandbox - // that should be measurable. This is deliberately unlike GetWorkloadStats, - // whose caller asserts knowledge the codes then answer. + // It serves each actor's latest sample from the ateom's own sampling timer, + // so a poll reads no sandbox and has no error path: an "available" ateom + // answers an empty samples list, and a workload with nothing measured yet + // (a boot, a restore, or a guest the last sweep did not reach) appears as a + // sample carrying its attribution with source = STATS_SOURCE_UNSPECIFIED -- + // "not measured", per the sample's own contract -- so even a workload that + // dies during boot is attributable, not anonymous. A sample is up to one + // sampling interval old. This is deliberately unlike GetWorkloadStats, whose + // caller asserts knowledge the codes then answer. // // Consumers MUST attribute each sample solely from the identity echoed // inside it, never from a mapping they hold: without an asserted uid, the // response is the only statement of who was measured. Like GetWorkloadStats - // it is a pure read, safe on a timer, and does not touch the lifecycle - // mutex. + // it is safe on a timer and does not touch the lifecycle mutex. GetActiveWorkloadStats(context.Context, *GetActiveWorkloadStatsRequest) (*GetActiveWorkloadStatsResponse, error) // TerminateWorkload stops and deletes container workloads and cleans up // network and bundle overlays on ateom. diff --git a/internal/protoredact/protoredact_test.go b/internal/protoredact/protoredact_test.go index d76b0df99c..cc1b946d4a 100644 --- a/internal/protoredact/protoredact_test.go +++ b/internal/protoredact/protoredact_test.go @@ -30,7 +30,7 @@ import ( "github.com/agent-substrate/substrate/internal/protoredact" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/agent-substrate/substrate/pkg/proto/credproviderpb" - objectstoresnapshotv1 "github.com/agent-substrate/substrate/pkg/proto/objectstoresnapshotpb/v1" + objectstorev1 "github.com/agent-substrate/substrate/pkg/proto/objectstorepb/v1" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/reflect/protodesc" "google.golang.org/protobuf/reflect/protoreflect" @@ -377,7 +377,7 @@ var ourProtoFiles = []protoreflect.FileDescriptor{ credproviderpb.File_credprovider_proto, glutton.File_glutton_proto, grpcechopb.File_grpcecho_proto, - objectstoresnapshotv1.File_objectstoresnapshot_proto, + objectstorev1.File_objectstore_proto, } // forEachField calls fn for every field of every message in ourProtoFiles, @@ -409,12 +409,13 @@ func isDebugRedact(fd protoreflect.FieldDescriptor) bool { // change is reviewed as a deliberate decision about what the logs may show. func TestDebugRedactFieldsArePinned(t *testing.T) { want := map[string]bool{ - "ateapi.EnvVar.value": true, - "ateapi.MintActorJWTResponse.actor_jwt": true, - "atelet.EnvEntry.value": true, - "credprovider.FetchSecretResponse.opaque_bytes": true, - "objectstoresnapshot.v1.FetchSnapshotRequest.actor_jwt": true, - "objectstoresnapshot.v1.UploadSnapshotRequest.actor_jwt": true, + "ateapi.EnvVar.value": true, + "ateapi.MintActorJWTResponse.actor_jwt": true, + "atelet.EnvEntry.value": true, + "ateom.ContainerSpec.env": true, + "credprovider.FetchSecretResponse.opaque_bytes": true, + "objectstore.v1.FetchSnapshotRequest.actor_jwt": true, + "objectstore.v1.UploadSnapshotRequest.actor_jwt": true, } got := map[string]bool{} forEachField(func(fd protoreflect.FieldDescriptor) { diff --git a/internal/resources/snapshot.go b/internal/resources/snapshot.go index ab4f932faf..42c01e68a1 100644 --- a/internal/resources/snapshot.go +++ b/internal/resources/snapshot.go @@ -52,6 +52,23 @@ func (p StoragePrefix) IsZero() bool { return p == StoragePrefix{} } func (p StoragePrefix) String() string { return p.uri } +// ParseStoragePrefix parses a storage prefix received as a string, such as a +// snapshot URI or an owner's prefix. It rejects a bare bucket, so a prefix +// can never name every object in a bucket. +func ParseStoragePrefix(uri string) (StoragePrefix, error) { + if err := ValidateSnapshotLocation(uri); err != nil { + return StoragePrefix{}, err + } + u, err := url.Parse(uri) + if err != nil { + return StoragePrefix{}, fmt.Errorf("invalid storage prefix %q: %v", uri, err) + } + if strings.Trim(u.Path, "/") == "" { + return StoragePrefix{}, fmt.Errorf("invalid storage prefix %q: missing path", uri) + } + return StoragePrefix{uri: uri}, nil +} + // SnapshotOwner is the resource whose objects a snapshot's prefix holds: the // Actor that took it, or the Tag that copied it. Deletion is expressed as // "collect everything under my own prefix", so an owner cannot name another's diff --git a/internal/resources/snapshot_test.go b/internal/resources/snapshot_test.go index 195ef8da90..e461025dd0 100644 --- a/internal/resources/snapshot_test.go +++ b/internal/resources/snapshot_test.go @@ -595,3 +595,27 @@ func TestSnapshotURIZeroValue(t *testing.T) { t.Errorf("%q reports IsZero", uri) } } + +func TestParseStoragePrefix(t *testing.T) { + for _, tc := range []struct { + uri string + wantErr bool + }{ + {uri: "gs://bucket/root/atespaces/team-a/actors/uid1"}, + {uri: "s3://bucket/root/atespaces/team-a/actors/uid1/snapshots/snap1"}, + {uri: "gs://bucket", wantErr: true}, + {uri: "gs://bucket/", wantErr: true}, + {uri: "gs:///root", wantErr: true}, + {uri: "gs://bucket/root?x=1", wantErr: true}, + {uri: "gs://user@bucket/root", wantErr: true}, + } { + got, err := ParseStoragePrefix(tc.uri) + if (err != nil) != tc.wantErr { + t.Errorf("ParseStoragePrefix(%q) error = %v, wantErr %v", tc.uri, err, tc.wantErr) + continue + } + if err == nil && got.String() != tc.uri { + t.Errorf("ParseStoragePrefix(%q) = %q", tc.uri, got) + } + } +} diff --git a/internal/resources/snapshotfiles.go b/internal/resources/snapshotfiles.go new file mode 100644 index 0000000000..4487344999 --- /dev/null +++ b/internal/resources/snapshotfiles.go @@ -0,0 +1,37 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package resources + +import ( + "fmt" + "path/filepath" +) + +// ValidateSnapshotFileNames requires each name to be a distinct plain file name +// in the checkpoint directory. Actual file access must still use os.Root so +// symlinks cannot escape that directory. +func ValidateSnapshotFileNames(files []string) error { + seen := make(map[string]bool, len(files)) + for i, name := range files { + switch { + case name != filepath.Base(name) || !filepath.IsLocal(name) || name == ".": + return fmt.Errorf("snapshotFiles[%d] %q is not a file name in the checkpoint directory", i, name) + case seen[name]: + return fmt.Errorf("snapshotFiles[%d] %q is duplicated", i, name) + } + seen[name] = true + } + return nil +} diff --git a/internal/resources/snapshotfiles_test.go b/internal/resources/snapshotfiles_test.go new file mode 100644 index 0000000000..cba3bb3d5c --- /dev/null +++ b/internal/resources/snapshotfiles_test.go @@ -0,0 +1,42 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package resources + +import "testing" + +func TestValidateSnapshotFileNames(t *testing.T) { + for _, tc := range []struct { + name string + files []string + wantErr bool + }{ + {name: "empty list", files: nil}, + {name: "plain names", files: []string{"manifest.json", "memory.img", "durable-dir.tar"}}, + {name: "empty name", files: []string{""}, wantErr: true}, + {name: "dot", files: []string{"."}, wantErr: true}, + {name: "dot dot", files: []string{".."}, wantErr: true}, + {name: "absolute", files: []string{"/etc/passwd"}, wantErr: true}, + {name: "nested", files: []string{"a/b"}, wantErr: true}, + {name: "escaping", files: []string{"../x"}, wantErr: true}, + {name: "duplicate", files: []string{"a", "b", "a"}, wantErr: true}, + } { + t.Run(tc.name, func(t *testing.T) { + err := ValidateSnapshotFileNames(tc.files) + if gotErr := err != nil; gotErr != tc.wantErr { + t.Errorf("ValidateSnapshotFileNames(%q) = %v, want error: %v", tc.files, err, tc.wantErr) + } + }) + } +} diff --git a/internal/resources/validate.go b/internal/resources/validate.go index dba7ee96e3..104f043435 100644 --- a/internal/resources/validate.go +++ b/internal/resources/validate.go @@ -88,13 +88,13 @@ func IsValidResourceName(name string) bool { return len(content.IsDNS1123Label(name)) == 0 } -// ValidateAteomUID rejects a target ateom pod UID that could escape the host +// ValidateWorkerPodUID rejects a worker pod UID that could escape the host // path built from it: the ateom control socket (.../ateoms//ateom.sock). // Kubernetes pod UIDs are UUIDs, which are valid DNS-1123 labels, so a label // check accepts every legitimate value while rejecting separators and "..". -func ValidateAteomUID(targetAteomUID string) error { - if errs := content.IsDNS1123Label(targetAteomUID); len(errs) > 0 { - return fmt.Errorf("invalid target ateom UID %q: %s", targetAteomUID, strings.Join(errs, "; ")) +func ValidateWorkerPodUID(workerPodUID string) error { + if errs := content.IsDNS1123Label(workerPodUID); len(errs) > 0 { + return fmt.Errorf("invalid worker pod UID %q: %s", workerPodUID, strings.Join(errs, "; ")) } return nil } @@ -120,6 +120,24 @@ func ValidateActorDirs(actorDirs *ateompb.ActorDirs, fldPath *field.Path) field. return errs } +// ValidateSnapshotFidelity rejects a checkpoint or restore request whose +// fidelity no runtime can serve. ROOTFS is defined in the enum but no sandbox +// runtime captures rootfs changes without memory yet, so it is refused here +// as well as at template admission. +func ValidateSnapshotFidelity(fidelity ateompb.SnapshotFidelity, fldPath *field.Path) field.ErrorList { + switch fidelity { + case ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY: + return nil + case ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED: + return field.ErrorList{field.Required(fldPath, "")} + case ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS: + return field.ErrorList{field.Invalid(fldPath, fidelity.String(), "ROOTFS fidelity is not supported yet")} + default: + return field.ErrorList{field.NotSupported(fldPath, fidelity, + []string{ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES.String(), ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY.String()})} + } +} + func validateAbsDir(dir string, fldPath *field.Path) field.ErrorList { if dir == "" { return field.ErrorList{field.Required(fldPath, "")} @@ -199,10 +217,7 @@ func ValidateRunscHash(sha256Hash string) error { // is a well-formed URI with a bucket, so a bad location fails fast instead of // deep inside an object-storage call. It deliberately does not restrict the // scheme: the storage layer only uses the host (bucket) and path, and which -// schemes are acceptable is a storage-backend policy, not a per-RPC one. The -// local paths used for snapshot upload/download are derived from the -// separately validated actor ref, not from this URI, so this is a sanity check -// rather than a path-traversal guard. +// schemes are acceptable is a storage-backend policy, not a per-RPC one. // // This validates the base that many snapshots share, not any one snapshot's // URI; SnapshotURI is the type for the latter, and it applies this check when @@ -223,6 +238,18 @@ func ValidateSnapshotLocation(location string) error { if u.Opaque != "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" { return fmt.Errorf("invalid snapshot location %q: must contain only a scheme, bucket, and path", location) } + // url.JoinPath cleans dot segments, and some backends trim spaces from + // segments (or windows, trailing dots) and treat '\' as a separator, so + // these could move snapshots outside of the location, e.g. into another + // atespace's prefix + if strings.Contains(u.Path, `\`) { + return fmt.Errorf(`invalid snapshot location %q: must not contain '\'`, location) + } + for segment := range strings.SplitSeq(u.Path, "/") { + if segment != "" && strings.Trim(segment, ". ") == "" { + return fmt.Errorf("invalid snapshot location %q: must not contain path segments of only '.' and ' '", location) + } + } return nil } diff --git a/internal/resources/validate_test.go b/internal/resources/validate_test.go index 8091868009..cec0e0864a 100644 --- a/internal/resources/validate_test.go +++ b/internal/resources/validate_test.go @@ -101,7 +101,7 @@ func TestIsValidResourceName(t *testing.T) { } } -func TestValidateAteomUID(t *testing.T) { +func TestValidateWorkerPodUID(t *testing.T) { tests := []struct { name string uid string @@ -116,8 +116,8 @@ func TestValidateAteomUID(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if err := ValidateAteomUID(tt.uid); (err != nil) != tt.wantErr { - t.Errorf("ValidateAteomUID(%q) err = %v, wantErr %v", tt.uid, err, tt.wantErr) + if err := ValidateWorkerPodUID(tt.uid); (err != nil) != tt.wantErr { + t.Errorf("ValidateWorkerPodUID(%q) err = %v, wantErr %v", tt.uid, err, tt.wantErr) } }) } @@ -195,6 +195,16 @@ func TestValidateSnapshotLocation(t *testing.T) { // Opaque form (no //) parses with an empty host, so it is rejected // on either the bucket or the opaque check. {"opaque", "gs:bucket/path", true}, + {"dot dot", "gs://bucket/team-a/../team-b", true}, + {"trailing dot dot", "gs://bucket/path/..", true}, + {"escaped dot dot", "gs://bucket/%2e%2e/path", true}, + {"dot", "gs://bucket/./path", true}, + {"triple dot", "gs://bucket/.../path", true}, + {"space padded dot dot", "gs://bucket/a/%20..%20/b", true}, + {"trailing space dot dot", "gs://bucket/a/..%20/b", true}, + {"backslash", `gs://bucket/team-a\..\team-b`, true}, + {"escaped backslash", "gs://bucket/a%5Cb", true}, + {"dots in a name", "gs://bucket/a..b/.c/d.", false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -314,6 +324,32 @@ func testActorDirs() *ateompb.ActorDirs { } } +func TestValidateSnapshotFidelity(t *testing.T) { + for _, tc := range []struct { + name string + fidelity ateompb.SnapshotFidelity + wantErr bool + }{ + {"volumes", ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, false}, + {"memory", ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, false}, + {"unspecified", ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED, true}, + {"rootfs not supported yet", ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS, true}, + {"outside the enum", ateompb.SnapshotFidelity(99), true}, + } { + t.Run(tc.name, func(t *testing.T) { + errs := ValidateSnapshotFidelity(tc.fidelity, field.NewPath("fidelity")) + if gotErr := len(errs) != 0; gotErr != tc.wantErr { + t.Fatalf("ValidateSnapshotFidelity(%v) = %v, wantErr %t", tc.fidelity, errs, tc.wantErr) + } + for _, e := range errs { + if e.Field != "fidelity" { + t.Errorf("error names field %q, want %q", e.Field, "fidelity") + } + } + }) + } +} + func TestValidateActorDirs(t *testing.T) { for _, tc := range []struct { name string diff --git a/internal/testcert/servertls.go b/internal/testcert/servertls.go new file mode 100644 index 0000000000..e5783c0371 --- /dev/null +++ b/internal/testcert/servertls.go @@ -0,0 +1,81 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testcert + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "encoding/pem" + "math/big" + "net" + "testing" + "time" + + "github.com/agent-substrate/substrate/internal/localca" +) + +// ServerTLS holds an origin key pair and the public CA trusted by its client. +type ServerTLS struct{ RootCA, Certificate, PrivateKey []byte } + +// NewServerTLS issues an origin certificate for the allocated Service IP. +// Only the public CA is returned; its signing key stays in this process. +func NewServerTLS(t *testing.T, ip net.IP) ServerTLS { + t.Helper() + ca, err := localca.GenerateCA("e2e-origin", localca.KeyTypeECDSAP256, 24*time.Hour) + if err != nil { + t.Fatalf("generating origin CA: %v", err) + } + return ServerTLSWithCA(t, ca, ip) +} + +// ServerTLSWithCA issues a Service IP and optional DNS certificate from a CA +// already trusted by the client or the MITM gateway's upstream TLS connection. +func ServerTLSWithCA(t *testing.T, ca *localca.CA, ip net.IP, dnsNames ...string) ServerTLS { + t.Helper() + if ip == nil { + t.Fatal("origin certificate requires a Service IP") + } + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatalf("generating origin key: %v", err) + } + serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) + if err != nil { + t.Fatalf("generating origin serial: %v", err) + } + leaf := &x509.Certificate{ + SerialNumber: serial, + IPAddresses: []net.IP{ip}, + DNSNames: dnsNames, + NotBefore: time.Now().Add(-time.Minute), NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + } + der, err := x509.CreateCertificate(rand.Reader, leaf, ca.RootCertificate, key.Public(), ca.SigningKey) + if err != nil { + t.Fatalf("issuing origin certificate: %v", err) + } + privateDER, err := x509.MarshalPKCS8PrivateKey(key) + if err != nil { + t.Fatalf("encoding origin key: %v", err) + } + return ServerTLS{ + RootCA: pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: ca.RootCertificate.Raw}), + Certificate: pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), + PrivateKey: pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: privateDER}), + } +} diff --git a/internal/testcert/servertls_test.go b/internal/testcert/servertls_test.go new file mode 100644 index 0000000000..fda0c12772 --- /dev/null +++ b/internal/testcert/servertls_test.go @@ -0,0 +1,86 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testcert + +import ( + "crypto/tls" + "crypto/x509" + "net" + "testing" + "time" + + "github.com/agent-substrate/substrate/internal/localca" +) + +func TestNewServerTLS(t *testing.T) { + material := NewServerTLS(t, net.ParseIP("10.0.0.7")) + pair, err := tls.X509KeyPair(material.Certificate, material.PrivateKey) + if err != nil { + t.Fatalf("loading serving key pair: %v", err) + } + leaf, err := x509.ParseCertificate(pair.Certificate[0]) + if err != nil { + t.Fatal(err) + } + roots := x509.NewCertPool() + if !roots.AppendCertsFromPEM(material.RootCA) { + t.Fatal("no CA certificate") + } + if _, err := leaf.Verify(x509.VerifyOptions{Roots: roots, DNSName: "10.0.0.7"}); err != nil { + t.Fatalf("verifying Service IP: %v", err) + } + if _, err := leaf.Verify(x509.VerifyOptions{Roots: roots, DNSName: "10.0.0.8"}); err == nil { + t.Fatal("accepted a different Service IP") + } + if _, err := leaf.Verify(x509.VerifyOptions{Roots: x509.NewCertPool(), DNSName: "10.0.0.7"}); err == nil { + t.Fatal("accepted an untrusted certificate") + } + if leaf.IsCA { + t.Fatal("origin received a CA certificate") + } +} + +func TestServerTLSWithCA(t *testing.T) { + ca, err := localca.GenerateCA("service-dns", localca.KeyTypeECDSAP256, time.Hour) + if err != nil { + t.Fatal(err) + } + const name = "https-origin.networking-test.svc.cluster.local" + material := ServerTLSWithCA(t, ca, net.ParseIP("10.0.0.7"), name) + pair, err := tls.X509KeyPair(material.Certificate, material.PrivateKey) + if err != nil { + t.Fatal(err) + } + leaf, err := x509.ParseCertificate(pair.Certificate[0]) + if err != nil { + t.Fatal(err) + } + roots := x509.NewCertPool() + roots.AddCert(ca.RootCertificate) + for _, host := range []string{name, "10.0.0.7"} { + if _, err := leaf.Verify(x509.VerifyOptions{Roots: roots, DNSName: host}); err != nil { + t.Errorf("verifying %s against supplied CA: %v", host, err) + } + } + if _, err := leaf.Verify(x509.VerifyOptions{Roots: roots, DNSName: "another.namespace.svc.cluster.local"}); err == nil { + t.Fatal("accepted a different Service DNS name") + } + if _, err := leaf.Verify(x509.VerifyOptions{Roots: x509.NewCertPool(), DNSName: name}); err == nil { + t.Fatal("accepted an untrusted Service DNS certificate") + } + if leaf.IsCA { + t.Fatal("origin received a CA certificate") + } +} diff --git a/internal/version/version.go b/internal/version/version.go index ba2db0d74e..dea37669e0 100644 --- a/internal/version/version.go +++ b/internal/version/version.go @@ -50,6 +50,7 @@ func init() { } } } + Version = info.Main.Path + ".Version=" + info.Main.Version } // String returns a human-readable single-line version summary, diff --git a/internal/volume/csi/tls_test.go b/internal/volume/csi/tls_test.go index aaa8edce05..5a56aac2c6 100644 --- a/internal/volume/csi/tls_test.go +++ b/internal/volume/csi/tls_test.go @@ -391,9 +391,10 @@ func TestCAPoolCache_HitAndFileChange(t *testing.T) { // Modify the file. writeFile(t, caPath, ca.certPEM()) - // Advance mtime explicitly; consecutive writes can share a filesystem tick. - modified := cache.fi.ModTime().Add(time.Second) - if err := os.Chtimes(caPath, modified, modified); err != nil { + // The cache intentionally keys on mtime and size; make the file change + // observable on filesystems whose timestamp granularity is coarse. + modifiedAt := time.Now().Add(time.Second) + if err := os.Chtimes(caPath, modifiedAt, modifiedAt); err != nil { t.Fatal(err) } diff --git a/manifests/ate-install/ate-api-server-envvars.yaml b/manifests/ate-install/ate-api-server-envvars.yaml index 647574752d..f733d225df 100644 --- a/manifests/ate-install/ate-api-server-envvars.yaml +++ b/manifests/ate-install/ate-api-server-envvars.yaml @@ -13,7 +13,7 @@ # limitations under the License. # DO NOT EDIT — generated from charts/substrate by hack/render-manifests.sh. -# Run `make helm-template` to regenerate. +# Run `hack/render-manifests.sh` to regenerate. apiVersion: v1 kind: ConfigMap @@ -22,3 +22,4 @@ metadata: namespace: ate-system data: ATE_API_POSTGRES_SCHEMA: "substrate" + ATE_API_ACTOR_JWT_ISSUER: "https://idp.ate-system.svc" diff --git a/manifests/ate-install/ate-api-server.yaml b/manifests/ate-install/ate-api-server.yaml index 90724822e1..2e325ff40c 100644 --- a/manifests/ate-install/ate-api-server.yaml +++ b/manifests/ate-install/ate-api-server.yaml @@ -83,6 +83,47 @@ spec: # the readinessProbe below reaching NotReady, plus propagation) + # --drain-timeout. terminationGracePeriodSeconds: 40 + # The control snapshot plugin deletes and copies external snapshots for + # ate-api-server, over a Unix socket in a pod-local emptyDir. A native + # sidecar starts before ate-api-server and stops after it. ate-api-server + # starts only once the startup probe sees the plugin serving. + initContainers: + - name: snapshot-plugin + image: ko://github.com/agent-substrate/substrate/cmd/snapshot-plugin + restartPolicy: Always + args: + - control + - --socket=/run/snapshot-plugin/control.sock + # The plugin listens on a Unix socket, which a gRPC probe cannot + # reach, and the image has no shell, so the binary checks itself. + startupProbe: + exec: + command: + - /ko-app/snapshot-plugin + - healthcheck + - --socket=/run/snapshot-plugin/control.sock + periodSeconds: 2 + failureThreshold: 30 + securityContext: + capabilities: + drop: + - ALL + env: + # The sidecar, not ate-api-server, holds the object storage settings: + # it copies external snapshots for tags and deletes the ones nothing + # refers to any more, so it needs the same backend atelet's sidecar + # reads and writes them through. Unset or "gcs" selects GCS with + # ambient credentials; "s3" with the AWS_* variables selects S3 (see + # the kind overlay). ate-api-server exits if ATE_STORAGE_BACKEND is + # set on it. + - name: ATE_STORAGE_BACKEND + value: "gcs" + envFrom: + - configMapRef: + name: ate-otel-config + volumeMounts: + - name: snapshot-plugin + mountPath: /run/snapshot-plugin containers: - name: ate-api-server image: ko://github.com/agent-substrate/substrate/cmd/ateapi @@ -95,6 +136,7 @@ spec: - --postgres-owner-role=@env - --postgres-schema=@env - --actor-id-jwt-pool=/run/actor-id-jwt-pool/pool.json + - --actor-jwt-issuer=@env - --actor-id-ca-pool=/run/actor-id-ca-pool/pool.json - --atelet-client-cred-bundle=/run/podidentity.podcert.ate.dev/credential-bundle.pem - --pod-identity-ca-certs=/run/podidentity.podcert.ate.dev/trust-bundle.pem @@ -107,6 +149,7 @@ spec: # Graceful shutdown knobs. The sum must fit within terminationGracePeriodSeconds. - --drain-delay=13s - --drain-timeout=15s + - --snapshot-plugin-socket=/run/snapshot-plugin/control.sock env: - name: POD_NAME valueFrom: @@ -122,11 +165,6 @@ spec: fieldPath: metadata.uid - name: OTEL_RESOURCE_ATTRIBUTES value: k8s.namespace.name=$(POD_NAMESPACE),k8s.pod.name=$(POD_NAME),k8s.pod.uid=$(POD_UID),service.instance.id=$(POD_UID) - # ate-api copies external snapshots for tags and deletes the ones - # nothing refers to any more, so it needs the same backend atelet reads - # and writes them through. - - name: ATE_STORAGE_BACKEND - value: "gcs" # OTLP exporter settings are shared by every control plane component; # see manifests/ate-install/ate-otel-config.yaml. Listed first so the # per-developer ConfigMap below can still override a key. @@ -147,6 +185,8 @@ spec: name: ate-api-server-secret-envvars optional: true volumeMounts: + - name: snapshot-plugin + mountPath: /run/snapshot-plugin # servicedns: the apiserver's own gRPC serving cert (DNS SAN api.ate-system.svc). - name: "servicedns" mountPath: "/run/servicedns.podcert.ate.dev" @@ -212,6 +252,8 @@ spec: initialDelaySeconds: 10 periodSeconds: 10 volumes: + - name: snapshot-plugin + emptyDir: {} - name: "servicedns" projected: sources: diff --git a/manifests/ate-install/ate-client.yaml b/manifests/ate-install/ate-client.yaml index e59bd53f8f..1774cc2e91 100644 --- a/manifests/ate-install/ate-client.yaml +++ b/manifests/ate-install/ate-client.yaml @@ -13,7 +13,7 @@ # limitations under the License. # DO NOT EDIT — generated from charts/substrate by hack/render-manifests.sh. -# Run `make helm-template` to regenerate. +# Run `hack/render-manifests.sh` to regenerate. apiVersion: v1 kind: ServiceAccount diff --git a/manifests/ate-install/atelet.yaml b/manifests/ate-install/atelet.yaml index 1d6556f636..7abddfc91f 100644 --- a/manifests/ate-install/atelet.yaml +++ b/manifests/ate-install/atelet.yaml @@ -147,6 +147,58 @@ spec: - key: ate.dev/sandboxClass operator: Exists effect: NoSchedule + # The node snapshot plugin moves external snapshots between this node and + # object storage for atelet, over a Unix socket in a pod-local emptyDir. + # A native sidecar (restartPolicy: Always) starts before atelet and stops + # after it, so uploads still in flight during atelet's drain finish. + # atelet starts only once the startup probe sees the plugin serving. + initContainers: + - name: snapshot-plugin + image: ko://github.com/agent-substrate/substrate/cmd/snapshot-plugin + restartPolicy: Always + args: + - node + - --socket=/run/snapshot-plugin/node.sock + - --root=/var/lib/ate + # The plugin listens on a Unix socket, which a gRPC probe cannot + # reach, and the image has no shell, so the binary checks itself. + startupProbe: + exec: + command: + - /ko-app/snapshot-plugin + - healthcheck + - --socket=/run/snapshot-plugin/node.sock + periodSeconds: 2 + failureThreshold: 30 + # Reads and writes snapshot files under the /var/lib/ate hostPath as + # root, like atelet; needs no capabilities. + securityContext: + runAsUser: 0 + runAsGroup: 0 + capabilities: + drop: + - ALL + # Same reservation as atelet and no limits: the plugin now does the + # zstd work atelet used to, and must not get less CPU for it. + resources: + requests: + cpu: 50m + memory: 128Mi + env: + # Object storage settings for snapshot transfers. Unset or "gcs" + # selects GCS with ambient credentials; "s3" with the AWS_* variables + # selects S3 (see the kind overlay). atelet keeps its own copy below + # for sandbox asset downloads, which do not go through the plugin. + - name: ATE_STORAGE_BACKEND + value: "gcs" + envFrom: + - configMapRef: + name: ate-otel-config + volumeMounts: + - name: run-ateom + mountPath: /var/lib/ate + - name: snapshot-plugin + mountPath: /run/snapshot-plugin containers: - name: atelet image: ko://github.com/agent-substrate/substrate/cmd/atelet @@ -164,6 +216,7 @@ spec: # terminationGracePeriodSeconds above. - --drain-delay=0s - --drain-timeout=5m + - --snapshot-plugin-socket=/run/snapshot-plugin/node.sock # atelet does no mounts, netlink, device, or namespace operations (those # live in the ateom worker pod) — it only reads/writes the # /var/lib/ate hostPath as root. CAP_DAC_OVERRIDE is for one thing: @@ -238,6 +291,8 @@ spec: volumeMounts: - name: run-ateom mountPath: /var/lib/ate + - name: snapshot-plugin + mountPath: /run/snapshot-plugin - name: podidentity mountPath: /run/podidentity.podcert.ate.dev - name: servicedns-ca @@ -257,6 +312,8 @@ spec: mountPath: /run/image-credential-provider/config.yaml readOnly: true volumes: + - name: snapshot-plugin + emptyDir: {} - name: run-ateom hostPath: path: /var/lib/ate diff --git a/manifests/ate-install/atenet-egress.yaml b/manifests/ate-install/atenet-egress.yaml index 3c988052b5..ffd6f0cff8 100644 --- a/manifests/ate-install/atenet-egress.yaml +++ b/manifests/ate-install/atenet-egress.yaml @@ -377,13 +377,12 @@ data: typed_config: "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager stat_prefix: mitm_http - # Absent this, Envoy answers an Upgrade request itself instead of - # proxying it, so every WebSocket through this leg fails at the - # handshake. Note it covers the HTTP/1.1 path only; WebSockets over - # h2 are RFC 8441 extended CONNECT and would additionally need - # http2_protocol_options.allow_connect. + # The egress request filter returns the explanatory 403. Keep + # the HCM default disabled as a defense if a request ever gets + # past that decision. upgrade_configs: - upgrade_type: websocket + enabled: false # Disable stream_idle_timeout as there is already an overall timeout in the outer tunnel in the HCM. stream_idle_timeout: 0s # This is the leg that knows where the traffic actually went. @@ -420,32 +419,15 @@ data: - name: all domains: ["*"] routes: - # Upgrade is what cannot cross an h2 upstream: forwarding one - # needs RFC 8441 extended CONNECT, which almost no origin - # advertises. So the default route below stays pinned to - # HTTP/1.1, which every origin accepts, rather than letting - # upstream ALPN choose -- ALPN takes h2 against nearly - # everything today, and every WebSocket then fails with a 502. - # - # gRPC is the one thing that default cannot serve: its status - # rides in trailers, which HTTP/1.1 cannot express. - # content-type is how it opts out. The prefix also catches - # application/grpc-web, which is deliberate -- - # egress_forward_proxy_grpc reads its protocol from upstream - # ALPN, so a grpc-web endpoint that only speaks HTTP/1.1 still - # gets HTTP/1.1. - # - # The cost of the pinned default is that non-gRPC h2 traffic - # is downgraded and its trailers dropped silently. Nothing - # sends those today. If that changes, invert this: match on - # the `upgrade` header, pin that route to HTTP/1.1, and let - # everything else negotiate. - # - # Each of the two exists once per dial the sidecar answers - # with (dev.ate.egress:dial), and there is no default: a - # request it did not answer for has no route. A by-name + # There is no default route: the initial route lookup cannot + # match before ext_proc supplies dev.ate.egress:dial. This + # lets the request filter return the unsupported-upgrade + # response before route-level upgrade rejection. A by-name # route also needs the answered name (dev.ate.egress:host); # without it the forward proxy would dial the Host's port. + # Native gRPC uses upstream ALPN to negotiate h2 and carry + # status in trailers. Other requests stay on HTTP/1.1; the + # content-type match keeps those protocol choices separate. - match: prefix: "/" headers: @@ -461,6 +443,9 @@ data: value: { present_match: true } route: cluster: egress_forward_proxy_grpc + upgrade_configs: + - upgrade_type: websocket + enabled: false timeout: 0s - match: prefix: "/" @@ -474,6 +459,9 @@ data: value: { string_match: { exact: address } } route: cluster: egress_original_dst_tls_grpc + upgrade_configs: + - upgrade_type: websocket + enabled: false timeout: 0s - match: prefix: "/" @@ -486,6 +474,9 @@ data: value: { present_match: true } route: cluster: egress_forward_proxy + upgrade_configs: + - upgrade_type: websocket + enabled: false # The route timeout runs until the response completes, which # for SSE or long-poll is never, so 0. timeout: 0s @@ -497,6 +488,9 @@ data: value: { string_match: { exact: address } } route: cluster: egress_original_dst_tls + upgrade_configs: + - upgrade_type: websocket + enabled: false timeout: 0s http_filters: # Every request on this chain is authorized against the actor's @@ -623,8 +617,12 @@ data: # answers it 426 before any filter runs. http_protocol_options: accept_http_10: true + # The egress request filter returns the explanatory 403. Keep + # the HCM default disabled as a defense if a request ever gets + # past that decision. upgrade_configs: - upgrade_type: websocket + enabled: false stream_idle_timeout: 0s access_log: - name: envoy.access_loggers.file @@ -670,6 +668,9 @@ data: value: { present_match: true } route: cluster: egress_forward_proxy_cleartext + upgrade_configs: + - upgrade_type: websocket + enabled: false timeout: 0s - match: prefix: "/" @@ -679,6 +680,9 @@ data: value: { string_match: { exact: address } } route: cluster: egress_original_dst_cleartext + upgrade_configs: + - upgrade_type: websocket + enabled: false timeout: 0s http_filters: # Every request on this chain is authorized against the actor's diff --git a/manifests/ate-install/atenet-router.yaml b/manifests/ate-install/atenet-router.yaml index fb1a551610..800ebf2ef7 100644 --- a/manifests/ate-install/atenet-router.yaml +++ b/manifests/ate-install/atenet-router.yaml @@ -265,9 +265,9 @@ spec: envFrom: - configMapRef: name: ate-otel-config + # The xDS server (--port-xds) binds 127.0.0.1 and serves only the Envoy + # in this pod, so it is deliberately not declared as a container port. ports: - - name: xds - containerPort: 18000 - name: extproc containerPort: 50051 - name: status diff --git a/manifests/ate-install/components/agentgateway-egress/kustomization.yaml b/manifests/ate-install/components/agentgateway-egress/kustomization.yaml index 6d3c32efd9..c78d320c52 100644 --- a/manifests/ate-install/components/agentgateway-egress/kustomization.yaml +++ b/manifests/ate-install/components/agentgateway-egress/kustomization.yaml @@ -59,6 +59,8 @@ patches: data: config.yaml: | frontendPolicies: + connect: + mode: tunnel accessLog: add: substrate.connect.authority: source.connectHeaders["host"] @@ -70,16 +72,20 @@ patches: key: /run/podidentity.podcert.ate.dev/credential-bundle.pem root: /run/servicedns-ca/trust-bundle.pem - binds: - - port: 8443 - tunnelProtocol: connect - listeners: - - protocol: HTTPS + # Keep the outer egress gateway on 8443, but bind it to this pod's + # address. Numeric CONNECT re-entry must select the wildcard + # internal bind instead of selecting this front door again. + gateways: + egress: + port: 8443 + bindAddress: $POD_IP + protocol: HTTPS tls: cert: /run/servicedns.podcert.ate.dev/credential-bundle.pem key: /run/servicedns.podcert.ate.dev/credential-bundle.pem root: /run/actor-id-ca-certs/ca.crt - routes: [] + + binds: - mode: internal protocol: AUTO listeners: @@ -92,7 +98,8 @@ patches: - backends: - dynamic: {} policies: - backendTLS: {} + backendTLS: + root: /etc/ssl/certs/ca-certificates.crt policies: substrateEgress: &egressPolicy #ATE_AGENTGATEWAY_CREDENTIAL_PROVIDERS diff --git a/manifests/ate-install/components/agentgateway/configmap.yaml b/manifests/ate-install/components/agentgateway/configmap.yaml index 1004c26c88..dfde305afa 100644 --- a/manifests/ate-install/components/agentgateway/configmap.yaml +++ b/manifests/ate-install/components/agentgateway/configmap.yaml @@ -69,6 +69,17 @@ data: - path: pathPrefix: / policies: &substrate-ingress + # Inspect every value before forwarding; substrateIngress reads only + # the first actor header. Apply this to HTTP, gRPC, and CONNECT traffic. + directResponse: + conditional: + - condition: >- + ('ate-target-actor' in source.connectHeaders && + size(source.connectHeaders.raw()['ate-target-actor']) != 1) || + ('ate-target-actor' in request.headers && + size(request.headers.raw()['ate-target-actor']) != 1) + status: 404 + body: Invalid ate-target-actor header substrateIngress: host: api.ate-system.svc:443 # AgentGateway only uses atunnel's CONNECT listener. @@ -138,6 +149,8 @@ data: config.yaml: | # yaml-language-server: $schema=https://agentgateway.dev/schema/config frontendPolicies: + connect: + mode: tunnel accessLog: add: substrate.connect.authority: source.connectHeaders["host"] @@ -149,18 +162,17 @@ data: key: /run/podidentity.podcert.ate.dev/credential-bundle.pem root: /run/servicedns-ca/trust-bundle.pem - binds: - # Authenticate the actor before accepting CONNECT. - - port: 8443 - tunnelProtocol: connect - listeners: - - protocol: HTTPS + gateways: + egress: + port: 8443 + bindAddress: $POD_IP + protocol: HTTPS tls: cert: /run/servicedns.podcert.ate.dev/credential-bundle.pem key: /run/servicedns.podcert.ate.dev/credential-bundle.pem root: /run/actor-id-ca-certs/ca.crt - # CONNECT re-enters the internal bind below. - routes: [] + + binds: # Route tunneled HTTP and policy-authorized TLS passthrough. - mode: internal diff --git a/manifests/ate-install/components/agentgateway/kustomization.yaml b/manifests/ate-install/components/agentgateway/kustomization.yaml index 42fd917a0c..4a2a2ccfeb 100644 --- a/manifests/ate-install/components/agentgateway/kustomization.yaml +++ b/manifests/ate-install/components/agentgateway/kustomization.yaml @@ -125,6 +125,10 @@ patches: env: - name: RUST_LOG value: debug + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP ports: - name: https containerPort: 8443 diff --git a/manifests/ate-install/generated/ate.dev_sandboxconfigs.yaml b/manifests/ate-install/generated/ate.dev_sandboxconfigs.yaml index a41ada36a9..7d3e919371 100644 --- a/manifests/ate-install/generated/ate.dev_sandboxconfigs.yaml +++ b/manifests/ate-install/generated/ate.dev_sandboxconfigs.yaml @@ -34,6 +34,9 @@ spec: - jsonPath: .spec.sandboxClass name: Class type: string + - jsonPath: .spec.defaultVersion + name: Default + type: string - jsonPath: .metadata.creationTimestamp name: Age type: date @@ -41,10 +44,10 @@ spec: schema: openAPIV3Schema: description: |- - SandboxConfig is cluster-scoped configuration describing the sandbox binaries - for a sandbox runtime family. It is referenced by an ActorTemplate's - sandbox_config.config_name (required) and decouples - sandbox binary selection from the workload definition. + SandboxConfig is cluster-scoped configuration describing the versions of the + sandbox binaries for a sandbox runtime family. It is referenced by an + ActorTemplate's sandbox_config.config_name (required) and decouples sandbox + binary selection from the workload definition. properties: apiVersion: description: |- @@ -66,60 +69,13 @@ spec: spec: description: spec defines the desired state of SandboxConfig properties: - assets: - additionalProperties: - additionalProperties: - description: |- - AssetFile is one content-addressed file that atelet fetches for a sandbox - runtime (e.g. the gVisor runsc binary, or a micro-VM kernel/firmware/config). - properties: - sha256: - description: |- - SHA256 is the lower-case hex SHA256 of the asset. It both names the cached - file (preventing collisions) and verifies the download's integrity. - pattern: ^[a-f0-9]{64}$ - type: string - url: - description: |- - URL is where to download the asset from (e.g. a gs:// URL). It may be - fetched anonymously or with credentials depending on atelet's - configuration. - minLength: 1 - type: string - required: - - sha256 - - url - type: object - type: object + defaultVersion: description: |- - Assets is the set of files atelet fetches for this runtime, keyed first by - architecture (GOARCH, e.g. "amd64", "arm64") and then by asset name. The - asset names are interpreted by the sandbox backend: gVisor expects a - "gvisor" asset (the release's gvisor.tar.zstd, which atelet extracts so - the gvisor-bin/ helpers sit next to runsc; a legacy bare-binary "runsc" - asset is still accepted); a micro-VM backend expects several (e.g. - "cloud-hypervisor", "kata-kernel", "kata-image"). The schema is - intentionally generic; per-class requirements are enforced by a - ValidatingAdmissionPolicy. - type: object - pauseImage: - description: |- - PauseImage is the container image used as the root sandbox container. - It holds the sandbox's namespaces and runs no workload code, so it is an - implementation detail of the sandbox rather than something actor authors - choose. Required for gvisor; not allowed for microvm, which runs no pause - container. It is captured in the snapshot manifest alongside the sandbox - binaries, so a restore always re-creates the sandbox from the same image - the snapshot was taken with. - - Typically, set it to [1] for on-gcp, and [2] for off-gcp - - - [1] gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da - - [2] registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4 + defaultVersion names the entry in versions that new sandboxes boot with. + It must name an existing, non-Disabled version. + maxLength: 63 + minLength: 1 type: string - x-kubernetes-validations: - - message: All images must include a digest - rule: self.contains('@') sandboxClass: default: gvisor description: |- @@ -130,13 +86,112 @@ spec: - gvisor - microvm type: string + versions: + description: versions are the sandbox versions this config offers, + keyed by name. + items: + description: |- + SandboxVersionConfig is one version of a SandboxConfig: the exact pause image + and asset set a sandbox boots with. + properties: + assets: + additionalProperties: + additionalProperties: + description: |- + AssetFile is one content-addressed file that atelet fetches for a sandbox + runtime (e.g. the gVisor runsc binary, or a micro-VM kernel/firmware/config). + properties: + sha256: + description: |- + SHA256 is the lower-case hex SHA256 of the asset. It both names the cached + file (preventing collisions) and verifies the download's integrity. + pattern: ^[a-f0-9]{64}$ + type: string + url: + description: |- + URL is where to download the asset from (e.g. a gs:// URL). It may be + fetched anonymously or with credentials depending on atelet's + configuration. + minLength: 1 + type: string + required: + - sha256 + - url + type: object + type: object + description: |- + Assets is the set of files atelet fetches for this runtime, keyed first by + architecture (GOARCH, e.g. "amd64", "arm64") and then by asset name. The + asset names are interpreted by the sandbox backend: gVisor expects a + "gvisor" asset (the release's gvisor.tar.zstd, which atelet extracts so + the gvisor-bin/ helpers sit next to runsc; a legacy bare-binary "runsc" + asset is still accepted); a micro-VM backend expects several (e.g. + "cloud-hypervisor", "kata-kernel", "kata-image"). The schema is + intentionally generic; per-class requirements are enforced by a + ValidatingAdmissionPolicy. + type: object + name: + description: |- + name identifies this version within its SandboxConfig. It is a DNS label + (lower-case alphanumerics and '-'). + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + pauseImage: + description: |- + PauseImage is the container image used as the root sandbox container. + It holds the sandbox's namespaces and runs no workload code, so it is an + implementation detail of the sandbox rather than something actor authors + choose. Required for gvisor; not allowed for microvm, which runs no pause + container. It is captured in the snapshot manifest alongside the sandbox + binaries, so a restore always re-creates the sandbox from the same image + the snapshot was taken with. + + Typically, set it to [1] for on-gcp, and [2] for off-gcp + + - [1] gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da + - [2] registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4 + maxLength: 512 + minLength: 1 + type: string + x-kubernetes-validations: + - message: All images must include a digest + rule: self.contains('@') + state: + default: Enabled + description: |- + state is whether this version may be used: Enabled (default) or + Disabled. + enum: + - Enabled + - Disabled + type: string + required: + - name + type: object + maxItems: 16 + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map required: + - defaultVersion - sandboxClass + - versions type: object x-kubernetes-validations: - - message: pauseImage is required for gvisor and not allowed for other - sandbox classes - rule: 'self.sandboxClass == ''gvisor'' ? has(self.pauseImage) : !has(self.pauseImage)' + - message: defaultVersion must name an entry in versions + rule: self.versions.exists(v, v.name == self.defaultVersion) + - message: the version named by defaultVersion must not be Disabled + rule: '!self.versions.exists(v, v.name == self.defaultVersion && has(v.state) + && v.state == ''Disabled'')' + - message: pauseImage is required on every version for gvisor and not + allowed for other sandbox classes + rule: 'self.sandboxClass == ''gvisor'' ? self.versions.all(v, has(v.pauseImage) + && size(v.pauseImage) > 0) : self.versions.all(v, !has(v.pauseImage) + || size(v.pauseImage) == 0)' required: - spec type: object diff --git a/manifests/ate-install/generated/ate.dev_workerpools.yaml b/manifests/ate-install/generated/ate.dev_workerpools.yaml index e6981dd899..cad0eb1319 100644 --- a/manifests/ate-install/generated/ate.dev_workerpools.yaml +++ b/manifests/ate-install/generated/ate.dev_workerpools.yaml @@ -73,20 +73,47 @@ spec: format: int32 minimum: 0 type: integer - sandboxClass: - default: gvisor + sandboxClasses: description: |- - SandboxClass selects the sandbox runtime family for this pool, which drives - the worker pod shape (KVM/vhost device mounts and node placement). The - concrete binary is still selected by WorkerImage. Defaults to gvisor. - The sandbox binaries themselves come from the SandboxConfig each - ActorTemplate names (required). + SandboxClasses lists the sandbox runtime families this pool runs. Exactly + one entry is allowed today. + items: + description: WorkerPoolSandboxClass is one sandbox runtime family + a WorkerPool runs. + properties: + configRef: + description: |- + ConfigRef names a cluster-scoped SandboxConfig for this sandbox class. + The referenced config's SandboxClass must match Name. Not consumed yet. + properties: + name: + description: Name is the SandboxConfig's metadata.name. + minLength: 1 + type: string + required: + - name + type: object + name: + description: |- + Name selects the sandbox runtime family, which drives the worker pod + shape (KVM/vhost device mounts and node placement). The concrete binary + is still selected by WorkerImage. The sandbox binaries themselves come + from the SandboxConfig each ActorTemplate names (required). - See Also: TODOs in ActorTemplate SandboxClass - enum: - - gvisor - - microvm - type: string + See Also: TODOs in ActorTemplate SandboxClass + enum: + - gvisor + - microvm + type: string + required: + - name + type: object + maxItems: 1 + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map template: description: Template holds optional metadata, scheduling, and resource settings for worker workloads. @@ -449,6 +476,7 @@ spec: type: string required: - replicas + - sandboxClasses - workerImage type: object status: diff --git a/manifests/ate-install/kind/ate-otel-config.yaml b/manifests/ate-install/kind/ate-otel-config.yaml index 52467280ba..ed2c8b9334 100644 --- a/manifests/ate-install/kind/ate-otel-config.yaml +++ b/manifests/ate-install/kind/ate-otel-config.yaml @@ -30,10 +30,10 @@ # the ateom worker pods it creates, which have no other way to receive them. # Kind only; production keeps the SDK defaults. # -# OTEL_LOGS_EXPORTER=otlp sends the actor events over OTLP instead of stdout; -# kind adds console to keep them in kubectl logs too. serverboot defaults it to -# none, so the base ConfigMap leaves it unset and every other environment keeps -# them on stdout. +# OTEL_LOGS_EXPORTER=otlp sends the actor events, ateapi's lifecycle records +# and the ateoms' usage samples, over OTLP instead of stdout; kind adds console +# to keep them in kubectl logs too. serverboot defaults it to none, so the base +# ConfigMap leaves it unset and every other environment keeps them on stdout. apiVersion: v1 kind: ConfigMap metadata: diff --git a/manifests/ate-install/kind/atelet/kustomization.yaml b/manifests/ate-install/kind/atelet/kustomization.yaml index 023b922ee3..0963cc3ebc 100644 --- a/manifests/ate-install/kind/atelet/kustomization.yaml +++ b/manifests/ate-install/kind/atelet/kustomization.yaml @@ -70,3 +70,22 @@ patches: $patch: delete - name: image-credential-provider-config $patch: delete + # The snapshot plugin transfers atelet's external snapshots, so it + # needs the same rustfs settings. atelet keeps its own copy for + # sandbox assets. + initContainers: + - name: snapshot-plugin + env: + - name: ATE_STORAGE_BACKEND + value: s3 + - name: AWS_REGION + value: us-east-1 + - name: AWS_ENDPOINT_URL + value: http://rustfs.ate-system.svc:9000 + - name: AWS_S3_USE_PATH_STYLE + value: "true" + # TODO: use a secret / identity management + - name: AWS_ACCESS_KEY_ID + value: rustfsadmin + - name: AWS_SECRET_ACCESS_KEY + value: rustfsadmin diff --git a/manifests/ate-install/kind/kustomization.yaml b/manifests/ate-install/kind/kustomization.yaml index b4feace682..f124e8ff9b 100644 --- a/manifests/ate-install/kind/kustomization.yaml +++ b/manifests/ate-install/kind/kustomization.yaml @@ -43,9 +43,9 @@ resources: # shared by every component via envFrom -- putting always_on there would pin the # router and the control plane to 100% too, undoing their per-component ratios. # -# The storage settings point ate-api at the same rustfs the ./atelet overlay -# points atelet at: both ends of an external snapshot's life have to agree on -# where it lives. +# The storage settings point ate-api-server's snapshot plugin at the same +# rustfs the ./atelet overlay points atelet's at: both ends of an external +# snapshot's life have to agree on where it lives. patches: - patch: |- apiVersion: apps/v1 @@ -61,6 +61,9 @@ patches: env: - name: OTEL_TRACES_SAMPLER value: parentbased_always_on + initContainers: + - name: snapshot-plugin + env: - name: ATE_STORAGE_BACKEND value: s3 - name: AWS_REGION diff --git a/manifests/ate-install/pod-certificate-controller.yaml b/manifests/ate-install/pod-certificate-controller.yaml index 4f4e9e36e6..23886e260e 100644 --- a/manifests/ate-install/pod-certificate-controller.yaml +++ b/manifests/ate-install/pod-certificate-controller.yaml @@ -13,7 +13,7 @@ # limitations under the License. # DO NOT EDIT — generated from charts/substrate-podcert by hack/render-manifests.sh. -# Run `make helm-template` to regenerate. +# Run `hack/render-manifests.sh` to regenerate. apiVersion: v1 kind: Namespace diff --git a/manifests/ate-install/role.yaml b/manifests/ate-install/role.yaml index 693eb48eba..502143729d 100644 --- a/manifests/ate-install/role.yaml +++ b/manifests/ate-install/role.yaml @@ -13,7 +13,7 @@ # limitations under the License. # DO NOT EDIT — generated from charts/substrate by hack/render-manifests.sh. -# Run `make helm-template` to regenerate. +# Run `hack/render-manifests.sh` to regenerate. apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole diff --git a/manifests/ate-install/rustfs.yaml b/manifests/ate-install/rustfs.yaml index 25bdc29eac..5ae6671b95 100644 --- a/manifests/ate-install/rustfs.yaml +++ b/manifests/ate-install/rustfs.yaml @@ -13,7 +13,7 @@ # limitations under the License. # DO NOT EDIT — generated from charts/substrate by hack/render-manifests.sh. -# Run `make helm-template` to regenerate. +# Run `hack/render-manifests.sh` to regenerate. apiVersion: v1 kind: ServiceAccount diff --git a/manifests/ate-install/sandboxconfig-gvisor.yaml b/manifests/ate-install/sandboxconfig-gvisor.yaml index 5ab2d1e251..77f3654bf9 100644 --- a/manifests/ate-install/sandboxconfig-gvisor.yaml +++ b/manifests/ate-install/sandboxconfig-gvisor.yaml @@ -17,23 +17,29 @@ # sandboxConfig.configName. atelet fetches the gVisor release tarball # (gvisor.tar.zstd: runsc plus the gvisor-bin/ helpers runsc requires next to # it) matching the worker node's architecture and extracts it locally. To pin a -# different release, edit the assets below or create another SandboxConfig and -# name it from the ActorTemplate. +# different release, add a version below and point defaultVersion at it, or +# create another SandboxConfig and name it from the ActorTemplate. apiVersion: ate.dev/v1alpha1 kind: SandboxConfig metadata: name: gvisor-default + annotations: + sandboxconfig.ate.dev/is-class-default: "true" spec: sandboxClass: gvisor - # The root sandbox container's image. On GCP, prefer the in-project mirror - # gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da. - pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" - assets: - amd64: - gvisor: - url: "gs://gvisor/releases/nightly/2026-09-02/x86_64/gvisor.tar.zstd" - sha256: "d547d81401461fd1c679c5c4fa0a6c2b8ef7dc3c22ce23c9e25dcc4c69cfd06f" - arm64: - gvisor: - url: "gs://gvisor/releases/nightly/2026-09-02/aarch64/gvisor.tar.zstd" - sha256: "a64916f9813ce7e4841a30480a599337f7dda07b421c6bf0123db2212aa7d1df" + defaultVersion: v1 + versions: + - name: v1 + state: Enabled + # The root sandbox container's image. On GCP, prefer the in-project mirror + # gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da. + pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" + assets: + amd64: + gvisor: + url: "gs://gvisor/releases/nightly/2026-09-02/x86_64/gvisor.tar.zstd" + sha256: "d547d81401461fd1c679c5c4fa0a6c2b8ef7dc3c22ce23c9e25dcc4c69cfd06f" + arm64: + gvisor: + url: "gs://gvisor/releases/nightly/2026-09-02/aarch64/gvisor.tar.zstd" + sha256: "a64916f9813ce7e4841a30480a599337f7dda07b421c6bf0123db2212aa7d1df" diff --git a/manifests/ate-install/sandboxconfig-validation.yaml b/manifests/ate-install/sandboxconfig-validation.yaml index 354acfecec..1e77e6af02 100644 --- a/manifests/ate-install/sandboxconfig-validation.yaml +++ b/manifests/ate-install/sandboxconfig-validation.yaml @@ -14,7 +14,8 @@ # Per-sandbox-class asset requirements for SandboxConfig. The CRD schema is # generic (any arch -> any asset name -> {url, sha256}); this policy enforces the -# requirements a given sandbox class actually needs, fail-closed at apply time. +# requirements a given sandbox class actually needs on every version under +# spec.versions, fail-closed at apply time. # (url/sha256 being required and well-formed is enforced by the CRD schema.) apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingAdmissionPolicy @@ -35,19 +36,20 @@ spec: # tarball. - expression: >- object.spec.sandboxClass != 'gvisor' || - (has(object.spec.assets) && size(object.spec.assets) > 0 && - object.spec.assets.all(arch, - 'gvisor' in object.spec.assets[arch] || 'runsc' in object.spec.assets[arch])) - message: "a gvisor SandboxConfig must define a 'gvisor' (release tarball) or legacy 'runsc' asset for every architecture under spec.assets" + object.spec.versions.all(v, + has(v.assets) && size(v.assets) > 0 && + v.assets.all(arch, 'gvisor' in v.assets[arch] || 'runsc' in v.assets[arch])) + message: "a gvisor SandboxConfig must define a 'gvisor' (release tarball) or legacy 'runsc' asset for every architecture of every version under spec.versions[].assets" # The micro-VM (cloud-hypervisor) runtime needs its asset set for every # architecture it advertises. - expression: >- object.spec.sandboxClass != 'microvm' || - (has(object.spec.assets) && size(object.spec.assets) > 0 && - object.spec.assets.all(arch, - ['cloud-hypervisor', 'virtiofsd', 'kata-kernel', 'kata-image'] - .all(name, name in object.spec.assets[arch]))) - message: "a microvm SandboxConfig must define cloud-hypervisor, virtiofsd, kata-kernel, and kata-image assets for every architecture under spec.assets" + object.spec.versions.all(v, + has(v.assets) && size(v.assets) > 0 && + v.assets.all(arch, + ['cloud-hypervisor', 'virtiofsd', 'kata-kernel', 'kata-image'] + .all(name, name in v.assets[arch]))) + message: "a microvm SandboxConfig must define cloud-hypervisor, virtiofsd, kata-kernel, and kata-image assets for every architecture of every version under spec.versions[].assets" --- apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingAdmissionPolicyBinding diff --git a/manifests/microvm/sandboxconfig-microvm.yaml.tmpl b/manifests/microvm/sandboxconfig-microvm.yaml.tmpl index b4975e79b0..9747a7c9c0 100644 --- a/manifests/microvm/sandboxconfig-microvm.yaml.tmpl +++ b/manifests/microvm/sandboxconfig-microvm.yaml.tmpl @@ -35,34 +35,40 @@ apiVersion: ate.dev/v1alpha1 kind: SandboxConfig metadata: name: microvm + annotations: + sandboxconfig.ate.dev/is-class-default: "true" spec: sandboxClass: microvm - assets: - arm64: - cloud-hypervisor: - url: "gs://${BUCKET_NAME}/kata-assets/cloud-hypervisor" - sha256: "f192b510eea1c710cbc439d716bb0573c223fc463dbe3e6523788a2b7ef62850" - # virtiofsd serves the overlay RO lower (virtio-fs). - virtiofsd: - url: "gs://${BUCKET_NAME}/kata-assets/virtiofsd" - sha256: "3aa9240045d2261930ef04138fe05cf288f94c867e63e5d0845758bac5e3d458" - kata-kernel: - url: "gs://${BUCKET_NAME}/kata-assets/vmlinux" - sha256: "4960f6cf781274eaa445fc9f88bb8c5ce951649278bc9caa13afb1e99cfe80e6" - kata-image: - url: "gs://${BUCKET_NAME}/kata-assets/rootfs.img" - sha256: "7e3bdd500eb92de17c377bd84f8ed57e50d6bb68c7ec08c88f5e1916cd49f30d" - # amd64 assets are kata 4.1.0 (assemble.sh ARCH=amd64). - amd64: - cloud-hypervisor: - url: "gs://${BUCKET_NAME}/kata-assets/cloud-hypervisor" - sha256: "448af3d4e59b22c2987f7df94c213ad40fb53a10d437e42b5ee6c4fce7c29ecc" - virtiofsd: - url: "gs://${BUCKET_NAME}/kata-assets/virtiofsd" - sha256: "15b2e72a78cc08a9bd8a6943e89fb69c88cb3cbeb63069efceade835342ac7d4" - kata-kernel: - url: "gs://${BUCKET_NAME}/kata-assets/vmlinux" - sha256: "8e9dbc3a6e4c26adb089d23d31201edebce905c7f3e31aa67b32bdd41df1b86f" - kata-image: - url: "gs://${BUCKET_NAME}/kata-assets/rootfs.img" - sha256: "96497f64da1de9c7473fef46c3d29ddd0805d334731cf9d903a21a5b2c33cefb" + defaultVersion: v1 + versions: + - name: v1 + state: Enabled + assets: + arm64: + cloud-hypervisor: + url: "gs://${BUCKET_NAME}/kata-assets/cloud-hypervisor" + sha256: "f192b510eea1c710cbc439d716bb0573c223fc463dbe3e6523788a2b7ef62850" + # virtiofsd serves the overlay RO lower (virtio-fs). + virtiofsd: + url: "gs://${BUCKET_NAME}/kata-assets/virtiofsd" + sha256: "3aa9240045d2261930ef04138fe05cf288f94c867e63e5d0845758bac5e3d458" + kata-kernel: + url: "gs://${BUCKET_NAME}/kata-assets/vmlinux" + sha256: "4960f6cf781274eaa445fc9f88bb8c5ce951649278bc9caa13afb1e99cfe80e6" + kata-image: + url: "gs://${BUCKET_NAME}/kata-assets/rootfs.img" + sha256: "7e3bdd500eb92de17c377bd84f8ed57e50d6bb68c7ec08c88f5e1916cd49f30d" + # amd64 assets are kata 4.1.0 (assemble.sh ARCH=amd64). + amd64: + cloud-hypervisor: + url: "gs://${BUCKET_NAME}/kata-assets/cloud-hypervisor" + sha256: "448af3d4e59b22c2987f7df94c213ad40fb53a10d437e42b5ee6c4fce7c29ecc" + virtiofsd: + url: "gs://${BUCKET_NAME}/kata-assets/virtiofsd" + sha256: "15b2e72a78cc08a9bd8a6943e89fb69c88cb3cbeb63069efceade835342ac7d4" + kata-kernel: + url: "gs://${BUCKET_NAME}/kata-assets/vmlinux" + sha256: "8e9dbc3a6e4c26adb089d23d31201edebce905c7f3e31aa67b32bdd41df1b86f" + kata-image: + url: "gs://${BUCKET_NAME}/kata-assets/rootfs.img" + sha256: "96497f64da1de9c7473fef46c3d29ddd0805d334731cf9d903a21a5b2c33cefb" diff --git a/pkg/api/v1alpha1/sandboxconfig_types.go b/pkg/api/v1alpha1/sandboxconfig_types.go index 3667cc034e..f88c4b7b3e 100644 --- a/pkg/api/v1alpha1/sandboxconfig_types.go +++ b/pkg/api/v1alpha1/sandboxconfig_types.go @@ -49,18 +49,42 @@ type AssetFile struct { SHA256 string `json:"sha256"` } -// SandboxConfigSpec is the desired state of a SandboxConfig. +// SandboxConfigClassDefaultAnnotation marks a SandboxConfig as a default for its +// sandboxClass when set to "true". More than one SandboxConfig of the same +// class may carry it; the newest one wins. +const SandboxConfigClassDefaultAnnotation = "sandboxconfig.ate.dev/is-class-default" + +// SandboxVersionState is whether a SandboxConfig version may be used. // -// +kubebuilder:validation:XValidation:rule="self.sandboxClass == 'gvisor' ? has(self.pauseImage) : !has(self.pauseImage)",message="pauseImage is required for gvisor and not allowed for other sandbox classes" -type SandboxConfigSpec struct { - // SandboxClass is the sandbox runtime family this config applies to. An - // ActorTemplate only uses SandboxConfigs whose SandboxClass matches its - // sandbox_config.sandbox_class. +// +kubebuilder:validation:Enum=Enabled;Disabled +type SandboxVersionState string + +const ( + // SandboxVersionStateEnabled marks a version as usable. Default. + SandboxVersionStateEnabled SandboxVersionState = "Enabled" + // SandboxVersionStateDisabled marks a version as unusable. The version + // named by defaultVersion cannot be disabled. + SandboxVersionStateDisabled SandboxVersionState = "Disabled" +) + +// SandboxVersionConfig is one version of a SandboxConfig: the exact pause image +// and asset set a sandbox boots with. +type SandboxVersionConfig struct { + // name identifies this version within its SandboxConfig. It is a DNS label + // (lower-case alphanumerics and '-'). // // +required - // +kubebuilder:validation:Enum=gvisor;microvm - // +kubebuilder:default=gvisor - SandboxClass SandboxClass `json:"sandboxClass"` + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=63 + // +kubebuilder:validation:Pattern=`^[a-z0-9]([-a-z0-9]*[a-z0-9])?$` + Name string `json:"name,omitempty"` + + // state is whether this version may be used: Enabled (default) or + // Disabled. + // + // +optional + // +kubebuilder:default=Enabled + State *SandboxVersionState `json:"state,omitempty"` // TODO: drop PauseImage once gVisor can run without a pause container: // https://github.com/google/gvisor/pull/13981 @@ -79,6 +103,8 @@ type SandboxConfigSpec struct { // - [2] registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4 // // +optional + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=512 // +kubebuilder:validation:XValidation:rule="self.contains('@')",message="All images must include a digest" PauseImage string `json:"pauseImage,omitempty"` @@ -96,10 +122,60 @@ type SandboxConfigSpec struct { Assets map[string]map[string]AssetFile `json:"assets,omitempty"` } -// SandboxConfig is cluster-scoped configuration describing the sandbox binaries -// for a sandbox runtime family. It is referenced by an ActorTemplate's -// sandbox_config.config_name (required) and decouples -// sandbox binary selection from the workload definition. +// SandboxConfigSpec is the desired state of a SandboxConfig. +// +// +kubebuilder:validation:XValidation:rule="self.versions.exists(v, v.name == self.defaultVersion)",message="defaultVersion must name an entry in versions" +// +kubebuilder:validation:XValidation:rule="!self.versions.exists(v, v.name == self.defaultVersion && has(v.state) && v.state == 'Disabled')",message="the version named by defaultVersion must not be Disabled" +// +kubebuilder:validation:XValidation:rule="self.sandboxClass == 'gvisor' ? self.versions.all(v, has(v.pauseImage) && size(v.pauseImage) > 0) : self.versions.all(v, !has(v.pauseImage) || size(v.pauseImage) == 0)",message="pauseImage is required on every version for gvisor and not allowed for other sandbox classes" +type SandboxConfigSpec struct { + // SandboxClass is the sandbox runtime family this config applies to. An + // ActorTemplate only uses SandboxConfigs whose SandboxClass matches its + // sandbox_config.sandbox_class. + // + // +required + // +kubebuilder:validation:Enum=gvisor;microvm + // +kubebuilder:default=gvisor + SandboxClass SandboxClass `json:"sandboxClass"` + + // defaultVersion names the entry in versions that new sandboxes boot with. + // It must name an existing, non-Disabled version. + // + // +required + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=63 + DefaultVersion string `json:"defaultVersion,omitempty"` + + // versions are the sandbox versions this config offers, keyed by name. + // + // +required + // +listType=map + // +listMapKey=name + // +kubebuilder:validation:MinItems=1 + // +kubebuilder:validation:MaxItems=16 + Versions []SandboxVersionConfig `json:"versions,omitempty"` +} + +// DefaultVersionConfig returns the entry of Versions named by DefaultVersion, +// and whether one exists. A Disabled entry is never returned: the CRD rejects +// a Disabled default, so a spec that reads back that way is treated as having +// no default version. +func (s *SandboxConfigSpec) DefaultVersionConfig() (*SandboxVersionConfig, bool) { + for i := range s.Versions { + if s.Versions[i].Name != s.DefaultVersion { + continue + } + if st := s.Versions[i].State; st != nil && *st == SandboxVersionStateDisabled { + return nil, false + } + return &s.Versions[i], true + } + return nil, false +} + +// SandboxConfig is cluster-scoped configuration describing the versions of the +// sandbox binaries for a sandbox runtime family. It is referenced by an +// ActorTemplate's sandbox_config.config_name (required) and decouples sandbox +// binary selection from the workload definition. // // +genclient // +genclient:nonNamespaced @@ -107,6 +183,7 @@ type SandboxConfigSpec struct { // +kubebuilder:object:root=true // +kubebuilder:resource:scope=Cluster,shortName=sandboxconfig // +kubebuilder:printcolumn:name="Class",type=string,JSONPath=`.spec.sandboxClass` +// +kubebuilder:printcolumn:name="Default",type=string,JSONPath=`.spec.defaultVersion` // +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` type SandboxConfig struct { metav1.TypeMeta `json:",inline"` diff --git a/pkg/api/v1alpha1/sandboxconfig_types_test.go b/pkg/api/v1alpha1/sandboxconfig_types_test.go new file mode 100644 index 0000000000..f5869ec696 --- /dev/null +++ b/pkg/api/v1alpha1/sandboxconfig_types_test.go @@ -0,0 +1,108 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1alpha1 + +import ( + "testing" + + "k8s.io/utils/ptr" +) + +func TestDefaultVersionConfig(t *testing.T) { + version := func(name string, state *SandboxVersionState) SandboxVersionConfig { + return SandboxVersionConfig{Name: name, State: state} + } + + tests := []struct { + name string + spec SandboxConfigSpec + wantName string // "" means no default is expected + }{{ + name: "default with nil state is returned", + spec: SandboxConfigSpec{ + DefaultVersion: "v1", + Versions: []SandboxVersionConfig{version("v1", nil)}, + }, + wantName: "v1", + }, { + name: "default explicitly Enabled is returned", + spec: SandboxConfigSpec{ + DefaultVersion: "v1", + Versions: []SandboxVersionConfig{version("v1", ptr.To(SandboxVersionStateEnabled))}, + }, + wantName: "v1", + }, { + name: "default picked out of several versions", + spec: SandboxConfigSpec{ + DefaultVersion: "v2", + Versions: []SandboxVersionConfig{ + version("v1", ptr.To(SandboxVersionStateDisabled)), + version("v2", nil), + version("v3", nil), + }, + }, + wantName: "v2", + }, { + name: "Disabled default is treated as missing", + spec: SandboxConfigSpec{ + DefaultVersion: "v1", + Versions: []SandboxVersionConfig{ + version("v1", ptr.To(SandboxVersionStateDisabled)), + version("v2", nil), + }, + }, + }, { + name: "defaultVersion names no entry", + spec: SandboxConfigSpec{ + DefaultVersion: "v9", + Versions: []SandboxVersionConfig{version("v1", nil)}, + }, + }, { + name: "no versions", + spec: SandboxConfigSpec{DefaultVersion: "v1"}, + }} + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, ok := tt.spec.DefaultVersionConfig() + if tt.wantName == "" { + if ok || got != nil { + t.Fatalf("DefaultVersionConfig() = %+v, %v; want nil, false", got, ok) + } + return + } + if !ok || got == nil { + t.Fatalf("DefaultVersionConfig() = %+v, %v; want %q, true", got, ok, tt.wantName) + } + if got.Name != tt.wantName { + t.Fatalf("DefaultVersionConfig().Name = %q; want %q", got.Name, tt.wantName) + } + // The pointer must alias the spec so callers see the real entry, + // not a copy. + if got != &tt.spec.Versions[indexOf(tt.spec.Versions, tt.wantName)] { + t.Fatalf("DefaultVersionConfig() returned a copy, want pointer into spec.Versions") + } + }) + } +} + +func indexOf(versions []SandboxVersionConfig, name string) int { + for i := range versions { + if versions[i].Name == name { + return i + } + } + return -1 +} diff --git a/pkg/api/v1alpha1/sandboxconfig_validation_test.go b/pkg/api/v1alpha1/sandboxconfig_validation_test.go index a1b4b4123c..5e819755c8 100644 --- a/pkg/api/v1alpha1/sandboxconfig_validation_test.go +++ b/pkg/api/v1alpha1/sandboxconfig_validation_test.go @@ -25,6 +25,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/yaml" ) @@ -37,27 +38,51 @@ const vapManifestPath = "../../../manifests/ate-install/sandboxconfig-validation const validPauseImage = "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" -// sandboxConfig returns a config with a pause image iff the class needs one. +// sandboxVersion returns a version with a pause image iff the class needs one. +func sandboxVersion(name string, class SandboxClass, assets map[string]map[string]AssetFile) SandboxVersionConfig { + v := SandboxVersionConfig{Name: name, Assets: assets} + if class == SandboxClassGvisor { + v.PauseImage = validPauseImage + } + return v +} + +// sandboxConfig returns a config with a single default version "v1". func sandboxConfig(name string, class SandboxClass, assets map[string]map[string]AssetFile) *SandboxConfig { - sc := &SandboxConfig{ + return &SandboxConfig{ ObjectMeta: metav1.ObjectMeta{Name: name}, Spec: SandboxConfigSpec{ - SandboxClass: class, - Assets: assets, + SandboxClass: class, + DefaultVersion: "v1", + Versions: []SandboxVersionConfig{sandboxVersion("v1", class, assets)}, }, } - if class == SandboxClassGvisor { - sc.Spec.PauseImage = validPauseImage - } - return sc } -// withPauseImage overrides the pause image on an otherwise-valid config. +// withPauseImage overrides the pause image of the first version on an +// otherwise-valid config. func withPauseImage(sc *SandboxConfig, image string) *SandboxConfig { - sc.Spec.PauseImage = image + sc.Spec.Versions[0].PauseImage = image + return sc +} + +// withVersions appends versions to a config. +func withVersions(sc *SandboxConfig, versions ...SandboxVersionConfig) *SandboxConfig { + sc.Spec.Versions = append(sc.Spec.Versions, versions...) + return sc +} + +// withDefaultVersion overrides the default version of a config. +func withDefaultVersion(sc *SandboxConfig, name string) *SandboxConfig { + sc.Spec.DefaultVersion = name return sc } +// gvisorAssets returns a valid gVisor asset set for amd64. +func gvisorAssets() map[string]map[string]AssetFile { + return map[string]map[string]AssetFile{"amd64": {"gvisor": gvisorAsset()}} +} + func runscAsset() AssetFile { return AssetFile{URL: "gs://bucket/runsc", SHA256: validSHA256} } func gvisorAsset() AssetFile { @@ -200,17 +225,106 @@ func TestSandboxConfigValidation(t *testing.T) { name: "gvisor missing pauseImage", sc: withPauseImage(sandboxConfig("bad-no-pause", SandboxClassGvisor, map[string]map[string]AssetFile{"amd64": {"gvisor": gvisorAsset()}}), ""), wantErr: true, - errMsg: "pauseImage is required for gvisor and not allowed", + errMsg: "pauseImage is required on every version for gvisor", }, { name: "microvm with pauseImage", sc: withPauseImage(sandboxConfig("bad-microvm-pause", SandboxClassMicroVM, map[string]map[string]AssetFile{"amd64": microVMAssets()}), validPauseImage), wantErr: true, - errMsg: "pauseImage is required for gvisor and not allowed", + errMsg: "pauseImage is required on every version for gvisor", }, { name: "unpinned pauseImage", sc: withPauseImage(sandboxConfig("bad-unpinned-pause", SandboxClassGvisor, map[string]map[string]AssetFile{"amd64": {"gvisor": gvisorAsset()}}), "registry.k8s.io/pause:3.10.2"), wantErr: true, errMsg: "All images must include a digest", + }, { + name: "valid gvisor with two versions", + sc: withVersions(sandboxConfig("ok-two-versions", SandboxClassGvisor, gvisorAssets()), sandboxVersion("v2", SandboxClassGvisor, gvisorAssets())), + wantErr: false, + }, { + name: "valid non-default disabled version", + sc: withVersions(sandboxConfig("ok-disabled-old", SandboxClassGvisor, gvisorAssets()), func() SandboxVersionConfig { + v := sandboxVersion("v0", SandboxClassGvisor, gvisorAssets()) + v.State = ptr.To(SandboxVersionStateDisabled) + return v + }()), + wantErr: false, + }, { + name: "defaultVersion not in versions", + sc: withDefaultVersion(sandboxConfig("bad-default-missing", SandboxClassGvisor, gvisorAssets()), "v9"), + wantErr: true, + errMsg: "defaultVersion must name an entry in versions", + }, { + name: "defaultVersion disabled", + sc: func() *SandboxConfig { + sc := sandboxConfig("bad-default-disabled", SandboxClassGvisor, gvisorAssets()) + sc.Spec.Versions[0].State = ptr.To(SandboxVersionStateDisabled) + return sc + }(), + wantErr: true, + errMsg: "must not be Disabled", + }, { + name: "no versions", + sc: &SandboxConfig{ObjectMeta: metav1.ObjectMeta{Name: "bad-no-versions"}, Spec: SandboxConfigSpec{SandboxClass: SandboxClassGvisor, DefaultVersion: "v1", Versions: []SandboxVersionConfig{}}}, + wantErr: true, + errMsg: "spec.versions", + }, { + name: "duplicate version names", + sc: withVersions(sandboxConfig("bad-dup-versions", SandboxClassGvisor, gvisorAssets()), sandboxVersion("v1", SandboxClassGvisor, gvisorAssets())), + wantErr: true, + errMsg: "Duplicate value", + }, { + name: "too many versions", + sc: func() *SandboxConfig { + sc := sandboxConfig("bad-many-versions", SandboxClassGvisor, gvisorAssets()) + for i := range 16 { + sc.Spec.Versions = append(sc.Spec.Versions, sandboxVersion(fmt.Sprintf("extra-%d", i), SandboxClassGvisor, gvisorAssets())) + } + return sc + }(), + wantErr: true, + errMsg: "spec.versions", + }, { + name: "version name not a DNS label", + sc: func() *SandboxConfig { + sc := sandboxConfig("bad-version-name", SandboxClassGvisor, gvisorAssets()) + sc.Spec.Versions[0].Name = "V_1" + sc.Spec.DefaultVersion = "V_1" + return sc + }(), + wantErr: true, + errMsg: "spec.versions[0].name", + }, { + name: "gvisor second version missing pauseImage", + sc: withVersions(sandboxConfig("bad-v2-no-pause", SandboxClassGvisor, gvisorAssets()), func() SandboxVersionConfig { + v := sandboxVersion("v2", SandboxClassGvisor, gvisorAssets()) + v.PauseImage = "" + return v + }()), + wantErr: true, + errMsg: "pauseImage is required on every version for gvisor", + }, { + name: "microvm second version with pauseImage", + sc: withVersions(sandboxConfig("bad-microvm-v2-pause", SandboxClassMicroVM, map[string]map[string]AssetFile{"amd64": microVMAssets()}), func() SandboxVersionConfig { + v := sandboxVersion("v2", SandboxClassMicroVM, map[string]map[string]AssetFile{"amd64": microVMAssets()}) + v.PauseImage = validPauseImage + return v + }()), + wantErr: true, + errMsg: "pauseImage is required on every version for gvisor", + }, { + name: "gvisor second version missing gvisor and runsc", + sc: withVersions(sandboxConfig("bad-v2-no-runsc", SandboxClassGvisor, gvisorAssets()), sandboxVersion("v2", SandboxClassGvisor, map[string]map[string]AssetFile{"amd64": {"notrunsc": runscAsset()}})), + wantErr: true, + errMsg: "runsc", + }, { + name: "microvm second version missing an asset", + sc: withVersions(sandboxConfig("bad-microvm-v2", SandboxClassMicroVM, map[string]map[string]AssetFile{"amd64": microVMAssets()}), sandboxVersion("v2", SandboxClassMicroVM, map[string]map[string]AssetFile{"amd64": func() map[string]AssetFile { + m := microVMAssets() + delete(m, "kata-kernel") + return m + }()})), + wantErr: true, + errMsg: "microvm SandboxConfig must define", }} for _, tt := range tests { @@ -233,3 +347,59 @@ func TestSandboxConfigValidation(t *testing.T) { }) } } + +func TestSandboxConfigUpdateRemovingDefaultVersion(t *testing.T) { + ctx := t.Context() + sc := withVersions(sandboxConfig("update-remove-default", SandboxClassGvisor, gvisorAssets()), sandboxVersion("v2", SandboxClassGvisor, gvisorAssets())) + if err := k8sClient.Create(ctx, sc); err != nil { + t.Fatalf("Create() unexpected error: %v", err) + } + t.Cleanup(func() { _ = k8sClient.Delete(ctx, sc) }) + + // Dropping the version defaultVersion names must be denied. + sc.Spec.Versions = sc.Spec.Versions[1:] + err := k8sClient.Update(ctx, sc) + if err == nil { + t.Fatalf("Update() removing the default version succeeded, want denied") + } + if !strings.Contains(err.Error(), "defaultVersion must name an entry in versions") { + t.Errorf("Update() error = %q, want it to mention defaultVersion", err.Error()) + } + + // Repointing defaultVersion in the same update is allowed. + sc.Spec.DefaultVersion = "v2" + if err := k8sClient.Update(ctx, sc); err != nil { + t.Fatalf("Update() repointing defaultVersion: %v", err) + } +} + +// TestShippedSandboxConfigManifests guards the SandboxConfigs the install ships +// against the CRD schema and the shipped ValidatingAdmissionPolicy. +func TestShippedSandboxConfigManifests(t *testing.T) { + ctx := t.Context() + applyVAP(t, ctx) + + for _, path := range []string{ + "../../../manifests/ate-install/sandboxconfig-gvisor.yaml", + "../../../manifests/microvm/sandboxconfig-microvm.yaml.tmpl", + } { + t.Run(path, func(t *testing.T) { + raw, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read manifest: %v", err) + } + raw = []byte(strings.ReplaceAll(string(raw), "${BUCKET_NAME}", "test-bucket")) + sc := &SandboxConfig{} + if err := yaml.UnmarshalStrict(raw, sc); err != nil { + t.Fatalf("decode manifest: %v", err) + } + if sc.Annotations[SandboxConfigClassDefaultAnnotation] != "true" { + t.Errorf("manifest is not annotated %s=true", SandboxConfigClassDefaultAnnotation) + } + if err := k8sClient.Create(ctx, sc); err != nil { + t.Fatalf("Create() unexpected error: %v", err) + } + t.Cleanup(func() { _ = k8sClient.Delete(ctx, sc) }) + }) + } +} diff --git a/pkg/api/v1alpha1/workerpool_types.go b/pkg/api/v1alpha1/workerpool_types.go index 0582acde26..65d7686231 100644 --- a/pkg/api/v1alpha1/workerpool_types.go +++ b/pkg/api/v1alpha1/workerpool_types.go @@ -101,18 +101,54 @@ type WorkerPoolSpec struct { // +optional Template *WorkerPoolPodTemplate `json:"template,omitempty"` - // SandboxClass selects the sandbox runtime family for this pool, which drives - // the worker pod shape (KVM/vhost device mounts and node placement). The - // concrete binary is still selected by WorkerImage. Defaults to gvisor. - // The sandbox binaries themselves come from the SandboxConfig each - // ActorTemplate names (required). + // SandboxClasses lists the sandbox runtime families this pool runs. Exactly + // one entry is allowed today. + // + // +required + // +kubebuilder:validation:MinItems=1 + // +kubebuilder:validation:MaxItems=1 + // +listType=map + // +listMapKey=name + SandboxClasses []WorkerPoolSandboxClass `json:"sandboxClasses,omitempty"` +} + +// WorkerPoolSandboxClass is one sandbox runtime family a WorkerPool runs. +type WorkerPoolSandboxClass struct { + // Name selects the sandbox runtime family, which drives the worker pod + // shape (KVM/vhost device mounts and node placement). The concrete binary + // is still selected by WorkerImage. The sandbox binaries themselves come + // from the SandboxConfig each ActorTemplate names (required). // // See Also: TODOs in ActorTemplate SandboxClass // - // +optional + // +required // +kubebuilder:validation:Enum=gvisor;microvm - // +kubebuilder:default=gvisor - SandboxClass SandboxClass `json:"sandboxClass,omitempty"` + Name SandboxClass `json:"name,omitempty"` + + // ConfigRef names a cluster-scoped SandboxConfig for this sandbox class. + // The referenced config's SandboxClass must match Name. Not consumed yet. + // + // +optional + ConfigRef *SandboxConfigReference `json:"configRef,omitempty"` +} + +// SandboxConfigReference names a cluster-scoped SandboxConfig. +type SandboxConfigReference struct { + // Name is the SandboxConfig's metadata.name. + // + // +required + // +kubebuilder:validation:MinLength=1 + Name string `json:"name,omitempty"` +} + +// DefaultSandboxClass returns the sandbox class of the pool's first +// SandboxClasses entry, or "" if there is none. +// Currently SandboxClasses is always a list of length 1. +func (s *WorkerPoolSpec) DefaultSandboxClass() SandboxClass { + if len(s.SandboxClasses) == 0 { + return "" + } + return s.SandboxClasses[0].Name } type WorkerPoolStatus struct { diff --git a/pkg/api/v1alpha1/workerpool_validation_test.go b/pkg/api/v1alpha1/workerpool_validation_test.go index 617e68d787..99d47c889c 100644 --- a/pkg/api/v1alpha1/workerpool_validation_test.go +++ b/pkg/api/v1alpha1/workerpool_validation_test.go @@ -42,8 +42,9 @@ func TestWorkerPoolValidation(t *testing.T) { Namespace: "default", }, Spec: WorkerPoolSpec{ - Replicas: 1, - WorkerImage: "ateom:latest", + Replicas: 1, + WorkerImage: "ateom:latest", + SandboxClasses: []WorkerPoolSandboxClass{{Name: SandboxClassGvisor}}, }, } @@ -70,6 +71,60 @@ func TestWorkerPoolValidation(t *testing.T) { }, wantErr: true, errMsg: "spec.workerImage: Invalid value: \"\": spec.workerImage in body should be at least 1 chars long", + }, { + name: "missing sandboxClasses", + mutate: func(wp *WorkerPool) { + wp.Spec.SandboxClasses = nil + }, + wantErr: true, + errMsg: "spec.sandboxClasses: Required value", + }, { + name: "empty sandboxClasses", + mutate: func(wp *WorkerPool) { + wp.Spec.SandboxClasses = []WorkerPoolSandboxClass{} + }, + wantErr: true, + errMsg: "spec.sandboxClasses", + }, { + name: "sandboxClass with empty name", + mutate: func(wp *WorkerPool) { + wp.Spec.SandboxClasses = []WorkerPoolSandboxClass{{}} + }, + wantErr: true, + errMsg: "spec.sandboxClasses[0].name", + }, { + name: "more than one sandboxClass", + mutate: func(wp *WorkerPool) { + wp.Spec.SandboxClasses = []WorkerPoolSandboxClass{{Name: SandboxClassGvisor}, {Name: SandboxClassMicroVM}} + }, + wantErr: true, + errMsg: "spec.sandboxClasses: Too many", + }, { + name: "unknown sandboxClass", + mutate: func(wp *WorkerPool) { + wp.Spec.SandboxClasses = []WorkerPoolSandboxClass{{Name: "kata"}} + }, + wantErr: true, + errMsg: "spec.sandboxClasses[0].name: Unsupported value", + }, { + name: "sandboxClass with configRef", + mutate: func(wp *WorkerPool) { + wp.Spec.SandboxClasses = []WorkerPoolSandboxClass{{ + Name: SandboxClassGvisor, + ConfigRef: &SandboxConfigReference{Name: "gvisor-default"}, + }} + }, + wantErr: false, + }, { + name: "configRef with empty name", + mutate: func(wp *WorkerPool) { + wp.Spec.SandboxClasses = []WorkerPoolSandboxClass{{ + Name: SandboxClassGvisor, + ConfigRef: &SandboxConfigReference{}, + }} + }, + wantErr: true, + errMsg: "spec.sandboxClasses[0].configRef.name", }, { name: "valid template", mutate: func(wp *WorkerPool) { @@ -190,7 +245,7 @@ func TestWorkerPoolValidation(t *testing.T) { // Devices are no longer interpreted: the limit only places the pod. name: "extended resource on a pool is not interpreted", mutate: func(wp *WorkerPool) { - wp.Spec.SandboxClass = SandboxClassMicroVM + wp.Spec.SandboxClasses = []WorkerPoolSandboxClass{{Name: SandboxClassMicroVM}} wp.Spec.Template = gpuTemplate(corev1.ResourceList{gpuResourceName: resource.MustParse("1")}, nil) }, wantErr: false, @@ -227,8 +282,9 @@ func TestWorkerPoolReservedMetadataUpdate(t *testing.T) { Namespace: "default", }, Spec: WorkerPoolSpec{ - Replicas: 1, - WorkerImage: "example.com/ateom:latest", + Replicas: 1, + WorkerImage: "example.com/ateom:latest", + SandboxClasses: []WorkerPoolSandboxClass{{Name: SandboxClassGvisor}}, }, } if err := k8sClient.Create(ctx, wp); err != nil { diff --git a/pkg/api/v1alpha1/zz_generated.deepcopy.go b/pkg/api/v1alpha1/zz_generated.deepcopy.go index 98051918d8..3cacd34349 100644 --- a/pkg/api/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/api/v1alpha1/zz_generated.deepcopy.go @@ -189,9 +189,51 @@ func (in *SandboxConfigList) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SandboxConfigReference) DeepCopyInto(out *SandboxConfigReference) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SandboxConfigReference. +func (in *SandboxConfigReference) DeepCopy() *SandboxConfigReference { + if in == nil { + return nil + } + out := new(SandboxConfigReference) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SandboxConfigSpec) DeepCopyInto(out *SandboxConfigSpec) { *out = *in + if in.Versions != nil { + in, out := &in.Versions, &out.Versions + *out = make([]SandboxVersionConfig, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SandboxConfigSpec. +func (in *SandboxConfigSpec) DeepCopy() *SandboxConfigSpec { + if in == nil { + return nil + } + out := new(SandboxConfigSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SandboxVersionConfig) DeepCopyInto(out *SandboxVersionConfig) { + *out = *in + if in.State != nil { + in, out := &in.State, &out.State + *out = new(SandboxVersionState) + **out = **in + } if in.Assets != nil { in, out := &in.Assets, &out.Assets *out = make(map[string]map[string]AssetFile, len(*in)) @@ -212,12 +254,12 @@ func (in *SandboxConfigSpec) DeepCopyInto(out *SandboxConfigSpec) { } } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SandboxConfigSpec. -func (in *SandboxConfigSpec) DeepCopy() *SandboxConfigSpec { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SandboxVersionConfig. +func (in *SandboxVersionConfig) DeepCopy() *SandboxVersionConfig { if in == nil { return nil } - out := new(SandboxConfigSpec) + out := new(SandboxVersionConfig) in.DeepCopyInto(out) return out } @@ -339,6 +381,26 @@ func (in *WorkerPoolPodTemplate) DeepCopy() *WorkerPoolPodTemplate { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *WorkerPoolSandboxClass) DeepCopyInto(out *WorkerPoolSandboxClass) { + *out = *in + if in.ConfigRef != nil { + in, out := &in.ConfigRef, &out.ConfigRef + *out = new(SandboxConfigReference) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WorkerPoolSandboxClass. +func (in *WorkerPoolSandboxClass) DeepCopy() *WorkerPoolSandboxClass { + if in == nil { + return nil + } + out := new(WorkerPoolSandboxClass) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *WorkerPoolSpec) DeepCopyInto(out *WorkerPoolSpec) { *out = *in @@ -347,6 +409,13 @@ func (in *WorkerPoolSpec) DeepCopyInto(out *WorkerPoolSpec) { *out = new(WorkerPoolPodTemplate) (*in).DeepCopyInto(*out) } + if in.SandboxClasses != nil { + in, out := &in.SandboxClasses, &out.SandboxClasses + *out = make([]WorkerPoolSandboxClass, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WorkerPoolSpec. diff --git a/pkg/objectstorage/env.go b/pkg/objectstorage/env.go new file mode 100644 index 0000000000..537c8d3169 --- /dev/null +++ b/pkg/objectstorage/env.go @@ -0,0 +1,65 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package objectstorage + +import ( + "context" + "fmt" + "log/slog" + "os" + + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/s3" +) + +// UsesS3 reports whether ATE_STORAGE_BACKEND selects S3. Any other value +// selects GCS, the default. +func UsesS3() bool { + return os.Getenv("ATE_STORAGE_BACKEND") == "s3" +} + +// NewS3ClientFromEnv builds an S3 client from the standard AWS environment +// variables. AWS_S3_USE_PATH_STYLE=true selects path-style addressing. +func NewS3ClientFromEnv(ctx context.Context) (*s3.Client, error) { + slog.InfoContext(ctx, "Using S3 storage backend") + // Depends on the standard AWS environment variables, which have to be set + // on every pod that uses this client. + cfg, err := config.LoadDefaultConfig(ctx) + if err != nil { + return nil, fmt.Errorf("loading S3 config: %w", err) + } + return s3.NewFromConfig(cfg, func(o *s3.Options) { + if usePathStyle := os.Getenv("AWS_S3_USE_PATH_STYLE"); usePathStyle == "true" { + o.UsePathStyle = true + } + }), nil +} + +// NewFromEnv builds the ObjectStorage for the backend UsesS3 selects. +func NewFromEnv(ctx context.Context) (ObjectStorage, error) { + if UsesS3() { + client, err := NewS3ClientFromEnv(ctx) + if err != nil { + return nil, err + } + return NewS3Client(client), nil + } + // GCS is currently the default, TODO: we assume workload identity / ADC + client, err := NewGCSClient(ctx) + if err != nil { + return nil, fmt.Errorf("creating GCS client: %w", err) + } + return client, nil +} diff --git a/pkg/objectstorage/env_test.go b/pkg/objectstorage/env_test.go new file mode 100644 index 0000000000..5b9b9e7e4e --- /dev/null +++ b/pkg/objectstorage/env_test.go @@ -0,0 +1,69 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package objectstorage + +import ( + "context" + "path/filepath" + "testing" +) + +func TestNewFromEnv(t *testing.T) { + for _, tc := range []struct { + name string + backend string + pathStyle string + wantS3 bool + wantPathStyle bool + }{ + {name: "unset selects GCS", backend: ""}, + {name: "unknown value selects GCS", backend: "gcs"}, + {name: "s3", backend: "s3", wantS3: true}, + {name: "s3 with path style", backend: "s3", pathStyle: "true", wantS3: true, wantPathStyle: true}, + {name: "s3 path style needs exactly true", backend: "s3", pathStyle: "1", wantS3: true}, + } { + t.Run(tc.name, func(t *testing.T) { + // GCS: the emulator host makes storage.NewClient skip credential lookup. + t.Setenv("STORAGE_EMULATOR_HOST", "127.0.0.1:1") + // S3: keep the developer's AWS config out of the test. + missing := filepath.Join(t.TempDir(), "missing") + t.Setenv("AWS_CONFIG_FILE", missing) + t.Setenv("AWS_SHARED_CREDENTIALS_FILE", missing) + t.Setenv("AWS_REGION", "us-east-1") + t.Setenv("ATE_STORAGE_BACKEND", tc.backend) + t.Setenv("AWS_S3_USE_PATH_STYLE", tc.pathStyle) + + got, err := NewFromEnv(context.Background()) + if err != nil { + t.Fatalf("NewFromEnv: %v", err) + } + switch c := got.(type) { + case *s3Client: + if !tc.wantS3 { + t.Fatalf("NewFromEnv returned an S3 client, want GCS") + } + if got := c.client.Options().UsePathStyle; got != tc.wantPathStyle { + t.Errorf("UsePathStyle = %v, want %v", got, tc.wantPathStyle) + } + case *gcsClient: + if tc.wantS3 { + t.Fatalf("NewFromEnv returned a GCS client, want S3") + } + default: + t.Fatalf("NewFromEnv returned %T", got) + } + }) + } +} diff --git a/pkg/proto/ateapipb/AGENTS.md b/pkg/proto/ateapipb/AGENTS.md index e34cbbcdef..3da3ae6db7 100644 --- a/pkg/proto/ateapipb/AGENTS.md +++ b/pkg/proto/ateapipb/AGENTS.md @@ -3,5 +3,6 @@ `ateapi.proto` defines Substrate's public API. - See `docs/api-style-guide.md` for the API style guide. +- See `docs/resource-model.md` for the resources and subresources the API exposes. Update it when you add, remove, or re-parent a resource. - See `docs/api-validation.md` for how to use declarative validation (DV) tags. - `*.pb.go` files are generated. Never edit them by hand. After changing `ateapi.proto`, run `hack/update/codegen.sh`, which also regenerates the DV code in `cmd/ateapi/internal/apivalidation`. diff --git a/pkg/proto/ateapipb/ateapi.pb.go b/pkg/proto/ateapipb/ateapi.pb.go index c73e198052..6f4145c96b 100644 --- a/pkg/proto/ateapipb/ateapi.pb.go +++ b/pkg/proto/ateapipb/ateapi.pb.go @@ -36,58 +36,225 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -type SnapshotContentScope int32 +// SnapshotOwner indicates the owner of a snapshot. +type SnapshotOwner int32 const ( - // Unspecified snapshot content scope. - SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_UNSPECIFIED SnapshotContentScope = 0 - // Captures process memory, root filesystem changes, and durable data. - SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL SnapshotContentScope = 1 - // Captures durable data without process memory or root filesystem changes. - SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA SnapshotContentScope = 2 // Keep this in sync with the maximums on fields of this type. + SnapshotOwner_SNAPSHOT_OWNER_UNSPECIFIED SnapshotOwner = 0 + // Snapshot captured directly from this Actor. + SnapshotOwner_SNAPSHOT_OWNER_ACTOR SnapshotOwner = 1 + // Snapshot owned by a Tag (also includes Golden Snapshot) + SnapshotOwner_SNAPSHOT_OWNER_TAG SnapshotOwner = 2 // Keep this in sync with Snapshot.owner's maximum. ) -// Enum value maps for SnapshotContentScope. +// Enum value maps for SnapshotOwner. var ( - SnapshotContentScope_name = map[int32]string{ - 0: "SNAPSHOT_CONTENT_SCOPE_UNSPECIFIED", - 1: "SNAPSHOT_CONTENT_SCOPE_FULL", - 2: "SNAPSHOT_CONTENT_SCOPE_DATA", + SnapshotOwner_name = map[int32]string{ + 0: "SNAPSHOT_OWNER_UNSPECIFIED", + 1: "SNAPSHOT_OWNER_ACTOR", + 2: "SNAPSHOT_OWNER_TAG", } - SnapshotContentScope_value = map[string]int32{ - "SNAPSHOT_CONTENT_SCOPE_UNSPECIFIED": 0, - "SNAPSHOT_CONTENT_SCOPE_FULL": 1, - "SNAPSHOT_CONTENT_SCOPE_DATA": 2, + SnapshotOwner_value = map[string]int32{ + "SNAPSHOT_OWNER_UNSPECIFIED": 0, + "SNAPSHOT_OWNER_ACTOR": 1, + "SNAPSHOT_OWNER_TAG": 2, } ) -func (x SnapshotContentScope) Enum() *SnapshotContentScope { - p := new(SnapshotContentScope) +func (x SnapshotOwner) Enum() *SnapshotOwner { + p := new(SnapshotOwner) *p = x return p } -func (x SnapshotContentScope) String() string { +func (x SnapshotOwner) String() string { return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) } -func (SnapshotContentScope) Descriptor() protoreflect.EnumDescriptor { +func (SnapshotOwner) Descriptor() protoreflect.EnumDescriptor { return file_ateapi_proto_enumTypes[0].Descriptor() } -func (SnapshotContentScope) Type() protoreflect.EnumType { +func (SnapshotOwner) Type() protoreflect.EnumType { return &file_ateapi_proto_enumTypes[0] } -func (x SnapshotContentScope) Number() protoreflect.EnumNumber { +func (x SnapshotOwner) Number() protoreflect.EnumNumber { return protoreflect.EnumNumber(x) } -// Deprecated: Use SnapshotContentScope.Descriptor instead. -func (SnapshotContentScope) EnumDescriptor() ([]byte, []int) { +// Deprecated: Use SnapshotOwner.Descriptor instead. +func (SnapshotOwner) EnumDescriptor() ([]byte, []int) { return file_ateapi_proto_rawDescGZIP(), []int{0} } +// SnapshotDurability indicates the durability level of the snapshot. +// TODO: potentially rename this to survivability level. Is it at the actor or snapshot level? +type SnapshotDurability int32 + +const ( + SnapshotDurability_SNAPSHOT_DURABILITY_UNSPECIFIED SnapshotDurability = 0 + // Node-local checkpoint on the worker node + // Survives node reboot. + SnapshotDurability_SNAPSHOT_DURABILITY_LOCAL SnapshotDurability = 1 + // Durable snapshot in external storage. + // Survives node crash and deletion. + SnapshotDurability_SNAPSHOT_DURABILITY_DURABLE SnapshotDurability = 2 // Keep this in sync with SnapshotStorage.durability's maximum. +) + +// Enum value maps for SnapshotDurability. +var ( + SnapshotDurability_name = map[int32]string{ + 0: "SNAPSHOT_DURABILITY_UNSPECIFIED", + 1: "SNAPSHOT_DURABILITY_LOCAL", + 2: "SNAPSHOT_DURABILITY_DURABLE", + } + SnapshotDurability_value = map[string]int32{ + "SNAPSHOT_DURABILITY_UNSPECIFIED": 0, + "SNAPSHOT_DURABILITY_LOCAL": 1, + "SNAPSHOT_DURABILITY_DURABLE": 2, + } +) + +func (x SnapshotDurability) Enum() *SnapshotDurability { + p := new(SnapshotDurability) + *p = x + return p +} + +func (x SnapshotDurability) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (SnapshotDurability) Descriptor() protoreflect.EnumDescriptor { + return file_ateapi_proto_enumTypes[1].Descriptor() +} + +func (SnapshotDurability) Type() protoreflect.EnumType { + return &file_ateapi_proto_enumTypes[1] +} + +func (x SnapshotDurability) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use SnapshotDurability.Descriptor instead. +func (SnapshotDurability) EnumDescriptor() ([]byte, []int) { + return file_ateapi_proto_rawDescGZIP(), []int{1} +} + +// SnapshotStorageStatus indicates the lifecycle stage of a snapshot storage entry. +type SnapshotStorageStatus int32 + +const ( + SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_UNSPECIFIED SnapshotStorageStatus = 0 + // Snapshot creation or promotion is in progress. + SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_IN_PROGRESS SnapshotStorageStatus = 1 + // Snapshot is complete and ready for use. + SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED SnapshotStorageStatus = 2 // Keep this in sync with SnapshotStorage.status's maximum. +) + +// Enum value maps for SnapshotStorageStatus. +var ( + SnapshotStorageStatus_name = map[int32]string{ + 0: "SNAPSHOT_STORAGE_STATUS_UNSPECIFIED", + 1: "SNAPSHOT_STORAGE_STATUS_IN_PROGRESS", + 2: "SNAPSHOT_STORAGE_STATUS_COMPLETED", + } + SnapshotStorageStatus_value = map[string]int32{ + "SNAPSHOT_STORAGE_STATUS_UNSPECIFIED": 0, + "SNAPSHOT_STORAGE_STATUS_IN_PROGRESS": 1, + "SNAPSHOT_STORAGE_STATUS_COMPLETED": 2, + } +) + +func (x SnapshotStorageStatus) Enum() *SnapshotStorageStatus { + p := new(SnapshotStorageStatus) + *p = x + return p +} + +func (x SnapshotStorageStatus) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (SnapshotStorageStatus) Descriptor() protoreflect.EnumDescriptor { + return file_ateapi_proto_enumTypes[2].Descriptor() +} + +func (SnapshotStorageStatus) Type() protoreflect.EnumType { + return &file_ateapi_proto_enumTypes[2] +} + +func (x SnapshotStorageStatus) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use SnapshotStorageStatus.Descriptor instead. +func (SnapshotStorageStatus) EnumDescriptor() ([]byte, []int) { + return file_ateapi_proto_rawDescGZIP(), []int{2} +} + +// SnapshotFidelity orders the state layers a snapshot can hold, lowest to +// highest. Each level includes the ones below it. +type SnapshotFidelity int32 + +const ( + SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED SnapshotFidelity = 0 + // Durable volumes only. The actor resumes by cold-booting its containers + // from the OCI image with the volumes restored. + SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES SnapshotFidelity = 1 + // Volumes plus the root filesystem changes made since boot. Not yet + // supported by any sandbox runtime; templates may not request it. + SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS SnapshotFidelity = 2 + // Volumes, root filesystem changes, and process memory. The actor resumes + // where it left off. + SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY SnapshotFidelity = 3 // Keep this in sync with the maximums on fields of this type. +) + +// Enum value maps for SnapshotFidelity. +var ( + SnapshotFidelity_name = map[int32]string{ + 0: "SNAPSHOT_FIDELITY_UNSPECIFIED", + 1: "SNAPSHOT_FIDELITY_VOLUMES", + 2: "SNAPSHOT_FIDELITY_ROOTFS", + 3: "SNAPSHOT_FIDELITY_MEMORY", + } + SnapshotFidelity_value = map[string]int32{ + "SNAPSHOT_FIDELITY_UNSPECIFIED": 0, + "SNAPSHOT_FIDELITY_VOLUMES": 1, + "SNAPSHOT_FIDELITY_ROOTFS": 2, + "SNAPSHOT_FIDELITY_MEMORY": 3, + } +) + +func (x SnapshotFidelity) Enum() *SnapshotFidelity { + p := new(SnapshotFidelity) + *p = x + return p +} + +func (x SnapshotFidelity) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (SnapshotFidelity) Descriptor() protoreflect.EnumDescriptor { + return file_ateapi_proto_enumTypes[3].Descriptor() +} + +func (SnapshotFidelity) Type() protoreflect.EnumType { + return &file_ateapi_proto_enumTypes[3] +} + +func (x SnapshotFidelity) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use SnapshotFidelity.Descriptor instead. +func (SnapshotFidelity) EnumDescriptor() ([]byte, []int) { + return file_ateapi_proto_rawDescGZIP(), []int{3} +} + type TagScope int32 const ( @@ -125,11 +292,11 @@ func (x TagScope) String() string { } func (TagScope) Descriptor() protoreflect.EnumDescriptor { - return file_ateapi_proto_enumTypes[1].Descriptor() + return file_ateapi_proto_enumTypes[4].Descriptor() } func (TagScope) Type() protoreflect.EnumType { - return &file_ateapi_proto_enumTypes[1] + return &file_ateapi_proto_enumTypes[4] } func (x TagScope) Number() protoreflect.EnumNumber { @@ -138,7 +305,7 @@ func (x TagScope) Number() protoreflect.EnumNumber { // Deprecated: Use TagScope.Descriptor instead. func (TagScope) EnumDescriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{1} + return file_ateapi_proto_rawDescGZIP(), []int{4} } type ActorState int32 @@ -195,11 +362,11 @@ func (x ActorState) String() string { } func (ActorState) Descriptor() protoreflect.EnumDescriptor { - return file_ateapi_proto_enumTypes[2].Descriptor() + return file_ateapi_proto_enumTypes[5].Descriptor() } func (ActorState) Type() protoreflect.EnumType { - return &file_ateapi_proto_enumTypes[2] + return &file_ateapi_proto_enumTypes[5] } func (x ActorState) Number() protoreflect.EnumNumber { @@ -208,7 +375,7 @@ func (x ActorState) Number() protoreflect.EnumNumber { // Deprecated: Use ActorState.Descriptor instead. func (ActorState) EnumDescriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{2} + return file_ateapi_proto_rawDescGZIP(), []int{5} } // SandboxClass selects the sandbox runtime family. Snapshots are not portable @@ -246,11 +413,11 @@ func (x SandboxClass) String() string { } func (SandboxClass) Descriptor() protoreflect.EnumDescriptor { - return file_ateapi_proto_enumTypes[3].Descriptor() + return file_ateapi_proto_enumTypes[6].Descriptor() } func (SandboxClass) Type() protoreflect.EnumType { - return &file_ateapi_proto_enumTypes[3] + return &file_ateapi_proto_enumTypes[6] } func (x SandboxClass) Number() protoreflect.EnumNumber { @@ -259,7 +426,7 @@ func (x SandboxClass) Number() protoreflect.EnumNumber { // Deprecated: Use SandboxClass.Descriptor instead. func (SandboxClass) EnumDescriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{3} + return file_ateapi_proto_rawDescGZIP(), []int{6} } // ActorMetadataField selects one identity field of the actor. @@ -299,11 +466,11 @@ func (x ActorMetadataField) String() string { } func (ActorMetadataField) Descriptor() protoreflect.EnumDescriptor { - return file_ateapi_proto_enumTypes[4].Descriptor() + return file_ateapi_proto_enumTypes[7].Descriptor() } func (ActorMetadataField) Type() protoreflect.EnumType { - return &file_ateapi_proto_enumTypes[4] + return &file_ateapi_proto_enumTypes[7] } func (x ActorMetadataField) Number() protoreflect.EnumNumber { @@ -312,7 +479,7 @@ func (x ActorMetadataField) Number() protoreflect.EnumNumber { // Deprecated: Use ActorMetadataField.Descriptor instead. func (ActorMetadataField) EnumDescriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{4} + return file_ateapi_proto_rawDescGZIP(), []int{7} } type WorkerState int32 @@ -350,11 +517,11 @@ func (x WorkerState) String() string { } func (WorkerState) Descriptor() protoreflect.EnumDescriptor { - return file_ateapi_proto_enumTypes[5].Descriptor() + return file_ateapi_proto_enumTypes[8].Descriptor() } func (WorkerState) Type() protoreflect.EnumType { - return &file_ateapi_proto_enumTypes[5] + return &file_ateapi_proto_enumTypes[8] } func (x WorkerState) Number() protoreflect.EnumNumber { @@ -363,7 +530,7 @@ func (x WorkerState) Number() protoreflect.EnumNumber { // Deprecated: Use WorkerState.Descriptor instead. func (WorkerState) EnumDescriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{5} + return file_ateapi_proto_rawDescGZIP(), []int{8} } type ExternalVolume_Status int32 @@ -405,11 +572,11 @@ func (x ExternalVolume_Status) String() string { } func (ExternalVolume_Status) Descriptor() protoreflect.EnumDescriptor { - return file_ateapi_proto_enumTypes[6].Descriptor() + return file_ateapi_proto_enumTypes[9].Descriptor() } func (ExternalVolume_Status) Type() protoreflect.EnumType { - return &file_ateapi_proto_enumTypes[6] + return &file_ateapi_proto_enumTypes[9] } func (x ExternalVolume_Status) Number() protoreflect.EnumNumber { @@ -418,47 +585,35 @@ func (x ExternalVolume_Status) Number() protoreflect.EnumNumber { // Deprecated: Use ExternalVolume_Status.Descriptor instead. func (ExternalVolume_Status) EnumDescriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{4, 0} + return file_ateapi_proto_rawDescGZIP(), []int{6, 0} } -// ExternalSnapshot addresses a snapshot held in object storage. -type ExternalSnapshot struct { +// ObjectSnapshot addresses a snapshot held in object storage. +type ObjectSnapshot struct { state protoimpl.MessageState `protogen:"open.v1"` // snapshot_uri addresses the snapshot's prefix in object storage. // // +k8s:required // +k8s:maxLength=2048 # the template's storage_location bound plus the owner prefix and snapshot name - SnapshotUri string `protobuf:"bytes,1,opt,name=snapshot_uri,json=snapshotUri,proto3" json:"snapshot_uri,omitempty"` - // content_scope is what the snapshot captured. - // - // +k8s:optional - // +k8s:minimum=1 - // +k8s:maximum=2 # keep this in sync with the SnapshotContentScope enum - ContentScope SnapshotContentScope `protobuf:"varint,2,opt,name=content_scope,json=contentScope,proto3,enum=ateapi.SnapshotContentScope" json:"content_scope,omitempty"` - // UID of the ActorTemplate whose sandbox this snapshot's guest state was - // captured from. - // - // +k8s:optional - // +k8s:format=k8s-uuid - ActorTemplateUid string `protobuf:"bytes,3,opt,name=actor_template_uid,json=actorTemplateUid,proto3" json:"actor_template_uid,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + SnapshotUri string `protobuf:"bytes,1,opt,name=snapshot_uri,json=snapshotUri,proto3" json:"snapshot_uri,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } -func (x *ExternalSnapshot) Reset() { - *x = ExternalSnapshot{} +func (x *ObjectSnapshot) Reset() { + *x = ObjectSnapshot{} mi := &file_ateapi_proto_msgTypes[0] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } -func (x *ExternalSnapshot) String() string { +func (x *ObjectSnapshot) String() string { return protoimpl.X.MessageStringOf(x) } -func (*ExternalSnapshot) ProtoMessage() {} +func (*ObjectSnapshot) ProtoMessage() {} -func (x *ExternalSnapshot) ProtoReflect() protoreflect.Message { +func (x *ObjectSnapshot) ProtoReflect() protoreflect.Message { mi := &file_ateapi_proto_msgTypes[0] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -470,48 +625,29 @@ func (x *ExternalSnapshot) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use ExternalSnapshot.ProtoReflect.Descriptor instead. -func (*ExternalSnapshot) Descriptor() ([]byte, []int) { +// Deprecated: Use ObjectSnapshot.ProtoReflect.Descriptor instead. +func (*ObjectSnapshot) Descriptor() ([]byte, []int) { return file_ateapi_proto_rawDescGZIP(), []int{0} } -func (x *ExternalSnapshot) GetSnapshotUri() string { +func (x *ObjectSnapshot) GetSnapshotUri() string { if x != nil { return x.SnapshotUri } return "" } -func (x *ExternalSnapshot) GetContentScope() SnapshotContentScope { - if x != nil { - return x.ContentScope - } - return SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_UNSPECIFIED -} - -func (x *ExternalSnapshot) GetActorTemplateUid() string { - if x != nil { - return x.ActorTemplateUid - } - return "" -} - // LocalSnapshot records information about a node-local snapshot. type LocalSnapshot struct { state protoimpl.MessageState `protogen:"open.v1"` // The name of the local checkpoint on the node. Checkpoint names are // server-generated UUIDs, but any resource name is valid here. // - // +k8s:optional - // +k8s:format=k8s-short-name - SnapshotName string `protobuf:"bytes,1,opt,name=snapshot_name,json=snapshotName,proto3" json:"snapshot_name,omitempty"` - // Scope the pause checkpoint captured (the template's onCommit at pause - // time). + // TODO: consider removing this since it's the same as the snapshot_id // - // +k8s:optional - // +k8s:minimum=1 - // +k8s:maximum=2 # keep this in sync with the SnapshotContentScope enum - ContentScope SnapshotContentScope `protobuf:"varint,2,opt,name=content_scope,json=contentScope,proto3,enum=ateapi.SnapshotContentScope" json:"content_scope,omitempty"` + // +k8s:required + // +k8s:format=k8s-short-name + SnapshotName string `protobuf:"bytes,1,opt,name=snapshot_name,json=snapshotName,proto3" json:"snapshot_name,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -553,11 +689,198 @@ func (x *LocalSnapshot) GetSnapshotName() string { return "" } -func (x *LocalSnapshot) GetContentScope() SnapshotContentScope { +// SnapshotStorage records the physical storage representation of a Snapshot. +type SnapshotStorage struct { + state protoimpl.MessageState `protogen:"open.v1"` + // durability is the durability tier of this storage entry. Only + // SNAPSHOT_DURABILITY_LOCAL and SNAPSHOT_DURABILITY_DURABLE are allowed. + // + // +k8s:required + // +k8s:minimum=1 + // +k8s:maximum=2 # keep this in sync with the SnapshotDurability enum + Durability SnapshotDurability `protobuf:"varint,1,opt,name=durability,proto3,enum=ateapi.SnapshotDurability" json:"durability,omitempty"` + // status is the lifecycle status of this storage representation. + // + // +k8s:required + // +k8s:minimum=1 + // +k8s:maximum=2 # keep this in sync with the SnapshotStorageStatus enum + Status SnapshotStorageStatus `protobuf:"varint,2,opt,name=status,proto3,enum=ateapi.SnapshotStorageStatus" json:"status,omitempty"` + // fidelity is what the snapshot holds. + // + // +k8s:required + // +k8s:minimum=1 + // +k8s:maximum=3 # keep this in sync with the SnapshotFidelity enum + Fidelity SnapshotFidelity `protobuf:"varint,3,opt,name=fidelity,proto3,enum=ateapi.SnapshotFidelity" json:"fidelity,omitempty"` + // Exactly one of local or object must be set. + // + // local snapshot is stored on the node where the actor was suspended. + // + // +k8s:optional + // +k8s:unionMember + Local *LocalSnapshot `protobuf:"bytes,4,opt,name=local,proto3" json:"local,omitempty"` + // object snapshot is stored in object storage. + // + // +k8s:optional + // +k8s:unionMember + Object *ObjectSnapshot `protobuf:"bytes,5,opt,name=object,proto3" json:"object,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SnapshotStorage) Reset() { + *x = SnapshotStorage{} + mi := &file_ateapi_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SnapshotStorage) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SnapshotStorage) ProtoMessage() {} + +func (x *SnapshotStorage) ProtoReflect() protoreflect.Message { + mi := &file_ateapi_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SnapshotStorage.ProtoReflect.Descriptor instead. +func (*SnapshotStorage) Descriptor() ([]byte, []int) { + return file_ateapi_proto_rawDescGZIP(), []int{2} +} + +func (x *SnapshotStorage) GetDurability() SnapshotDurability { + if x != nil { + return x.Durability + } + return SnapshotDurability_SNAPSHOT_DURABILITY_UNSPECIFIED +} + +func (x *SnapshotStorage) GetStatus() SnapshotStorageStatus { + if x != nil { + return x.Status + } + return SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_UNSPECIFIED +} + +func (x *SnapshotStorage) GetFidelity() SnapshotFidelity { + if x != nil { + return x.Fidelity + } + return SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED +} + +func (x *SnapshotStorage) GetLocal() *LocalSnapshot { + if x != nil { + return x.Local + } + return nil +} + +func (x *SnapshotStorage) GetObject() *ObjectSnapshot { + if x != nil { + return x.Object + } + return nil +} + +// Snapshot represents one generation of an Actor's or Tag's snapshot state. +type Snapshot struct { + state protoimpl.MessageState `protogen:"open.v1"` + // generation is the monotonically increasing generation number of this + // snapshot, starting at 1. + // + // +k8s:required + // +k8s:minimum=1 + Generation int32 `protobuf:"varint,1,opt,name=generation,proto3" json:"generation,omitempty"` + // owner identifies the origin and ownership of this snapshot. + // + // +k8s:required + // +k8s:minimum=1 + // +k8s:maximum=2 # keep this in sync with the SnapshotOwner enum + Owner SnapshotOwner `protobuf:"varint,2,opt,name=owner,proto3,enum=ateapi.SnapshotOwner" json:"owner,omitempty"` + // UID of the ActorTemplate whose sandbox this snapshot's guest state was + // captured from. + // + // +k8s:optional + // +k8s:format=k8s-uuid + ActorTemplateUid string `protobuf:"bytes,3,opt,name=actor_template_uid,json=actorTemplateUid,proto3" json:"actor_template_uid,omitempty"` + // storage lists the storage representations of this snapshot by durability. + // Only SNAPSHOT_DURABILITY_LOCAL and SNAPSHOT_DURABILITY_DURABLE are allowed. + // + // +k8s:required + // +k8s:maxItems=2 + // +k8s:listType=map + // +k8s:listMapKey=durability + Storage []*SnapshotStorage `protobuf:"bytes,4,rep,name=storage,proto3" json:"storage,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Snapshot) Reset() { + *x = Snapshot{} + mi := &file_ateapi_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Snapshot) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Snapshot) ProtoMessage() {} + +func (x *Snapshot) ProtoReflect() protoreflect.Message { + mi := &file_ateapi_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Snapshot.ProtoReflect.Descriptor instead. +func (*Snapshot) Descriptor() ([]byte, []int) { + return file_ateapi_proto_rawDescGZIP(), []int{3} +} + +func (x *Snapshot) GetGeneration() int32 { + if x != nil { + return x.Generation + } + return 0 +} + +func (x *Snapshot) GetOwner() SnapshotOwner { + if x != nil { + return x.Owner + } + return SnapshotOwner_SNAPSHOT_OWNER_UNSPECIFIED +} + +func (x *Snapshot) GetActorTemplateUid() string { + if x != nil { + return x.ActorTemplateUid + } + return "" +} + +func (x *Snapshot) GetStorage() []*SnapshotStorage { if x != nil { - return x.ContentScope + return x.Storage } - return SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_UNSPECIFIED + return nil } // Selector matches worker pools by label. @@ -579,7 +902,7 @@ type Selector struct { func (x *Selector) Reset() { *x = Selector{} - mi := &file_ateapi_proto_msgTypes[2] + mi := &file_ateapi_proto_msgTypes[4] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -591,7 +914,7 @@ func (x *Selector) String() string { func (*Selector) ProtoMessage() {} func (x *Selector) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[2] + mi := &file_ateapi_proto_msgTypes[4] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -604,7 +927,7 @@ func (x *Selector) ProtoReflect() protoreflect.Message { // Deprecated: Use Selector.ProtoReflect.Descriptor instead. func (*Selector) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{2} + return file_ateapi_proto_rawDescGZIP(), []int{4} } func (x *Selector) GetMatchLabels() map[string]string { @@ -675,7 +998,7 @@ type ResourceMetadata struct { func (x *ResourceMetadata) Reset() { *x = ResourceMetadata{} - mi := &file_ateapi_proto_msgTypes[3] + mi := &file_ateapi_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -687,7 +1010,7 @@ func (x *ResourceMetadata) String() string { func (*ResourceMetadata) ProtoMessage() {} func (x *ResourceMetadata) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[3] + mi := &file_ateapi_proto_msgTypes[5] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -700,7 +1023,7 @@ func (x *ResourceMetadata) ProtoReflect() protoreflect.Message { // Deprecated: Use ResourceMetadata.ProtoReflect.Descriptor instead. func (*ResourceMetadata) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{3} + return file_ateapi_proto_rawDescGZIP(), []int{5} } func (x *ResourceMetadata) GetAtespace() string { @@ -755,7 +1078,7 @@ type ExternalVolume struct { // +k8s:update=NoModify // +k8s:update=NoUnset # set-once; immutable would reject the create // ratchet's nil->set when the final object is validated as an update - VolumeName string `protobuf:"bytes,1,opt,name=volume_name,json=volumeName,proto3" json:"volume_name,omitempty"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` // The globally unique volume_id returned from the storage system. // This will be initially empty during volume creation. Its format is the // storage system's own, so it is only bounded and checked for control @@ -776,7 +1099,7 @@ type ExternalVolume struct { // +k8s:required // +k8s:maxLength=253 // +k8s:update=NoModify - // +k8s:update=NoUnset # set-once, like volume_name above + // +k8s:update=NoUnset # set-once, like name above // +k8s:customValidation VolumeType string `protobuf:"bytes,3,opt,name=volume_type,json=volumeType,proto3" json:"volume_type,omitempty"` // +k8s:optional @@ -799,7 +1122,7 @@ type ExternalVolume struct { func (x *ExternalVolume) Reset() { *x = ExternalVolume{} - mi := &file_ateapi_proto_msgTypes[4] + mi := &file_ateapi_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -811,7 +1134,7 @@ func (x *ExternalVolume) String() string { func (*ExternalVolume) ProtoMessage() {} func (x *ExternalVolume) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[4] + mi := &file_ateapi_proto_msgTypes[6] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -824,12 +1147,12 @@ func (x *ExternalVolume) ProtoReflect() protoreflect.Message { // Deprecated: Use ExternalVolume.ProtoReflect.Descriptor instead. func (*ExternalVolume) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{4} + return file_ateapi_proto_rawDescGZIP(), []int{6} } -func (x *ExternalVolume) GetVolumeName() string { +func (x *ExternalVolume) GetName() string { if x != nil { - return x.VolumeName + return x.Name } return "" } @@ -900,7 +1223,7 @@ type Actor struct { func (x *Actor) Reset() { *x = Actor{} - mi := &file_ateapi_proto_msgTypes[5] + mi := &file_ateapi_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -912,7 +1235,7 @@ func (x *Actor) String() string { func (*Actor) ProtoMessage() {} func (x *Actor) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[5] + mi := &file_ateapi_proto_msgTypes[7] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -925,7 +1248,7 @@ func (x *Actor) ProtoReflect() protoreflect.Message { // Deprecated: Use Actor.ProtoReflect.Descriptor instead. func (*Actor) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{5} + return file_ateapi_proto_rawDescGZIP(), []int{7} } func (x *Actor) GetMetadata() *ResourceMetadata { @@ -1003,7 +1326,7 @@ type EgressPolicy struct { func (x *EgressPolicy) Reset() { *x = EgressPolicy{} - mi := &file_ateapi_proto_msgTypes[6] + mi := &file_ateapi_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1015,7 +1338,7 @@ func (x *EgressPolicy) String() string { func (*EgressPolicy) ProtoMessage() {} func (x *EgressPolicy) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[6] + mi := &file_ateapi_proto_msgTypes[8] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1028,7 +1351,7 @@ func (x *EgressPolicy) ProtoReflect() protoreflect.Message { // Deprecated: Use EgressPolicy.ProtoReflect.Descriptor instead. func (*EgressPolicy) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{6} + return file_ateapi_proto_rawDescGZIP(), []int{8} } func (x *EgressPolicy) GetMetadata() *ResourceMetadata { @@ -1078,7 +1401,7 @@ type EgressRule struct { func (x *EgressRule) Reset() { *x = EgressRule{} - mi := &file_ateapi_proto_msgTypes[7] + mi := &file_ateapi_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1090,7 +1413,7 @@ func (x *EgressRule) String() string { func (*EgressRule) ProtoMessage() {} func (x *EgressRule) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[7] + mi := &file_ateapi_proto_msgTypes[9] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1103,7 +1426,7 @@ func (x *EgressRule) ProtoReflect() protoreflect.Message { // Deprecated: Use EgressRule.ProtoReflect.Descriptor instead. func (*EgressRule) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{7} + return file_ateapi_proto_rawDescGZIP(), []int{9} } func (x *EgressRule) GetHttp() *HTTPRule { @@ -1171,7 +1494,7 @@ type HTTPRule struct { func (x *HTTPRule) Reset() { *x = HTTPRule{} - mi := &file_ateapi_proto_msgTypes[8] + mi := &file_ateapi_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1183,7 +1506,7 @@ func (x *HTTPRule) String() string { func (*HTTPRule) ProtoMessage() {} func (x *HTTPRule) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[8] + mi := &file_ateapi_proto_msgTypes[10] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1196,7 +1519,7 @@ func (x *HTTPRule) ProtoReflect() protoreflect.Message { // Deprecated: Use HTTPRule.ProtoReflect.Descriptor instead. func (*HTTPRule) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{8} + return file_ateapi_proto_rawDescGZIP(), []int{10} } func (x *HTTPRule) GetHostnames() []string { @@ -1256,7 +1579,7 @@ type HTTPSRule struct { func (x *HTTPSRule) Reset() { *x = HTTPSRule{} - mi := &file_ateapi_proto_msgTypes[9] + mi := &file_ateapi_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1268,7 +1591,7 @@ func (x *HTTPSRule) String() string { func (*HTTPSRule) ProtoMessage() {} func (x *HTTPSRule) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[9] + mi := &file_ateapi_proto_msgTypes[11] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1281,7 +1604,7 @@ func (x *HTTPSRule) ProtoReflect() protoreflect.Message { // Deprecated: Use HTTPSRule.ProtoReflect.Descriptor instead. func (*HTTPSRule) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{9} + return file_ateapi_proto_rawDescGZIP(), []int{11} } func (x *HTTPSRule) GetHostnames() []string { @@ -1332,7 +1655,7 @@ type TLSPassthroughRule struct { func (x *TLSPassthroughRule) Reset() { *x = TLSPassthroughRule{} - mi := &file_ateapi_proto_msgTypes[10] + mi := &file_ateapi_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1344,7 +1667,7 @@ func (x *TLSPassthroughRule) String() string { func (*TLSPassthroughRule) ProtoMessage() {} func (x *TLSPassthroughRule) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[10] + mi := &file_ateapi_proto_msgTypes[12] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1357,7 +1680,7 @@ func (x *TLSPassthroughRule) ProtoReflect() protoreflect.Message { // Deprecated: Use TLSPassthroughRule.ProtoReflect.Descriptor instead. func (*TLSPassthroughRule) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{10} + return file_ateapi_proto_rawDescGZIP(), []int{12} } func (x *TLSPassthroughRule) GetHostnames() []string { @@ -1398,7 +1721,7 @@ type Ports struct { func (x *Ports) Reset() { *x = Ports{} - mi := &file_ateapi_proto_msgTypes[11] + mi := &file_ateapi_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1410,7 +1733,7 @@ func (x *Ports) String() string { func (*Ports) ProtoMessage() {} func (x *Ports) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[11] + mi := &file_ateapi_proto_msgTypes[13] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1423,7 +1746,7 @@ func (x *Ports) ProtoReflect() protoreflect.Message { // Deprecated: Use Ports.ProtoReflect.Descriptor instead. func (*Ports) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{11} + return file_ateapi_proto_rawDescGZIP(), []int{13} } func (x *Ports) GetAll() *AllPorts { @@ -1449,7 +1772,7 @@ type AllPorts struct { func (x *AllPorts) Reset() { *x = AllPorts{} - mi := &file_ateapi_proto_msgTypes[12] + mi := &file_ateapi_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1461,7 +1784,7 @@ func (x *AllPorts) String() string { func (*AllPorts) ProtoMessage() {} func (x *AllPorts) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[12] + mi := &file_ateapi_proto_msgTypes[14] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1474,7 +1797,7 @@ func (x *AllPorts) ProtoReflect() protoreflect.Message { // Deprecated: Use AllPorts.ProtoReflect.Descriptor instead. func (*AllPorts) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{12} + return file_ateapi_proto_rawDescGZIP(), []int{14} } // HttpRuleEffects contains effects applied by a matching HTTP or HTTPS rule. @@ -1508,7 +1831,7 @@ type HttpRuleEffects struct { func (x *HttpRuleEffects) Reset() { *x = HttpRuleEffects{} - mi := &file_ateapi_proto_msgTypes[13] + mi := &file_ateapi_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1520,7 +1843,7 @@ func (x *HttpRuleEffects) String() string { func (*HttpRuleEffects) ProtoMessage() {} func (x *HttpRuleEffects) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[13] + mi := &file_ateapi_proto_msgTypes[15] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1533,7 +1856,7 @@ func (x *HttpRuleEffects) ProtoReflect() protoreflect.Message { // Deprecated: Use HttpRuleEffects.ProtoReflect.Descriptor instead. func (*HttpRuleEffects) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{13} + return file_ateapi_proto_rawDescGZIP(), []int{15} } func (x *HttpRuleEffects) GetReplaceHeaders() []*CredentialHeader { @@ -1583,7 +1906,7 @@ type CredentialHeader struct { func (x *CredentialHeader) Reset() { *x = CredentialHeader{} - mi := &file_ateapi_proto_msgTypes[14] + mi := &file_ateapi_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1595,7 +1918,7 @@ func (x *CredentialHeader) String() string { func (*CredentialHeader) ProtoMessage() {} func (x *CredentialHeader) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[14] + mi := &file_ateapi_proto_msgTypes[16] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1608,7 +1931,7 @@ func (x *CredentialHeader) ProtoReflect() protoreflect.Message { // Deprecated: Use CredentialHeader.ProtoReflect.Descriptor instead. func (*CredentialHeader) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{14} + return file_ateapi_proto_rawDescGZIP(), []int{16} } func (x *CredentialHeader) GetHeader() string { @@ -1665,7 +1988,7 @@ type ActorJWTSource struct { func (x *ActorJWTSource) Reset() { *x = ActorJWTSource{} - mi := &file_ateapi_proto_msgTypes[15] + mi := &file_ateapi_proto_msgTypes[17] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1677,7 +2000,7 @@ func (x *ActorJWTSource) String() string { func (*ActorJWTSource) ProtoMessage() {} func (x *ActorJWTSource) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[15] + mi := &file_ateapi_proto_msgTypes[17] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1690,7 +2013,7 @@ func (x *ActorJWTSource) ProtoReflect() protoreflect.Message { // Deprecated: Use ActorJWTSource.ProtoReflect.Descriptor instead. func (*ActorJWTSource) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{15} + return file_ateapi_proto_rawDescGZIP(), []int{17} } func (x *ActorJWTSource) GetAudiences() []string { @@ -1721,22 +2044,6 @@ type ActorStatus struct { // +k8s:optional // +k8s:update=NoModify # can be set and cleared, but not changed in place WorkerAssignment *WorkerAssignment `protobuf:"bytes,2,opt,name=worker_assignment,json=workerAssignment,proto3" json:"worker_assignment,omitempty"` - // in_progress_snapshot_uri is the URI in object storage of the durable - // snapshot the Actor is currently taking. - // - // +k8s:optional - // +k8s:maxLength=2048 # the template's storage_location bound plus the owner prefix and snapshot name - InProgressSnapshotUri string `protobuf:"bytes,3,opt,name=in_progress_snapshot_uri,json=inProgressSnapshotUri,proto3" json:"in_progress_snapshot_uri,omitempty"` - // external_snapshot is the Actor's current external snapshot. - // If the Actor was created from a Tag this is the tag's snapshot, borrowed - // until the Actor's first suspend writes one of its own. Otherwise it is - // unset until the Actor is first suspended. - // +k8s:optional - ExternalSnapshot *ExternalSnapshot `protobuf:"bytes,4,opt,name=external_snapshot,json=externalSnapshot,proto3" json:"external_snapshot,omitempty"` - // Node-local state used only while the Actor is paused. - // - // +k8s:optional - LocalSnapshot *LocalSnapshot `protobuf:"bytes,5,opt,name=local_snapshot,json=localSnapshot,proto3" json:"local_snapshot,omitempty"` // Volumes attached to the actor. These volumes only live as long as the actor. // They are deleted when the actor is deleted. Each template volume // appears at most once. @@ -1744,14 +2051,8 @@ type ActorStatus struct { // +k8s:optional // +k8s:maxItems=32 # matches the template's volumes bound // +k8s:listType=map - // +k8s:listMapKey=volume_name - ActorVolumes []*ExternalVolume `protobuf:"bytes,7,rep,name=actor_volumes,json=actorVolumes,proto3" json:"actor_volumes,omitempty"` - // The name of the in-progress node-local checkpoint. Like durable snapshot - // names, these are server-generated UUIDs. - // - // +k8s:optional - // +k8s:format=k8s-short-name - InProgressLocalSnapshotName string `protobuf:"bytes,8,opt,name=in_progress_local_snapshot_name,json=inProgressLocalSnapshotName,proto3" json:"in_progress_local_snapshot_name,omitempty"` + // +k8s:listMapKey=name + ExternalVolumes []*ExternalVolume `protobuf:"bytes,7,rep,name=external_volumes,json=externalVolumes,proto3" json:"external_volumes,omitempty"` // crash records why and when the Actor entered CRASHED. It is set with the // CRASHED state and cleared when a revert returns the Actor to SUSPENDED. // @@ -1763,14 +2064,32 @@ type ActorStatus struct { // // +k8s:optional // +k8s:format=k8s-long-name - AssignedNode string `protobuf:"bytes,10,opt,name=assigned_node,json=assignedNode,proto3" json:"assigned_node,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + AssignedNode string `protobuf:"bytes,10,opt,name=assigned_node,json=assignedNode,proto3" json:"assigned_node,omitempty"` + // snapshots holds the Actor's active and in-progress snapshot generations. + // + // +k8s:optional + // +k8s:maxItems=16 + // +k8s:listType=atomic + // +k8s:unique=map + // +k8s:listMapKey=generation + Snapshots []*Snapshot `protobuf:"bytes,11,rep,name=snapshots,proto3" json:"snapshots,omitempty"` + // last_assigned_generation tracks the last snapshot generation assigned for + // this Actor during a Suspend or Pause request. + // Even during Revert, the generation is not decreased to keep track of snapshots + // that have become stale and need to be cleaned up. + // + // +k8s:optional + // +k8s:alpha(since: "0.0")=+k8s:monotonic # TODO: get rid of alpha prefix + // +k8s:minimum=0 + // +k8s:update=NoUnset + LastAssignedGeneration int32 `protobuf:"varint,12,opt,name=last_assigned_generation,json=lastAssignedGeneration,proto3" json:"last_assigned_generation,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *ActorStatus) Reset() { *x = ActorStatus{} - mi := &file_ateapi_proto_msgTypes[16] + mi := &file_ateapi_proto_msgTypes[18] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1782,7 +2101,7 @@ func (x *ActorStatus) String() string { func (*ActorStatus) ProtoMessage() {} func (x *ActorStatus) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[16] + mi := &file_ateapi_proto_msgTypes[18] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1795,7 +2114,7 @@ func (x *ActorStatus) ProtoReflect() protoreflect.Message { // Deprecated: Use ActorStatus.ProtoReflect.Descriptor instead. func (*ActorStatus) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{16} + return file_ateapi_proto_rawDescGZIP(), []int{18} } func (x *ActorStatus) GetState() ActorState { @@ -1812,53 +2131,39 @@ func (x *ActorStatus) GetWorkerAssignment() *WorkerAssignment { return nil } -func (x *ActorStatus) GetInProgressSnapshotUri() string { - if x != nil { - return x.InProgressSnapshotUri - } - return "" -} - -func (x *ActorStatus) GetExternalSnapshot() *ExternalSnapshot { - if x != nil { - return x.ExternalSnapshot - } - return nil -} - -func (x *ActorStatus) GetLocalSnapshot() *LocalSnapshot { +func (x *ActorStatus) GetExternalVolumes() []*ExternalVolume { if x != nil { - return x.LocalSnapshot + return x.ExternalVolumes } return nil } -func (x *ActorStatus) GetActorVolumes() []*ExternalVolume { +func (x *ActorStatus) GetCrash() *ActorCrash { if x != nil { - return x.ActorVolumes + return x.Crash } return nil } -func (x *ActorStatus) GetInProgressLocalSnapshotName() string { +func (x *ActorStatus) GetAssignedNode() string { if x != nil { - return x.InProgressLocalSnapshotName + return x.AssignedNode } return "" } -func (x *ActorStatus) GetCrash() *ActorCrash { +func (x *ActorStatus) GetSnapshots() []*Snapshot { if x != nil { - return x.Crash + return x.Snapshots } return nil } -func (x *ActorStatus) GetAssignedNode() string { +func (x *ActorStatus) GetLastAssignedGeneration() int32 { if x != nil { - return x.AssignedNode + return x.LastAssignedGeneration } - return "" + return 0 } // ActorCrash describes the failure that moved an Actor to CRASHED. @@ -1879,7 +2184,7 @@ type ActorCrash struct { func (x *ActorCrash) Reset() { *x = ActorCrash{} - mi := &file_ateapi_proto_msgTypes[17] + mi := &file_ateapi_proto_msgTypes[19] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1891,7 +2196,7 @@ func (x *ActorCrash) String() string { func (*ActorCrash) ProtoMessage() {} func (x *ActorCrash) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[17] + mi := &file_ateapi_proto_msgTypes[19] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1904,7 +2209,7 @@ func (x *ActorCrash) ProtoReflect() protoreflect.Message { // Deprecated: Use ActorCrash.ProtoReflect.Descriptor instead. func (*ActorCrash) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{17} + return file_ateapi_proto_rawDescGZIP(), []int{19} } func (x *ActorCrash) GetMessage() string { @@ -1985,7 +2290,7 @@ type WorkerAssignment struct { func (x *WorkerAssignment) Reset() { *x = WorkerAssignment{} - mi := &file_ateapi_proto_msgTypes[18] + mi := &file_ateapi_proto_msgTypes[20] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1997,7 +2302,7 @@ func (x *WorkerAssignment) String() string { func (*WorkerAssignment) ProtoMessage() {} func (x *WorkerAssignment) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[18] + mi := &file_ateapi_proto_msgTypes[20] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2010,7 +2315,7 @@ func (x *WorkerAssignment) ProtoReflect() protoreflect.Message { // Deprecated: Use WorkerAssignment.ProtoReflect.Descriptor instead. func (*WorkerAssignment) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{18} + return file_ateapi_proto_rawDescGZIP(), []int{20} } func (x *WorkerAssignment) GetWorker() *ObjectRef { @@ -2074,33 +2379,26 @@ func (x *WorkerAssignment) GetWorkerEpoch() int64 { // the rest is immutable provenance. type TagStatus struct { state protoimpl.MessageState `protogen:"open.v1"` - // snapshot is the external snapshot this tag points at. Unset while the tag - // is still being created, and immutable once set: a tag never moves between - // snapshots. A tag is usable only once this is set. + // snapshot is the external snapshot this tag points at. Its storage entry is + // IN_PROGRESS while the tag is being created and transitions to COMPLETED + // once the copy finishes: a tag never moves between snapshots and is usable + // only once COMPLETED. // - // +k8s:optional - Snapshot *ExternalSnapshot `protobuf:"bytes,1,opt,name=snapshot,proto3" json:"snapshot,omitempty"` + // +k8s:required + Snapshot *Snapshot `protobuf:"bytes,1,opt,name=snapshot,proto3" json:"snapshot,omitempty"` // UID of the ActorTemplate the snapshot was taken under. Actors can only be // seeded from a tag under the same template. // // +k8s:required // +k8s:format=k8s-uuid ActorTemplateUid string `protobuf:"bytes,2,opt,name=actor_template_uid,json=actorTemplateUid,proto3" json:"actor_template_uid,omitempty"` - // storage_location is the base object-storage URI for this tag's snapshot. - // Set by the server from the source actor's template when the tag is created - // and immutable thereafter. The full snapshot URI is available in - // snapshot.snapshot_uri once tag creation completes. - // - // +k8s:required - // +k8s:maxLength=1024 - StorageLocation string `protobuf:"bytes,3,opt,name=storage_location,json=storageLocation,proto3" json:"storage_location,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *TagStatus) Reset() { *x = TagStatus{} - mi := &file_ateapi_proto_msgTypes[19] + mi := &file_ateapi_proto_msgTypes[21] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2112,7 +2410,7 @@ func (x *TagStatus) String() string { func (*TagStatus) ProtoMessage() {} func (x *TagStatus) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[19] + mi := &file_ateapi_proto_msgTypes[21] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2125,10 +2423,10 @@ func (x *TagStatus) ProtoReflect() protoreflect.Message { // Deprecated: Use TagStatus.ProtoReflect.Descriptor instead. func (*TagStatus) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{19} + return file_ateapi_proto_rawDescGZIP(), []int{21} } -func (x *TagStatus) GetSnapshot() *ExternalSnapshot { +func (x *TagStatus) GetSnapshot() *Snapshot { if x != nil { return x.Snapshot } @@ -2142,13 +2440,6 @@ func (x *TagStatus) GetActorTemplateUid() string { return "" } -func (x *TagStatus) GetStorageLocation() string { - if x != nil { - return x.StorageLocation - } - return "" -} - // Tag is an Atespace-owned, stable name for one external snapshot. The // external snapshot pointed by a tag is a copy of an actor snapshot. The // snapshot is copied when the tag is created, and deleted when the tag is. Its @@ -2184,7 +2475,7 @@ type Tag struct { func (x *Tag) Reset() { *x = Tag{} - mi := &file_ateapi_proto_msgTypes[20] + mi := &file_ateapi_proto_msgTypes[22] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2196,7 +2487,7 @@ func (x *Tag) String() string { func (*Tag) ProtoMessage() {} func (x *Tag) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[20] + mi := &file_ateapi_proto_msgTypes[22] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2209,7 +2500,7 @@ func (x *Tag) ProtoReflect() protoreflect.Message { // Deprecated: Use Tag.ProtoReflect.Descriptor instead. func (*Tag) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{20} + return file_ateapi_proto_rawDescGZIP(), []int{22} } func (x *Tag) GetMetadata() *ResourceMetadata { @@ -2255,7 +2546,7 @@ type Atespace struct { func (x *Atespace) Reset() { *x = Atespace{} - mi := &file_ateapi_proto_msgTypes[21] + mi := &file_ateapi_proto_msgTypes[23] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2267,7 +2558,7 @@ func (x *Atespace) String() string { func (*Atespace) ProtoMessage() {} func (x *Atespace) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[21] + mi := &file_ateapi_proto_msgTypes[23] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2280,7 +2571,7 @@ func (x *Atespace) ProtoReflect() protoreflect.Message { // Deprecated: Use Atespace.ProtoReflect.Descriptor instead. func (*Atespace) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{21} + return file_ateapi_proto_rawDescGZIP(), []int{23} } func (x *Atespace) GetMetadata() *ResourceMetadata { @@ -2313,7 +2604,7 @@ type ObjectRef struct { func (x *ObjectRef) Reset() { *x = ObjectRef{} - mi := &file_ateapi_proto_msgTypes[22] + mi := &file_ateapi_proto_msgTypes[24] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2325,7 +2616,7 @@ func (x *ObjectRef) String() string { func (*ObjectRef) ProtoMessage() {} func (x *ObjectRef) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[22] + mi := &file_ateapi_proto_msgTypes[24] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2338,7 +2629,7 @@ func (x *ObjectRef) ProtoReflect() protoreflect.Message { // Deprecated: Use ObjectRef.ProtoReflect.Descriptor instead. func (*ObjectRef) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{22} + return file_ateapi_proto_rawDescGZIP(), []int{24} } func (x *ObjectRef) GetAtespace() string { @@ -2395,7 +2686,7 @@ type ActorTemplate struct { func (x *ActorTemplate) Reset() { *x = ActorTemplate{} - mi := &file_ateapi_proto_msgTypes[23] + mi := &file_ateapi_proto_msgTypes[25] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2407,7 +2698,7 @@ func (x *ActorTemplate) String() string { func (*ActorTemplate) ProtoMessage() {} func (x *ActorTemplate) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[23] + mi := &file_ateapi_proto_msgTypes[25] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2420,7 +2711,7 @@ func (x *ActorTemplate) ProtoReflect() protoreflect.Message { // Deprecated: Use ActorTemplate.ProtoReflect.Descriptor instead. func (*ActorTemplate) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{23} + return file_ateapi_proto_rawDescGZIP(), []int{25} } func (x *ActorTemplate) GetMetadata() *ResourceMetadata { @@ -2496,7 +2787,7 @@ type Resources struct { func (x *Resources) Reset() { *x = Resources{} - mi := &file_ateapi_proto_msgTypes[24] + mi := &file_ateapi_proto_msgTypes[26] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2508,7 +2799,7 @@ func (x *Resources) String() string { func (*Resources) ProtoMessage() {} func (x *Resources) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[24] + mi := &file_ateapi_proto_msgTypes[26] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2521,7 +2812,7 @@ func (x *Resources) ProtoReflect() protoreflect.Message { // Deprecated: Use Resources.ProtoReflect.Descriptor instead. func (*Resources) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{24} + return file_ateapi_proto_rawDescGZIP(), []int{26} } func (x *Resources) GetLimits() []*Limits { @@ -2549,7 +2840,7 @@ type Limits struct { func (x *Limits) Reset() { *x = Limits{} - mi := &file_ateapi_proto_msgTypes[25] + mi := &file_ateapi_proto_msgTypes[27] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2561,7 +2852,7 @@ func (x *Limits) String() string { func (*Limits) ProtoMessage() {} func (x *Limits) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[25] + mi := &file_ateapi_proto_msgTypes[27] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2574,7 +2865,7 @@ func (x *Limits) ProtoReflect() protoreflect.Message { // Deprecated: Use Limits.ProtoReflect.Descriptor instead. func (*Limits) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{25} + return file_ateapi_proto_rawDescGZIP(), []int{27} } func (x *Limits) GetName() string { @@ -2615,7 +2906,7 @@ type GoldenSnapshotStatus struct { func (x *GoldenSnapshotStatus) Reset() { *x = GoldenSnapshotStatus{} - mi := &file_ateapi_proto_msgTypes[26] + mi := &file_ateapi_proto_msgTypes[28] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2627,7 +2918,7 @@ func (x *GoldenSnapshotStatus) String() string { func (*GoldenSnapshotStatus) ProtoMessage() {} func (x *GoldenSnapshotStatus) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[26] + mi := &file_ateapi_proto_msgTypes[28] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2640,7 +2931,7 @@ func (x *GoldenSnapshotStatus) ProtoReflect() protoreflect.Message { // Deprecated: Use GoldenSnapshotStatus.ProtoReflect.Descriptor instead. func (*GoldenSnapshotStatus) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{26} + return file_ateapi_proto_rawDescGZIP(), []int{28} } func (x *GoldenSnapshotStatus) GetGoldenTag() *ObjectRef { @@ -2674,7 +2965,7 @@ type ActorTemplateStatus struct { func (x *ActorTemplateStatus) Reset() { *x = ActorTemplateStatus{} - mi := &file_ateapi_proto_msgTypes[27] + mi := &file_ateapi_proto_msgTypes[29] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2686,7 +2977,7 @@ func (x *ActorTemplateStatus) String() string { func (*ActorTemplateStatus) ProtoMessage() {} func (x *ActorTemplateStatus) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[27] + mi := &file_ateapi_proto_msgTypes[29] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2699,7 +2990,7 @@ func (x *ActorTemplateStatus) ProtoReflect() protoreflect.Message { // Deprecated: Use ActorTemplateStatus.ProtoReflect.Descriptor instead. func (*ActorTemplateStatus) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{27} + return file_ateapi_proto_rawDescGZIP(), []int{29} } func (x *ActorTemplateStatus) GetGoldenSnapshotStatus() *GoldenSnapshotStatus { @@ -2730,7 +3021,7 @@ type SandboxConfig struct { func (x *SandboxConfig) Reset() { *x = SandboxConfig{} - mi := &file_ateapi_proto_msgTypes[28] + mi := &file_ateapi_proto_msgTypes[30] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2742,7 +3033,7 @@ func (x *SandboxConfig) String() string { func (*SandboxConfig) ProtoMessage() {} func (x *SandboxConfig) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[28] + mi := &file_ateapi_proto_msgTypes[30] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2755,7 +3046,7 @@ func (x *SandboxConfig) ProtoReflect() protoreflect.Message { // Deprecated: Use SandboxConfig.ProtoReflect.Descriptor instead. func (*SandboxConfig) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{28} + return file_ateapi_proto_rawDescGZIP(), []int{30} } func (x *SandboxConfig) GetSandboxClass() SandboxClass { @@ -2773,33 +3064,37 @@ func (x *SandboxConfig) GetConfigName() string { } // SnapshotConfig selects what actor snapshots capture and where they are -// stored. A FULL snapshot resumes from its own content. A DATA snapshot -// resumes by starting the containers afresh from the OCI image, with the -// durable-dir volumes populated from the snapshot. +// stored. type SnapshotConfig struct { state protoimpl.MessageState `protogen:"open.v1"` - // on_commit defines the scope of every snapshot taken of an actor, both - // the node-local one captured on pause and the one uploaded on suspend. - // Defaults to FULL when unset. - // - // +k8s:required - // +k8s:minimum=1 - // +k8s:maximum=2 # keep this in sync with the SnapshotContentScope enum - OnCommit SnapshotContentScope `protobuf:"varint,2,opt,name=on_commit,json=onCommit,proto3,enum=ateapi.SnapshotContentScope" json:"on_commit,omitempty"` // storage_location is the base object-storage URI snapshots of actors on // this version are stored under. Required. // // +k8s:required // +k8s:maxLength=1024 // +k8s:customValidation # Validate URI - StorageLocation string `protobuf:"bytes,4,opt,name=storage_location,json=storageLocation,proto3" json:"storage_location,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + StorageLocation string `protobuf:"bytes,1,opt,name=storage_location,json=storageLocation,proto3" json:"storage_location,omitempty"` + // preferred_fidelity is the highest fidelity the system packages when it + // snapshots the actor, on pause (node-local) and on suspend (uploaded). + // It is a best-effort ceiling, not a guarantee: the system never packages + // more than this level, and it may package less when a layer cannot be + // captured or is too expensive to capture at the time, for example memory + // under host pressure. A process resumed in place on its worker keeps its + // memory regardless of this setting. ROOTFS is not supported yet and is + // rejected. Defaults to MEMORY when unset. + // + // +k8s:required + // +k8s:minimum=1 + // +k8s:maximum=3 # keep this in sync with the SnapshotFidelity enum + // +k8s:customValidation # ROOTFS is not supported yet + PreferredFidelity SnapshotFidelity `protobuf:"varint,2,opt,name=preferred_fidelity,json=preferredFidelity,proto3,enum=ateapi.SnapshotFidelity" json:"preferred_fidelity,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *SnapshotConfig) Reset() { *x = SnapshotConfig{} - mi := &file_ateapi_proto_msgTypes[29] + mi := &file_ateapi_proto_msgTypes[31] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2811,7 +3106,7 @@ func (x *SnapshotConfig) String() string { func (*SnapshotConfig) ProtoMessage() {} func (x *SnapshotConfig) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[29] + mi := &file_ateapi_proto_msgTypes[31] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2824,21 +3119,21 @@ func (x *SnapshotConfig) ProtoReflect() protoreflect.Message { // Deprecated: Use SnapshotConfig.ProtoReflect.Descriptor instead. func (*SnapshotConfig) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{29} + return file_ateapi_proto_rawDescGZIP(), []int{31} } -func (x *SnapshotConfig) GetOnCommit() SnapshotContentScope { +func (x *SnapshotConfig) GetStorageLocation() string { if x != nil { - return x.OnCommit + return x.StorageLocation } - return SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_UNSPECIFIED + return "" } -func (x *SnapshotConfig) GetStorageLocation() string { +func (x *SnapshotConfig) GetPreferredFidelity() SnapshotFidelity { if x != nil { - return x.StorageLocation + return x.PreferredFidelity } - return "" + return SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED } // Container is a single application container of an ActorTemplate. @@ -2911,7 +3206,7 @@ type Container struct { func (x *Container) Reset() { *x = Container{} - mi := &file_ateapi_proto_msgTypes[30] + mi := &file_ateapi_proto_msgTypes[32] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2923,7 +3218,7 @@ func (x *Container) String() string { func (*Container) ProtoMessage() {} func (x *Container) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[30] + mi := &file_ateapi_proto_msgTypes[32] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2936,7 +3231,7 @@ func (x *Container) ProtoReflect() protoreflect.Message { // Deprecated: Use Container.ProtoReflect.Descriptor instead. func (*Container) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{30} + return file_ateapi_proto_rawDescGZIP(), []int{32} } func (x *Container) GetName() string { @@ -3016,7 +3311,7 @@ type SecurityContext struct { func (x *SecurityContext) Reset() { *x = SecurityContext{} - mi := &file_ateapi_proto_msgTypes[31] + mi := &file_ateapi_proto_msgTypes[33] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3028,7 +3323,7 @@ func (x *SecurityContext) String() string { func (*SecurityContext) ProtoMessage() {} func (x *SecurityContext) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[31] + mi := &file_ateapi_proto_msgTypes[33] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3041,7 +3336,7 @@ func (x *SecurityContext) ProtoReflect() protoreflect.Message { // Deprecated: Use SecurityContext.ProtoReflect.Descriptor instead. func (*SecurityContext) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{31} + return file_ateapi_proto_rawDescGZIP(), []int{33} } func (x *SecurityContext) GetCapabilities() *Capabilities { @@ -3079,7 +3374,7 @@ type Capabilities struct { func (x *Capabilities) Reset() { *x = Capabilities{} - mi := &file_ateapi_proto_msgTypes[32] + mi := &file_ateapi_proto_msgTypes[34] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3091,7 +3386,7 @@ func (x *Capabilities) String() string { func (*Capabilities) ProtoMessage() {} func (x *Capabilities) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[32] + mi := &file_ateapi_proto_msgTypes[34] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3104,7 +3399,7 @@ func (x *Capabilities) ProtoReflect() protoreflect.Message { // Deprecated: Use Capabilities.ProtoReflect.Descriptor instead. func (*Capabilities) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{32} + return file_ateapi_proto_rawDescGZIP(), []int{34} } func (x *Capabilities) GetAdd() []string { @@ -3142,7 +3437,7 @@ type EnvVar struct { func (x *EnvVar) Reset() { *x = EnvVar{} - mi := &file_ateapi_proto_msgTypes[33] + mi := &file_ateapi_proto_msgTypes[35] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3154,7 +3449,7 @@ func (x *EnvVar) String() string { func (*EnvVar) ProtoMessage() {} func (x *EnvVar) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[33] + mi := &file_ateapi_proto_msgTypes[35] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3167,7 +3462,7 @@ func (x *EnvVar) ProtoReflect() protoreflect.Message { // Deprecated: Use EnvVar.ProtoReflect.Descriptor instead. func (*EnvVar) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{33} + return file_ateapi_proto_rawDescGZIP(), []int{35} } func (x *EnvVar) GetName() string { @@ -3207,7 +3502,7 @@ type ContainerWakeupProbe struct { func (x *ContainerWakeupProbe) Reset() { *x = ContainerWakeupProbe{} - mi := &file_ateapi_proto_msgTypes[34] + mi := &file_ateapi_proto_msgTypes[36] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3219,7 +3514,7 @@ func (x *ContainerWakeupProbe) String() string { func (*ContainerWakeupProbe) ProtoMessage() {} func (x *ContainerWakeupProbe) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[34] + mi := &file_ateapi_proto_msgTypes[36] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3232,7 +3527,7 @@ func (x *ContainerWakeupProbe) ProtoReflect() protoreflect.Message { // Deprecated: Use ContainerWakeupProbe.ProtoReflect.Descriptor instead. func (*ContainerWakeupProbe) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{34} + return file_ateapi_proto_rawDescGZIP(), []int{36} } func (x *ContainerWakeupProbe) GetHttpGet() *HTTPGetAction { @@ -3272,7 +3567,7 @@ type HTTPGetAction struct { func (x *HTTPGetAction) Reset() { *x = HTTPGetAction{} - mi := &file_ateapi_proto_msgTypes[35] + mi := &file_ateapi_proto_msgTypes[37] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3284,7 +3579,7 @@ func (x *HTTPGetAction) String() string { func (*HTTPGetAction) ProtoMessage() {} func (x *HTTPGetAction) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[35] + mi := &file_ateapi_proto_msgTypes[37] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3297,7 +3592,7 @@ func (x *HTTPGetAction) ProtoReflect() protoreflect.Message { // Deprecated: Use HTTPGetAction.ProtoReflect.Descriptor instead. func (*HTTPGetAction) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{35} + return file_ateapi_proto_rawDescGZIP(), []int{37} } func (x *HTTPGetAction) GetPath() string { @@ -3344,7 +3639,7 @@ type Volume struct { func (x *Volume) Reset() { *x = Volume{} - mi := &file_ateapi_proto_msgTypes[36] + mi := &file_ateapi_proto_msgTypes[38] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3356,7 +3651,7 @@ func (x *Volume) String() string { func (*Volume) ProtoMessage() {} func (x *Volume) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[36] + mi := &file_ateapi_proto_msgTypes[38] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3369,7 +3664,7 @@ func (x *Volume) ProtoReflect() protoreflect.Message { // Deprecated: Use Volume.ProtoReflect.Descriptor instead. func (*Volume) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{36} + return file_ateapi_proto_rawDescGZIP(), []int{38} } func (x *Volume) GetName() string { @@ -3425,7 +3720,7 @@ type ImageVolumeSource struct { func (x *ImageVolumeSource) Reset() { *x = ImageVolumeSource{} - mi := &file_ateapi_proto_msgTypes[37] + mi := &file_ateapi_proto_msgTypes[39] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3437,7 +3732,7 @@ func (x *ImageVolumeSource) String() string { func (*ImageVolumeSource) ProtoMessage() {} func (x *ImageVolumeSource) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[37] + mi := &file_ateapi_proto_msgTypes[39] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3450,7 +3745,7 @@ func (x *ImageVolumeSource) ProtoReflect() protoreflect.Message { // Deprecated: Use ImageVolumeSource.ProtoReflect.Descriptor instead. func (*ImageVolumeSource) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{37} + return file_ateapi_proto_rawDescGZIP(), []int{39} } func (x *ImageVolumeSource) GetReference() string { @@ -3470,7 +3765,7 @@ type DurableDirVolumeSource struct { func (x *DurableDirVolumeSource) Reset() { *x = DurableDirVolumeSource{} - mi := &file_ateapi_proto_msgTypes[38] + mi := &file_ateapi_proto_msgTypes[40] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3482,7 +3777,7 @@ func (x *DurableDirVolumeSource) String() string { func (*DurableDirVolumeSource) ProtoMessage() {} func (x *DurableDirVolumeSource) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[38] + mi := &file_ateapi_proto_msgTypes[40] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3495,7 +3790,7 @@ func (x *DurableDirVolumeSource) ProtoReflect() protoreflect.Message { // Deprecated: Use DurableDirVolumeSource.ProtoReflect.Descriptor instead. func (*DurableDirVolumeSource) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{38} + return file_ateapi_proto_rawDescGZIP(), []int{40} } // ExternalVolumeTemplate provisions an external volume per actor; the volume @@ -3521,7 +3816,7 @@ type ExternalVolumeTemplate struct { func (x *ExternalVolumeTemplate) Reset() { *x = ExternalVolumeTemplate{} - mi := &file_ateapi_proto_msgTypes[39] + mi := &file_ateapi_proto_msgTypes[41] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3533,7 +3828,7 @@ func (x *ExternalVolumeTemplate) String() string { func (*ExternalVolumeTemplate) ProtoMessage() {} func (x *ExternalVolumeTemplate) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[39] + mi := &file_ateapi_proto_msgTypes[41] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3546,7 +3841,7 @@ func (x *ExternalVolumeTemplate) ProtoReflect() protoreflect.Message { // Deprecated: Use ExternalVolumeTemplate.ProtoReflect.Descriptor instead. func (*ExternalVolumeTemplate) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{39} + return file_ateapi_proto_rawDescGZIP(), []int{41} } func (x *ExternalVolumeTemplate) GetCapacity() string { @@ -3583,7 +3878,7 @@ type SystemInfoVolumeSource struct { func (x *SystemInfoVolumeSource) Reset() { *x = SystemInfoVolumeSource{} - mi := &file_ateapi_proto_msgTypes[40] + mi := &file_ateapi_proto_msgTypes[42] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3595,7 +3890,7 @@ func (x *SystemInfoVolumeSource) String() string { func (*SystemInfoVolumeSource) ProtoMessage() {} func (x *SystemInfoVolumeSource) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[40] + mi := &file_ateapi_proto_msgTypes[42] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3608,7 +3903,7 @@ func (x *SystemInfoVolumeSource) ProtoReflect() protoreflect.Message { // Deprecated: Use SystemInfoVolumeSource.ProtoReflect.Descriptor instead. func (*SystemInfoVolumeSource) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{40} + return file_ateapi_proto_rawDescGZIP(), []int{42} } func (x *SystemInfoVolumeSource) GetDataSources() []*SystemInfoDataSource { @@ -3634,7 +3929,7 @@ type SystemInfoDataSource struct { func (x *SystemInfoDataSource) Reset() { *x = SystemInfoDataSource{} - mi := &file_ateapi_proto_msgTypes[41] + mi := &file_ateapi_proto_msgTypes[43] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3646,7 +3941,7 @@ func (x *SystemInfoDataSource) String() string { func (*SystemInfoDataSource) ProtoMessage() {} func (x *SystemInfoDataSource) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[41] + mi := &file_ateapi_proto_msgTypes[43] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3659,7 +3954,7 @@ func (x *SystemInfoDataSource) ProtoReflect() protoreflect.Message { // Deprecated: Use SystemInfoDataSource.ProtoReflect.Descriptor instead. func (*SystemInfoDataSource) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{41} + return file_ateapi_proto_rawDescGZIP(), []int{43} } func (x *SystemInfoDataSource) GetActorMetadata() *ActorMetadataDataSource { @@ -3696,7 +3991,7 @@ type ActorMetadataDataSource struct { func (x *ActorMetadataDataSource) Reset() { *x = ActorMetadataDataSource{} - mi := &file_ateapi_proto_msgTypes[42] + mi := &file_ateapi_proto_msgTypes[44] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3708,7 +4003,7 @@ func (x *ActorMetadataDataSource) String() string { func (*ActorMetadataDataSource) ProtoMessage() {} func (x *ActorMetadataDataSource) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[42] + mi := &file_ateapi_proto_msgTypes[44] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3721,7 +4016,7 @@ func (x *ActorMetadataDataSource) ProtoReflect() protoreflect.Message { // Deprecated: Use ActorMetadataDataSource.ProtoReflect.Descriptor instead. func (*ActorMetadataDataSource) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{42} + return file_ateapi_proto_rawDescGZIP(), []int{44} } func (x *ActorMetadataDataSource) GetItems() []*ActorMetadataItem { @@ -3753,7 +4048,7 @@ type ActorMetadataItem struct { func (x *ActorMetadataItem) Reset() { *x = ActorMetadataItem{} - mi := &file_ateapi_proto_msgTypes[43] + mi := &file_ateapi_proto_msgTypes[45] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3765,7 +4060,7 @@ func (x *ActorMetadataItem) String() string { func (*ActorMetadataItem) ProtoMessage() {} func (x *ActorMetadataItem) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[43] + mi := &file_ateapi_proto_msgTypes[45] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3778,7 +4073,7 @@ func (x *ActorMetadataItem) ProtoReflect() protoreflect.Message { // Deprecated: Use ActorMetadataItem.ProtoReflect.Descriptor instead. func (*ActorMetadataItem) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{43} + return file_ateapi_proto_rawDescGZIP(), []int{45} } func (x *ActorMetadataItem) GetField() ActorMetadataField { @@ -3830,7 +4125,7 @@ type TrustBundleDataSource struct { func (x *TrustBundleDataSource) Reset() { *x = TrustBundleDataSource{} - mi := &file_ateapi_proto_msgTypes[44] + mi := &file_ateapi_proto_msgTypes[46] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3842,7 +4137,7 @@ func (x *TrustBundleDataSource) String() string { func (*TrustBundleDataSource) ProtoMessage() {} func (x *TrustBundleDataSource) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[44] + mi := &file_ateapi_proto_msgTypes[46] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3855,7 +4150,7 @@ func (x *TrustBundleDataSource) ProtoReflect() protoreflect.Message { // Deprecated: Use TrustBundleDataSource.ProtoReflect.Descriptor instead. func (*TrustBundleDataSource) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{44} + return file_ateapi_proto_rawDescGZIP(), []int{46} } func (x *TrustBundleDataSource) GetNames() []string { @@ -3894,7 +4189,7 @@ type VolumeMount struct { func (x *VolumeMount) Reset() { *x = VolumeMount{} - mi := &file_ateapi_proto_msgTypes[45] + mi := &file_ateapi_proto_msgTypes[47] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3906,7 +4201,7 @@ func (x *VolumeMount) String() string { func (*VolumeMount) ProtoMessage() {} func (x *VolumeMount) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[45] + mi := &file_ateapi_proto_msgTypes[47] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3919,7 +4214,7 @@ func (x *VolumeMount) ProtoReflect() protoreflect.Message { // Deprecated: Use VolumeMount.ProtoReflect.Descriptor instead. func (*VolumeMount) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{45} + return file_ateapi_proto_rawDescGZIP(), []int{47} } func (x *VolumeMount) GetName() string { @@ -3948,7 +4243,7 @@ type CreateAtespaceRequest struct { func (x *CreateAtespaceRequest) Reset() { *x = CreateAtespaceRequest{} - mi := &file_ateapi_proto_msgTypes[46] + mi := &file_ateapi_proto_msgTypes[48] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3960,7 +4255,7 @@ func (x *CreateAtespaceRequest) String() string { func (*CreateAtespaceRequest) ProtoMessage() {} func (x *CreateAtespaceRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[46] + mi := &file_ateapi_proto_msgTypes[48] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3973,7 +4268,7 @@ func (x *CreateAtespaceRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use CreateAtespaceRequest.ProtoReflect.Descriptor instead. func (*CreateAtespaceRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{46} + return file_ateapi_proto_rawDescGZIP(), []int{48} } func (x *CreateAtespaceRequest) GetAtespace() *Atespace { @@ -3994,7 +4289,7 @@ type GetAtespaceRequest struct { func (x *GetAtespaceRequest) Reset() { *x = GetAtespaceRequest{} - mi := &file_ateapi_proto_msgTypes[47] + mi := &file_ateapi_proto_msgTypes[49] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4006,7 +4301,7 @@ func (x *GetAtespaceRequest) String() string { func (*GetAtespaceRequest) ProtoMessage() {} func (x *GetAtespaceRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[47] + mi := &file_ateapi_proto_msgTypes[49] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4019,7 +4314,7 @@ func (x *GetAtespaceRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetAtespaceRequest.ProtoReflect.Descriptor instead. func (*GetAtespaceRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{47} + return file_ateapi_proto_rawDescGZIP(), []int{49} } func (x *GetAtespaceRequest) GetAtespace() *ObjectRef { @@ -4050,7 +4345,7 @@ type ListAtespacesRequest struct { func (x *ListAtespacesRequest) Reset() { *x = ListAtespacesRequest{} - mi := &file_ateapi_proto_msgTypes[48] + mi := &file_ateapi_proto_msgTypes[50] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4062,7 +4357,7 @@ func (x *ListAtespacesRequest) String() string { func (*ListAtespacesRequest) ProtoMessage() {} func (x *ListAtespacesRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[48] + mi := &file_ateapi_proto_msgTypes[50] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4075,7 +4370,7 @@ func (x *ListAtespacesRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ListAtespacesRequest.ProtoReflect.Descriptor instead. func (*ListAtespacesRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{48} + return file_ateapi_proto_rawDescGZIP(), []int{50} } func (x *ListAtespacesRequest) GetPageSize() int32 { @@ -4104,7 +4399,7 @@ type ListAtespacesResponse struct { func (x *ListAtespacesResponse) Reset() { *x = ListAtespacesResponse{} - mi := &file_ateapi_proto_msgTypes[49] + mi := &file_ateapi_proto_msgTypes[51] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4116,7 +4411,7 @@ func (x *ListAtespacesResponse) String() string { func (*ListAtespacesResponse) ProtoMessage() {} func (x *ListAtespacesResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[49] + mi := &file_ateapi_proto_msgTypes[51] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4129,7 +4424,7 @@ func (x *ListAtespacesResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ListAtespacesResponse.ProtoReflect.Descriptor instead. func (*ListAtespacesResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{49} + return file_ateapi_proto_rawDescGZIP(), []int{51} } func (x *ListAtespacesResponse) GetAtespaces() []*Atespace { @@ -4161,7 +4456,7 @@ type DeleteAtespaceRequest struct { func (x *DeleteAtespaceRequest) Reset() { *x = DeleteAtespaceRequest{} - mi := &file_ateapi_proto_msgTypes[50] + mi := &file_ateapi_proto_msgTypes[52] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4173,7 +4468,7 @@ func (x *DeleteAtespaceRequest) String() string { func (*DeleteAtespaceRequest) ProtoMessage() {} func (x *DeleteAtespaceRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[50] + mi := &file_ateapi_proto_msgTypes[52] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4186,7 +4481,7 @@ func (x *DeleteAtespaceRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use DeleteAtespaceRequest.ProtoReflect.Descriptor instead. func (*DeleteAtespaceRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{50} + return file_ateapi_proto_rawDescGZIP(), []int{52} } func (x *DeleteAtespaceRequest) GetAtespace() *ObjectRef { @@ -4217,7 +4512,7 @@ type CreateActorTemplateRequest struct { func (x *CreateActorTemplateRequest) Reset() { *x = CreateActorTemplateRequest{} - mi := &file_ateapi_proto_msgTypes[51] + mi := &file_ateapi_proto_msgTypes[53] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4229,7 +4524,7 @@ func (x *CreateActorTemplateRequest) String() string { func (*CreateActorTemplateRequest) ProtoMessage() {} func (x *CreateActorTemplateRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[51] + mi := &file_ateapi_proto_msgTypes[53] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4242,7 +4537,7 @@ func (x *CreateActorTemplateRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use CreateActorTemplateRequest.ProtoReflect.Descriptor instead. func (*CreateActorTemplateRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{51} + return file_ateapi_proto_rawDescGZIP(), []int{53} } func (x *CreateActorTemplateRequest) GetActorTemplate() *ActorTemplate { @@ -4263,7 +4558,7 @@ type GetActorTemplateRequest struct { func (x *GetActorTemplateRequest) Reset() { *x = GetActorTemplateRequest{} - mi := &file_ateapi_proto_msgTypes[52] + mi := &file_ateapi_proto_msgTypes[54] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4275,7 +4570,7 @@ func (x *GetActorTemplateRequest) String() string { func (*GetActorTemplateRequest) ProtoMessage() {} func (x *GetActorTemplateRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[52] + mi := &file_ateapi_proto_msgTypes[54] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4288,7 +4583,7 @@ func (x *GetActorTemplateRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetActorTemplateRequest.ProtoReflect.Descriptor instead. func (*GetActorTemplateRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{52} + return file_ateapi_proto_rawDescGZIP(), []int{54} } func (x *GetActorTemplateRequest) GetActorTemplate() *ObjectRef { @@ -4325,7 +4620,7 @@ type ListActorTemplatesRequest struct { func (x *ListActorTemplatesRequest) Reset() { *x = ListActorTemplatesRequest{} - mi := &file_ateapi_proto_msgTypes[53] + mi := &file_ateapi_proto_msgTypes[55] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4337,7 +4632,7 @@ func (x *ListActorTemplatesRequest) String() string { func (*ListActorTemplatesRequest) ProtoMessage() {} func (x *ListActorTemplatesRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[53] + mi := &file_ateapi_proto_msgTypes[55] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4350,7 +4645,7 @@ func (x *ListActorTemplatesRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ListActorTemplatesRequest.ProtoReflect.Descriptor instead. func (*ListActorTemplatesRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{53} + return file_ateapi_proto_rawDescGZIP(), []int{55} } func (x *ListActorTemplatesRequest) GetAtespace() string { @@ -4387,7 +4682,7 @@ type ListActorTemplatesResponse struct { func (x *ListActorTemplatesResponse) Reset() { *x = ListActorTemplatesResponse{} - mi := &file_ateapi_proto_msgTypes[54] + mi := &file_ateapi_proto_msgTypes[56] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4399,7 +4694,7 @@ func (x *ListActorTemplatesResponse) String() string { func (*ListActorTemplatesResponse) ProtoMessage() {} func (x *ListActorTemplatesResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[54] + mi := &file_ateapi_proto_msgTypes[56] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4412,7 +4707,7 @@ func (x *ListActorTemplatesResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ListActorTemplatesResponse.ProtoReflect.Descriptor instead. func (*ListActorTemplatesResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{54} + return file_ateapi_proto_rawDescGZIP(), []int{56} } func (x *ListActorTemplatesResponse) GetActorTemplates() []*ActorTemplate { @@ -4444,7 +4739,7 @@ type DeleteActorTemplateRequest struct { func (x *DeleteActorTemplateRequest) Reset() { *x = DeleteActorTemplateRequest{} - mi := &file_ateapi_proto_msgTypes[55] + mi := &file_ateapi_proto_msgTypes[57] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4456,7 +4751,7 @@ func (x *DeleteActorTemplateRequest) String() string { func (*DeleteActorTemplateRequest) ProtoMessage() {} func (x *DeleteActorTemplateRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[55] + mi := &file_ateapi_proto_msgTypes[57] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4469,7 +4764,7 @@ func (x *DeleteActorTemplateRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use DeleteActorTemplateRequest.ProtoReflect.Descriptor instead. func (*DeleteActorTemplateRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{55} + return file_ateapi_proto_rawDescGZIP(), []int{57} } func (x *DeleteActorTemplateRequest) GetActorTemplate() *ObjectRef { @@ -4497,7 +4792,7 @@ type GetActorRequest struct { func (x *GetActorRequest) Reset() { *x = GetActorRequest{} - mi := &file_ateapi_proto_msgTypes[56] + mi := &file_ateapi_proto_msgTypes[58] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4509,7 +4804,7 @@ func (x *GetActorRequest) String() string { func (*GetActorRequest) ProtoMessage() {} func (x *GetActorRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[56] + mi := &file_ateapi_proto_msgTypes[58] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4522,7 +4817,7 @@ func (x *GetActorRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetActorRequest.ProtoReflect.Descriptor instead. func (*GetActorRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{56} + return file_ateapi_proto_rawDescGZIP(), []int{58} } func (x *GetActorRequest) GetActor() *ObjectRef { @@ -4545,7 +4840,7 @@ type CreateActorRequest struct { func (x *CreateActorRequest) Reset() { *x = CreateActorRequest{} - mi := &file_ateapi_proto_msgTypes[57] + mi := &file_ateapi_proto_msgTypes[59] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4557,7 +4852,7 @@ func (x *CreateActorRequest) String() string { func (*CreateActorRequest) ProtoMessage() {} func (x *CreateActorRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[57] + mi := &file_ateapi_proto_msgTypes[59] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4570,7 +4865,7 @@ func (x *CreateActorRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use CreateActorRequest.ProtoReflect.Descriptor instead. func (*CreateActorRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{57} + return file_ateapi_proto_rawDescGZIP(), []int{59} } func (x *CreateActorRequest) GetActor() *Actor { @@ -4601,7 +4896,7 @@ type UpdateActorRequest struct { func (x *UpdateActorRequest) Reset() { *x = UpdateActorRequest{} - mi := &file_ateapi_proto_msgTypes[58] + mi := &file_ateapi_proto_msgTypes[60] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4613,7 +4908,7 @@ func (x *UpdateActorRequest) String() string { func (*UpdateActorRequest) ProtoMessage() {} func (x *UpdateActorRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[58] + mi := &file_ateapi_proto_msgTypes[60] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4626,7 +4921,7 @@ func (x *UpdateActorRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use UpdateActorRequest.ProtoReflect.Descriptor instead. func (*UpdateActorRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{58} + return file_ateapi_proto_rawDescGZIP(), []int{60} } func (x *UpdateActorRequest) GetActor() *Actor { @@ -4647,7 +4942,7 @@ type SuspendActorRequest struct { func (x *SuspendActorRequest) Reset() { *x = SuspendActorRequest{} - mi := &file_ateapi_proto_msgTypes[59] + mi := &file_ateapi_proto_msgTypes[61] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4659,7 +4954,7 @@ func (x *SuspendActorRequest) String() string { func (*SuspendActorRequest) ProtoMessage() {} func (x *SuspendActorRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[59] + mi := &file_ateapi_proto_msgTypes[61] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4672,7 +4967,7 @@ func (x *SuspendActorRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use SuspendActorRequest.ProtoReflect.Descriptor instead. func (*SuspendActorRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{59} + return file_ateapi_proto_rawDescGZIP(), []int{61} } func (x *SuspendActorRequest) GetActor() *ObjectRef { @@ -4691,7 +4986,7 @@ type SuspendActorResponse struct { func (x *SuspendActorResponse) Reset() { *x = SuspendActorResponse{} - mi := &file_ateapi_proto_msgTypes[60] + mi := &file_ateapi_proto_msgTypes[62] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4703,7 +4998,7 @@ func (x *SuspendActorResponse) String() string { func (*SuspendActorResponse) ProtoMessage() {} func (x *SuspendActorResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[60] + mi := &file_ateapi_proto_msgTypes[62] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4716,7 +5011,7 @@ func (x *SuspendActorResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use SuspendActorResponse.ProtoReflect.Descriptor instead. func (*SuspendActorResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{60} + return file_ateapi_proto_rawDescGZIP(), []int{62} } func (x *SuspendActorResponse) GetActor() *Actor { @@ -4737,7 +5032,7 @@ type PauseActorRequest struct { func (x *PauseActorRequest) Reset() { *x = PauseActorRequest{} - mi := &file_ateapi_proto_msgTypes[61] + mi := &file_ateapi_proto_msgTypes[63] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4749,7 +5044,7 @@ func (x *PauseActorRequest) String() string { func (*PauseActorRequest) ProtoMessage() {} func (x *PauseActorRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[61] + mi := &file_ateapi_proto_msgTypes[63] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4762,7 +5057,7 @@ func (x *PauseActorRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use PauseActorRequest.ProtoReflect.Descriptor instead. func (*PauseActorRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{61} + return file_ateapi_proto_rawDescGZIP(), []int{63} } func (x *PauseActorRequest) GetActor() *ObjectRef { @@ -4781,7 +5076,7 @@ type PauseActorResponse struct { func (x *PauseActorResponse) Reset() { *x = PauseActorResponse{} - mi := &file_ateapi_proto_msgTypes[62] + mi := &file_ateapi_proto_msgTypes[64] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4793,7 +5088,7 @@ func (x *PauseActorResponse) String() string { func (*PauseActorResponse) ProtoMessage() {} func (x *PauseActorResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[62] + mi := &file_ateapi_proto_msgTypes[64] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4806,7 +5101,7 @@ func (x *PauseActorResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use PauseActorResponse.ProtoReflect.Descriptor instead. func (*PauseActorResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{62} + return file_ateapi_proto_rawDescGZIP(), []int{64} } func (x *PauseActorResponse) GetActor() *Actor { @@ -4827,7 +5122,7 @@ type ResumeActorRequest struct { func (x *ResumeActorRequest) Reset() { *x = ResumeActorRequest{} - mi := &file_ateapi_proto_msgTypes[63] + mi := &file_ateapi_proto_msgTypes[65] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4839,7 +5134,7 @@ func (x *ResumeActorRequest) String() string { func (*ResumeActorRequest) ProtoMessage() {} func (x *ResumeActorRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[63] + mi := &file_ateapi_proto_msgTypes[65] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4852,7 +5147,7 @@ func (x *ResumeActorRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ResumeActorRequest.ProtoReflect.Descriptor instead. func (*ResumeActorRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{63} + return file_ateapi_proto_rawDescGZIP(), []int{65} } func (x *ResumeActorRequest) GetActor() *ObjectRef { @@ -4874,7 +5169,7 @@ type ResumeActorResponse struct { func (x *ResumeActorResponse) Reset() { *x = ResumeActorResponse{} - mi := &file_ateapi_proto_msgTypes[64] + mi := &file_ateapi_proto_msgTypes[66] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4886,7 +5181,7 @@ func (x *ResumeActorResponse) String() string { func (*ResumeActorResponse) ProtoMessage() {} func (x *ResumeActorResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[64] + mi := &file_ateapi_proto_msgTypes[66] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4899,7 +5194,7 @@ func (x *ResumeActorResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ResumeActorResponse.ProtoReflect.Descriptor instead. func (*ResumeActorResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{64} + return file_ateapi_proto_rawDescGZIP(), []int{66} } func (x *ResumeActorResponse) GetActor() *Actor { @@ -4929,7 +5224,7 @@ type RevertActorRequest struct { func (x *RevertActorRequest) Reset() { *x = RevertActorRequest{} - mi := &file_ateapi_proto_msgTypes[65] + mi := &file_ateapi_proto_msgTypes[67] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4941,7 +5236,7 @@ func (x *RevertActorRequest) String() string { func (*RevertActorRequest) ProtoMessage() {} func (x *RevertActorRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[65] + mi := &file_ateapi_proto_msgTypes[67] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4954,7 +5249,7 @@ func (x *RevertActorRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use RevertActorRequest.ProtoReflect.Descriptor instead. func (*RevertActorRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{65} + return file_ateapi_proto_rawDescGZIP(), []int{67} } func (x *RevertActorRequest) GetActor() *ObjectRef { @@ -4976,7 +5271,7 @@ type RevertActorResponse struct { func (x *RevertActorResponse) Reset() { *x = RevertActorResponse{} - mi := &file_ateapi_proto_msgTypes[66] + mi := &file_ateapi_proto_msgTypes[68] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4988,7 +5283,7 @@ func (x *RevertActorResponse) String() string { func (*RevertActorResponse) ProtoMessage() {} func (x *RevertActorResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[66] + mi := &file_ateapi_proto_msgTypes[68] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5001,7 +5296,7 @@ func (x *RevertActorResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RevertActorResponse.ProtoReflect.Descriptor instead. func (*RevertActorResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{66} + return file_ateapi_proto_rawDescGZIP(), []int{68} } func (x *RevertActorResponse) GetActor() *Actor { @@ -5030,7 +5325,7 @@ type DeleteActorRequest struct { func (x *DeleteActorRequest) Reset() { *x = DeleteActorRequest{} - mi := &file_ateapi_proto_msgTypes[67] + mi := &file_ateapi_proto_msgTypes[69] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5042,7 +5337,7 @@ func (x *DeleteActorRequest) String() string { func (*DeleteActorRequest) ProtoMessage() {} func (x *DeleteActorRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[67] + mi := &file_ateapi_proto_msgTypes[69] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5055,7 +5350,7 @@ func (x *DeleteActorRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use DeleteActorRequest.ProtoReflect.Descriptor instead. func (*DeleteActorRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{67} + return file_ateapi_proto_rawDescGZIP(), []int{69} } func (x *DeleteActorRequest) GetActor() *ObjectRef { @@ -5093,7 +5388,7 @@ type GetActorEgressPolicyRequest struct { func (x *GetActorEgressPolicyRequest) Reset() { *x = GetActorEgressPolicyRequest{} - mi := &file_ateapi_proto_msgTypes[68] + mi := &file_ateapi_proto_msgTypes[70] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5105,7 +5400,7 @@ func (x *GetActorEgressPolicyRequest) String() string { func (*GetActorEgressPolicyRequest) ProtoMessage() {} func (x *GetActorEgressPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[68] + mi := &file_ateapi_proto_msgTypes[70] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5118,7 +5413,7 @@ func (x *GetActorEgressPolicyRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetActorEgressPolicyRequest.ProtoReflect.Descriptor instead. func (*GetActorEgressPolicyRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{68} + return file_ateapi_proto_rawDescGZIP(), []int{70} } func (x *GetActorEgressPolicyRequest) GetActor() *ObjectRef { @@ -5149,7 +5444,7 @@ type CreateActorEgressPolicyRequest struct { func (x *CreateActorEgressPolicyRequest) Reset() { *x = CreateActorEgressPolicyRequest{} - mi := &file_ateapi_proto_msgTypes[69] + mi := &file_ateapi_proto_msgTypes[71] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5161,7 +5456,7 @@ func (x *CreateActorEgressPolicyRequest) String() string { func (*CreateActorEgressPolicyRequest) ProtoMessage() {} func (x *CreateActorEgressPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[69] + mi := &file_ateapi_proto_msgTypes[71] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5174,7 +5469,7 @@ func (x *CreateActorEgressPolicyRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use CreateActorEgressPolicyRequest.ProtoReflect.Descriptor instead. func (*CreateActorEgressPolicyRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{69} + return file_ateapi_proto_rawDescGZIP(), []int{71} } func (x *CreateActorEgressPolicyRequest) GetActor() *ObjectRef { @@ -5212,7 +5507,7 @@ type UpdateActorEgressPolicyRequest struct { func (x *UpdateActorEgressPolicyRequest) Reset() { *x = UpdateActorEgressPolicyRequest{} - mi := &file_ateapi_proto_msgTypes[70] + mi := &file_ateapi_proto_msgTypes[72] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5224,7 +5519,7 @@ func (x *UpdateActorEgressPolicyRequest) String() string { func (*UpdateActorEgressPolicyRequest) ProtoMessage() {} func (x *UpdateActorEgressPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[70] + mi := &file_ateapi_proto_msgTypes[72] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5237,7 +5532,7 @@ func (x *UpdateActorEgressPolicyRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use UpdateActorEgressPolicyRequest.ProtoReflect.Descriptor instead. func (*UpdateActorEgressPolicyRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{70} + return file_ateapi_proto_rawDescGZIP(), []int{72} } func (x *UpdateActorEgressPolicyRequest) GetActor() *ObjectRef { @@ -5272,7 +5567,7 @@ type DeleteActorEgressPolicyRequest struct { func (x *DeleteActorEgressPolicyRequest) Reset() { *x = DeleteActorEgressPolicyRequest{} - mi := &file_ateapi_proto_msgTypes[71] + mi := &file_ateapi_proto_msgTypes[73] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5284,7 +5579,7 @@ func (x *DeleteActorEgressPolicyRequest) String() string { func (*DeleteActorEgressPolicyRequest) ProtoMessage() {} func (x *DeleteActorEgressPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[71] + mi := &file_ateapi_proto_msgTypes[73] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5297,7 +5592,7 @@ func (x *DeleteActorEgressPolicyRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use DeleteActorEgressPolicyRequest.ProtoReflect.Descriptor instead. func (*DeleteActorEgressPolicyRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{71} + return file_ateapi_proto_rawDescGZIP(), []int{73} } func (x *DeleteActorEgressPolicyRequest) GetActor() *ObjectRef { @@ -5325,7 +5620,7 @@ type GetTagRequest struct { func (x *GetTagRequest) Reset() { *x = GetTagRequest{} - mi := &file_ateapi_proto_msgTypes[72] + mi := &file_ateapi_proto_msgTypes[74] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5337,7 +5632,7 @@ func (x *GetTagRequest) String() string { func (*GetTagRequest) ProtoMessage() {} func (x *GetTagRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[72] + mi := &file_ateapi_proto_msgTypes[74] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5350,7 +5645,7 @@ func (x *GetTagRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetTagRequest.ProtoReflect.Descriptor instead. func (*GetTagRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{72} + return file_ateapi_proto_rawDescGZIP(), []int{74} } func (x *GetTagRequest) GetTag() *ObjectRef { @@ -5376,7 +5671,7 @@ type MintActorJWTRequest struct { // +k8s:maxItems=16 # guardrail; tokens realistically bind a handful of audiences // +k8s:listType=set // +k8s:eachVal=+k8s:maxLength=512 # audiences are caller-defined URIs; bound only - Audience []string `protobuf:"bytes,1,rep,name=audience,proto3" json:"audience,omitempty"` + Audiences []string `protobuf:"bytes,1,rep,name=audiences,proto3" json:"audiences,omitempty"` // Lifetime of the JWT, in seconds. // // +k8s:required @@ -5389,7 +5684,7 @@ type MintActorJWTRequest struct { func (x *MintActorJWTRequest) Reset() { *x = MintActorJWTRequest{} - mi := &file_ateapi_proto_msgTypes[73] + mi := &file_ateapi_proto_msgTypes[75] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5401,7 +5696,7 @@ func (x *MintActorJWTRequest) String() string { func (*MintActorJWTRequest) ProtoMessage() {} func (x *MintActorJWTRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[73] + mi := &file_ateapi_proto_msgTypes[75] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5414,7 +5709,7 @@ func (x *MintActorJWTRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use MintActorJWTRequest.ProtoReflect.Descriptor instead. func (*MintActorJWTRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{73} + return file_ateapi_proto_rawDescGZIP(), []int{75} } func (x *MintActorJWTRequest) GetActor() *ObjectRef { @@ -5424,9 +5719,9 @@ func (x *MintActorJWTRequest) GetActor() *ObjectRef { return nil } -func (x *MintActorJWTRequest) GetAudience() []string { +func (x *MintActorJWTRequest) GetAudiences() []string { if x != nil { - return x.Audience + return x.Audiences } return nil } @@ -5464,7 +5759,7 @@ type MintActorJWTResponse struct { func (x *MintActorJWTResponse) Reset() { *x = MintActorJWTResponse{} - mi := &file_ateapi_proto_msgTypes[74] + mi := &file_ateapi_proto_msgTypes[76] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5476,7 +5771,7 @@ func (x *MintActorJWTResponse) String() string { func (*MintActorJWTResponse) ProtoMessage() {} func (x *MintActorJWTResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[74] + mi := &file_ateapi_proto_msgTypes[76] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5489,7 +5784,7 @@ func (x *MintActorJWTResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use MintActorJWTResponse.ProtoReflect.Descriptor instead. func (*MintActorJWTResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{74} + return file_ateapi_proto_rawDescGZIP(), []int{76} } func (x *MintActorJWTResponse) GetActorJwt() string { @@ -5534,7 +5829,7 @@ type MintActorCertificateRequest struct { func (x *MintActorCertificateRequest) Reset() { *x = MintActorCertificateRequest{} - mi := &file_ateapi_proto_msgTypes[75] + mi := &file_ateapi_proto_msgTypes[77] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5546,7 +5841,7 @@ func (x *MintActorCertificateRequest) String() string { func (*MintActorCertificateRequest) ProtoMessage() {} func (x *MintActorCertificateRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[75] + mi := &file_ateapi_proto_msgTypes[77] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5559,7 +5854,7 @@ func (x *MintActorCertificateRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use MintActorCertificateRequest.ProtoReflect.Descriptor instead. func (*MintActorCertificateRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{75} + return file_ateapi_proto_rawDescGZIP(), []int{77} } func (x *MintActorCertificateRequest) GetActor() *ObjectRef { @@ -5596,7 +5891,7 @@ type MintActorCertificateResponse struct { func (x *MintActorCertificateResponse) Reset() { *x = MintActorCertificateResponse{} - mi := &file_ateapi_proto_msgTypes[76] + mi := &file_ateapi_proto_msgTypes[78] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5608,7 +5903,7 @@ func (x *MintActorCertificateResponse) String() string { func (*MintActorCertificateResponse) ProtoMessage() {} func (x *MintActorCertificateResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[76] + mi := &file_ateapi_proto_msgTypes[78] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5621,7 +5916,7 @@ func (x *MintActorCertificateResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use MintActorCertificateResponse.ProtoReflect.Descriptor instead. func (*MintActorCertificateResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{76} + return file_ateapi_proto_rawDescGZIP(), []int{78} } func (x *MintActorCertificateResponse) GetActorCertificates() [][]byte { @@ -5657,7 +5952,7 @@ type ListTagsRequest struct { func (x *ListTagsRequest) Reset() { *x = ListTagsRequest{} - mi := &file_ateapi_proto_msgTypes[77] + mi := &file_ateapi_proto_msgTypes[79] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5669,7 +5964,7 @@ func (x *ListTagsRequest) String() string { func (*ListTagsRequest) ProtoMessage() {} func (x *ListTagsRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[77] + mi := &file_ateapi_proto_msgTypes[79] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5682,7 +5977,7 @@ func (x *ListTagsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ListTagsRequest.ProtoReflect.Descriptor instead. func (*ListTagsRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{77} + return file_ateapi_proto_rawDescGZIP(), []int{79} } func (x *ListTagsRequest) GetAtespace() string { @@ -5716,7 +6011,7 @@ type ListTagsResponse struct { func (x *ListTagsResponse) Reset() { *x = ListTagsResponse{} - mi := &file_ateapi_proto_msgTypes[78] + mi := &file_ateapi_proto_msgTypes[80] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5728,7 +6023,7 @@ func (x *ListTagsResponse) String() string { func (*ListTagsResponse) ProtoMessage() {} func (x *ListTagsResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[78] + mi := &file_ateapi_proto_msgTypes[80] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5741,7 +6036,7 @@ func (x *ListTagsResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ListTagsResponse.ProtoReflect.Descriptor instead. func (*ListTagsResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{78} + return file_ateapi_proto_rawDescGZIP(), []int{80} } func (x *ListTagsResponse) GetTags() []*Tag { @@ -5786,7 +6081,7 @@ type CreateTagRequest struct { func (x *CreateTagRequest) Reset() { *x = CreateTagRequest{} - mi := &file_ateapi_proto_msgTypes[79] + mi := &file_ateapi_proto_msgTypes[81] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5798,7 +6093,7 @@ func (x *CreateTagRequest) String() string { func (*CreateTagRequest) ProtoMessage() {} func (x *CreateTagRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[79] + mi := &file_ateapi_proto_msgTypes[81] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5811,7 +6106,7 @@ func (x *CreateTagRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use CreateTagRequest.ProtoReflect.Descriptor instead. func (*CreateTagRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{79} + return file_ateapi_proto_rawDescGZIP(), []int{81} } func (x *CreateTagRequest) GetTag() *Tag { @@ -5841,7 +6136,7 @@ type UpdateTagRequest struct { func (x *UpdateTagRequest) Reset() { *x = UpdateTagRequest{} - mi := &file_ateapi_proto_msgTypes[80] + mi := &file_ateapi_proto_msgTypes[82] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5853,7 +6148,7 @@ func (x *UpdateTagRequest) String() string { func (*UpdateTagRequest) ProtoMessage() {} func (x *UpdateTagRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[80] + mi := &file_ateapi_proto_msgTypes[82] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5866,7 +6161,7 @@ func (x *UpdateTagRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use UpdateTagRequest.ProtoReflect.Descriptor instead. func (*UpdateTagRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{80} + return file_ateapi_proto_rawDescGZIP(), []int{82} } func (x *UpdateTagRequest) GetTag() *Tag { @@ -5891,7 +6186,7 @@ type DeleteTagRequest struct { func (x *DeleteTagRequest) Reset() { *x = DeleteTagRequest{} - mi := &file_ateapi_proto_msgTypes[81] + mi := &file_ateapi_proto_msgTypes[83] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5903,7 +6198,7 @@ func (x *DeleteTagRequest) String() string { func (*DeleteTagRequest) ProtoMessage() {} func (x *DeleteTagRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[81] + mi := &file_ateapi_proto_msgTypes[83] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5916,7 +6211,7 @@ func (x *DeleteTagRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use DeleteTagRequest.ProtoReflect.Descriptor instead. func (*DeleteTagRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{81} + return file_ateapi_proto_rawDescGZIP(), []int{83} } func (x *DeleteTagRequest) GetTag() *ObjectRef { @@ -5955,7 +6250,7 @@ type DeleteOptions struct { func (x *DeleteOptions) Reset() { *x = DeleteOptions{} - mi := &file_ateapi_proto_msgTypes[82] + mi := &file_ateapi_proto_msgTypes[84] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5967,7 +6262,7 @@ func (x *DeleteOptions) String() string { func (*DeleteOptions) ProtoMessage() {} func (x *DeleteOptions) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[82] + mi := &file_ateapi_proto_msgTypes[84] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5980,7 +6275,7 @@ func (x *DeleteOptions) ProtoReflect() protoreflect.Message { // Deprecated: Use DeleteOptions.ProtoReflect.Descriptor instead. func (*DeleteOptions) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{82} + return file_ateapi_proto_rawDescGZIP(), []int{84} } func (x *DeleteOptions) GetVersion() int64 { @@ -6024,7 +6319,7 @@ type ListWorkerActorAssignmentsRequest struct { func (x *ListWorkerActorAssignmentsRequest) Reset() { *x = ListWorkerActorAssignmentsRequest{} - mi := &file_ateapi_proto_msgTypes[83] + mi := &file_ateapi_proto_msgTypes[85] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6036,7 +6331,7 @@ func (x *ListWorkerActorAssignmentsRequest) String() string { func (*ListWorkerActorAssignmentsRequest) ProtoMessage() {} func (x *ListWorkerActorAssignmentsRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[83] + mi := &file_ateapi_proto_msgTypes[85] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6049,7 +6344,7 @@ func (x *ListWorkerActorAssignmentsRequest) ProtoReflect() protoreflect.Message // Deprecated: Use ListWorkerActorAssignmentsRequest.ProtoReflect.Descriptor instead. func (*ListWorkerActorAssignmentsRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{83} + return file_ateapi_proto_rawDescGZIP(), []int{85} } func (x *ListWorkerActorAssignmentsRequest) GetWorker() *ObjectRef { @@ -6086,7 +6381,7 @@ type ListWorkerActorAssignmentsResponse struct { func (x *ListWorkerActorAssignmentsResponse) Reset() { *x = ListWorkerActorAssignmentsResponse{} - mi := &file_ateapi_proto_msgTypes[84] + mi := &file_ateapi_proto_msgTypes[86] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6098,7 +6393,7 @@ func (x *ListWorkerActorAssignmentsResponse) String() string { func (*ListWorkerActorAssignmentsResponse) ProtoMessage() {} func (x *ListWorkerActorAssignmentsResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[84] + mi := &file_ateapi_proto_msgTypes[86] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6111,7 +6406,7 @@ func (x *ListWorkerActorAssignmentsResponse) ProtoReflect() protoreflect.Message // Deprecated: Use ListWorkerActorAssignmentsResponse.ProtoReflect.Descriptor instead. func (*ListWorkerActorAssignmentsResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{84} + return file_ateapi_proto_rawDescGZIP(), []int{86} } func (x *ListWorkerActorAssignmentsResponse) GetActorAssignments() []*ActorAssignment { @@ -6149,7 +6444,7 @@ type ListWorkersRequest struct { func (x *ListWorkersRequest) Reset() { *x = ListWorkersRequest{} - mi := &file_ateapi_proto_msgTypes[85] + mi := &file_ateapi_proto_msgTypes[87] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6161,7 +6456,7 @@ func (x *ListWorkersRequest) String() string { func (*ListWorkersRequest) ProtoMessage() {} func (x *ListWorkersRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[85] + mi := &file_ateapi_proto_msgTypes[87] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6174,7 +6469,7 @@ func (x *ListWorkersRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ListWorkersRequest.ProtoReflect.Descriptor instead. func (*ListWorkersRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{85} + return file_ateapi_proto_rawDescGZIP(), []int{87} } func (x *ListWorkersRequest) GetPageSize() int32 { @@ -6203,7 +6498,7 @@ type ListWorkersResponse struct { func (x *ListWorkersResponse) Reset() { *x = ListWorkersResponse{} - mi := &file_ateapi_proto_msgTypes[86] + mi := &file_ateapi_proto_msgTypes[88] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6215,7 +6510,7 @@ func (x *ListWorkersResponse) String() string { func (*ListWorkersResponse) ProtoMessage() {} func (x *ListWorkersResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[86] + mi := &file_ateapi_proto_msgTypes[88] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6228,7 +6523,7 @@ func (x *ListWorkersResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ListWorkersResponse.ProtoReflect.Descriptor instead. func (*ListWorkersResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{86} + return file_ateapi_proto_rawDescGZIP(), []int{88} } func (x *ListWorkersResponse) GetWorkers() []*Worker { @@ -6258,7 +6553,7 @@ type GetWorkerRequest struct { func (x *GetWorkerRequest) Reset() { *x = GetWorkerRequest{} - mi := &file_ateapi_proto_msgTypes[87] + mi := &file_ateapi_proto_msgTypes[89] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6270,7 +6565,7 @@ func (x *GetWorkerRequest) String() string { func (*GetWorkerRequest) ProtoMessage() {} func (x *GetWorkerRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[87] + mi := &file_ateapi_proto_msgTypes[89] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6283,7 +6578,7 @@ func (x *GetWorkerRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetWorkerRequest.ProtoReflect.Descriptor instead. func (*GetWorkerRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{87} + return file_ateapi_proto_rawDescGZIP(), []int{89} } func (x *GetWorkerRequest) GetWorker() *ObjectRef { @@ -6305,7 +6600,7 @@ type CreateWorkerRequest struct { func (x *CreateWorkerRequest) Reset() { *x = CreateWorkerRequest{} - mi := &file_ateapi_proto_msgTypes[88] + mi := &file_ateapi_proto_msgTypes[90] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6317,7 +6612,7 @@ func (x *CreateWorkerRequest) String() string { func (*CreateWorkerRequest) ProtoMessage() {} func (x *CreateWorkerRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[88] + mi := &file_ateapi_proto_msgTypes[90] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6330,7 +6625,7 @@ func (x *CreateWorkerRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use CreateWorkerRequest.ProtoReflect.Descriptor instead. func (*CreateWorkerRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{88} + return file_ateapi_proto_rawDescGZIP(), []int{90} } func (x *CreateWorkerRequest) GetWorker() *Worker { @@ -6365,7 +6660,7 @@ type UpdateWorkerRequest struct { func (x *UpdateWorkerRequest) Reset() { *x = UpdateWorkerRequest{} - mi := &file_ateapi_proto_msgTypes[89] + mi := &file_ateapi_proto_msgTypes[91] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6377,7 +6672,7 @@ func (x *UpdateWorkerRequest) String() string { func (*UpdateWorkerRequest) ProtoMessage() {} func (x *UpdateWorkerRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[89] + mi := &file_ateapi_proto_msgTypes[91] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6390,7 +6685,7 @@ func (x *UpdateWorkerRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use UpdateWorkerRequest.ProtoReflect.Descriptor instead. func (*UpdateWorkerRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{89} + return file_ateapi_proto_rawDescGZIP(), []int{91} } func (x *UpdateWorkerRequest) GetWorker() *Worker { @@ -6417,7 +6712,7 @@ type DeleteWorkerRequest struct { func (x *DeleteWorkerRequest) Reset() { *x = DeleteWorkerRequest{} - mi := &file_ateapi_proto_msgTypes[90] + mi := &file_ateapi_proto_msgTypes[92] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6429,7 +6724,7 @@ func (x *DeleteWorkerRequest) String() string { func (*DeleteWorkerRequest) ProtoMessage() {} func (x *DeleteWorkerRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[90] + mi := &file_ateapi_proto_msgTypes[92] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6442,7 +6737,7 @@ func (x *DeleteWorkerRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use DeleteWorkerRequest.ProtoReflect.Descriptor instead. func (*DeleteWorkerRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{90} + return file_ateapi_proto_rawDescGZIP(), []int{92} } func (x *DeleteWorkerRequest) GetWorker() *ObjectRef { @@ -6472,7 +6767,7 @@ type DrainWorkerRequest struct { func (x *DrainWorkerRequest) Reset() { *x = DrainWorkerRequest{} - mi := &file_ateapi_proto_msgTypes[91] + mi := &file_ateapi_proto_msgTypes[93] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6484,7 +6779,7 @@ func (x *DrainWorkerRequest) String() string { func (*DrainWorkerRequest) ProtoMessage() {} func (x *DrainWorkerRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[91] + mi := &file_ateapi_proto_msgTypes[93] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6497,7 +6792,7 @@ func (x *DrainWorkerRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use DrainWorkerRequest.ProtoReflect.Descriptor instead. func (*DrainWorkerRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{91} + return file_ateapi_proto_rawDescGZIP(), []int{93} } func (x *DrainWorkerRequest) GetWorker() *ObjectRef { @@ -6535,7 +6830,7 @@ type ListActorsRequest struct { func (x *ListActorsRequest) Reset() { *x = ListActorsRequest{} - mi := &file_ateapi_proto_msgTypes[92] + mi := &file_ateapi_proto_msgTypes[94] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6547,7 +6842,7 @@ func (x *ListActorsRequest) String() string { func (*ListActorsRequest) ProtoMessage() {} func (x *ListActorsRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[92] + mi := &file_ateapi_proto_msgTypes[94] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6560,7 +6855,7 @@ func (x *ListActorsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ListActorsRequest.ProtoReflect.Descriptor instead. func (*ListActorsRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{92} + return file_ateapi_proto_rawDescGZIP(), []int{94} } func (x *ListActorsRequest) GetAtespace() string { @@ -6596,7 +6891,7 @@ type ListActorsResponse struct { func (x *ListActorsResponse) Reset() { *x = ListActorsResponse{} - mi := &file_ateapi_proto_msgTypes[93] + mi := &file_ateapi_proto_msgTypes[95] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6608,7 +6903,7 @@ func (x *ListActorsResponse) String() string { func (*ListActorsResponse) ProtoMessage() {} func (x *ListActorsResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[93] + mi := &file_ateapi_proto_msgTypes[95] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6621,7 +6916,7 @@ func (x *ListActorsResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ListActorsResponse.ProtoReflect.Descriptor instead. func (*ListActorsResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{93} + return file_ateapi_proto_rawDescGZIP(), []int{95} } func (x *ListActorsResponse) GetActors() []*Actor { @@ -6687,8 +6982,7 @@ type Worker struct { // +k8s:customValidation # until `format=k8s-ip` is supported // +k8s:immutable Ips []string `protobuf:"bytes,7,rep,name=ips,proto3" json:"ips,omitempty"` - // sandbox_class mirrors the WorkerPool's sandboxClass; its values are the - // CRD's own vocabulary, so it is only bounded, not validated. + // sandbox_class mirrors the WorkerPool's default sandbox class // // +k8s:optional // +k8s:maxLength=63 @@ -6729,7 +7023,7 @@ type Worker struct { func (x *Worker) Reset() { *x = Worker{} - mi := &file_ateapi_proto_msgTypes[94] + mi := &file_ateapi_proto_msgTypes[96] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6741,7 +7035,7 @@ func (x *Worker) String() string { func (*Worker) ProtoMessage() {} func (x *Worker) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[94] + mi := &file_ateapi_proto_msgTypes[96] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6754,7 +7048,7 @@ func (x *Worker) ProtoReflect() protoreflect.Message { // Deprecated: Use Worker.ProtoReflect.Descriptor instead. func (*Worker) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{94} + return file_ateapi_proto_rawDescGZIP(), []int{96} } func (x *Worker) GetMetadata() *ResourceMetadata { @@ -6864,17 +7158,26 @@ type WorkerStatus struct { // +k8s:optional // +k8s:minimum=0 ObservedEpoch int64 `protobuf:"varint,4,opt,name=observed_epoch,json=observedEpoch,proto3" json:"observed_epoch,omitempty"` - // Hardware identity reported by the Worker at registration. + // Runtime that new and cold-booted actors start under, as reported by the + // Worker at registration. Snapshots matching default_runtime can also + // warm-boot on this worker. // // +k8s:optional - Hardware *HardwareIdentity `protobuf:"bytes,5,opt,name=hardware,proto3" json:"hardware,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + DefaultRuntime *SandboxRuntime `protobuf:"bytes,5,opt,name=default_runtime,json=defaultRuntime,proto3" json:"default_runtime,omitempty"` + // Additional runtimes (excluding default_runtime) that this worker can + // restore snapshots from right now. + // + // +k8s:optional + // +k8s:maxItems=32 + // +k8s:listType=atomic + RestorableRuntimes []*SandboxRuntime `protobuf:"bytes,6,rep,name=restorable_runtimes,json=restorableRuntimes,proto3" json:"restorable_runtimes,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *WorkerStatus) Reset() { *x = WorkerStatus{} - mi := &file_ateapi_proto_msgTypes[95] + mi := &file_ateapi_proto_msgTypes[97] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6886,7 +7189,7 @@ func (x *WorkerStatus) String() string { func (*WorkerStatus) ProtoMessage() {} func (x *WorkerStatus) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[95] + mi := &file_ateapi_proto_msgTypes[97] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6899,7 +7202,7 @@ func (x *WorkerStatus) ProtoReflect() protoreflect.Message { // Deprecated: Use WorkerStatus.ProtoReflect.Descriptor instead. func (*WorkerStatus) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{95} + return file_ateapi_proto_rawDescGZIP(), []int{97} } func (x *WorkerStatus) GetState() WorkerState { @@ -6930,44 +7233,55 @@ func (x *WorkerStatus) GetObservedEpoch() int64 { return 0 } -func (x *WorkerStatus) GetHardware() *HardwareIdentity { +func (x *WorkerStatus) GetDefaultRuntime() *SandboxRuntime { + if x != nil { + return x.DefaultRuntime + } + return nil +} + +func (x *WorkerStatus) GetRestorableRuntimes() []*SandboxRuntime { if x != nil { - return x.Hardware + return x.RestorableRuntimes } return nil } -// HardwareIdentity identifies the hardware of a Worker or the Worker a -// snapshot was captured on. -type HardwareIdentity struct { +// SandboxRuntime describes a sandbox runtime and its compatibility identity as +// reported by a Worker or recorded on a snapshot. +type SandboxRuntime struct { state protoimpl.MessageState `protogen:"open.v1"` - // Key-value pairs describing hardware properties (currently - // "architecture"; TODO: add "cpu_vendor", "cpu_model", etc.). + // e.g. "gvisor", "microvm" // - // +k8s:optional - // +k8s:maxProperties=32 - // +k8s:eachKey=+k8s:maxLength=128 - // +k8s:eachVal=+k8s:maxLength=256 - Attributes map[string]string `protobuf:"bytes,1,rep,name=attributes,proto3" json:"attributes,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + // +k8s:required + // +k8s:maxLength=63 + SandboxClass string `protobuf:"bytes,1,opt,name=sandbox_class,json=sandboxClass,proto3" json:"sandbox_class,omitempty"` + // Primary compatibility version stamped onto an actor's snapshot when a + // checkpoint is taken on this runtime. Snapshots matching `version` + // can warm-boot on this runtime. + // TODO: Add support for additional compatibility versions. + // + // +k8s:required + Version *VersionedSandboxCompat `protobuf:"bytes,2,opt,name=version,proto3" json:"version,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } -func (x *HardwareIdentity) Reset() { - *x = HardwareIdentity{} - mi := &file_ateapi_proto_msgTypes[96] +func (x *SandboxRuntime) Reset() { + *x = SandboxRuntime{} + mi := &file_ateapi_proto_msgTypes[98] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } -func (x *HardwareIdentity) String() string { +func (x *SandboxRuntime) String() string { return protoimpl.X.MessageStringOf(x) } -func (*HardwareIdentity) ProtoMessage() {} +func (*SandboxRuntime) ProtoMessage() {} -func (x *HardwareIdentity) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[96] +func (x *SandboxRuntime) ProtoReflect() protoreflect.Message { + mi := &file_ateapi_proto_msgTypes[98] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6978,18 +7292,159 @@ func (x *HardwareIdentity) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use HardwareIdentity.ProtoReflect.Descriptor instead. -func (*HardwareIdentity) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{96} +// Deprecated: Use SandboxRuntime.ProtoReflect.Descriptor instead. +func (*SandboxRuntime) Descriptor() ([]byte, []int) { + return file_ateapi_proto_rawDescGZIP(), []int{98} +} + +func (x *SandboxRuntime) GetSandboxClass() string { + if x != nil { + return x.SandboxClass + } + return "" +} + +func (x *SandboxRuntime) GetVersion() *VersionedSandboxCompat { + if x != nil { + return x.Version + } + return nil +} + +// VersionedSandboxCompat holds the compatibility schema version and the +// key-value compatibility attributes for a SandboxRuntime. +type VersionedSandboxCompat struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Compatibility schema version (e.g. "v1") defining which runtime and + // hardware attributes are recorded and how they are computed. + // + // +k8s:required + // +k8s:maxLength=64 + SchemaVersion string `protobuf:"bytes,1,opt,name=schema_version,json=schemaVersion,proto3" json:"schema_version,omitempty"` + // Key-value compatibility attributes defined by this schema version, covering + // both runtime assets (e.g. "gvisor_asset_hash") and effective guest hardware + // features after masking (e.g. "architecture", "cpu_features"). + // Matched against a snapshot under the same schema_version: every attribute + // on the snapshot must be present on the worker with an equal value; keys + // only the worker has are ignored. Bump schema_version when a key's meaning + // changes or older snapshots must no longer match. + // Order as sent does not matter: ate-api-server sorts attributes by key + // before recording them. + // + // +k8s:required + // +k8s:maxItems=32 + // +k8s:listType=map + // +k8s:listMapKey=key + Attributes []*AttributeEntry `protobuf:"bytes,2,rep,name=attributes,proto3" json:"attributes,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *VersionedSandboxCompat) Reset() { + *x = VersionedSandboxCompat{} + mi := &file_ateapi_proto_msgTypes[99] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *VersionedSandboxCompat) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*VersionedSandboxCompat) ProtoMessage() {} + +func (x *VersionedSandboxCompat) ProtoReflect() protoreflect.Message { + mi := &file_ateapi_proto_msgTypes[99] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use VersionedSandboxCompat.ProtoReflect.Descriptor instead. +func (*VersionedSandboxCompat) Descriptor() ([]byte, []int) { + return file_ateapi_proto_rawDescGZIP(), []int{99} +} + +func (x *VersionedSandboxCompat) GetSchemaVersion() string { + if x != nil { + return x.SchemaVersion + } + return "" } -func (x *HardwareIdentity) GetAttributes() map[string]string { +func (x *VersionedSandboxCompat) GetAttributes() []*AttributeEntry { if x != nil { return x.Attributes } return nil } +// AttributeEntry is a single key-value compatibility attribute. +type AttributeEntry struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Attribute name (e.g. "architecture", "cpu_features", "gvisor_asset_hash"). + // + // +k8s:required + // +k8s:maxLength=128 + Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` + // Attribute value corresponding to key. + // + // +k8s:required + // +k8s:maxLength=256 + Value string `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *AttributeEntry) Reset() { + *x = AttributeEntry{} + mi := &file_ateapi_proto_msgTypes[100] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *AttributeEntry) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AttributeEntry) ProtoMessage() {} + +func (x *AttributeEntry) ProtoReflect() protoreflect.Message { + mi := &file_ateapi_proto_msgTypes[100] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AttributeEntry.ProtoReflect.Descriptor instead. +func (*AttributeEntry) Descriptor() ([]byte, []int) { + return file_ateapi_proto_rawDescGZIP(), []int{100} +} + +func (x *AttributeEntry) GetKey() string { + if x != nil { + return x.Key + } + return "" +} + +func (x *AttributeEntry) GetValue() string { + if x != nil { + return x.Value + } + return "" +} + // WorkerResources represents a set of resources. type WorkerResources struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -7012,7 +7467,7 @@ type WorkerResources struct { func (x *WorkerResources) Reset() { *x = WorkerResources{} - mi := &file_ateapi_proto_msgTypes[97] + mi := &file_ateapi_proto_msgTypes[101] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7024,7 +7479,7 @@ func (x *WorkerResources) String() string { func (*WorkerResources) ProtoMessage() {} func (x *WorkerResources) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[97] + mi := &file_ateapi_proto_msgTypes[101] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7037,7 +7492,7 @@ func (x *WorkerResources) ProtoReflect() protoreflect.Message { // Deprecated: Use WorkerResources.ProtoReflect.Descriptor instead. func (*WorkerResources) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{97} + return file_ateapi_proto_rawDescGZIP(), []int{101} } func (x *WorkerResources) GetResources() *Resources { @@ -7090,7 +7545,7 @@ type ActorAssignment struct { func (x *ActorAssignment) Reset() { *x = ActorAssignment{} - mi := &file_ateapi_proto_msgTypes[98] + mi := &file_ateapi_proto_msgTypes[102] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7102,7 +7557,7 @@ func (x *ActorAssignment) String() string { func (*ActorAssignment) ProtoMessage() {} func (x *ActorAssignment) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[98] + mi := &file_ateapi_proto_msgTypes[102] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7115,7 +7570,7 @@ func (x *ActorAssignment) ProtoReflect() protoreflect.Message { // Deprecated: Use ActorAssignment.ProtoReflect.Descriptor instead. func (*ActorAssignment) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{98} + return file_ateapi_proto_rawDescGZIP(), []int{102} } func (x *ActorAssignment) GetMetadata() *ResourceMetadata { @@ -7167,18 +7622,26 @@ type RegisterWorkerRequest struct { // // +k8s:required Capacity *WorkerResources `protobuf:"bytes,2,opt,name=capacity,proto3" json:"capacity,omitempty"` - // Hardware identity reported by the Worker. Like capacity, this replaces - // what is recorded on the Worker. + // Runtime that new and cold-booted actors start under. + // Snapshots matching default_runtime can also warm-boot on this worker. + // Like capacity, this replaces what is recorded on the Worker. // // +k8s:required - Hardware *HardwareIdentity `protobuf:"bytes,3,opt,name=hardware,proto3" json:"hardware,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + DefaultRuntime *SandboxRuntime `protobuf:"bytes,3,opt,name=default_runtime,json=defaultRuntime,proto3" json:"default_runtime,omitempty"` + // Additional runtimes (excluding default_runtime) that this worker can + // restore snapshots from right now. + // + // +k8s:optional + // +k8s:maxItems=32 + // +k8s:listType=atomic + RestorableRuntimes []*SandboxRuntime `protobuf:"bytes,4,rep,name=restorable_runtimes,json=restorableRuntimes,proto3" json:"restorable_runtimes,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *RegisterWorkerRequest) Reset() { *x = RegisterWorkerRequest{} - mi := &file_ateapi_proto_msgTypes[99] + mi := &file_ateapi_proto_msgTypes[103] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7190,7 +7653,7 @@ func (x *RegisterWorkerRequest) String() string { func (*RegisterWorkerRequest) ProtoMessage() {} func (x *RegisterWorkerRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[99] + mi := &file_ateapi_proto_msgTypes[103] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7203,7 +7666,7 @@ func (x *RegisterWorkerRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use RegisterWorkerRequest.ProtoReflect.Descriptor instead. func (*RegisterWorkerRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{99} + return file_ateapi_proto_rawDescGZIP(), []int{103} } func (x *RegisterWorkerRequest) GetWorker() *ObjectRef { @@ -7220,9 +7683,16 @@ func (x *RegisterWorkerRequest) GetCapacity() *WorkerResources { return nil } -func (x *RegisterWorkerRequest) GetHardware() *HardwareIdentity { +func (x *RegisterWorkerRequest) GetDefaultRuntime() *SandboxRuntime { if x != nil { - return x.Hardware + return x.DefaultRuntime + } + return nil +} + +func (x *RegisterWorkerRequest) GetRestorableRuntimes() []*SandboxRuntime { + if x != nil { + return x.RestorableRuntimes } return nil } @@ -7237,7 +7707,7 @@ type RegisterWorkerResponse struct { func (x *RegisterWorkerResponse) Reset() { *x = RegisterWorkerResponse{} - mi := &file_ateapi_proto_msgTypes[100] + mi := &file_ateapi_proto_msgTypes[104] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7249,7 +7719,7 @@ func (x *RegisterWorkerResponse) String() string { func (*RegisterWorkerResponse) ProtoMessage() {} func (x *RegisterWorkerResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[100] + mi := &file_ateapi_proto_msgTypes[104] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7262,7 +7732,7 @@ func (x *RegisterWorkerResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RegisterWorkerResponse.ProtoReflect.Descriptor instead. func (*RegisterWorkerResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{100} + return file_ateapi_proto_rawDescGZIP(), []int{104} } func (x *RegisterWorkerResponse) GetWorker() *Worker { @@ -7301,7 +7771,7 @@ type MintAteomActorCertificateRequest struct { func (x *MintAteomActorCertificateRequest) Reset() { *x = MintAteomActorCertificateRequest{} - mi := &file_ateapi_proto_msgTypes[101] + mi := &file_ateapi_proto_msgTypes[105] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7313,7 +7783,7 @@ func (x *MintAteomActorCertificateRequest) String() string { func (*MintAteomActorCertificateRequest) ProtoMessage() {} func (x *MintAteomActorCertificateRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[101] + mi := &file_ateapi_proto_msgTypes[105] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7326,7 +7796,7 @@ func (x *MintAteomActorCertificateRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use MintAteomActorCertificateRequest.ProtoReflect.Descriptor instead. func (*MintAteomActorCertificateRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{101} + return file_ateapi_proto_rawDescGZIP(), []int{105} } func (x *MintAteomActorCertificateRequest) GetActor() *ObjectRef { @@ -7363,7 +7833,7 @@ type MintAteomActorCertificateResponse struct { func (x *MintAteomActorCertificateResponse) Reset() { *x = MintAteomActorCertificateResponse{} - mi := &file_ateapi_proto_msgTypes[102] + mi := &file_ateapi_proto_msgTypes[106] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7375,7 +7845,7 @@ func (x *MintAteomActorCertificateResponse) String() string { func (*MintAteomActorCertificateResponse) ProtoMessage() {} func (x *MintAteomActorCertificateResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[102] + mi := &file_ateapi_proto_msgTypes[106] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7388,7 +7858,7 @@ func (x *MintAteomActorCertificateResponse) ProtoReflect() protoreflect.Message // Deprecated: Use MintAteomActorCertificateResponse.ProtoReflect.Descriptor instead. func (*MintAteomActorCertificateResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{102} + return file_ateapi_proto_rawDescGZIP(), []int{106} } func (x *MintAteomActorCertificateResponse) GetActorCertificates() [][]byte { @@ -7426,7 +7896,7 @@ type RequestActorSuspendRequest struct { func (x *RequestActorSuspendRequest) Reset() { *x = RequestActorSuspendRequest{} - mi := &file_ateapi_proto_msgTypes[103] + mi := &file_ateapi_proto_msgTypes[107] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7438,7 +7908,7 @@ func (x *RequestActorSuspendRequest) String() string { func (*RequestActorSuspendRequest) ProtoMessage() {} func (x *RequestActorSuspendRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[103] + mi := &file_ateapi_proto_msgTypes[107] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7451,7 +7921,7 @@ func (x *RequestActorSuspendRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use RequestActorSuspendRequest.ProtoReflect.Descriptor instead. func (*RequestActorSuspendRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{103} + return file_ateapi_proto_rawDescGZIP(), []int{107} } func (x *RequestActorSuspendRequest) GetWorker() *ObjectRef { @@ -7485,7 +7955,7 @@ type RequestActorSuspendResponse struct { func (x *RequestActorSuspendResponse) Reset() { *x = RequestActorSuspendResponse{} - mi := &file_ateapi_proto_msgTypes[104] + mi := &file_ateapi_proto_msgTypes[108] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7497,7 +7967,7 @@ func (x *RequestActorSuspendResponse) String() string { func (*RequestActorSuspendResponse) ProtoMessage() {} func (x *RequestActorSuspendResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[104] + mi := &file_ateapi_proto_msgTypes[108] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7510,7 +7980,7 @@ func (x *RequestActorSuspendResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RequestActorSuspendResponse.ProtoReflect.Descriptor instead. func (*RequestActorSuspendResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{104} + return file_ateapi_proto_rawDescGZIP(), []int{108} } func (x *RequestActorSuspendResponse) GetActor() *Actor { @@ -7545,7 +8015,7 @@ type AccessPolicy struct { func (x *AccessPolicy) Reset() { *x = AccessPolicy{} - mi := &file_ateapi_proto_msgTypes[105] + mi := &file_ateapi_proto_msgTypes[109] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7557,7 +8027,7 @@ func (x *AccessPolicy) String() string { func (*AccessPolicy) ProtoMessage() {} func (x *AccessPolicy) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[105] + mi := &file_ateapi_proto_msgTypes[109] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7570,7 +8040,7 @@ func (x *AccessPolicy) ProtoReflect() protoreflect.Message { // Deprecated: Use AccessPolicy.ProtoReflect.Descriptor instead. func (*AccessPolicy) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{105} + return file_ateapi_proto_rawDescGZIP(), []int{109} } func (x *AccessPolicy) GetMetadata() *ResourceMetadata { @@ -7612,7 +8082,7 @@ type Binding struct { func (x *Binding) Reset() { *x = Binding{} - mi := &file_ateapi_proto_msgTypes[106] + mi := &file_ateapi_proto_msgTypes[110] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7624,7 +8094,7 @@ func (x *Binding) String() string { func (*Binding) ProtoMessage() {} func (x *Binding) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[106] + mi := &file_ateapi_proto_msgTypes[110] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7637,7 +8107,7 @@ func (x *Binding) ProtoReflect() protoreflect.Message { // Deprecated: Use Binding.ProtoReflect.Descriptor instead. func (*Binding) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{106} + return file_ateapi_proto_rawDescGZIP(), []int{110} } func (x *Binding) GetRole() string { @@ -7663,7 +8133,7 @@ type GetGlobalAccessPolicyRequest struct { func (x *GetGlobalAccessPolicyRequest) Reset() { *x = GetGlobalAccessPolicyRequest{} - mi := &file_ateapi_proto_msgTypes[107] + mi := &file_ateapi_proto_msgTypes[111] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7675,7 +8145,7 @@ func (x *GetGlobalAccessPolicyRequest) String() string { func (*GetGlobalAccessPolicyRequest) ProtoMessage() {} func (x *GetGlobalAccessPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[107] + mi := &file_ateapi_proto_msgTypes[111] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7688,7 +8158,7 @@ func (x *GetGlobalAccessPolicyRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetGlobalAccessPolicyRequest.ProtoReflect.Descriptor instead. func (*GetGlobalAccessPolicyRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{107} + return file_ateapi_proto_rawDescGZIP(), []int{111} } // CreateGlobalAccessPolicyRequest creates the deployment-wide global access policy singleton. @@ -7705,7 +8175,7 @@ type CreateGlobalAccessPolicyRequest struct { func (x *CreateGlobalAccessPolicyRequest) Reset() { *x = CreateGlobalAccessPolicyRequest{} - mi := &file_ateapi_proto_msgTypes[108] + mi := &file_ateapi_proto_msgTypes[112] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7717,7 +8187,7 @@ func (x *CreateGlobalAccessPolicyRequest) String() string { func (*CreateGlobalAccessPolicyRequest) ProtoMessage() {} func (x *CreateGlobalAccessPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[108] + mi := &file_ateapi_proto_msgTypes[112] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7730,7 +8200,7 @@ func (x *CreateGlobalAccessPolicyRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use CreateGlobalAccessPolicyRequest.ProtoReflect.Descriptor instead. func (*CreateGlobalAccessPolicyRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{108} + return file_ateapi_proto_rawDescGZIP(), []int{112} } func (x *CreateGlobalAccessPolicyRequest) GetAccessPolicy() *AccessPolicy { @@ -7754,7 +8224,7 @@ type UpdateGlobalAccessPolicyRequest struct { func (x *UpdateGlobalAccessPolicyRequest) Reset() { *x = UpdateGlobalAccessPolicyRequest{} - mi := &file_ateapi_proto_msgTypes[109] + mi := &file_ateapi_proto_msgTypes[113] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7766,7 +8236,7 @@ func (x *UpdateGlobalAccessPolicyRequest) String() string { func (*UpdateGlobalAccessPolicyRequest) ProtoMessage() {} func (x *UpdateGlobalAccessPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[109] + mi := &file_ateapi_proto_msgTypes[113] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7779,7 +8249,7 @@ func (x *UpdateGlobalAccessPolicyRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use UpdateGlobalAccessPolicyRequest.ProtoReflect.Descriptor instead. func (*UpdateGlobalAccessPolicyRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{109} + return file_ateapi_proto_rawDescGZIP(), []int{113} } func (x *UpdateGlobalAccessPolicyRequest) GetAccessPolicy() *AccessPolicy { @@ -7803,7 +8273,7 @@ type GetAtespaceAccessPolicyRequest struct { func (x *GetAtespaceAccessPolicyRequest) Reset() { *x = GetAtespaceAccessPolicyRequest{} - mi := &file_ateapi_proto_msgTypes[110] + mi := &file_ateapi_proto_msgTypes[114] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7815,7 +8285,7 @@ func (x *GetAtespaceAccessPolicyRequest) String() string { func (*GetAtespaceAccessPolicyRequest) ProtoMessage() {} func (x *GetAtespaceAccessPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[110] + mi := &file_ateapi_proto_msgTypes[114] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7828,7 +8298,7 @@ func (x *GetAtespaceAccessPolicyRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetAtespaceAccessPolicyRequest.ProtoReflect.Descriptor instead. func (*GetAtespaceAccessPolicyRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{110} + return file_ateapi_proto_rawDescGZIP(), []int{114} } func (x *GetAtespaceAccessPolicyRequest) GetAtespace() *ObjectRef { @@ -7857,7 +8327,7 @@ type CreateAtespaceAccessPolicyRequest struct { func (x *CreateAtespaceAccessPolicyRequest) Reset() { *x = CreateAtespaceAccessPolicyRequest{} - mi := &file_ateapi_proto_msgTypes[111] + mi := &file_ateapi_proto_msgTypes[115] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7869,7 +8339,7 @@ func (x *CreateAtespaceAccessPolicyRequest) String() string { func (*CreateAtespaceAccessPolicyRequest) ProtoMessage() {} func (x *CreateAtespaceAccessPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[111] + mi := &file_ateapi_proto_msgTypes[115] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7882,7 +8352,7 @@ func (x *CreateAtespaceAccessPolicyRequest) ProtoReflect() protoreflect.Message // Deprecated: Use CreateAtespaceAccessPolicyRequest.ProtoReflect.Descriptor instead. func (*CreateAtespaceAccessPolicyRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{111} + return file_ateapi_proto_rawDescGZIP(), []int{115} } func (x *CreateAtespaceAccessPolicyRequest) GetAtespace() *ObjectRef { @@ -7918,7 +8388,7 @@ type UpdateAtespaceAccessPolicyRequest struct { func (x *UpdateAtespaceAccessPolicyRequest) Reset() { *x = UpdateAtespaceAccessPolicyRequest{} - mi := &file_ateapi_proto_msgTypes[112] + mi := &file_ateapi_proto_msgTypes[116] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7930,7 +8400,7 @@ func (x *UpdateAtespaceAccessPolicyRequest) String() string { func (*UpdateAtespaceAccessPolicyRequest) ProtoMessage() {} func (x *UpdateAtespaceAccessPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[112] + mi := &file_ateapi_proto_msgTypes[116] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7943,7 +8413,7 @@ func (x *UpdateAtespaceAccessPolicyRequest) ProtoReflect() protoreflect.Message // Deprecated: Use UpdateAtespaceAccessPolicyRequest.ProtoReflect.Descriptor instead. func (*UpdateAtespaceAccessPolicyRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{112} + return file_ateapi_proto_rawDescGZIP(), []int{116} } func (x *UpdateAtespaceAccessPolicyRequest) GetAtespace() *ObjectRef { @@ -7978,7 +8448,7 @@ type DeleteAtespaceAccessPolicyRequest struct { func (x *DeleteAtespaceAccessPolicyRequest) Reset() { *x = DeleteAtespaceAccessPolicyRequest{} - mi := &file_ateapi_proto_msgTypes[113] + mi := &file_ateapi_proto_msgTypes[117] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7990,7 +8460,7 @@ func (x *DeleteAtespaceAccessPolicyRequest) String() string { func (*DeleteAtespaceAccessPolicyRequest) ProtoMessage() {} func (x *DeleteAtespaceAccessPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[113] + mi := &file_ateapi_proto_msgTypes[117] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8003,7 +8473,7 @@ func (x *DeleteAtespaceAccessPolicyRequest) ProtoReflect() protoreflect.Message // Deprecated: Use DeleteAtespaceAccessPolicyRequest.ProtoReflect.Descriptor instead. func (*DeleteAtespaceAccessPolicyRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{113} + return file_ateapi_proto_rawDescGZIP(), []int{117} } func (x *DeleteAtespaceAccessPolicyRequest) GetAtespace() *ObjectRef { @@ -8024,14 +8494,26 @@ var File_ateapi_proto protoreflect.FileDescriptor const file_ateapi_proto_rawDesc = "" + "\n" + - "\fateapi.proto\x12\x06ateapi\x1a\x1fgoogle/protobuf/timestamp.proto\"\xa6\x01\n" + - "\x10ExternalSnapshot\x12!\n" + - "\fsnapshot_uri\x18\x01 \x01(\tR\vsnapshotUri\x12A\n" + - "\rcontent_scope\x18\x02 \x01(\x0e2\x1c.ateapi.SnapshotContentScopeR\fcontentScope\x12,\n" + - "\x12actor_template_uid\x18\x03 \x01(\tR\x10actorTemplateUid\"w\n" + + "\fateapi.proto\x12\x06ateapi\x1a\x1fgoogle/protobuf/timestamp.proto\"3\n" + + "\x0eObjectSnapshot\x12!\n" + + "\fsnapshot_uri\x18\x01 \x01(\tR\vsnapshotUri\"4\n" + "\rLocalSnapshot\x12#\n" + - "\rsnapshot_name\x18\x01 \x01(\tR\fsnapshotName\x12A\n" + - "\rcontent_scope\x18\x02 \x01(\x0e2\x1c.ateapi.SnapshotContentScopeR\fcontentScope\"\x90\x01\n" + + "\rsnapshot_name\x18\x01 \x01(\tR\fsnapshotName\"\x97\x02\n" + + "\x0fSnapshotStorage\x12:\n" + + "\n" + + "durability\x18\x01 \x01(\x0e2\x1a.ateapi.SnapshotDurabilityR\n" + + "durability\x125\n" + + "\x06status\x18\x02 \x01(\x0e2\x1d.ateapi.SnapshotStorageStatusR\x06status\x124\n" + + "\bfidelity\x18\x03 \x01(\x0e2\x18.ateapi.SnapshotFidelityR\bfidelity\x12+\n" + + "\x05local\x18\x04 \x01(\v2\x15.ateapi.LocalSnapshotR\x05local\x12.\n" + + "\x06object\x18\x05 \x01(\v2\x16.ateapi.ObjectSnapshotR\x06object\"\xb8\x01\n" + + "\bSnapshot\x12\x1e\n" + + "\n" + + "generation\x18\x01 \x01(\x05R\n" + + "generation\x12+\n" + + "\x05owner\x18\x02 \x01(\x0e2\x15.ateapi.SnapshotOwnerR\x05owner\x12,\n" + + "\x12actor_template_uid\x18\x03 \x01(\tR\x10actorTemplateUid\x121\n" + + "\astorage\x18\x04 \x03(\v2\x17.ateapi.SnapshotStorageR\astorage\"\x90\x01\n" + "\bSelector\x12D\n" + "\fmatch_labels\x18\x01 \x03(\v2!.ateapi.Selector.MatchLabelsEntryR\vmatchLabels\x1a>\n" + "\x10MatchLabelsEntry\x12\x10\n" + @@ -8045,10 +8527,9 @@ const file_ateapi_proto_rawDesc = "" + "\vcreate_time\x18\x05 \x01(\v2\x1a.google.protobuf.TimestampR\n" + "createTime\x12;\n" + "\vupdate_time\x18\x06 \x01(\v2\x1a.google.protobuf.TimestampR\n" + - "updateTime\"\xa8\x03\n" + - "\x0eExternalVolume\x12\x1f\n" + - "\vvolume_name\x18\x01 \x01(\tR\n" + - "volumeName\x12*\n" + + "updateTime\"\x9b\x03\n" + + "\x0eExternalVolume\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x12*\n" + "\x11storage_volume_id\x18\x02 \x01(\tR\x0fstorageVolumeId\x12\x1f\n" + "\vvolume_type\x18\x03 \x01(\tR\n" + "volumeType\x125\n" + @@ -8102,18 +8583,16 @@ const file_ateapi_proto_rawDesc = "" + "\tactor_jwt\x18\x04 \x01(\v2\x16.ateapi.ActorJWTSourceR\bactorJwt\"]\n" + "\x0eActorJWTSource\x12\x1c\n" + "\taudiences\x18\x01 \x03(\tR\taudiences\x12-\n" + - "\x12expiration_seconds\x18\x02 \x01(\x03R\x11expirationSeconds\"\x8e\x04\n" + + "\x12expiration_seconds\x18\x02 \x01(\x03R\x11expirationSeconds\"\xfa\x02\n" + "\vActorStatus\x12(\n" + "\x05state\x18\x01 \x01(\x0e2\x12.ateapi.ActorStateR\x05state\x12E\n" + - "\x11worker_assignment\x18\x02 \x01(\v2\x18.ateapi.WorkerAssignmentR\x10workerAssignment\x127\n" + - "\x18in_progress_snapshot_uri\x18\x03 \x01(\tR\x15inProgressSnapshotUri\x12E\n" + - "\x11external_snapshot\x18\x04 \x01(\v2\x18.ateapi.ExternalSnapshotR\x10externalSnapshot\x12<\n" + - "\x0elocal_snapshot\x18\x05 \x01(\v2\x15.ateapi.LocalSnapshotR\rlocalSnapshot\x12;\n" + - "\ractor_volumes\x18\a \x03(\v2\x16.ateapi.ExternalVolumeR\factorVolumes\x12D\n" + - "\x1fin_progress_local_snapshot_name\x18\b \x01(\tR\x1binProgressLocalSnapshotName\x12(\n" + + "\x11worker_assignment\x18\x02 \x01(\v2\x18.ateapi.WorkerAssignmentR\x10workerAssignment\x12A\n" + + "\x10external_volumes\x18\a \x03(\v2\x16.ateapi.ExternalVolumeR\x0fexternalVolumes\x12(\n" + "\x05crash\x18\t \x01(\v2\x12.ateapi.ActorCrashR\x05crash\x12#\n" + "\rassigned_node\x18\n" + - " \x01(\tR\fassignedNode\"a\n" + + " \x01(\tR\fassignedNode\x12.\n" + + "\tsnapshots\x18\v \x03(\v2\x10.ateapi.SnapshotR\tsnapshots\x128\n" + + "\x18last_assigned_generation\x18\f \x01(\x05R\x16lastAssignedGeneration\"a\n" + "\n" + "ActorCrash\x12\x18\n" + "\amessage\x18\x01 \x01(\tR\amessage\x129\n" + @@ -8129,11 +8608,10 @@ const file_ateapi_proto_rawDesc = "" + "\x0eworker_pod_uid\x18\x04 \x01(\tR\fworkerPodUid\x12$\n" + "\x0eworker_pod_ips\x18\x05 \x03(\tR\fworkerPodIps\x12\x1b\n" + "\tnode_name\x18\a \x01(\tR\bnodeName\x12!\n" + - "\fworker_epoch\x18\b \x01(\x03R\vworkerEpoch\"\x9a\x01\n" + - "\tTagStatus\x124\n" + - "\bsnapshot\x18\x01 \x01(\v2\x18.ateapi.ExternalSnapshotR\bsnapshot\x12,\n" + - "\x12actor_template_uid\x18\x02 \x01(\tR\x10actorTemplateUid\x12)\n" + - "\x10storage_location\x18\x03 \x01(\tR\x0fstorageLocation\"\xc4\x01\n" + + "\fworker_epoch\x18\b \x01(\x03R\vworkerEpoch\"g\n" + + "\tTagStatus\x12,\n" + + "\bsnapshot\x18\x01 \x01(\v2\x10.ateapi.SnapshotR\bsnapshot\x12,\n" + + "\x12actor_template_uid\x18\x02 \x01(\tR\x10actorTemplateUid\"\xc4\x01\n" + "\x03Tag\x124\n" + "\bmetadata\x18\x01 \x01(\v2\x18.ateapi.ResourceMetadataR\bmetadata\x12)\n" + "\x06status\x18\x02 \x01(\v2\x11.ateapi.TagStatusR\x06status\x12&\n" + @@ -8170,10 +8648,10 @@ const file_ateapi_proto_rawDesc = "" + "\rSandboxConfig\x129\n" + "\rsandbox_class\x18\x01 \x01(\x0e2\x14.ateapi.SandboxClassR\fsandboxClass\x12\x1f\n" + "\vconfig_name\x18\x02 \x01(\tR\n" + - "configName\"v\n" + - "\x0eSnapshotConfig\x129\n" + - "\ton_commit\x18\x02 \x01(\x0e2\x1c.ateapi.SnapshotContentScopeR\bonCommit\x12)\n" + - "\x10storage_location\x18\x04 \x01(\tR\x0fstorageLocation\"\xf5\x02\n" + + "configName\"\x84\x01\n" + + "\x0eSnapshotConfig\x12)\n" + + "\x10storage_location\x18\x01 \x01(\tR\x0fstorageLocation\x12G\n" + + "\x12preferred_fidelity\x18\x02 \x01(\x0e2\x18.ateapi.SnapshotFidelityR\x11preferredFidelity\"\xf5\x02\n" + "\tContainer\x12\x12\n" + "\x04name\x18\x01 \x01(\tR\x04name\x12\x14\n" + "\x05image\x18\x02 \x01(\tR\x05image\x12\x18\n" + @@ -8297,10 +8775,10 @@ const file_ateapi_proto_rawDesc = "" + "\x05actor\x18\x01 \x01(\v2\x11.ateapi.ObjectRefR\x05actor\x12/\n" + "\aoptions\x18\x02 \x01(\v2\x15.ateapi.DeleteOptionsR\aoptions\"4\n" + "\rGetTagRequest\x12#\n" + - "\x03tag\x18\x01 \x01(\v2\x11.ateapi.ObjectRefR\x03tag\"\x89\x01\n" + + "\x03tag\x18\x01 \x01(\v2\x11.ateapi.ObjectRefR\x03tag\"\x8b\x01\n" + "\x13MintActorJWTRequest\x12'\n" + - "\x05actor\x18\x05 \x01(\v2\x11.ateapi.ObjectRefR\x05actor\x12\x1a\n" + - "\baudience\x18\x01 \x03(\tR\baudience\x12-\n" + + "\x05actor\x18\x05 \x01(\v2\x11.ateapi.ObjectRefR\x05actor\x12\x1c\n" + + "\taudiences\x18\x01 \x03(\tR\taudiences\x12-\n" + "\x12expiration_seconds\x18\b \x01(\x03R\x11expirationSeconds\"s\n" + "\x14MintActorJWTResponse\x12 \n" + "\tactor_jwt\x18\x01 \x01(\tB\x03\x80\x01\x01R\bactorJwt\x129\n" + @@ -8380,20 +8858,25 @@ const file_ateapi_proto_rawDesc = "" + "\x06status\x18\v \x01(\v2\x14.ateapi.WorkerStatusR\x06status\x1a9\n" + "\vLabelsEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + - "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\x82\x02\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\xd6\x02\n" + "\fWorkerStatus\x12)\n" + "\x05state\x18\x01 \x01(\x0e2\x13.ateapi.WorkerStateR\x05state\x123\n" + "\bcapacity\x18\x02 \x01(\v2\x17.ateapi.WorkerResourcesR\bcapacity\x125\n" + "\tallocated\x18\x03 \x01(\v2\x17.ateapi.WorkerResourcesR\tallocated\x12%\n" + - "\x0eobserved_epoch\x18\x04 \x01(\x03R\robservedEpoch\x124\n" + - "\bhardware\x18\x05 \x01(\v2\x18.ateapi.HardwareIdentityR\bhardware\"\x9b\x01\n" + - "\x10HardwareIdentity\x12H\n" + + "\x0eobserved_epoch\x18\x04 \x01(\x03R\robservedEpoch\x12?\n" + + "\x0fdefault_runtime\x18\x05 \x01(\v2\x16.ateapi.SandboxRuntimeR\x0edefaultRuntime\x12G\n" + + "\x13restorable_runtimes\x18\x06 \x03(\v2\x16.ateapi.SandboxRuntimeR\x12restorableRuntimes\"o\n" + + "\x0eSandboxRuntime\x12#\n" + + "\rsandbox_class\x18\x01 \x01(\tR\fsandboxClass\x128\n" + + "\aversion\x18\x02 \x01(\v2\x1e.ateapi.VersionedSandboxCompatR\aversion\"w\n" + + "\x16VersionedSandboxCompat\x12%\n" + + "\x0eschema_version\x18\x01 \x01(\tR\rschemaVersion\x126\n" + "\n" + - "attributes\x18\x01 \x03(\v2(.ateapi.HardwareIdentity.AttributesEntryR\n" + - "attributes\x1a=\n" + - "\x0fAttributesEntry\x12\x10\n" + + "attributes\x18\x02 \x03(\v2\x16.ateapi.AttributeEntryR\n" + + "attributes\"8\n" + + "\x0eAttributeEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + - "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"Z\n" + + "\x05value\x18\x02 \x01(\tR\x05value\"Z\n" + "\x0fWorkerResources\x12/\n" + "\tresources\x18\x01 \x01(\v2\x11.ateapi.ResourcesR\tresources\x12\x16\n" + "\x06actors\x18\x02 \x01(\x05R\x06actors\"\xe1\x01\n" + @@ -8402,11 +8885,12 @@ const file_ateapi_proto_rawDesc = "" + "\x05actor\x18\x02 \x01(\v2\x11.ateapi.ObjectRefR\x05actor\x12\x1b\n" + "\tactor_uid\x18\x03 \x01(\tR\bactorUid\x12/\n" + "\tresources\x18\x05 \x01(\v2\x11.ateapi.ResourcesR\tresources\x12!\n" + - "\fworker_epoch\x18\a \x01(\x03R\vworkerEpoch\"\xad\x01\n" + + "\fworker_epoch\x18\a \x01(\x03R\vworkerEpoch\"\x81\x02\n" + "\x15RegisterWorkerRequest\x12)\n" + "\x06worker\x18\x01 \x01(\v2\x11.ateapi.ObjectRefR\x06worker\x123\n" + - "\bcapacity\x18\x02 \x01(\v2\x17.ateapi.WorkerResourcesR\bcapacity\x124\n" + - "\bhardware\x18\x03 \x01(\v2\x18.ateapi.HardwareIdentityR\bhardware\"@\n" + + "\bcapacity\x18\x02 \x01(\v2\x17.ateapi.WorkerResourcesR\bcapacity\x12?\n" + + "\x0fdefault_runtime\x18\x03 \x01(\v2\x16.ateapi.SandboxRuntimeR\x0edefaultRuntime\x12G\n" + + "\x13restorable_runtimes\x18\x04 \x03(\v2\x16.ateapi.SandboxRuntimeR\x12restorableRuntimes\"@\n" + "\x16RegisterWorkerResponse\x12&\n" + "\x06worker\x18\x01 \x01(\v2\x0e.ateapi.WorkerR\x06worker\"\xa8\x01\n" + " MintAteomActorCertificateRequest\x12'\n" + @@ -8442,11 +8926,24 @@ const file_ateapi_proto_rawDesc = "" + "\raccess_policy\x18\x02 \x01(\v2\x14.ateapi.AccessPolicyR\faccessPolicy\"\x83\x01\n" + "!DeleteAtespaceAccessPolicyRequest\x12-\n" + "\batespace\x18\x01 \x01(\v2\x11.ateapi.ObjectRefR\batespace\x12/\n" + - "\aoptions\x18\x02 \x01(\v2\x15.ateapi.DeleteOptionsR\aoptions*\x80\x01\n" + - "\x14SnapshotContentScope\x12&\n" + - "\"SNAPSHOT_CONTENT_SCOPE_UNSPECIFIED\x10\x00\x12\x1f\n" + - "\x1bSNAPSHOT_CONTENT_SCOPE_FULL\x10\x01\x12\x1f\n" + - "\x1bSNAPSHOT_CONTENT_SCOPE_DATA\x10\x02*V\n" + + "\aoptions\x18\x02 \x01(\v2\x15.ateapi.DeleteOptionsR\aoptions*a\n" + + "\rSnapshotOwner\x12\x1e\n" + + "\x1aSNAPSHOT_OWNER_UNSPECIFIED\x10\x00\x12\x18\n" + + "\x14SNAPSHOT_OWNER_ACTOR\x10\x01\x12\x16\n" + + "\x12SNAPSHOT_OWNER_TAG\x10\x02*y\n" + + "\x12SnapshotDurability\x12#\n" + + "\x1fSNAPSHOT_DURABILITY_UNSPECIFIED\x10\x00\x12\x1d\n" + + "\x19SNAPSHOT_DURABILITY_LOCAL\x10\x01\x12\x1f\n" + + "\x1bSNAPSHOT_DURABILITY_DURABLE\x10\x02*\x90\x01\n" + + "\x15SnapshotStorageStatus\x12'\n" + + "#SNAPSHOT_STORAGE_STATUS_UNSPECIFIED\x10\x00\x12'\n" + + "#SNAPSHOT_STORAGE_STATUS_IN_PROGRESS\x10\x01\x12%\n" + + "!SNAPSHOT_STORAGE_STATUS_COMPLETED\x10\x02*\x90\x01\n" + + "\x10SnapshotFidelity\x12!\n" + + "\x1dSNAPSHOT_FIDELITY_UNSPECIFIED\x10\x00\x12\x1d\n" + + "\x19SNAPSHOT_FIDELITY_VOLUMES\x10\x01\x12\x1c\n" + + "\x18SNAPSHOT_FIDELITY_ROOTFS\x10\x02\x12\x1c\n" + + "\x18SNAPSHOT_FIDELITY_MEMORY\x10\x03*V\n" + "\bTagScope\x12\x19\n" + "\x15TAG_SCOPE_UNSPECIFIED\x10\x00\x12\x16\n" + "\x12TAG_SCOPE_ATESPACE\x10\x01\x12\x17\n" + @@ -8538,381 +9035,394 @@ func file_ateapi_proto_rawDescGZIP() []byte { return file_ateapi_proto_rawDescData } -var file_ateapi_proto_enumTypes = make([]protoimpl.EnumInfo, 7) -var file_ateapi_proto_msgTypes = make([]protoimpl.MessageInfo, 118) +var file_ateapi_proto_enumTypes = make([]protoimpl.EnumInfo, 10) +var file_ateapi_proto_msgTypes = make([]protoimpl.MessageInfo, 121) var file_ateapi_proto_goTypes = []any{ - (SnapshotContentScope)(0), // 0: ateapi.SnapshotContentScope - (TagScope)(0), // 1: ateapi.TagScope - (ActorState)(0), // 2: ateapi.ActorState - (SandboxClass)(0), // 3: ateapi.SandboxClass - (ActorMetadataField)(0), // 4: ateapi.ActorMetadataField - (WorkerState)(0), // 5: ateapi.WorkerState - (ExternalVolume_Status)(0), // 6: ateapi.ExternalVolume.Status - (*ExternalSnapshot)(nil), // 7: ateapi.ExternalSnapshot - (*LocalSnapshot)(nil), // 8: ateapi.LocalSnapshot - (*Selector)(nil), // 9: ateapi.Selector - (*ResourceMetadata)(nil), // 10: ateapi.ResourceMetadata - (*ExternalVolume)(nil), // 11: ateapi.ExternalVolume - (*Actor)(nil), // 12: ateapi.Actor - (*EgressPolicy)(nil), // 13: ateapi.EgressPolicy - (*EgressRule)(nil), // 14: ateapi.EgressRule - (*HTTPRule)(nil), // 15: ateapi.HTTPRule - (*HTTPSRule)(nil), // 16: ateapi.HTTPSRule - (*TLSPassthroughRule)(nil), // 17: ateapi.TLSPassthroughRule - (*Ports)(nil), // 18: ateapi.Ports - (*AllPorts)(nil), // 19: ateapi.AllPorts - (*HttpRuleEffects)(nil), // 20: ateapi.HttpRuleEffects - (*CredentialHeader)(nil), // 21: ateapi.CredentialHeader - (*ActorJWTSource)(nil), // 22: ateapi.ActorJWTSource - (*ActorStatus)(nil), // 23: ateapi.ActorStatus - (*ActorCrash)(nil), // 24: ateapi.ActorCrash - (*WorkerAssignment)(nil), // 25: ateapi.WorkerAssignment - (*TagStatus)(nil), // 26: ateapi.TagStatus - (*Tag)(nil), // 27: ateapi.Tag - (*Atespace)(nil), // 28: ateapi.Atespace - (*ObjectRef)(nil), // 29: ateapi.ObjectRef - (*ActorTemplate)(nil), // 30: ateapi.ActorTemplate - (*Resources)(nil), // 31: ateapi.Resources - (*Limits)(nil), // 32: ateapi.Limits - (*GoldenSnapshotStatus)(nil), // 33: ateapi.GoldenSnapshotStatus - (*ActorTemplateStatus)(nil), // 34: ateapi.ActorTemplateStatus - (*SandboxConfig)(nil), // 35: ateapi.SandboxConfig - (*SnapshotConfig)(nil), // 36: ateapi.SnapshotConfig - (*Container)(nil), // 37: ateapi.Container - (*SecurityContext)(nil), // 38: ateapi.SecurityContext - (*Capabilities)(nil), // 39: ateapi.Capabilities - (*EnvVar)(nil), // 40: ateapi.EnvVar - (*ContainerWakeupProbe)(nil), // 41: ateapi.ContainerWakeupProbe - (*HTTPGetAction)(nil), // 42: ateapi.HTTPGetAction - (*Volume)(nil), // 43: ateapi.Volume - (*ImageVolumeSource)(nil), // 44: ateapi.ImageVolumeSource - (*DurableDirVolumeSource)(nil), // 45: ateapi.DurableDirVolumeSource - (*ExternalVolumeTemplate)(nil), // 46: ateapi.ExternalVolumeTemplate - (*SystemInfoVolumeSource)(nil), // 47: ateapi.SystemInfoVolumeSource - (*SystemInfoDataSource)(nil), // 48: ateapi.SystemInfoDataSource - (*ActorMetadataDataSource)(nil), // 49: ateapi.ActorMetadataDataSource - (*ActorMetadataItem)(nil), // 50: ateapi.ActorMetadataItem - (*TrustBundleDataSource)(nil), // 51: ateapi.TrustBundleDataSource - (*VolumeMount)(nil), // 52: ateapi.VolumeMount - (*CreateAtespaceRequest)(nil), // 53: ateapi.CreateAtespaceRequest - (*GetAtespaceRequest)(nil), // 54: ateapi.GetAtespaceRequest - (*ListAtespacesRequest)(nil), // 55: ateapi.ListAtespacesRequest - (*ListAtespacesResponse)(nil), // 56: ateapi.ListAtespacesResponse - (*DeleteAtespaceRequest)(nil), // 57: ateapi.DeleteAtespaceRequest - (*CreateActorTemplateRequest)(nil), // 58: ateapi.CreateActorTemplateRequest - (*GetActorTemplateRequest)(nil), // 59: ateapi.GetActorTemplateRequest - (*ListActorTemplatesRequest)(nil), // 60: ateapi.ListActorTemplatesRequest - (*ListActorTemplatesResponse)(nil), // 61: ateapi.ListActorTemplatesResponse - (*DeleteActorTemplateRequest)(nil), // 62: ateapi.DeleteActorTemplateRequest - (*GetActorRequest)(nil), // 63: ateapi.GetActorRequest - (*CreateActorRequest)(nil), // 64: ateapi.CreateActorRequest - (*UpdateActorRequest)(nil), // 65: ateapi.UpdateActorRequest - (*SuspendActorRequest)(nil), // 66: ateapi.SuspendActorRequest - (*SuspendActorResponse)(nil), // 67: ateapi.SuspendActorResponse - (*PauseActorRequest)(nil), // 68: ateapi.PauseActorRequest - (*PauseActorResponse)(nil), // 69: ateapi.PauseActorResponse - (*ResumeActorRequest)(nil), // 70: ateapi.ResumeActorRequest - (*ResumeActorResponse)(nil), // 71: ateapi.ResumeActorResponse - (*RevertActorRequest)(nil), // 72: ateapi.RevertActorRequest - (*RevertActorResponse)(nil), // 73: ateapi.RevertActorResponse - (*DeleteActorRequest)(nil), // 74: ateapi.DeleteActorRequest - (*GetActorEgressPolicyRequest)(nil), // 75: ateapi.GetActorEgressPolicyRequest - (*CreateActorEgressPolicyRequest)(nil), // 76: ateapi.CreateActorEgressPolicyRequest - (*UpdateActorEgressPolicyRequest)(nil), // 77: ateapi.UpdateActorEgressPolicyRequest - (*DeleteActorEgressPolicyRequest)(nil), // 78: ateapi.DeleteActorEgressPolicyRequest - (*GetTagRequest)(nil), // 79: ateapi.GetTagRequest - (*MintActorJWTRequest)(nil), // 80: ateapi.MintActorJWTRequest - (*MintActorJWTResponse)(nil), // 81: ateapi.MintActorJWTResponse - (*MintActorCertificateRequest)(nil), // 82: ateapi.MintActorCertificateRequest - (*MintActorCertificateResponse)(nil), // 83: ateapi.MintActorCertificateResponse - (*ListTagsRequest)(nil), // 84: ateapi.ListTagsRequest - (*ListTagsResponse)(nil), // 85: ateapi.ListTagsResponse - (*CreateTagRequest)(nil), // 86: ateapi.CreateTagRequest - (*UpdateTagRequest)(nil), // 87: ateapi.UpdateTagRequest - (*DeleteTagRequest)(nil), // 88: ateapi.DeleteTagRequest - (*DeleteOptions)(nil), // 89: ateapi.DeleteOptions - (*ListWorkerActorAssignmentsRequest)(nil), // 90: ateapi.ListWorkerActorAssignmentsRequest - (*ListWorkerActorAssignmentsResponse)(nil), // 91: ateapi.ListWorkerActorAssignmentsResponse - (*ListWorkersRequest)(nil), // 92: ateapi.ListWorkersRequest - (*ListWorkersResponse)(nil), // 93: ateapi.ListWorkersResponse - (*GetWorkerRequest)(nil), // 94: ateapi.GetWorkerRequest - (*CreateWorkerRequest)(nil), // 95: ateapi.CreateWorkerRequest - (*UpdateWorkerRequest)(nil), // 96: ateapi.UpdateWorkerRequest - (*DeleteWorkerRequest)(nil), // 97: ateapi.DeleteWorkerRequest - (*DrainWorkerRequest)(nil), // 98: ateapi.DrainWorkerRequest - (*ListActorsRequest)(nil), // 99: ateapi.ListActorsRequest - (*ListActorsResponse)(nil), // 100: ateapi.ListActorsResponse - (*Worker)(nil), // 101: ateapi.Worker - (*WorkerStatus)(nil), // 102: ateapi.WorkerStatus - (*HardwareIdentity)(nil), // 103: ateapi.HardwareIdentity - (*WorkerResources)(nil), // 104: ateapi.WorkerResources - (*ActorAssignment)(nil), // 105: ateapi.ActorAssignment - (*RegisterWorkerRequest)(nil), // 106: ateapi.RegisterWorkerRequest - (*RegisterWorkerResponse)(nil), // 107: ateapi.RegisterWorkerResponse - (*MintAteomActorCertificateRequest)(nil), // 108: ateapi.MintAteomActorCertificateRequest - (*MintAteomActorCertificateResponse)(nil), // 109: ateapi.MintAteomActorCertificateResponse - (*RequestActorSuspendRequest)(nil), // 110: ateapi.RequestActorSuspendRequest - (*RequestActorSuspendResponse)(nil), // 111: ateapi.RequestActorSuspendResponse - (*AccessPolicy)(nil), // 112: ateapi.AccessPolicy - (*Binding)(nil), // 113: ateapi.Binding - (*GetGlobalAccessPolicyRequest)(nil), // 114: ateapi.GetGlobalAccessPolicyRequest - (*CreateGlobalAccessPolicyRequest)(nil), // 115: ateapi.CreateGlobalAccessPolicyRequest - (*UpdateGlobalAccessPolicyRequest)(nil), // 116: ateapi.UpdateGlobalAccessPolicyRequest - (*GetAtespaceAccessPolicyRequest)(nil), // 117: ateapi.GetAtespaceAccessPolicyRequest - (*CreateAtespaceAccessPolicyRequest)(nil), // 118: ateapi.CreateAtespaceAccessPolicyRequest - (*UpdateAtespaceAccessPolicyRequest)(nil), // 119: ateapi.UpdateAtespaceAccessPolicyRequest - (*DeleteAtespaceAccessPolicyRequest)(nil), // 120: ateapi.DeleteAtespaceAccessPolicyRequest - nil, // 121: ateapi.Selector.MatchLabelsEntry - nil, // 122: ateapi.ExternalVolume.VolumeContextEntry - nil, // 123: ateapi.Worker.LabelsEntry - nil, // 124: ateapi.HardwareIdentity.AttributesEntry - (*timestamppb.Timestamp)(nil), // 125: google.protobuf.Timestamp + (SnapshotOwner)(0), // 0: ateapi.SnapshotOwner + (SnapshotDurability)(0), // 1: ateapi.SnapshotDurability + (SnapshotStorageStatus)(0), // 2: ateapi.SnapshotStorageStatus + (SnapshotFidelity)(0), // 3: ateapi.SnapshotFidelity + (TagScope)(0), // 4: ateapi.TagScope + (ActorState)(0), // 5: ateapi.ActorState + (SandboxClass)(0), // 6: ateapi.SandboxClass + (ActorMetadataField)(0), // 7: ateapi.ActorMetadataField + (WorkerState)(0), // 8: ateapi.WorkerState + (ExternalVolume_Status)(0), // 9: ateapi.ExternalVolume.Status + (*ObjectSnapshot)(nil), // 10: ateapi.ObjectSnapshot + (*LocalSnapshot)(nil), // 11: ateapi.LocalSnapshot + (*SnapshotStorage)(nil), // 12: ateapi.SnapshotStorage + (*Snapshot)(nil), // 13: ateapi.Snapshot + (*Selector)(nil), // 14: ateapi.Selector + (*ResourceMetadata)(nil), // 15: ateapi.ResourceMetadata + (*ExternalVolume)(nil), // 16: ateapi.ExternalVolume + (*Actor)(nil), // 17: ateapi.Actor + (*EgressPolicy)(nil), // 18: ateapi.EgressPolicy + (*EgressRule)(nil), // 19: ateapi.EgressRule + (*HTTPRule)(nil), // 20: ateapi.HTTPRule + (*HTTPSRule)(nil), // 21: ateapi.HTTPSRule + (*TLSPassthroughRule)(nil), // 22: ateapi.TLSPassthroughRule + (*Ports)(nil), // 23: ateapi.Ports + (*AllPorts)(nil), // 24: ateapi.AllPorts + (*HttpRuleEffects)(nil), // 25: ateapi.HttpRuleEffects + (*CredentialHeader)(nil), // 26: ateapi.CredentialHeader + (*ActorJWTSource)(nil), // 27: ateapi.ActorJWTSource + (*ActorStatus)(nil), // 28: ateapi.ActorStatus + (*ActorCrash)(nil), // 29: ateapi.ActorCrash + (*WorkerAssignment)(nil), // 30: ateapi.WorkerAssignment + (*TagStatus)(nil), // 31: ateapi.TagStatus + (*Tag)(nil), // 32: ateapi.Tag + (*Atespace)(nil), // 33: ateapi.Atespace + (*ObjectRef)(nil), // 34: ateapi.ObjectRef + (*ActorTemplate)(nil), // 35: ateapi.ActorTemplate + (*Resources)(nil), // 36: ateapi.Resources + (*Limits)(nil), // 37: ateapi.Limits + (*GoldenSnapshotStatus)(nil), // 38: ateapi.GoldenSnapshotStatus + (*ActorTemplateStatus)(nil), // 39: ateapi.ActorTemplateStatus + (*SandboxConfig)(nil), // 40: ateapi.SandboxConfig + (*SnapshotConfig)(nil), // 41: ateapi.SnapshotConfig + (*Container)(nil), // 42: ateapi.Container + (*SecurityContext)(nil), // 43: ateapi.SecurityContext + (*Capabilities)(nil), // 44: ateapi.Capabilities + (*EnvVar)(nil), // 45: ateapi.EnvVar + (*ContainerWakeupProbe)(nil), // 46: ateapi.ContainerWakeupProbe + (*HTTPGetAction)(nil), // 47: ateapi.HTTPGetAction + (*Volume)(nil), // 48: ateapi.Volume + (*ImageVolumeSource)(nil), // 49: ateapi.ImageVolumeSource + (*DurableDirVolumeSource)(nil), // 50: ateapi.DurableDirVolumeSource + (*ExternalVolumeTemplate)(nil), // 51: ateapi.ExternalVolumeTemplate + (*SystemInfoVolumeSource)(nil), // 52: ateapi.SystemInfoVolumeSource + (*SystemInfoDataSource)(nil), // 53: ateapi.SystemInfoDataSource + (*ActorMetadataDataSource)(nil), // 54: ateapi.ActorMetadataDataSource + (*ActorMetadataItem)(nil), // 55: ateapi.ActorMetadataItem + (*TrustBundleDataSource)(nil), // 56: ateapi.TrustBundleDataSource + (*VolumeMount)(nil), // 57: ateapi.VolumeMount + (*CreateAtespaceRequest)(nil), // 58: ateapi.CreateAtespaceRequest + (*GetAtespaceRequest)(nil), // 59: ateapi.GetAtespaceRequest + (*ListAtespacesRequest)(nil), // 60: ateapi.ListAtespacesRequest + (*ListAtespacesResponse)(nil), // 61: ateapi.ListAtespacesResponse + (*DeleteAtespaceRequest)(nil), // 62: ateapi.DeleteAtespaceRequest + (*CreateActorTemplateRequest)(nil), // 63: ateapi.CreateActorTemplateRequest + (*GetActorTemplateRequest)(nil), // 64: ateapi.GetActorTemplateRequest + (*ListActorTemplatesRequest)(nil), // 65: ateapi.ListActorTemplatesRequest + (*ListActorTemplatesResponse)(nil), // 66: ateapi.ListActorTemplatesResponse + (*DeleteActorTemplateRequest)(nil), // 67: ateapi.DeleteActorTemplateRequest + (*GetActorRequest)(nil), // 68: ateapi.GetActorRequest + (*CreateActorRequest)(nil), // 69: ateapi.CreateActorRequest + (*UpdateActorRequest)(nil), // 70: ateapi.UpdateActorRequest + (*SuspendActorRequest)(nil), // 71: ateapi.SuspendActorRequest + (*SuspendActorResponse)(nil), // 72: ateapi.SuspendActorResponse + (*PauseActorRequest)(nil), // 73: ateapi.PauseActorRequest + (*PauseActorResponse)(nil), // 74: ateapi.PauseActorResponse + (*ResumeActorRequest)(nil), // 75: ateapi.ResumeActorRequest + (*ResumeActorResponse)(nil), // 76: ateapi.ResumeActorResponse + (*RevertActorRequest)(nil), // 77: ateapi.RevertActorRequest + (*RevertActorResponse)(nil), // 78: ateapi.RevertActorResponse + (*DeleteActorRequest)(nil), // 79: ateapi.DeleteActorRequest + (*GetActorEgressPolicyRequest)(nil), // 80: ateapi.GetActorEgressPolicyRequest + (*CreateActorEgressPolicyRequest)(nil), // 81: ateapi.CreateActorEgressPolicyRequest + (*UpdateActorEgressPolicyRequest)(nil), // 82: ateapi.UpdateActorEgressPolicyRequest + (*DeleteActorEgressPolicyRequest)(nil), // 83: ateapi.DeleteActorEgressPolicyRequest + (*GetTagRequest)(nil), // 84: ateapi.GetTagRequest + (*MintActorJWTRequest)(nil), // 85: ateapi.MintActorJWTRequest + (*MintActorJWTResponse)(nil), // 86: ateapi.MintActorJWTResponse + (*MintActorCertificateRequest)(nil), // 87: ateapi.MintActorCertificateRequest + (*MintActorCertificateResponse)(nil), // 88: ateapi.MintActorCertificateResponse + (*ListTagsRequest)(nil), // 89: ateapi.ListTagsRequest + (*ListTagsResponse)(nil), // 90: ateapi.ListTagsResponse + (*CreateTagRequest)(nil), // 91: ateapi.CreateTagRequest + (*UpdateTagRequest)(nil), // 92: ateapi.UpdateTagRequest + (*DeleteTagRequest)(nil), // 93: ateapi.DeleteTagRequest + (*DeleteOptions)(nil), // 94: ateapi.DeleteOptions + (*ListWorkerActorAssignmentsRequest)(nil), // 95: ateapi.ListWorkerActorAssignmentsRequest + (*ListWorkerActorAssignmentsResponse)(nil), // 96: ateapi.ListWorkerActorAssignmentsResponse + (*ListWorkersRequest)(nil), // 97: ateapi.ListWorkersRequest + (*ListWorkersResponse)(nil), // 98: ateapi.ListWorkersResponse + (*GetWorkerRequest)(nil), // 99: ateapi.GetWorkerRequest + (*CreateWorkerRequest)(nil), // 100: ateapi.CreateWorkerRequest + (*UpdateWorkerRequest)(nil), // 101: ateapi.UpdateWorkerRequest + (*DeleteWorkerRequest)(nil), // 102: ateapi.DeleteWorkerRequest + (*DrainWorkerRequest)(nil), // 103: ateapi.DrainWorkerRequest + (*ListActorsRequest)(nil), // 104: ateapi.ListActorsRequest + (*ListActorsResponse)(nil), // 105: ateapi.ListActorsResponse + (*Worker)(nil), // 106: ateapi.Worker + (*WorkerStatus)(nil), // 107: ateapi.WorkerStatus + (*SandboxRuntime)(nil), // 108: ateapi.SandboxRuntime + (*VersionedSandboxCompat)(nil), // 109: ateapi.VersionedSandboxCompat + (*AttributeEntry)(nil), // 110: ateapi.AttributeEntry + (*WorkerResources)(nil), // 111: ateapi.WorkerResources + (*ActorAssignment)(nil), // 112: ateapi.ActorAssignment + (*RegisterWorkerRequest)(nil), // 113: ateapi.RegisterWorkerRequest + (*RegisterWorkerResponse)(nil), // 114: ateapi.RegisterWorkerResponse + (*MintAteomActorCertificateRequest)(nil), // 115: ateapi.MintAteomActorCertificateRequest + (*MintAteomActorCertificateResponse)(nil), // 116: ateapi.MintAteomActorCertificateResponse + (*RequestActorSuspendRequest)(nil), // 117: ateapi.RequestActorSuspendRequest + (*RequestActorSuspendResponse)(nil), // 118: ateapi.RequestActorSuspendResponse + (*AccessPolicy)(nil), // 119: ateapi.AccessPolicy + (*Binding)(nil), // 120: ateapi.Binding + (*GetGlobalAccessPolicyRequest)(nil), // 121: ateapi.GetGlobalAccessPolicyRequest + (*CreateGlobalAccessPolicyRequest)(nil), // 122: ateapi.CreateGlobalAccessPolicyRequest + (*UpdateGlobalAccessPolicyRequest)(nil), // 123: ateapi.UpdateGlobalAccessPolicyRequest + (*GetAtespaceAccessPolicyRequest)(nil), // 124: ateapi.GetAtespaceAccessPolicyRequest + (*CreateAtespaceAccessPolicyRequest)(nil), // 125: ateapi.CreateAtespaceAccessPolicyRequest + (*UpdateAtespaceAccessPolicyRequest)(nil), // 126: ateapi.UpdateAtespaceAccessPolicyRequest + (*DeleteAtespaceAccessPolicyRequest)(nil), // 127: ateapi.DeleteAtespaceAccessPolicyRequest + nil, // 128: ateapi.Selector.MatchLabelsEntry + nil, // 129: ateapi.ExternalVolume.VolumeContextEntry + nil, // 130: ateapi.Worker.LabelsEntry + (*timestamppb.Timestamp)(nil), // 131: google.protobuf.Timestamp } var file_ateapi_proto_depIdxs = []int32{ - 0, // 0: ateapi.ExternalSnapshot.content_scope:type_name -> ateapi.SnapshotContentScope - 0, // 1: ateapi.LocalSnapshot.content_scope:type_name -> ateapi.SnapshotContentScope - 121, // 2: ateapi.Selector.match_labels:type_name -> ateapi.Selector.MatchLabelsEntry - 125, // 3: ateapi.ResourceMetadata.create_time:type_name -> google.protobuf.Timestamp - 125, // 4: ateapi.ResourceMetadata.update_time:type_name -> google.protobuf.Timestamp - 6, // 5: ateapi.ExternalVolume.status:type_name -> ateapi.ExternalVolume.Status - 122, // 6: ateapi.ExternalVolume.volume_context:type_name -> ateapi.ExternalVolume.VolumeContextEntry - 10, // 7: ateapi.Actor.metadata:type_name -> ateapi.ResourceMetadata - 29, // 8: ateapi.Actor.actor_template:type_name -> ateapi.ObjectRef - 9, // 9: ateapi.Actor.worker_selector:type_name -> ateapi.Selector - 29, // 10: ateapi.Actor.source_tag:type_name -> ateapi.ObjectRef - 23, // 11: ateapi.Actor.status:type_name -> ateapi.ActorStatus - 10, // 12: ateapi.EgressPolicy.metadata:type_name -> ateapi.ResourceMetadata - 14, // 13: ateapi.EgressPolicy.rules:type_name -> ateapi.EgressRule - 15, // 14: ateapi.EgressRule.http:type_name -> ateapi.HTTPRule - 16, // 15: ateapi.EgressRule.https:type_name -> ateapi.HTTPSRule - 17, // 16: ateapi.EgressRule.tls_passthrough:type_name -> ateapi.TLSPassthroughRule - 18, // 17: ateapi.HTTPRule.ports:type_name -> ateapi.Ports - 20, // 18: ateapi.HTTPRule.effects:type_name -> ateapi.HttpRuleEffects - 18, // 19: ateapi.HTTPSRule.ports:type_name -> ateapi.Ports - 20, // 20: ateapi.HTTPSRule.effects:type_name -> ateapi.HttpRuleEffects - 18, // 21: ateapi.TLSPassthroughRule.ports:type_name -> ateapi.Ports - 19, // 22: ateapi.Ports.all:type_name -> ateapi.AllPorts - 21, // 23: ateapi.HttpRuleEffects.replace_headers:type_name -> ateapi.CredentialHeader - 22, // 24: ateapi.CredentialHeader.actor_jwt:type_name -> ateapi.ActorJWTSource - 2, // 25: ateapi.ActorStatus.state:type_name -> ateapi.ActorState - 25, // 26: ateapi.ActorStatus.worker_assignment:type_name -> ateapi.WorkerAssignment - 7, // 27: ateapi.ActorStatus.external_snapshot:type_name -> ateapi.ExternalSnapshot - 8, // 28: ateapi.ActorStatus.local_snapshot:type_name -> ateapi.LocalSnapshot - 11, // 29: ateapi.ActorStatus.actor_volumes:type_name -> ateapi.ExternalVolume - 24, // 30: ateapi.ActorStatus.crash:type_name -> ateapi.ActorCrash - 125, // 31: ateapi.ActorCrash.crash_time:type_name -> google.protobuf.Timestamp - 29, // 32: ateapi.WorkerAssignment.worker:type_name -> ateapi.ObjectRef - 7, // 33: ateapi.TagStatus.snapshot:type_name -> ateapi.ExternalSnapshot - 10, // 34: ateapi.Tag.metadata:type_name -> ateapi.ResourceMetadata - 26, // 35: ateapi.Tag.status:type_name -> ateapi.TagStatus - 1, // 36: ateapi.Tag.scope:type_name -> ateapi.TagScope - 29, // 37: ateapi.Tag.source_actor:type_name -> ateapi.ObjectRef - 10, // 38: ateapi.Atespace.metadata:type_name -> ateapi.ResourceMetadata - 10, // 39: ateapi.ActorTemplate.metadata:type_name -> ateapi.ResourceMetadata - 9, // 40: ateapi.ActorTemplate.worker_selector:type_name -> ateapi.Selector - 37, // 41: ateapi.ActorTemplate.containers:type_name -> ateapi.Container - 43, // 42: ateapi.ActorTemplate.volumes:type_name -> ateapi.Volume - 36, // 43: ateapi.ActorTemplate.snapshot_config:type_name -> ateapi.SnapshotConfig - 35, // 44: ateapi.ActorTemplate.sandbox_config:type_name -> ateapi.SandboxConfig - 31, // 45: ateapi.ActorTemplate.resources:type_name -> ateapi.Resources - 34, // 46: ateapi.ActorTemplate.status:type_name -> ateapi.ActorTemplateStatus - 32, // 47: ateapi.Resources.limits:type_name -> ateapi.Limits - 29, // 48: ateapi.GoldenSnapshotStatus.golden_tag:type_name -> ateapi.ObjectRef - 125, // 49: ateapi.GoldenSnapshotStatus.take_golden_snapshot_at:type_name -> google.protobuf.Timestamp - 33, // 50: ateapi.ActorTemplateStatus.golden_snapshot_status:type_name -> ateapi.GoldenSnapshotStatus - 3, // 51: ateapi.SandboxConfig.sandbox_class:type_name -> ateapi.SandboxClass - 0, // 52: ateapi.SnapshotConfig.on_commit:type_name -> ateapi.SnapshotContentScope - 40, // 53: ateapi.Container.env:type_name -> ateapi.EnvVar - 41, // 54: ateapi.Container.wakeup_probe:type_name -> ateapi.ContainerWakeupProbe - 52, // 55: ateapi.Container.volume_mounts:type_name -> ateapi.VolumeMount - 38, // 56: ateapi.Container.security_context:type_name -> ateapi.SecurityContext - 31, // 57: ateapi.Container.resources:type_name -> ateapi.Resources - 39, // 58: ateapi.SecurityContext.capabilities:type_name -> ateapi.Capabilities - 42, // 59: ateapi.ContainerWakeupProbe.http_get:type_name -> ateapi.HTTPGetAction - 45, // 60: ateapi.Volume.durable_dir:type_name -> ateapi.DurableDirVolumeSource - 46, // 61: ateapi.Volume.external_volume_template:type_name -> ateapi.ExternalVolumeTemplate - 47, // 62: ateapi.Volume.system_info:type_name -> ateapi.SystemInfoVolumeSource - 44, // 63: ateapi.Volume.image:type_name -> ateapi.ImageVolumeSource - 48, // 64: ateapi.SystemInfoVolumeSource.data_sources:type_name -> ateapi.SystemInfoDataSource - 49, // 65: ateapi.SystemInfoDataSource.actor_metadata:type_name -> ateapi.ActorMetadataDataSource - 51, // 66: ateapi.SystemInfoDataSource.trust_bundle:type_name -> ateapi.TrustBundleDataSource - 50, // 67: ateapi.ActorMetadataDataSource.items:type_name -> ateapi.ActorMetadataItem - 4, // 68: ateapi.ActorMetadataItem.field:type_name -> ateapi.ActorMetadataField - 28, // 69: ateapi.CreateAtespaceRequest.atespace:type_name -> ateapi.Atespace - 29, // 70: ateapi.GetAtespaceRequest.atespace:type_name -> ateapi.ObjectRef - 28, // 71: ateapi.ListAtespacesResponse.atespaces:type_name -> ateapi.Atespace - 29, // 72: ateapi.DeleteAtespaceRequest.atespace:type_name -> ateapi.ObjectRef - 89, // 73: ateapi.DeleteAtespaceRequest.options:type_name -> ateapi.DeleteOptions - 30, // 74: ateapi.CreateActorTemplateRequest.actor_template:type_name -> ateapi.ActorTemplate - 29, // 75: ateapi.GetActorTemplateRequest.actor_template:type_name -> ateapi.ObjectRef - 30, // 76: ateapi.ListActorTemplatesResponse.actor_templates:type_name -> ateapi.ActorTemplate - 29, // 77: ateapi.DeleteActorTemplateRequest.actor_template:type_name -> ateapi.ObjectRef - 89, // 78: ateapi.DeleteActorTemplateRequest.options:type_name -> ateapi.DeleteOptions - 29, // 79: ateapi.GetActorRequest.actor:type_name -> ateapi.ObjectRef - 12, // 80: ateapi.CreateActorRequest.actor:type_name -> ateapi.Actor - 12, // 81: ateapi.UpdateActorRequest.actor:type_name -> ateapi.Actor - 29, // 82: ateapi.SuspendActorRequest.actor:type_name -> ateapi.ObjectRef - 12, // 83: ateapi.SuspendActorResponse.actor:type_name -> ateapi.Actor - 29, // 84: ateapi.PauseActorRequest.actor:type_name -> ateapi.ObjectRef - 12, // 85: ateapi.PauseActorResponse.actor:type_name -> ateapi.Actor - 29, // 86: ateapi.ResumeActorRequest.actor:type_name -> ateapi.ObjectRef - 12, // 87: ateapi.ResumeActorResponse.actor:type_name -> ateapi.Actor - 29, // 88: ateapi.RevertActorRequest.actor:type_name -> ateapi.ObjectRef - 12, // 89: ateapi.RevertActorResponse.actor:type_name -> ateapi.Actor - 29, // 90: ateapi.DeleteActorRequest.actor:type_name -> ateapi.ObjectRef - 89, // 91: ateapi.DeleteActorRequest.options:type_name -> ateapi.DeleteOptions - 29, // 92: ateapi.GetActorEgressPolicyRequest.actor:type_name -> ateapi.ObjectRef - 29, // 93: ateapi.CreateActorEgressPolicyRequest.actor:type_name -> ateapi.ObjectRef - 13, // 94: ateapi.CreateActorEgressPolicyRequest.egress_policy:type_name -> ateapi.EgressPolicy - 29, // 95: ateapi.UpdateActorEgressPolicyRequest.actor:type_name -> ateapi.ObjectRef - 13, // 96: ateapi.UpdateActorEgressPolicyRequest.egress_policy:type_name -> ateapi.EgressPolicy - 29, // 97: ateapi.DeleteActorEgressPolicyRequest.actor:type_name -> ateapi.ObjectRef - 89, // 98: ateapi.DeleteActorEgressPolicyRequest.options:type_name -> ateapi.DeleteOptions - 29, // 99: ateapi.GetTagRequest.tag:type_name -> ateapi.ObjectRef - 29, // 100: ateapi.MintActorJWTRequest.actor:type_name -> ateapi.ObjectRef - 125, // 101: ateapi.MintActorJWTResponse.expires_at:type_name -> google.protobuf.Timestamp - 29, // 102: ateapi.MintActorCertificateRequest.actor:type_name -> ateapi.ObjectRef - 27, // 103: ateapi.ListTagsResponse.tags:type_name -> ateapi.Tag - 27, // 104: ateapi.CreateTagRequest.tag:type_name -> ateapi.Tag - 27, // 105: ateapi.UpdateTagRequest.tag:type_name -> ateapi.Tag - 29, // 106: ateapi.DeleteTagRequest.tag:type_name -> ateapi.ObjectRef - 89, // 107: ateapi.DeleteTagRequest.options:type_name -> ateapi.DeleteOptions - 29, // 108: ateapi.ListWorkerActorAssignmentsRequest.worker:type_name -> ateapi.ObjectRef - 105, // 109: ateapi.ListWorkerActorAssignmentsResponse.actor_assignments:type_name -> ateapi.ActorAssignment - 101, // 110: ateapi.ListWorkersResponse.workers:type_name -> ateapi.Worker - 29, // 111: ateapi.GetWorkerRequest.worker:type_name -> ateapi.ObjectRef - 101, // 112: ateapi.CreateWorkerRequest.worker:type_name -> ateapi.Worker - 101, // 113: ateapi.UpdateWorkerRequest.worker:type_name -> ateapi.Worker - 29, // 114: ateapi.DeleteWorkerRequest.worker:type_name -> ateapi.ObjectRef - 89, // 115: ateapi.DeleteWorkerRequest.options:type_name -> ateapi.DeleteOptions - 29, // 116: ateapi.DrainWorkerRequest.worker:type_name -> ateapi.ObjectRef - 12, // 117: ateapi.ListActorsResponse.actors:type_name -> ateapi.Actor - 10, // 118: ateapi.Worker.metadata:type_name -> ateapi.ResourceMetadata - 123, // 119: ateapi.Worker.labels:type_name -> ateapi.Worker.LabelsEntry - 102, // 120: ateapi.Worker.status:type_name -> ateapi.WorkerStatus - 5, // 121: ateapi.WorkerStatus.state:type_name -> ateapi.WorkerState - 104, // 122: ateapi.WorkerStatus.capacity:type_name -> ateapi.WorkerResources - 104, // 123: ateapi.WorkerStatus.allocated:type_name -> ateapi.WorkerResources - 103, // 124: ateapi.WorkerStatus.hardware:type_name -> ateapi.HardwareIdentity - 124, // 125: ateapi.HardwareIdentity.attributes:type_name -> ateapi.HardwareIdentity.AttributesEntry - 31, // 126: ateapi.WorkerResources.resources:type_name -> ateapi.Resources - 10, // 127: ateapi.ActorAssignment.metadata:type_name -> ateapi.ResourceMetadata - 29, // 128: ateapi.ActorAssignment.actor:type_name -> ateapi.ObjectRef - 31, // 129: ateapi.ActorAssignment.resources:type_name -> ateapi.Resources - 29, // 130: ateapi.RegisterWorkerRequest.worker:type_name -> ateapi.ObjectRef - 104, // 131: ateapi.RegisterWorkerRequest.capacity:type_name -> ateapi.WorkerResources - 103, // 132: ateapi.RegisterWorkerRequest.hardware:type_name -> ateapi.HardwareIdentity - 101, // 133: ateapi.RegisterWorkerResponse.worker:type_name -> ateapi.Worker - 29, // 134: ateapi.MintAteomActorCertificateRequest.actor:type_name -> ateapi.ObjectRef - 29, // 135: ateapi.RequestActorSuspendRequest.worker:type_name -> ateapi.ObjectRef - 29, // 136: ateapi.RequestActorSuspendRequest.actor:type_name -> ateapi.ObjectRef - 12, // 137: ateapi.RequestActorSuspendResponse.actor:type_name -> ateapi.Actor - 10, // 138: ateapi.AccessPolicy.metadata:type_name -> ateapi.ResourceMetadata - 113, // 139: ateapi.AccessPolicy.bindings:type_name -> ateapi.Binding - 112, // 140: ateapi.CreateGlobalAccessPolicyRequest.access_policy:type_name -> ateapi.AccessPolicy - 112, // 141: ateapi.UpdateGlobalAccessPolicyRequest.access_policy:type_name -> ateapi.AccessPolicy - 29, // 142: ateapi.GetAtespaceAccessPolicyRequest.atespace:type_name -> ateapi.ObjectRef - 29, // 143: ateapi.CreateAtespaceAccessPolicyRequest.atespace:type_name -> ateapi.ObjectRef - 112, // 144: ateapi.CreateAtespaceAccessPolicyRequest.access_policy:type_name -> ateapi.AccessPolicy - 29, // 145: ateapi.UpdateAtespaceAccessPolicyRequest.atespace:type_name -> ateapi.ObjectRef - 112, // 146: ateapi.UpdateAtespaceAccessPolicyRequest.access_policy:type_name -> ateapi.AccessPolicy - 29, // 147: ateapi.DeleteAtespaceAccessPolicyRequest.atespace:type_name -> ateapi.ObjectRef - 89, // 148: ateapi.DeleteAtespaceAccessPolicyRequest.options:type_name -> ateapi.DeleteOptions - 63, // 149: ateapi.Control.GetActor:input_type -> ateapi.GetActorRequest - 64, // 150: ateapi.Control.CreateActor:input_type -> ateapi.CreateActorRequest - 65, // 151: ateapi.Control.UpdateActor:input_type -> ateapi.UpdateActorRequest - 66, // 152: ateapi.Control.SuspendActor:input_type -> ateapi.SuspendActorRequest - 68, // 153: ateapi.Control.PauseActor:input_type -> ateapi.PauseActorRequest - 70, // 154: ateapi.Control.ResumeActor:input_type -> ateapi.ResumeActorRequest - 72, // 155: ateapi.Control.RevertActor:input_type -> ateapi.RevertActorRequest - 74, // 156: ateapi.Control.DeleteActor:input_type -> ateapi.DeleteActorRequest - 75, // 157: ateapi.Control.GetActorEgressPolicy:input_type -> ateapi.GetActorEgressPolicyRequest - 76, // 158: ateapi.Control.CreateActorEgressPolicy:input_type -> ateapi.CreateActorEgressPolicyRequest - 77, // 159: ateapi.Control.UpdateActorEgressPolicy:input_type -> ateapi.UpdateActorEgressPolicyRequest - 78, // 160: ateapi.Control.DeleteActorEgressPolicy:input_type -> ateapi.DeleteActorEgressPolicyRequest - 80, // 161: ateapi.Control.MintActorJWT:input_type -> ateapi.MintActorJWTRequest - 82, // 162: ateapi.Control.MintActorCertificate:input_type -> ateapi.MintActorCertificateRequest - 86, // 163: ateapi.Control.CreateTag:input_type -> ateapi.CreateTagRequest - 79, // 164: ateapi.Control.GetTag:input_type -> ateapi.GetTagRequest - 84, // 165: ateapi.Control.ListTags:input_type -> ateapi.ListTagsRequest - 87, // 166: ateapi.Control.UpdateTag:input_type -> ateapi.UpdateTagRequest - 88, // 167: ateapi.Control.DeleteTag:input_type -> ateapi.DeleteTagRequest - 92, // 168: ateapi.Control.ListWorkers:input_type -> ateapi.ListWorkersRequest - 94, // 169: ateapi.Control.GetWorker:input_type -> ateapi.GetWorkerRequest - 95, // 170: ateapi.Control.CreateWorker:input_type -> ateapi.CreateWorkerRequest - 96, // 171: ateapi.Control.UpdateWorker:input_type -> ateapi.UpdateWorkerRequest - 97, // 172: ateapi.Control.DeleteWorker:input_type -> ateapi.DeleteWorkerRequest - 98, // 173: ateapi.Control.DrainWorker:input_type -> ateapi.DrainWorkerRequest - 90, // 174: ateapi.Control.ListWorkerActorAssignments:input_type -> ateapi.ListWorkerActorAssignmentsRequest - 99, // 175: ateapi.Control.ListActors:input_type -> ateapi.ListActorsRequest - 53, // 176: ateapi.Control.CreateAtespace:input_type -> ateapi.CreateAtespaceRequest - 54, // 177: ateapi.Control.GetAtespace:input_type -> ateapi.GetAtespaceRequest - 55, // 178: ateapi.Control.ListAtespaces:input_type -> ateapi.ListAtespacesRequest - 57, // 179: ateapi.Control.DeleteAtespace:input_type -> ateapi.DeleteAtespaceRequest - 58, // 180: ateapi.Control.CreateActorTemplate:input_type -> ateapi.CreateActorTemplateRequest - 59, // 181: ateapi.Control.GetActorTemplate:input_type -> ateapi.GetActorTemplateRequest - 60, // 182: ateapi.Control.ListActorTemplates:input_type -> ateapi.ListActorTemplatesRequest - 62, // 183: ateapi.Control.DeleteActorTemplate:input_type -> ateapi.DeleteActorTemplateRequest - 114, // 184: ateapi.Control.GetGlobalAccessPolicy:input_type -> ateapi.GetGlobalAccessPolicyRequest - 115, // 185: ateapi.Control.CreateGlobalAccessPolicy:input_type -> ateapi.CreateGlobalAccessPolicyRequest - 116, // 186: ateapi.Control.UpdateGlobalAccessPolicy:input_type -> ateapi.UpdateGlobalAccessPolicyRequest - 117, // 187: ateapi.Control.GetAtespaceAccessPolicy:input_type -> ateapi.GetAtespaceAccessPolicyRequest - 118, // 188: ateapi.Control.CreateAtespaceAccessPolicy:input_type -> ateapi.CreateAtespaceAccessPolicyRequest - 119, // 189: ateapi.Control.UpdateAtespaceAccessPolicy:input_type -> ateapi.UpdateAtespaceAccessPolicyRequest - 120, // 190: ateapi.Control.DeleteAtespaceAccessPolicy:input_type -> ateapi.DeleteAtespaceAccessPolicyRequest - 106, // 191: ateapi.WorkerService.RegisterWorker:input_type -> ateapi.RegisterWorkerRequest - 108, // 192: ateapi.WorkerService.MintAteomActorCertificate:input_type -> ateapi.MintAteomActorCertificateRequest - 110, // 193: ateapi.WorkerService.RequestActorSuspend:input_type -> ateapi.RequestActorSuspendRequest - 12, // 194: ateapi.Control.GetActor:output_type -> ateapi.Actor - 12, // 195: ateapi.Control.CreateActor:output_type -> ateapi.Actor - 12, // 196: ateapi.Control.UpdateActor:output_type -> ateapi.Actor - 67, // 197: ateapi.Control.SuspendActor:output_type -> ateapi.SuspendActorResponse - 69, // 198: ateapi.Control.PauseActor:output_type -> ateapi.PauseActorResponse - 71, // 199: ateapi.Control.ResumeActor:output_type -> ateapi.ResumeActorResponse - 73, // 200: ateapi.Control.RevertActor:output_type -> ateapi.RevertActorResponse - 12, // 201: ateapi.Control.DeleteActor:output_type -> ateapi.Actor - 13, // 202: ateapi.Control.GetActorEgressPolicy:output_type -> ateapi.EgressPolicy - 13, // 203: ateapi.Control.CreateActorEgressPolicy:output_type -> ateapi.EgressPolicy - 13, // 204: ateapi.Control.UpdateActorEgressPolicy:output_type -> ateapi.EgressPolicy - 13, // 205: ateapi.Control.DeleteActorEgressPolicy:output_type -> ateapi.EgressPolicy - 81, // 206: ateapi.Control.MintActorJWT:output_type -> ateapi.MintActorJWTResponse - 83, // 207: ateapi.Control.MintActorCertificate:output_type -> ateapi.MintActorCertificateResponse - 27, // 208: ateapi.Control.CreateTag:output_type -> ateapi.Tag - 27, // 209: ateapi.Control.GetTag:output_type -> ateapi.Tag - 85, // 210: ateapi.Control.ListTags:output_type -> ateapi.ListTagsResponse - 27, // 211: ateapi.Control.UpdateTag:output_type -> ateapi.Tag - 27, // 212: ateapi.Control.DeleteTag:output_type -> ateapi.Tag - 93, // 213: ateapi.Control.ListWorkers:output_type -> ateapi.ListWorkersResponse - 101, // 214: ateapi.Control.GetWorker:output_type -> ateapi.Worker - 101, // 215: ateapi.Control.CreateWorker:output_type -> ateapi.Worker - 101, // 216: ateapi.Control.UpdateWorker:output_type -> ateapi.Worker - 101, // 217: ateapi.Control.DeleteWorker:output_type -> ateapi.Worker - 101, // 218: ateapi.Control.DrainWorker:output_type -> ateapi.Worker - 91, // 219: ateapi.Control.ListWorkerActorAssignments:output_type -> ateapi.ListWorkerActorAssignmentsResponse - 100, // 220: ateapi.Control.ListActors:output_type -> ateapi.ListActorsResponse - 28, // 221: ateapi.Control.CreateAtespace:output_type -> ateapi.Atespace - 28, // 222: ateapi.Control.GetAtespace:output_type -> ateapi.Atespace - 56, // 223: ateapi.Control.ListAtespaces:output_type -> ateapi.ListAtespacesResponse - 28, // 224: ateapi.Control.DeleteAtespace:output_type -> ateapi.Atespace - 30, // 225: ateapi.Control.CreateActorTemplate:output_type -> ateapi.ActorTemplate - 30, // 226: ateapi.Control.GetActorTemplate:output_type -> ateapi.ActorTemplate - 61, // 227: ateapi.Control.ListActorTemplates:output_type -> ateapi.ListActorTemplatesResponse - 30, // 228: ateapi.Control.DeleteActorTemplate:output_type -> ateapi.ActorTemplate - 112, // 229: ateapi.Control.GetGlobalAccessPolicy:output_type -> ateapi.AccessPolicy - 112, // 230: ateapi.Control.CreateGlobalAccessPolicy:output_type -> ateapi.AccessPolicy - 112, // 231: ateapi.Control.UpdateGlobalAccessPolicy:output_type -> ateapi.AccessPolicy - 112, // 232: ateapi.Control.GetAtespaceAccessPolicy:output_type -> ateapi.AccessPolicy - 112, // 233: ateapi.Control.CreateAtespaceAccessPolicy:output_type -> ateapi.AccessPolicy - 112, // 234: ateapi.Control.UpdateAtespaceAccessPolicy:output_type -> ateapi.AccessPolicy - 112, // 235: ateapi.Control.DeleteAtespaceAccessPolicy:output_type -> ateapi.AccessPolicy - 107, // 236: ateapi.WorkerService.RegisterWorker:output_type -> ateapi.RegisterWorkerResponse - 109, // 237: ateapi.WorkerService.MintAteomActorCertificate:output_type -> ateapi.MintAteomActorCertificateResponse - 111, // 238: ateapi.WorkerService.RequestActorSuspend:output_type -> ateapi.RequestActorSuspendResponse - 194, // [194:239] is the sub-list for method output_type - 149, // [149:194] is the sub-list for method input_type - 149, // [149:149] is the sub-list for extension type_name - 149, // [149:149] is the sub-list for extension extendee - 0, // [0:149] is the sub-list for field type_name + 1, // 0: ateapi.SnapshotStorage.durability:type_name -> ateapi.SnapshotDurability + 2, // 1: ateapi.SnapshotStorage.status:type_name -> ateapi.SnapshotStorageStatus + 3, // 2: ateapi.SnapshotStorage.fidelity:type_name -> ateapi.SnapshotFidelity + 11, // 3: ateapi.SnapshotStorage.local:type_name -> ateapi.LocalSnapshot + 10, // 4: ateapi.SnapshotStorage.object:type_name -> ateapi.ObjectSnapshot + 0, // 5: ateapi.Snapshot.owner:type_name -> ateapi.SnapshotOwner + 12, // 6: ateapi.Snapshot.storage:type_name -> ateapi.SnapshotStorage + 128, // 7: ateapi.Selector.match_labels:type_name -> ateapi.Selector.MatchLabelsEntry + 131, // 8: ateapi.ResourceMetadata.create_time:type_name -> google.protobuf.Timestamp + 131, // 9: ateapi.ResourceMetadata.update_time:type_name -> google.protobuf.Timestamp + 9, // 10: ateapi.ExternalVolume.status:type_name -> ateapi.ExternalVolume.Status + 129, // 11: ateapi.ExternalVolume.volume_context:type_name -> ateapi.ExternalVolume.VolumeContextEntry + 15, // 12: ateapi.Actor.metadata:type_name -> ateapi.ResourceMetadata + 34, // 13: ateapi.Actor.actor_template:type_name -> ateapi.ObjectRef + 14, // 14: ateapi.Actor.worker_selector:type_name -> ateapi.Selector + 34, // 15: ateapi.Actor.source_tag:type_name -> ateapi.ObjectRef + 28, // 16: ateapi.Actor.status:type_name -> ateapi.ActorStatus + 15, // 17: ateapi.EgressPolicy.metadata:type_name -> ateapi.ResourceMetadata + 19, // 18: ateapi.EgressPolicy.rules:type_name -> ateapi.EgressRule + 20, // 19: ateapi.EgressRule.http:type_name -> ateapi.HTTPRule + 21, // 20: ateapi.EgressRule.https:type_name -> ateapi.HTTPSRule + 22, // 21: ateapi.EgressRule.tls_passthrough:type_name -> ateapi.TLSPassthroughRule + 23, // 22: ateapi.HTTPRule.ports:type_name -> ateapi.Ports + 25, // 23: ateapi.HTTPRule.effects:type_name -> ateapi.HttpRuleEffects + 23, // 24: ateapi.HTTPSRule.ports:type_name -> ateapi.Ports + 25, // 25: ateapi.HTTPSRule.effects:type_name -> ateapi.HttpRuleEffects + 23, // 26: ateapi.TLSPassthroughRule.ports:type_name -> ateapi.Ports + 24, // 27: ateapi.Ports.all:type_name -> ateapi.AllPorts + 26, // 28: ateapi.HttpRuleEffects.replace_headers:type_name -> ateapi.CredentialHeader + 27, // 29: ateapi.CredentialHeader.actor_jwt:type_name -> ateapi.ActorJWTSource + 5, // 30: ateapi.ActorStatus.state:type_name -> ateapi.ActorState + 30, // 31: ateapi.ActorStatus.worker_assignment:type_name -> ateapi.WorkerAssignment + 16, // 32: ateapi.ActorStatus.external_volumes:type_name -> ateapi.ExternalVolume + 29, // 33: ateapi.ActorStatus.crash:type_name -> ateapi.ActorCrash + 13, // 34: ateapi.ActorStatus.snapshots:type_name -> ateapi.Snapshot + 131, // 35: ateapi.ActorCrash.crash_time:type_name -> google.protobuf.Timestamp + 34, // 36: ateapi.WorkerAssignment.worker:type_name -> ateapi.ObjectRef + 13, // 37: ateapi.TagStatus.snapshot:type_name -> ateapi.Snapshot + 15, // 38: ateapi.Tag.metadata:type_name -> ateapi.ResourceMetadata + 31, // 39: ateapi.Tag.status:type_name -> ateapi.TagStatus + 4, // 40: ateapi.Tag.scope:type_name -> ateapi.TagScope + 34, // 41: ateapi.Tag.source_actor:type_name -> ateapi.ObjectRef + 15, // 42: ateapi.Atespace.metadata:type_name -> ateapi.ResourceMetadata + 15, // 43: ateapi.ActorTemplate.metadata:type_name -> ateapi.ResourceMetadata + 14, // 44: ateapi.ActorTemplate.worker_selector:type_name -> ateapi.Selector + 42, // 45: ateapi.ActorTemplate.containers:type_name -> ateapi.Container + 48, // 46: ateapi.ActorTemplate.volumes:type_name -> ateapi.Volume + 41, // 47: ateapi.ActorTemplate.snapshot_config:type_name -> ateapi.SnapshotConfig + 40, // 48: ateapi.ActorTemplate.sandbox_config:type_name -> ateapi.SandboxConfig + 36, // 49: ateapi.ActorTemplate.resources:type_name -> ateapi.Resources + 39, // 50: ateapi.ActorTemplate.status:type_name -> ateapi.ActorTemplateStatus + 37, // 51: ateapi.Resources.limits:type_name -> ateapi.Limits + 34, // 52: ateapi.GoldenSnapshotStatus.golden_tag:type_name -> ateapi.ObjectRef + 131, // 53: ateapi.GoldenSnapshotStatus.take_golden_snapshot_at:type_name -> google.protobuf.Timestamp + 38, // 54: ateapi.ActorTemplateStatus.golden_snapshot_status:type_name -> ateapi.GoldenSnapshotStatus + 6, // 55: ateapi.SandboxConfig.sandbox_class:type_name -> ateapi.SandboxClass + 3, // 56: ateapi.SnapshotConfig.preferred_fidelity:type_name -> ateapi.SnapshotFidelity + 45, // 57: ateapi.Container.env:type_name -> ateapi.EnvVar + 46, // 58: ateapi.Container.wakeup_probe:type_name -> ateapi.ContainerWakeupProbe + 57, // 59: ateapi.Container.volume_mounts:type_name -> ateapi.VolumeMount + 43, // 60: ateapi.Container.security_context:type_name -> ateapi.SecurityContext + 36, // 61: ateapi.Container.resources:type_name -> ateapi.Resources + 44, // 62: ateapi.SecurityContext.capabilities:type_name -> ateapi.Capabilities + 47, // 63: ateapi.ContainerWakeupProbe.http_get:type_name -> ateapi.HTTPGetAction + 50, // 64: ateapi.Volume.durable_dir:type_name -> ateapi.DurableDirVolumeSource + 51, // 65: ateapi.Volume.external_volume_template:type_name -> ateapi.ExternalVolumeTemplate + 52, // 66: ateapi.Volume.system_info:type_name -> ateapi.SystemInfoVolumeSource + 49, // 67: ateapi.Volume.image:type_name -> ateapi.ImageVolumeSource + 53, // 68: ateapi.SystemInfoVolumeSource.data_sources:type_name -> ateapi.SystemInfoDataSource + 54, // 69: ateapi.SystemInfoDataSource.actor_metadata:type_name -> ateapi.ActorMetadataDataSource + 56, // 70: ateapi.SystemInfoDataSource.trust_bundle:type_name -> ateapi.TrustBundleDataSource + 55, // 71: ateapi.ActorMetadataDataSource.items:type_name -> ateapi.ActorMetadataItem + 7, // 72: ateapi.ActorMetadataItem.field:type_name -> ateapi.ActorMetadataField + 33, // 73: ateapi.CreateAtespaceRequest.atespace:type_name -> ateapi.Atespace + 34, // 74: ateapi.GetAtespaceRequest.atespace:type_name -> ateapi.ObjectRef + 33, // 75: ateapi.ListAtespacesResponse.atespaces:type_name -> ateapi.Atespace + 34, // 76: ateapi.DeleteAtespaceRequest.atespace:type_name -> ateapi.ObjectRef + 94, // 77: ateapi.DeleteAtespaceRequest.options:type_name -> ateapi.DeleteOptions + 35, // 78: ateapi.CreateActorTemplateRequest.actor_template:type_name -> ateapi.ActorTemplate + 34, // 79: ateapi.GetActorTemplateRequest.actor_template:type_name -> ateapi.ObjectRef + 35, // 80: ateapi.ListActorTemplatesResponse.actor_templates:type_name -> ateapi.ActorTemplate + 34, // 81: ateapi.DeleteActorTemplateRequest.actor_template:type_name -> ateapi.ObjectRef + 94, // 82: ateapi.DeleteActorTemplateRequest.options:type_name -> ateapi.DeleteOptions + 34, // 83: ateapi.GetActorRequest.actor:type_name -> ateapi.ObjectRef + 17, // 84: ateapi.CreateActorRequest.actor:type_name -> ateapi.Actor + 17, // 85: ateapi.UpdateActorRequest.actor:type_name -> ateapi.Actor + 34, // 86: ateapi.SuspendActorRequest.actor:type_name -> ateapi.ObjectRef + 17, // 87: ateapi.SuspendActorResponse.actor:type_name -> ateapi.Actor + 34, // 88: ateapi.PauseActorRequest.actor:type_name -> ateapi.ObjectRef + 17, // 89: ateapi.PauseActorResponse.actor:type_name -> ateapi.Actor + 34, // 90: ateapi.ResumeActorRequest.actor:type_name -> ateapi.ObjectRef + 17, // 91: ateapi.ResumeActorResponse.actor:type_name -> ateapi.Actor + 34, // 92: ateapi.RevertActorRequest.actor:type_name -> ateapi.ObjectRef + 17, // 93: ateapi.RevertActorResponse.actor:type_name -> ateapi.Actor + 34, // 94: ateapi.DeleteActorRequest.actor:type_name -> ateapi.ObjectRef + 94, // 95: ateapi.DeleteActorRequest.options:type_name -> ateapi.DeleteOptions + 34, // 96: ateapi.GetActorEgressPolicyRequest.actor:type_name -> ateapi.ObjectRef + 34, // 97: ateapi.CreateActorEgressPolicyRequest.actor:type_name -> ateapi.ObjectRef + 18, // 98: ateapi.CreateActorEgressPolicyRequest.egress_policy:type_name -> ateapi.EgressPolicy + 34, // 99: ateapi.UpdateActorEgressPolicyRequest.actor:type_name -> ateapi.ObjectRef + 18, // 100: ateapi.UpdateActorEgressPolicyRequest.egress_policy:type_name -> ateapi.EgressPolicy + 34, // 101: ateapi.DeleteActorEgressPolicyRequest.actor:type_name -> ateapi.ObjectRef + 94, // 102: ateapi.DeleteActorEgressPolicyRequest.options:type_name -> ateapi.DeleteOptions + 34, // 103: ateapi.GetTagRequest.tag:type_name -> ateapi.ObjectRef + 34, // 104: ateapi.MintActorJWTRequest.actor:type_name -> ateapi.ObjectRef + 131, // 105: ateapi.MintActorJWTResponse.expires_at:type_name -> google.protobuf.Timestamp + 34, // 106: ateapi.MintActorCertificateRequest.actor:type_name -> ateapi.ObjectRef + 32, // 107: ateapi.ListTagsResponse.tags:type_name -> ateapi.Tag + 32, // 108: ateapi.CreateTagRequest.tag:type_name -> ateapi.Tag + 32, // 109: ateapi.UpdateTagRequest.tag:type_name -> ateapi.Tag + 34, // 110: ateapi.DeleteTagRequest.tag:type_name -> ateapi.ObjectRef + 94, // 111: ateapi.DeleteTagRequest.options:type_name -> ateapi.DeleteOptions + 34, // 112: ateapi.ListWorkerActorAssignmentsRequest.worker:type_name -> ateapi.ObjectRef + 112, // 113: ateapi.ListWorkerActorAssignmentsResponse.actor_assignments:type_name -> ateapi.ActorAssignment + 106, // 114: ateapi.ListWorkersResponse.workers:type_name -> ateapi.Worker + 34, // 115: ateapi.GetWorkerRequest.worker:type_name -> ateapi.ObjectRef + 106, // 116: ateapi.CreateWorkerRequest.worker:type_name -> ateapi.Worker + 106, // 117: ateapi.UpdateWorkerRequest.worker:type_name -> ateapi.Worker + 34, // 118: ateapi.DeleteWorkerRequest.worker:type_name -> ateapi.ObjectRef + 94, // 119: ateapi.DeleteWorkerRequest.options:type_name -> ateapi.DeleteOptions + 34, // 120: ateapi.DrainWorkerRequest.worker:type_name -> ateapi.ObjectRef + 17, // 121: ateapi.ListActorsResponse.actors:type_name -> ateapi.Actor + 15, // 122: ateapi.Worker.metadata:type_name -> ateapi.ResourceMetadata + 130, // 123: ateapi.Worker.labels:type_name -> ateapi.Worker.LabelsEntry + 107, // 124: ateapi.Worker.status:type_name -> ateapi.WorkerStatus + 8, // 125: ateapi.WorkerStatus.state:type_name -> ateapi.WorkerState + 111, // 126: ateapi.WorkerStatus.capacity:type_name -> ateapi.WorkerResources + 111, // 127: ateapi.WorkerStatus.allocated:type_name -> ateapi.WorkerResources + 108, // 128: ateapi.WorkerStatus.default_runtime:type_name -> ateapi.SandboxRuntime + 108, // 129: ateapi.WorkerStatus.restorable_runtimes:type_name -> ateapi.SandboxRuntime + 109, // 130: ateapi.SandboxRuntime.version:type_name -> ateapi.VersionedSandboxCompat + 110, // 131: ateapi.VersionedSandboxCompat.attributes:type_name -> ateapi.AttributeEntry + 36, // 132: ateapi.WorkerResources.resources:type_name -> ateapi.Resources + 15, // 133: ateapi.ActorAssignment.metadata:type_name -> ateapi.ResourceMetadata + 34, // 134: ateapi.ActorAssignment.actor:type_name -> ateapi.ObjectRef + 36, // 135: ateapi.ActorAssignment.resources:type_name -> ateapi.Resources + 34, // 136: ateapi.RegisterWorkerRequest.worker:type_name -> ateapi.ObjectRef + 111, // 137: ateapi.RegisterWorkerRequest.capacity:type_name -> ateapi.WorkerResources + 108, // 138: ateapi.RegisterWorkerRequest.default_runtime:type_name -> ateapi.SandboxRuntime + 108, // 139: ateapi.RegisterWorkerRequest.restorable_runtimes:type_name -> ateapi.SandboxRuntime + 106, // 140: ateapi.RegisterWorkerResponse.worker:type_name -> ateapi.Worker + 34, // 141: ateapi.MintAteomActorCertificateRequest.actor:type_name -> ateapi.ObjectRef + 34, // 142: ateapi.RequestActorSuspendRequest.worker:type_name -> ateapi.ObjectRef + 34, // 143: ateapi.RequestActorSuspendRequest.actor:type_name -> ateapi.ObjectRef + 17, // 144: ateapi.RequestActorSuspendResponse.actor:type_name -> ateapi.Actor + 15, // 145: ateapi.AccessPolicy.metadata:type_name -> ateapi.ResourceMetadata + 120, // 146: ateapi.AccessPolicy.bindings:type_name -> ateapi.Binding + 119, // 147: ateapi.CreateGlobalAccessPolicyRequest.access_policy:type_name -> ateapi.AccessPolicy + 119, // 148: ateapi.UpdateGlobalAccessPolicyRequest.access_policy:type_name -> ateapi.AccessPolicy + 34, // 149: ateapi.GetAtespaceAccessPolicyRequest.atespace:type_name -> ateapi.ObjectRef + 34, // 150: ateapi.CreateAtespaceAccessPolicyRequest.atespace:type_name -> ateapi.ObjectRef + 119, // 151: ateapi.CreateAtespaceAccessPolicyRequest.access_policy:type_name -> ateapi.AccessPolicy + 34, // 152: ateapi.UpdateAtespaceAccessPolicyRequest.atespace:type_name -> ateapi.ObjectRef + 119, // 153: ateapi.UpdateAtespaceAccessPolicyRequest.access_policy:type_name -> ateapi.AccessPolicy + 34, // 154: ateapi.DeleteAtespaceAccessPolicyRequest.atespace:type_name -> ateapi.ObjectRef + 94, // 155: ateapi.DeleteAtespaceAccessPolicyRequest.options:type_name -> ateapi.DeleteOptions + 68, // 156: ateapi.Control.GetActor:input_type -> ateapi.GetActorRequest + 69, // 157: ateapi.Control.CreateActor:input_type -> ateapi.CreateActorRequest + 70, // 158: ateapi.Control.UpdateActor:input_type -> ateapi.UpdateActorRequest + 71, // 159: ateapi.Control.SuspendActor:input_type -> ateapi.SuspendActorRequest + 73, // 160: ateapi.Control.PauseActor:input_type -> ateapi.PauseActorRequest + 75, // 161: ateapi.Control.ResumeActor:input_type -> ateapi.ResumeActorRequest + 77, // 162: ateapi.Control.RevertActor:input_type -> ateapi.RevertActorRequest + 79, // 163: ateapi.Control.DeleteActor:input_type -> ateapi.DeleteActorRequest + 80, // 164: ateapi.Control.GetActorEgressPolicy:input_type -> ateapi.GetActorEgressPolicyRequest + 81, // 165: ateapi.Control.CreateActorEgressPolicy:input_type -> ateapi.CreateActorEgressPolicyRequest + 82, // 166: ateapi.Control.UpdateActorEgressPolicy:input_type -> ateapi.UpdateActorEgressPolicyRequest + 83, // 167: ateapi.Control.DeleteActorEgressPolicy:input_type -> ateapi.DeleteActorEgressPolicyRequest + 85, // 168: ateapi.Control.MintActorJWT:input_type -> ateapi.MintActorJWTRequest + 87, // 169: ateapi.Control.MintActorCertificate:input_type -> ateapi.MintActorCertificateRequest + 91, // 170: ateapi.Control.CreateTag:input_type -> ateapi.CreateTagRequest + 84, // 171: ateapi.Control.GetTag:input_type -> ateapi.GetTagRequest + 89, // 172: ateapi.Control.ListTags:input_type -> ateapi.ListTagsRequest + 92, // 173: ateapi.Control.UpdateTag:input_type -> ateapi.UpdateTagRequest + 93, // 174: ateapi.Control.DeleteTag:input_type -> ateapi.DeleteTagRequest + 97, // 175: ateapi.Control.ListWorkers:input_type -> ateapi.ListWorkersRequest + 99, // 176: ateapi.Control.GetWorker:input_type -> ateapi.GetWorkerRequest + 100, // 177: ateapi.Control.CreateWorker:input_type -> ateapi.CreateWorkerRequest + 101, // 178: ateapi.Control.UpdateWorker:input_type -> ateapi.UpdateWorkerRequest + 102, // 179: ateapi.Control.DeleteWorker:input_type -> ateapi.DeleteWorkerRequest + 103, // 180: ateapi.Control.DrainWorker:input_type -> ateapi.DrainWorkerRequest + 95, // 181: ateapi.Control.ListWorkerActorAssignments:input_type -> ateapi.ListWorkerActorAssignmentsRequest + 104, // 182: ateapi.Control.ListActors:input_type -> ateapi.ListActorsRequest + 58, // 183: ateapi.Control.CreateAtespace:input_type -> ateapi.CreateAtespaceRequest + 59, // 184: ateapi.Control.GetAtespace:input_type -> ateapi.GetAtespaceRequest + 60, // 185: ateapi.Control.ListAtespaces:input_type -> ateapi.ListAtespacesRequest + 62, // 186: ateapi.Control.DeleteAtespace:input_type -> ateapi.DeleteAtespaceRequest + 63, // 187: ateapi.Control.CreateActorTemplate:input_type -> ateapi.CreateActorTemplateRequest + 64, // 188: ateapi.Control.GetActorTemplate:input_type -> ateapi.GetActorTemplateRequest + 65, // 189: ateapi.Control.ListActorTemplates:input_type -> ateapi.ListActorTemplatesRequest + 67, // 190: ateapi.Control.DeleteActorTemplate:input_type -> ateapi.DeleteActorTemplateRequest + 121, // 191: ateapi.Control.GetGlobalAccessPolicy:input_type -> ateapi.GetGlobalAccessPolicyRequest + 122, // 192: ateapi.Control.CreateGlobalAccessPolicy:input_type -> ateapi.CreateGlobalAccessPolicyRequest + 123, // 193: ateapi.Control.UpdateGlobalAccessPolicy:input_type -> ateapi.UpdateGlobalAccessPolicyRequest + 124, // 194: ateapi.Control.GetAtespaceAccessPolicy:input_type -> ateapi.GetAtespaceAccessPolicyRequest + 125, // 195: ateapi.Control.CreateAtespaceAccessPolicy:input_type -> ateapi.CreateAtespaceAccessPolicyRequest + 126, // 196: ateapi.Control.UpdateAtespaceAccessPolicy:input_type -> ateapi.UpdateAtespaceAccessPolicyRequest + 127, // 197: ateapi.Control.DeleteAtespaceAccessPolicy:input_type -> ateapi.DeleteAtespaceAccessPolicyRequest + 113, // 198: ateapi.WorkerService.RegisterWorker:input_type -> ateapi.RegisterWorkerRequest + 115, // 199: ateapi.WorkerService.MintAteomActorCertificate:input_type -> ateapi.MintAteomActorCertificateRequest + 117, // 200: ateapi.WorkerService.RequestActorSuspend:input_type -> ateapi.RequestActorSuspendRequest + 17, // 201: ateapi.Control.GetActor:output_type -> ateapi.Actor + 17, // 202: ateapi.Control.CreateActor:output_type -> ateapi.Actor + 17, // 203: ateapi.Control.UpdateActor:output_type -> ateapi.Actor + 72, // 204: ateapi.Control.SuspendActor:output_type -> ateapi.SuspendActorResponse + 74, // 205: ateapi.Control.PauseActor:output_type -> ateapi.PauseActorResponse + 76, // 206: ateapi.Control.ResumeActor:output_type -> ateapi.ResumeActorResponse + 78, // 207: ateapi.Control.RevertActor:output_type -> ateapi.RevertActorResponse + 17, // 208: ateapi.Control.DeleteActor:output_type -> ateapi.Actor + 18, // 209: ateapi.Control.GetActorEgressPolicy:output_type -> ateapi.EgressPolicy + 18, // 210: ateapi.Control.CreateActorEgressPolicy:output_type -> ateapi.EgressPolicy + 18, // 211: ateapi.Control.UpdateActorEgressPolicy:output_type -> ateapi.EgressPolicy + 18, // 212: ateapi.Control.DeleteActorEgressPolicy:output_type -> ateapi.EgressPolicy + 86, // 213: ateapi.Control.MintActorJWT:output_type -> ateapi.MintActorJWTResponse + 88, // 214: ateapi.Control.MintActorCertificate:output_type -> ateapi.MintActorCertificateResponse + 32, // 215: ateapi.Control.CreateTag:output_type -> ateapi.Tag + 32, // 216: ateapi.Control.GetTag:output_type -> ateapi.Tag + 90, // 217: ateapi.Control.ListTags:output_type -> ateapi.ListTagsResponse + 32, // 218: ateapi.Control.UpdateTag:output_type -> ateapi.Tag + 32, // 219: ateapi.Control.DeleteTag:output_type -> ateapi.Tag + 98, // 220: ateapi.Control.ListWorkers:output_type -> ateapi.ListWorkersResponse + 106, // 221: ateapi.Control.GetWorker:output_type -> ateapi.Worker + 106, // 222: ateapi.Control.CreateWorker:output_type -> ateapi.Worker + 106, // 223: ateapi.Control.UpdateWorker:output_type -> ateapi.Worker + 106, // 224: ateapi.Control.DeleteWorker:output_type -> ateapi.Worker + 106, // 225: ateapi.Control.DrainWorker:output_type -> ateapi.Worker + 96, // 226: ateapi.Control.ListWorkerActorAssignments:output_type -> ateapi.ListWorkerActorAssignmentsResponse + 105, // 227: ateapi.Control.ListActors:output_type -> ateapi.ListActorsResponse + 33, // 228: ateapi.Control.CreateAtespace:output_type -> ateapi.Atespace + 33, // 229: ateapi.Control.GetAtespace:output_type -> ateapi.Atespace + 61, // 230: ateapi.Control.ListAtespaces:output_type -> ateapi.ListAtespacesResponse + 33, // 231: ateapi.Control.DeleteAtespace:output_type -> ateapi.Atespace + 35, // 232: ateapi.Control.CreateActorTemplate:output_type -> ateapi.ActorTemplate + 35, // 233: ateapi.Control.GetActorTemplate:output_type -> ateapi.ActorTemplate + 66, // 234: ateapi.Control.ListActorTemplates:output_type -> ateapi.ListActorTemplatesResponse + 35, // 235: ateapi.Control.DeleteActorTemplate:output_type -> ateapi.ActorTemplate + 119, // 236: ateapi.Control.GetGlobalAccessPolicy:output_type -> ateapi.AccessPolicy + 119, // 237: ateapi.Control.CreateGlobalAccessPolicy:output_type -> ateapi.AccessPolicy + 119, // 238: ateapi.Control.UpdateGlobalAccessPolicy:output_type -> ateapi.AccessPolicy + 119, // 239: ateapi.Control.GetAtespaceAccessPolicy:output_type -> ateapi.AccessPolicy + 119, // 240: ateapi.Control.CreateAtespaceAccessPolicy:output_type -> ateapi.AccessPolicy + 119, // 241: ateapi.Control.UpdateAtespaceAccessPolicy:output_type -> ateapi.AccessPolicy + 119, // 242: ateapi.Control.DeleteAtespaceAccessPolicy:output_type -> ateapi.AccessPolicy + 114, // 243: ateapi.WorkerService.RegisterWorker:output_type -> ateapi.RegisterWorkerResponse + 116, // 244: ateapi.WorkerService.MintAteomActorCertificate:output_type -> ateapi.MintAteomActorCertificateResponse + 118, // 245: ateapi.WorkerService.RequestActorSuspend:output_type -> ateapi.RequestActorSuspendResponse + 201, // [201:246] is the sub-list for method output_type + 156, // [156:201] is the sub-list for method input_type + 156, // [156:156] is the sub-list for extension type_name + 156, // [156:156] is the sub-list for extension extendee + 0, // [0:156] is the sub-list for field type_name } func init() { file_ateapi_proto_init() } @@ -8925,8 +9435,8 @@ func file_ateapi_proto_init() { File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_ateapi_proto_rawDesc), len(file_ateapi_proto_rawDesc)), - NumEnums: 7, - NumMessages: 118, + NumEnums: 10, + NumMessages: 121, NumExtensions: 0, NumServices: 2, }, diff --git a/pkg/proto/ateapipb/ateapi.proto b/pkg/proto/ateapipb/ateapi.proto index c552b21614..138cb6967c 100644 --- a/pkg/proto/ateapipb/ateapi.proto +++ b/pkg/proto/ateapipb/ateapi.proto @@ -171,27 +171,13 @@ service Control { rpc DeleteAtespaceAccessPolicy(DeleteAtespaceAccessPolicyRequest) returns (AccessPolicy) {} } -// ExternalSnapshot addresses a snapshot held in object storage. -message ExternalSnapshot { +// ObjectSnapshot addresses a snapshot held in object storage. +message ObjectSnapshot { // snapshot_uri addresses the snapshot's prefix in object storage. // // +k8s:required // +k8s:maxLength=2048 # the template's storage_location bound plus the owner prefix and snapshot name string snapshot_uri = 1; - - // content_scope is what the snapshot captured. - // - // +k8s:optional - // +k8s:minimum=1 - // +k8s:maximum=2 # keep this in sync with the SnapshotContentScope enum - SnapshotContentScope content_scope = 2; - - // UID of the ActorTemplate whose sandbox this snapshot's guest state was - // captured from. - // - // +k8s:optional - // +k8s:format=k8s-uuid - string actor_template_uid = 3; } // LocalSnapshot records information about a node-local snapshot. @@ -199,26 +185,131 @@ message LocalSnapshot { // The name of the local checkpoint on the node. Checkpoint names are // server-generated UUIDs, but any resource name is valid here. // - // +k8s:optional + // TODO: consider removing this since it's the same as the snapshot_id + // + // +k8s:required // +k8s:format=k8s-short-name string snapshot_name = 1; +} + +// SnapshotOwner indicates the owner of a snapshot. +enum SnapshotOwner { + SNAPSHOT_OWNER_UNSPECIFIED = 0; + // Snapshot captured directly from this Actor. + SNAPSHOT_OWNER_ACTOR = 1; + // Snapshot owned by a Tag (also includes Golden Snapshot) + SNAPSHOT_OWNER_TAG = 2; + // Keep this in sync with Snapshot.owner's maximum. +} - // Scope the pause checkpoint captured (the template's onCommit at pause - // time). +// SnapshotDurability indicates the durability level of the snapshot. +// TODO: potentially rename this to survivability level. Is it at the actor or snapshot level? +enum SnapshotDurability { + SNAPSHOT_DURABILITY_UNSPECIFIED = 0; + // Node-local checkpoint on the worker node + // Survives node reboot. + SNAPSHOT_DURABILITY_LOCAL = 1; + // Durable snapshot in external storage. + // Survives node crash and deletion. + SNAPSHOT_DURABILITY_DURABLE = 2; + // Keep this in sync with SnapshotStorage.durability's maximum. +} + +// SnapshotStorageStatus indicates the lifecycle stage of a snapshot storage entry. +enum SnapshotStorageStatus { + SNAPSHOT_STORAGE_STATUS_UNSPECIFIED = 0; + // Snapshot creation or promotion is in progress. + SNAPSHOT_STORAGE_STATUS_IN_PROGRESS = 1; + // Snapshot is complete and ready for use. + SNAPSHOT_STORAGE_STATUS_COMPLETED = 2; + // Keep this in sync with SnapshotStorage.status's maximum. +} + +// SnapshotStorage records the physical storage representation of a Snapshot. +message SnapshotStorage { + // durability is the durability tier of this storage entry. Only + // SNAPSHOT_DURABILITY_LOCAL and SNAPSHOT_DURABILITY_DURABLE are allowed. // - // +k8s:optional + // +k8s:required + // +k8s:minimum=1 + // +k8s:maximum=2 # keep this in sync with the SnapshotDurability enum + SnapshotDurability durability = 1; + + // status is the lifecycle status of this storage representation. + // + // +k8s:required // +k8s:minimum=1 - // +k8s:maximum=2 # keep this in sync with the SnapshotContentScope enum - SnapshotContentScope content_scope = 2; + // +k8s:maximum=2 # keep this in sync with the SnapshotStorageStatus enum + SnapshotStorageStatus status = 2; + + // fidelity is what the snapshot holds. + // + // +k8s:required + // +k8s:minimum=1 + // +k8s:maximum=3 # keep this in sync with the SnapshotFidelity enum + SnapshotFidelity fidelity = 3; + + // Exactly one of local or object must be set. + // + // local snapshot is stored on the node where the actor was suspended. + // + // +k8s:optional + // +k8s:unionMember + LocalSnapshot local = 4; + + // object snapshot is stored in object storage. + // + // +k8s:optional + // +k8s:unionMember + ObjectSnapshot object = 5; } -enum SnapshotContentScope { - // Unspecified snapshot content scope. - SNAPSHOT_CONTENT_SCOPE_UNSPECIFIED = 0; - // Captures process memory, root filesystem changes, and durable data. - SNAPSHOT_CONTENT_SCOPE_FULL = 1; - // Captures durable data without process memory or root filesystem changes. - SNAPSHOT_CONTENT_SCOPE_DATA = 2; +// Snapshot represents one generation of an Actor's or Tag's snapshot state. +message Snapshot { + // generation is the monotonically increasing generation number of this + // snapshot, starting at 1. + // + // +k8s:required + // +k8s:minimum=1 + int32 generation = 1; + + // owner identifies the origin and ownership of this snapshot. + // + // +k8s:required + // +k8s:minimum=1 + // +k8s:maximum=2 # keep this in sync with the SnapshotOwner enum + SnapshotOwner owner = 2; + + // UID of the ActorTemplate whose sandbox this snapshot's guest state was + // captured from. + // + // +k8s:optional + // +k8s:format=k8s-uuid + string actor_template_uid = 3; + + // storage lists the storage representations of this snapshot by durability. + // Only SNAPSHOT_DURABILITY_LOCAL and SNAPSHOT_DURABILITY_DURABLE are allowed. + // + // +k8s:required + // +k8s:maxItems=2 + // +k8s:listType=map + // +k8s:listMapKey=durability + repeated SnapshotStorage storage = 4; +} + +// SnapshotFidelity orders the state layers a snapshot can hold, lowest to +// highest. Each level includes the ones below it. +enum SnapshotFidelity { + SNAPSHOT_FIDELITY_UNSPECIFIED = 0; + // Durable volumes only. The actor resumes by cold-booting its containers + // from the OCI image with the volumes restored. + SNAPSHOT_FIDELITY_VOLUMES = 1; + // Volumes plus the root filesystem changes made since boot. Not yet + // supported by any sandbox runtime; templates may not request it. + SNAPSHOT_FIDELITY_ROOTFS = 2; + // Volumes, root filesystem changes, and process memory. The actor resumes + // where it left off. + SNAPSHOT_FIDELITY_MEMORY = 3; // Keep this in sync with the maximums on fields of this type. } @@ -316,7 +407,7 @@ message ExternalVolume { // +k8s:update=NoModify // +k8s:update=NoUnset # set-once; immutable would reject the create // ratchet's nil->set when the final object is validated as an update - string volume_name = 1; + string name = 1; // The globally unique volume_id returned from the storage system. // This will be initially empty during volume creation. Its format is the @@ -339,7 +430,7 @@ message ExternalVolume { // +k8s:required // +k8s:maxLength=253 // +k8s:update=NoModify - // +k8s:update=NoUnset # set-once, like volume_name above + // +k8s:update=NoUnset # set-once, like name above // +k8s:customValidation string volume_type = 3; @@ -710,25 +801,6 @@ message ActorStatus { // +k8s:update=NoModify # can be set and cleared, but not changed in place WorkerAssignment worker_assignment = 2; - // in_progress_snapshot_uri is the URI in object storage of the durable - // snapshot the Actor is currently taking. - // - // +k8s:optional - // +k8s:maxLength=2048 # the template's storage_location bound plus the owner prefix and snapshot name - string in_progress_snapshot_uri = 3; - - // external_snapshot is the Actor's current external snapshot. - // If the Actor was created from a Tag this is the tag's snapshot, borrowed - // until the Actor's first suspend writes one of its own. Otherwise it is - // unset until the Actor is first suspended. - // +k8s:optional - ExternalSnapshot external_snapshot = 4; - - // Node-local state used only while the Actor is paused. - // - // +k8s:optional - LocalSnapshot local_snapshot = 5; - // Volumes attached to the actor. These volumes only live as long as the actor. // They are deleted when the actor is deleted. Each template volume // appears at most once. @@ -736,15 +808,8 @@ message ActorStatus { // +k8s:optional // +k8s:maxItems=32 # matches the template's volumes bound // +k8s:listType=map - // +k8s:listMapKey=volume_name - repeated ExternalVolume actor_volumes = 7; - - // The name of the in-progress node-local checkpoint. Like durable snapshot - // names, these are server-generated UUIDs. - // - // +k8s:optional - // +k8s:format=k8s-short-name - string in_progress_local_snapshot_name = 8; + // +k8s:listMapKey=name + repeated ExternalVolume external_volumes = 7; // crash records why and when the Actor entered CRASHED. It is set with the // CRASHED state and cleared when a revert returns the Actor to SUSPENDED. @@ -759,6 +824,26 @@ message ActorStatus { // +k8s:optional // +k8s:format=k8s-long-name string assigned_node = 10; + + // snapshots holds the Actor's active and in-progress snapshot generations. + // + // +k8s:optional + // +k8s:maxItems=16 + // +k8s:listType=atomic + // +k8s:unique=map + // +k8s:listMapKey=generation + repeated Snapshot snapshots = 11; + + // last_assigned_generation tracks the last snapshot generation assigned for + // this Actor during a Suspend or Pause request. + // Even during Revert, the generation is not decreased to keep track of snapshots + // that have become stale and need to be cleaned up. + // + // +k8s:optional + // +k8s:alpha(since: "0.0")=+k8s:monotonic # TODO: get rid of alpha prefix + // +k8s:minimum=0 + // +k8s:update=NoUnset + int32 last_assigned_generation = 12; } // ActorCrash describes the failure that moved an Actor to CRASHED. @@ -845,12 +930,13 @@ message WorkerAssignment { // The tag is fixed to one external snapshot once its creation completes, so // the rest is immutable provenance. message TagStatus { - // snapshot is the external snapshot this tag points at. Unset while the tag - // is still being created, and immutable once set: a tag never moves between - // snapshots. A tag is usable only once this is set. + // snapshot is the external snapshot this tag points at. Its storage entry is + // IN_PROGRESS while the tag is being created and transitions to COMPLETED + // once the copy finishes: a tag never moves between snapshots and is usable + // only once COMPLETED. // - // +k8s:optional - ExternalSnapshot snapshot = 1; + // +k8s:required + Snapshot snapshot = 1; // UID of the ActorTemplate the snapshot was taken under. Actors can only be // seeded from a tag under the same template. @@ -858,15 +944,6 @@ message TagStatus { // +k8s:required // +k8s:format=k8s-uuid string actor_template_uid = 2; - - // storage_location is the base object-storage URI for this tag's snapshot. - // Set by the server from the source actor's template when the tag is created - // and immutable thereafter. The full snapshot URI is available in - // snapshot.snapshot_uri once tag creation completes. - // - // +k8s:required - // +k8s:maxLength=1024 - string storage_location = 3; } // Tag is an Atespace-owned, stable name for one external snapshot. The @@ -1052,26 +1129,30 @@ message SandboxConfig { } // SnapshotConfig selects what actor snapshots capture and where they are -// stored. A FULL snapshot resumes from its own content. A DATA snapshot -// resumes by starting the containers afresh from the OCI image, with the -// durable-dir volumes populated from the snapshot. +// stored. message SnapshotConfig { - // on_commit defines the scope of every snapshot taken of an actor, both - // the node-local one captured on pause and the one uploaded on suspend. - // Defaults to FULL when unset. - // - // +k8s:required - // +k8s:minimum=1 - // +k8s:maximum=2 # keep this in sync with the SnapshotContentScope enum - SnapshotContentScope on_commit = 2; - // storage_location is the base object-storage URI snapshots of actors on // this version are stored under. Required. // // +k8s:required // +k8s:maxLength=1024 // +k8s:customValidation # Validate URI - string storage_location = 4; + string storage_location = 1; + + // preferred_fidelity is the highest fidelity the system packages when it + // snapshots the actor, on pause (node-local) and on suspend (uploaded). + // It is a best-effort ceiling, not a guarantee: the system never packages + // more than this level, and it may package less when a layer cannot be + // captured or is too expensive to capture at the time, for example memory + // under host pressure. A process resumed in place on its worker keeps its + // memory regardless of this setting. ROOTFS is not supported yet and is + // rejected. Defaults to MEMORY when unset. + // + // +k8s:required + // +k8s:minimum=1 + // +k8s:maximum=3 # keep this in sync with the SnapshotFidelity enum + // +k8s:customValidation # ROOTFS is not supported yet + SnapshotFidelity preferred_fidelity = 2; } // Container is a single application container of an ActorTemplate. @@ -1703,7 +1784,7 @@ message MintActorJWTRequest { // +k8s:maxItems=16 # guardrail; tokens realistically bind a handful of audiences // +k8s:listType=set // +k8s:eachVal=+k8s:maxLength=512 # audiences are caller-defined URIs; bound only - repeated string audience = 1; + repeated string audiences = 1; // Lifetime of the JWT, in seconds. // @@ -2060,8 +2141,7 @@ message Worker { // +k8s:immutable repeated string ips = 7; - // sandbox_class mirrors the WorkerPool's sandboxClass; its values are the - // CRD's own vocabulary, so it is only bounded, not validated. + // sandbox_class mirrors the WorkerPool's default sandbox class // // +k8s:optional // +k8s:maxLength=63 @@ -2143,23 +2223,83 @@ message WorkerStatus { // +k8s:minimum=0 int64 observed_epoch = 4; - // Hardware identity reported by the Worker at registration. + // Runtime that new and cold-booted actors start under, as reported by the + // Worker at registration. Snapshots matching default_runtime can also + // warm-boot on this worker. // // +k8s:optional - HardwareIdentity hardware = 5; -} + SandboxRuntime default_runtime = 5; -// HardwareIdentity identifies the hardware of a Worker or the Worker a -// snapshot was captured on. -message HardwareIdentity { - // Key-value pairs describing hardware properties (currently - // "architecture"; TODO: add "cpu_vendor", "cpu_model", etc.). + // Additional runtimes (excluding default_runtime) that this worker can + // restore snapshots from right now. // // +k8s:optional - // +k8s:maxProperties=32 - // +k8s:eachKey=+k8s:maxLength=128 - // +k8s:eachVal=+k8s:maxLength=256 - map attributes = 1; + // +k8s:maxItems=32 + // +k8s:listType=atomic + repeated SandboxRuntime restorable_runtimes = 6; +} + +// SandboxRuntime describes a sandbox runtime and its compatibility identity as +// reported by a Worker or recorded on a snapshot. +message SandboxRuntime { + // e.g. "gvisor", "microvm" + // + // +k8s:required + // +k8s:maxLength=63 + string sandbox_class = 1; + + // Primary compatibility version stamped onto an actor's snapshot when a + // checkpoint is taken on this runtime. Snapshots matching `version` + // can warm-boot on this runtime. + // TODO: Add support for additional compatibility versions. + // + // +k8s:required + VersionedSandboxCompat version = 2; + + // TODO: Add an informational SandboxConfig version name (e.g. "gvisor-2") + // once workers report one. +} + +// VersionedSandboxCompat holds the compatibility schema version and the +// key-value compatibility attributes for a SandboxRuntime. +message VersionedSandboxCompat { + // Compatibility schema version (e.g. "v1") defining which runtime and + // hardware attributes are recorded and how they are computed. + // + // +k8s:required + // +k8s:maxLength=64 + string schema_version = 1; + + // Key-value compatibility attributes defined by this schema version, covering + // both runtime assets (e.g. "gvisor_asset_hash") and effective guest hardware + // features after masking (e.g. "architecture", "cpu_features"). + // Matched against a snapshot under the same schema_version: every attribute + // on the snapshot must be present on the worker with an equal value; keys + // only the worker has are ignored. Bump schema_version when a key's meaning + // changes or older snapshots must no longer match. + // Order as sent does not matter: ate-api-server sorts attributes by key + // before recording them. + // + // +k8s:required + // +k8s:maxItems=32 + // +k8s:listType=map + // +k8s:listMapKey=key + repeated AttributeEntry attributes = 2; +} + +// AttributeEntry is a single key-value compatibility attribute. +message AttributeEntry { + // Attribute name (e.g. "architecture", "cpu_features", "gvisor_asset_hash"). + // + // +k8s:required + // +k8s:maxLength=128 + string key = 1; + + // Attribute value corresponding to key. + // + // +k8s:required + // +k8s:maxLength=256 + string value = 2; } // WorkerResources represents a set of resources. @@ -2220,15 +2360,15 @@ message ActorAssignment { // authorization: Control is the client-facing API, while these RPCs are served // only to an atelet, and only for the Workers on its own node. service WorkerService { - // RegisterWorker records what a Worker can hold and its hardware identity in - // one write, so a Worker is never schedulable without hardware to match - // snapshots against. Capacity and hardware are the Worker's to report rather - // than the control plane's to infer: they are what the ateom can actually - // supply and expose, only its node can observe them, and a fleet may run - // mixed ateom versions. + // RegisterWorker records what a Worker can hold and its supported sandbox + // runtimes in one write, so a Worker is never schedulable without runtime + // compatibility information to match snapshots against. Capacity and + // runtimes are the Worker's to report rather than the control plane's to + // infer: they are what the worker can actually supply and expose, only its + // node can observe them, and a fleet may run mixed versions. // // atelet calls this with its own client certificate, as it does for - // MintCert. Idempotent: re-sending the same capacity and hardware is not a + // MintCert. Idempotent: re-sending the same capacity and runtimes is not a // write. rpc RegisterWorker(RegisterWorkerRequest) returns (RegisterWorkerResponse); @@ -2275,11 +2415,20 @@ message RegisterWorkerRequest { // +k8s:required WorkerResources capacity = 2; - // Hardware identity reported by the Worker. Like capacity, this replaces - // what is recorded on the Worker. + // Runtime that new and cold-booted actors start under. + // Snapshots matching default_runtime can also warm-boot on this worker. + // Like capacity, this replaces what is recorded on the Worker. // // +k8s:required - HardwareIdentity hardware = 3; + SandboxRuntime default_runtime = 3; + + // Additional runtimes (excluding default_runtime) that this worker can + // restore snapshots from right now. + // + // +k8s:optional + // +k8s:maxItems=32 + // +k8s:listType=atomic + repeated SandboxRuntime restorable_runtimes = 4; } message RegisterWorkerResponse { diff --git a/pkg/proto/ateapipb/ateapi_grpc.pb.go b/pkg/proto/ateapipb/ateapi_grpc.pb.go index 549fdb2550..682ebb0e46 100644 --- a/pkg/proto/ateapipb/ateapi_grpc.pb.go +++ b/pkg/proto/ateapipb/ateapi_grpc.pb.go @@ -1839,15 +1839,15 @@ const ( // authorization: Control is the client-facing API, while these RPCs are served // only to an atelet, and only for the Workers on its own node. type WorkerServiceClient interface { - // RegisterWorker records what a Worker can hold and its hardware identity in - // one write, so a Worker is never schedulable without hardware to match - // snapshots against. Capacity and hardware are the Worker's to report rather - // than the control plane's to infer: they are what the ateom can actually - // supply and expose, only its node can observe them, and a fleet may run - // mixed ateom versions. + // RegisterWorker records what a Worker can hold and its supported sandbox + // runtimes in one write, so a Worker is never schedulable without runtime + // compatibility information to match snapshots against. Capacity and + // runtimes are the Worker's to report rather than the control plane's to + // infer: they are what the worker can actually supply and expose, only its + // node can observe them, and a fleet may run mixed versions. // // atelet calls this with its own client certificate, as it does for - // MintCert. Idempotent: re-sending the same capacity and hardware is not a + // MintCert. Idempotent: re-sending the same capacity and runtimes is not a // write. RegisterWorker(ctx context.Context, in *RegisterWorkerRequest, opts ...grpc.CallOption) (*RegisterWorkerResponse, error) // Create a Substrate-issued SPIFFE certificate that asserts an ateom acting @@ -1924,15 +1924,15 @@ func (c *workerServiceClient) RequestActorSuspend(ctx context.Context, in *Reque // authorization: Control is the client-facing API, while these RPCs are served // only to an atelet, and only for the Workers on its own node. type WorkerServiceServer interface { - // RegisterWorker records what a Worker can hold and its hardware identity in - // one write, so a Worker is never schedulable without hardware to match - // snapshots against. Capacity and hardware are the Worker's to report rather - // than the control plane's to infer: they are what the ateom can actually - // supply and expose, only its node can observe them, and a fleet may run - // mixed ateom versions. + // RegisterWorker records what a Worker can hold and its supported sandbox + // runtimes in one write, so a Worker is never schedulable without runtime + // compatibility information to match snapshots against. Capacity and + // runtimes are the Worker's to report rather than the control plane's to + // infer: they are what the worker can actually supply and expose, only its + // node can observe them, and a fleet may run mixed versions. // // atelet calls this with its own client certificate, as it does for - // MintCert. Idempotent: re-sending the same capacity and hardware is not a + // MintCert. Idempotent: re-sending the same capacity and runtimes is not a // write. RegisterWorker(context.Context, *RegisterWorkerRequest) (*RegisterWorkerResponse, error) // Create a Substrate-issued SPIFFE certificate that asserts an ateom acting diff --git a/pkg/proto/objectstoresnapshotpb/v1/gen.go b/pkg/proto/objectstorepb/v1/gen.go similarity index 94% rename from pkg/proto/objectstoresnapshotpb/v1/gen.go rename to pkg/proto/objectstorepb/v1/gen.go index 8baf4835c3..25a454562d 100644 --- a/pkg/proto/objectstoresnapshotpb/v1/gen.go +++ b/pkg/proto/objectstorepb/v1/gen.go @@ -12,6 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -package objectstoresnapshotv1 +package objectstorev1 -//go:generate bash -c "../../../../hack/protoc.sh --plugin=protoc-gen-go=$(bash ../../../../hack/run-tool.sh --print-bin-path protoc-gen-go) --plugin=protoc-gen-go-grpc=$(bash ../../../../hack/run-tool.sh --print-bin-path protoc-gen-go-grpc) --go_out=paths=source_relative:. --go-grpc_out=paths=source_relative:. objectstoresnapshot.proto" +//go:generate bash -c "../../../../hack/protoc.sh --plugin=protoc-gen-go=$(bash ../../../../hack/run-tool.sh --print-bin-path protoc-gen-go) --plugin=protoc-gen-go-grpc=$(bash ../../../../hack/run-tool.sh --print-bin-path protoc-gen-go-grpc) --go_out=paths=source_relative:. --go-grpc_out=paths=source_relative:. objectstore.proto" diff --git a/pkg/proto/objectstoresnapshotpb/v1/objectstoresnapshot.pb.go b/pkg/proto/objectstorepb/v1/objectstore.pb.go similarity index 71% rename from pkg/proto/objectstoresnapshotpb/v1/objectstoresnapshot.pb.go rename to pkg/proto/objectstorepb/v1/objectstore.pb.go index eff7f79c82..bc1a9d680a 100644 --- a/pkg/proto/objectstoresnapshotpb/v1/objectstoresnapshot.pb.go +++ b/pkg/proto/objectstorepb/v1/objectstore.pb.go @@ -16,9 +16,9 @@ // versions: // protoc-gen-go v1.36.11-devel // protoc v4.25.3 -// source: objectstoresnapshot.proto +// source: objectstore.proto -package objectstoresnapshotv1 +package objectstorev1 import ( protoreflect "google.golang.org/protobuf/reflect/protoreflect" @@ -55,7 +55,7 @@ type FetchSnapshotRequest struct { func (x *FetchSnapshotRequest) Reset() { *x = FetchSnapshotRequest{} - mi := &file_objectstoresnapshot_proto_msgTypes[0] + mi := &file_objectstore_proto_msgTypes[0] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -67,7 +67,7 @@ func (x *FetchSnapshotRequest) String() string { func (*FetchSnapshotRequest) ProtoMessage() {} func (x *FetchSnapshotRequest) ProtoReflect() protoreflect.Message { - mi := &file_objectstoresnapshot_proto_msgTypes[0] + mi := &file_objectstore_proto_msgTypes[0] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -80,7 +80,7 @@ func (x *FetchSnapshotRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use FetchSnapshotRequest.ProtoReflect.Descriptor instead. func (*FetchSnapshotRequest) Descriptor() ([]byte, []int) { - return file_objectstoresnapshot_proto_rawDescGZIP(), []int{0} + return file_objectstore_proto_rawDescGZIP(), []int{0} } func (x *FetchSnapshotRequest) GetSnapshotUri() string { @@ -119,7 +119,7 @@ type FetchSnapshotResponse struct { func (x *FetchSnapshotResponse) Reset() { *x = FetchSnapshotResponse{} - mi := &file_objectstoresnapshot_proto_msgTypes[1] + mi := &file_objectstore_proto_msgTypes[1] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -131,7 +131,7 @@ func (x *FetchSnapshotResponse) String() string { func (*FetchSnapshotResponse) ProtoMessage() {} func (x *FetchSnapshotResponse) ProtoReflect() protoreflect.Message { - mi := &file_objectstoresnapshot_proto_msgTypes[1] + mi := &file_objectstore_proto_msgTypes[1] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -144,7 +144,7 @@ func (x *FetchSnapshotResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use FetchSnapshotResponse.ProtoReflect.Descriptor instead. func (*FetchSnapshotResponse) Descriptor() ([]byte, []int) { - return file_objectstoresnapshot_proto_rawDescGZIP(), []int{1} + return file_objectstore_proto_rawDescGZIP(), []int{1} } // UploadSnapshotRequest asks the plugin to upload snapshot files. @@ -166,7 +166,7 @@ type UploadSnapshotRequest struct { func (x *UploadSnapshotRequest) Reset() { *x = UploadSnapshotRequest{} - mi := &file_objectstoresnapshot_proto_msgTypes[2] + mi := &file_objectstore_proto_msgTypes[2] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -178,7 +178,7 @@ func (x *UploadSnapshotRequest) String() string { func (*UploadSnapshotRequest) ProtoMessage() {} func (x *UploadSnapshotRequest) ProtoReflect() protoreflect.Message { - mi := &file_objectstoresnapshot_proto_msgTypes[2] + mi := &file_objectstore_proto_msgTypes[2] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -191,7 +191,7 @@ func (x *UploadSnapshotRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use UploadSnapshotRequest.ProtoReflect.Descriptor instead. func (*UploadSnapshotRequest) Descriptor() ([]byte, []int) { - return file_objectstoresnapshot_proto_rawDescGZIP(), []int{2} + return file_objectstore_proto_rawDescGZIP(), []int{2} } func (x *UploadSnapshotRequest) GetSnapshotUri() string { @@ -230,7 +230,7 @@ type UploadSnapshotResponse struct { func (x *UploadSnapshotResponse) Reset() { *x = UploadSnapshotResponse{} - mi := &file_objectstoresnapshot_proto_msgTypes[3] + mi := &file_objectstore_proto_msgTypes[3] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -242,7 +242,7 @@ func (x *UploadSnapshotResponse) String() string { func (*UploadSnapshotResponse) ProtoMessage() {} func (x *UploadSnapshotResponse) ProtoReflect() protoreflect.Message { - mi := &file_objectstoresnapshot_proto_msgTypes[3] + mi := &file_objectstore_proto_msgTypes[3] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -255,7 +255,7 @@ func (x *UploadSnapshotResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use UploadSnapshotResponse.ProtoReflect.Descriptor instead. func (*UploadSnapshotResponse) Descriptor() ([]byte, []int) { - return file_objectstoresnapshot_proto_rawDescGZIP(), []int{3} + return file_objectstore_proto_rawDescGZIP(), []int{3} } // CleanupSnapshotRequest names the snapshots to delete. @@ -271,7 +271,7 @@ type CleanupSnapshotRequest struct { func (x *CleanupSnapshotRequest) Reset() { *x = CleanupSnapshotRequest{} - mi := &file_objectstoresnapshot_proto_msgTypes[4] + mi := &file_objectstore_proto_msgTypes[4] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -283,7 +283,7 @@ func (x *CleanupSnapshotRequest) String() string { func (*CleanupSnapshotRequest) ProtoMessage() {} func (x *CleanupSnapshotRequest) ProtoReflect() protoreflect.Message { - mi := &file_objectstoresnapshot_proto_msgTypes[4] + mi := &file_objectstore_proto_msgTypes[4] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -296,7 +296,7 @@ func (x *CleanupSnapshotRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use CleanupSnapshotRequest.ProtoReflect.Descriptor instead. func (*CleanupSnapshotRequest) Descriptor() ([]byte, []int) { - return file_objectstoresnapshot_proto_rawDescGZIP(), []int{4} + return file_objectstore_proto_rawDescGZIP(), []int{4} } func (x *CleanupSnapshotRequest) GetSnapshotUri() string { @@ -314,7 +314,7 @@ type CleanupSnapshotResponse struct { func (x *CleanupSnapshotResponse) Reset() { *x = CleanupSnapshotResponse{} - mi := &file_objectstoresnapshot_proto_msgTypes[5] + mi := &file_objectstore_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -326,7 +326,7 @@ func (x *CleanupSnapshotResponse) String() string { func (*CleanupSnapshotResponse) ProtoMessage() {} func (x *CleanupSnapshotResponse) ProtoReflect() protoreflect.Message { - mi := &file_objectstoresnapshot_proto_msgTypes[5] + mi := &file_objectstore_proto_msgTypes[5] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -339,7 +339,7 @@ func (x *CleanupSnapshotResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use CleanupSnapshotResponse.ProtoReflect.Descriptor instead. func (*CleanupSnapshotResponse) Descriptor() ([]byte, []int) { - return file_objectstoresnapshot_proto_rawDescGZIP(), []int{5} + return file_objectstore_proto_rawDescGZIP(), []int{5} } // CopySnapshotRequest names the source and destination snapshots. @@ -355,7 +355,7 @@ type CopySnapshotRequest struct { func (x *CopySnapshotRequest) Reset() { *x = CopySnapshotRequest{} - mi := &file_objectstoresnapshot_proto_msgTypes[6] + mi := &file_objectstore_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -367,7 +367,7 @@ func (x *CopySnapshotRequest) String() string { func (*CopySnapshotRequest) ProtoMessage() {} func (x *CopySnapshotRequest) ProtoReflect() protoreflect.Message { - mi := &file_objectstoresnapshot_proto_msgTypes[6] + mi := &file_objectstore_proto_msgTypes[6] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -380,7 +380,7 @@ func (x *CopySnapshotRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use CopySnapshotRequest.ProtoReflect.Descriptor instead. func (*CopySnapshotRequest) Descriptor() ([]byte, []int) { - return file_objectstoresnapshot_proto_rawDescGZIP(), []int{6} + return file_objectstore_proto_rawDescGZIP(), []int{6} } func (x *CopySnapshotRequest) GetSrcUri() string { @@ -405,7 +405,7 @@ type CopySnapshotResponse struct { func (x *CopySnapshotResponse) Reset() { *x = CopySnapshotResponse{} - mi := &file_objectstoresnapshot_proto_msgTypes[7] + mi := &file_objectstore_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -417,7 +417,7 @@ func (x *CopySnapshotResponse) String() string { func (*CopySnapshotResponse) ProtoMessage() {} func (x *CopySnapshotResponse) ProtoReflect() protoreflect.Message { - mi := &file_objectstoresnapshot_proto_msgTypes[7] + mi := &file_objectstore_proto_msgTypes[7] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -430,14 +430,14 @@ func (x *CopySnapshotResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use CopySnapshotResponse.ProtoReflect.Descriptor instead. func (*CopySnapshotResponse) Descriptor() ([]byte, []int) { - return file_objectstoresnapshot_proto_rawDescGZIP(), []int{7} + return file_objectstore_proto_rawDescGZIP(), []int{7} } -var File_objectstoresnapshot_proto protoreflect.FileDescriptor +var File_objectstore_proto protoreflect.FileDescriptor -const file_objectstoresnapshot_proto_rawDesc = "" + +const file_objectstore_proto_rawDesc = "" + "\n" + - "\x19objectstoresnapshot.proto\x12\x16objectstoresnapshot.v1\"\x90\x01\n" + + "\x11objectstore.proto\x12\x0eobjectstore.v1\"\x90\x01\n" + "\x14FetchSnapshotRequest\x12!\n" + "\fsnapshot_uri\x18\x01 \x01(\tR\vsnapshotUri\x12 \n" + "\tactor_jwt\x18\x02 \x01(\tB\x03\x80\x01\x01R\bactorJwt\x12\x1d\n" + @@ -458,46 +458,46 @@ const file_objectstoresnapshot_proto_rawDesc = "" + "\x13CopySnapshotRequest\x12\x17\n" + "\asrc_uri\x18\x01 \x01(\tR\x06srcUri\x12\x17\n" + "\adst_uri\x18\x02 \x01(\tR\x06dstUri\"\x16\n" + - "\x14CopySnapshotResponse2\xf1\x01\n" + - "\fNodeProvider\x12n\n" + - "\rFetchSnapshot\x12,.objectstoresnapshot.v1.FetchSnapshotRequest\x1a-.objectstoresnapshot.v1.FetchSnapshotResponse\"\x00\x12q\n" + - "\x0eUploadSnapshot\x12-.objectstoresnapshot.v1.UploadSnapshotRequest\x1a..objectstoresnapshot.v1.UploadSnapshotResponse\"\x002\xf4\x01\n" + - "\x0fControlProvider\x12t\n" + - "\x0fCleanupSnapshot\x12..objectstoresnapshot.v1.CleanupSnapshotRequest\x1a/.objectstoresnapshot.v1.CleanupSnapshotResponse\"\x00\x12k\n" + - "\fCopySnapshot\x12+.objectstoresnapshot.v1.CopySnapshotRequest\x1a,.objectstoresnapshot.v1.CopySnapshotResponse\"\x00B_Z]github.com/agent-substrate/substrate/pkg/proto/objectstoresnapshotpb/v1;objectstoresnapshotv1b\x06proto3" + "\x14CopySnapshotResponse2\xd1\x01\n" + + "\fNodeProvider\x12^\n" + + "\rFetchSnapshot\x12$.objectstore.v1.FetchSnapshotRequest\x1a%.objectstore.v1.FetchSnapshotResponse\"\x00\x12a\n" + + "\x0eUploadSnapshot\x12%.objectstore.v1.UploadSnapshotRequest\x1a&.objectstore.v1.UploadSnapshotResponse\"\x002\xd4\x01\n" + + "\x0fControlProvider\x12d\n" + + "\x0fCleanupSnapshot\x12&.objectstore.v1.CleanupSnapshotRequest\x1a'.objectstore.v1.CleanupSnapshotResponse\"\x00\x12[\n" + + "\fCopySnapshot\x12#.objectstore.v1.CopySnapshotRequest\x1a$.objectstore.v1.CopySnapshotResponse\"\x00BOZMgithub.com/agent-substrate/substrate/pkg/proto/objectstorepb/v1;objectstorev1b\x06proto3" var ( - file_objectstoresnapshot_proto_rawDescOnce sync.Once - file_objectstoresnapshot_proto_rawDescData []byte + file_objectstore_proto_rawDescOnce sync.Once + file_objectstore_proto_rawDescData []byte ) -func file_objectstoresnapshot_proto_rawDescGZIP() []byte { - file_objectstoresnapshot_proto_rawDescOnce.Do(func() { - file_objectstoresnapshot_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_objectstoresnapshot_proto_rawDesc), len(file_objectstoresnapshot_proto_rawDesc))) +func file_objectstore_proto_rawDescGZIP() []byte { + file_objectstore_proto_rawDescOnce.Do(func() { + file_objectstore_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_objectstore_proto_rawDesc), len(file_objectstore_proto_rawDesc))) }) - return file_objectstoresnapshot_proto_rawDescData -} - -var file_objectstoresnapshot_proto_msgTypes = make([]protoimpl.MessageInfo, 8) -var file_objectstoresnapshot_proto_goTypes = []any{ - (*FetchSnapshotRequest)(nil), // 0: objectstoresnapshot.v1.FetchSnapshotRequest - (*FetchSnapshotResponse)(nil), // 1: objectstoresnapshot.v1.FetchSnapshotResponse - (*UploadSnapshotRequest)(nil), // 2: objectstoresnapshot.v1.UploadSnapshotRequest - (*UploadSnapshotResponse)(nil), // 3: objectstoresnapshot.v1.UploadSnapshotResponse - (*CleanupSnapshotRequest)(nil), // 4: objectstoresnapshot.v1.CleanupSnapshotRequest - (*CleanupSnapshotResponse)(nil), // 5: objectstoresnapshot.v1.CleanupSnapshotResponse - (*CopySnapshotRequest)(nil), // 6: objectstoresnapshot.v1.CopySnapshotRequest - (*CopySnapshotResponse)(nil), // 7: objectstoresnapshot.v1.CopySnapshotResponse -} -var file_objectstoresnapshot_proto_depIdxs = []int32{ - 0, // 0: objectstoresnapshot.v1.NodeProvider.FetchSnapshot:input_type -> objectstoresnapshot.v1.FetchSnapshotRequest - 2, // 1: objectstoresnapshot.v1.NodeProvider.UploadSnapshot:input_type -> objectstoresnapshot.v1.UploadSnapshotRequest - 4, // 2: objectstoresnapshot.v1.ControlProvider.CleanupSnapshot:input_type -> objectstoresnapshot.v1.CleanupSnapshotRequest - 6, // 3: objectstoresnapshot.v1.ControlProvider.CopySnapshot:input_type -> objectstoresnapshot.v1.CopySnapshotRequest - 1, // 4: objectstoresnapshot.v1.NodeProvider.FetchSnapshot:output_type -> objectstoresnapshot.v1.FetchSnapshotResponse - 3, // 5: objectstoresnapshot.v1.NodeProvider.UploadSnapshot:output_type -> objectstoresnapshot.v1.UploadSnapshotResponse - 5, // 6: objectstoresnapshot.v1.ControlProvider.CleanupSnapshot:output_type -> objectstoresnapshot.v1.CleanupSnapshotResponse - 7, // 7: objectstoresnapshot.v1.ControlProvider.CopySnapshot:output_type -> objectstoresnapshot.v1.CopySnapshotResponse + return file_objectstore_proto_rawDescData +} + +var file_objectstore_proto_msgTypes = make([]protoimpl.MessageInfo, 8) +var file_objectstore_proto_goTypes = []any{ + (*FetchSnapshotRequest)(nil), // 0: objectstore.v1.FetchSnapshotRequest + (*FetchSnapshotResponse)(nil), // 1: objectstore.v1.FetchSnapshotResponse + (*UploadSnapshotRequest)(nil), // 2: objectstore.v1.UploadSnapshotRequest + (*UploadSnapshotResponse)(nil), // 3: objectstore.v1.UploadSnapshotResponse + (*CleanupSnapshotRequest)(nil), // 4: objectstore.v1.CleanupSnapshotRequest + (*CleanupSnapshotResponse)(nil), // 5: objectstore.v1.CleanupSnapshotResponse + (*CopySnapshotRequest)(nil), // 6: objectstore.v1.CopySnapshotRequest + (*CopySnapshotResponse)(nil), // 7: objectstore.v1.CopySnapshotResponse +} +var file_objectstore_proto_depIdxs = []int32{ + 0, // 0: objectstore.v1.NodeProvider.FetchSnapshot:input_type -> objectstore.v1.FetchSnapshotRequest + 2, // 1: objectstore.v1.NodeProvider.UploadSnapshot:input_type -> objectstore.v1.UploadSnapshotRequest + 4, // 2: objectstore.v1.ControlProvider.CleanupSnapshot:input_type -> objectstore.v1.CleanupSnapshotRequest + 6, // 3: objectstore.v1.ControlProvider.CopySnapshot:input_type -> objectstore.v1.CopySnapshotRequest + 1, // 4: objectstore.v1.NodeProvider.FetchSnapshot:output_type -> objectstore.v1.FetchSnapshotResponse + 3, // 5: objectstore.v1.NodeProvider.UploadSnapshot:output_type -> objectstore.v1.UploadSnapshotResponse + 5, // 6: objectstore.v1.ControlProvider.CleanupSnapshot:output_type -> objectstore.v1.CleanupSnapshotResponse + 7, // 7: objectstore.v1.ControlProvider.CopySnapshot:output_type -> objectstore.v1.CopySnapshotResponse 4, // [4:8] is the sub-list for method output_type 0, // [0:4] is the sub-list for method input_type 0, // [0:0] is the sub-list for extension type_name @@ -505,26 +505,26 @@ var file_objectstoresnapshot_proto_depIdxs = []int32{ 0, // [0:0] is the sub-list for field type_name } -func init() { file_objectstoresnapshot_proto_init() } -func file_objectstoresnapshot_proto_init() { - if File_objectstoresnapshot_proto != nil { +func init() { file_objectstore_proto_init() } +func file_objectstore_proto_init() { + if File_objectstore_proto != nil { return } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: unsafe.Slice(unsafe.StringData(file_objectstoresnapshot_proto_rawDesc), len(file_objectstoresnapshot_proto_rawDesc)), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_objectstore_proto_rawDesc), len(file_objectstore_proto_rawDesc)), NumEnums: 0, NumMessages: 8, NumExtensions: 0, NumServices: 2, }, - GoTypes: file_objectstoresnapshot_proto_goTypes, - DependencyIndexes: file_objectstoresnapshot_proto_depIdxs, - MessageInfos: file_objectstoresnapshot_proto_msgTypes, + GoTypes: file_objectstore_proto_goTypes, + DependencyIndexes: file_objectstore_proto_depIdxs, + MessageInfos: file_objectstore_proto_msgTypes, }.Build() - File_objectstoresnapshot_proto = out.File - file_objectstoresnapshot_proto_goTypes = nil - file_objectstoresnapshot_proto_depIdxs = nil + File_objectstore_proto = out.File + file_objectstore_proto_goTypes = nil + file_objectstore_proto_depIdxs = nil } diff --git a/pkg/proto/objectstoresnapshotpb/v1/objectstoresnapshot.proto b/pkg/proto/objectstorepb/v1/objectstore.proto similarity index 98% rename from pkg/proto/objectstoresnapshotpb/v1/objectstoresnapshot.proto rename to pkg/proto/objectstorepb/v1/objectstore.proto index b99ff868b9..6e20d27d6c 100644 --- a/pkg/proto/objectstoresnapshotpb/v1/objectstoresnapshot.proto +++ b/pkg/proto/objectstorepb/v1/objectstore.proto @@ -14,9 +14,9 @@ syntax = "proto3"; -package objectstoresnapshot.v1; +package objectstore.v1; -option go_package = "github.com/agent-substrate/substrate/pkg/proto/objectstoresnapshotpb/v1;objectstoresnapshotv1"; +option go_package = "github.com/agent-substrate/substrate/pkg/proto/objectstorepb/v1;objectstorev1"; // NodeProvider is the plugin API an object storage platform implements // on every node. atelet calls it to move actor snapshots between the node and diff --git a/pkg/proto/objectstoresnapshotpb/v1/objectstoresnapshot_grpc.pb.go b/pkg/proto/objectstorepb/v1/objectstore_grpc.pb.go similarity index 96% rename from pkg/proto/objectstoresnapshotpb/v1/objectstoresnapshot_grpc.pb.go rename to pkg/proto/objectstorepb/v1/objectstore_grpc.pb.go index e9066c3a3b..acda323a34 100644 --- a/pkg/proto/objectstoresnapshotpb/v1/objectstoresnapshot_grpc.pb.go +++ b/pkg/proto/objectstorepb/v1/objectstore_grpc.pb.go @@ -16,9 +16,9 @@ // versions: // - protoc-gen-go-grpc v1.6.2 // - protoc v4.25.3 -// source: objectstoresnapshot.proto +// source: objectstore.proto -package objectstoresnapshotv1 +package objectstorev1 import ( context "context" @@ -33,8 +33,8 @@ import ( const _ = grpc.SupportPackageIsVersion9 const ( - NodeProvider_FetchSnapshot_FullMethodName = "/objectstoresnapshot.v1.NodeProvider/FetchSnapshot" - NodeProvider_UploadSnapshot_FullMethodName = "/objectstoresnapshot.v1.NodeProvider/UploadSnapshot" + NodeProvider_FetchSnapshot_FullMethodName = "/objectstore.v1.NodeProvider/FetchSnapshot" + NodeProvider_UploadSnapshot_FullMethodName = "/objectstore.v1.NodeProvider/UploadSnapshot" ) // NodeProviderClient is the client API for NodeProvider service. @@ -206,7 +206,7 @@ func _NodeProvider_UploadSnapshot_Handler(srv interface{}, ctx context.Context, // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) var NodeProvider_ServiceDesc = grpc.ServiceDesc{ - ServiceName: "objectstoresnapshot.v1.NodeProvider", + ServiceName: "objectstore.v1.NodeProvider", HandlerType: (*NodeProviderServer)(nil), Methods: []grpc.MethodDesc{ { @@ -219,12 +219,12 @@ var NodeProvider_ServiceDesc = grpc.ServiceDesc{ }, }, Streams: []grpc.StreamDesc{}, - Metadata: "objectstoresnapshot.proto", + Metadata: "objectstore.proto", } const ( - ControlProvider_CleanupSnapshot_FullMethodName = "/objectstoresnapshot.v1.ControlProvider/CleanupSnapshot" - ControlProvider_CopySnapshot_FullMethodName = "/objectstoresnapshot.v1.ControlProvider/CopySnapshot" + ControlProvider_CleanupSnapshot_FullMethodName = "/objectstore.v1.ControlProvider/CleanupSnapshot" + ControlProvider_CopySnapshot_FullMethodName = "/objectstore.v1.ControlProvider/CopySnapshot" ) // ControlProviderClient is the client API for ControlProvider service. @@ -364,7 +364,7 @@ func _ControlProvider_CopySnapshot_Handler(srv interface{}, ctx context.Context, // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) var ControlProvider_ServiceDesc = grpc.ServiceDesc{ - ServiceName: "objectstoresnapshot.v1.ControlProvider", + ServiceName: "objectstore.v1.ControlProvider", HandlerType: (*ControlProviderServer)(nil), Methods: []grpc.MethodDesc{ { @@ -377,5 +377,5 @@ var ControlProvider_ServiceDesc = grpc.ServiceDesc{ }, }, Streams: []grpc.StreamDesc{}, - Metadata: "objectstoresnapshot.proto", + Metadata: "objectstore.proto", } diff --git a/tools/apitool/exemptions.json b/tools/apitool/exemptions.json index 6d77cf51c0..804256c62f 100644 --- a/tools/apitool/exemptions.json +++ b/tools/apitool/exemptions.json @@ -324,11 +324,6 @@ "subject": "ateapi.ResumeActorResponse.actor", "message": "field has no doc comment" }, - { - "rule": "documented", - "subject": "ateapi.SnapshotContentScope", - "message": "enum has no doc comment" - }, { "rule": "documented", "subject": "ateapi.SuspendActorRequest", diff --git a/tools/apitool/internal/model/model.go b/tools/apitool/internal/model/model.go index 32c5dff669..7a664e341e 100644 --- a/tools/apitool/internal/model/model.go +++ b/tools/apitool/internal/model/model.go @@ -132,13 +132,13 @@ type Field struct { // Repeated is true for a `repeated` field, not a map - see // MapValueKind for those. For example, true for - // `repeated ExternalVolume actor_volumes = 7;` on ActorStatus; false + // `repeated ExternalVolume external_volumes = 7;` on ActorStatus; false // for ResourceMetadata.atespace. Repeated bool // TypeDisplay is always a ready-to-render string. For example, "string" // for ResourceMetadata.name, "int64" for ResourceMetadata.version, - // "repeated ExternalVolume" for ActorStatus.actor_volumes, + // "repeated ExternalVolume" for ActorStatus.external_volumes, // "map" for SandboxAssets.assets, "ActorState" for // ActorStatus.state, or "google.protobuf.Timestamp" for // ResourceMetadata.create_time. diff --git a/tools/setup-gcp/README.md b/tools/setup-gcp/README.md index 9d86f5a1e9..c675dc3526 100644 --- a/tools/setup-gcp/README.md +++ b/tools/setup-gcp/README.md @@ -96,34 +96,6 @@ Filestore CSI driver disabled). > podcertificate ClusterTrustBundles to be ready" and `kubectl get > clustertrustbundles` reports the resource type is not served. -> [!WARNING] -> **Turn node auto-upgrade off on any node pool that runs workers, and do not -> use spot or preemptible nodes for them.** When a worker pod is deleted, -> `SIGTERM` is forwarded into the actor's containers and the control plane keeps -> accepting a suspend for 30 minutes. An actor suspended inside that -> window keeps its state. One still awake when the window closes is moved to -> `ACTOR_STATE_CRASHED` with its worker assignment cleared, and `CRASHED` is -> terminal: `resume` and `suspend` are both refused, there is no recover verb, -> and the snapshot the actor still holds cannot be used to start it. The only -> way out is to delete the actor and create a new one, which loses its state. -> -> Auto-upgrade is the trigger to plan for, because GKE enables it by default and -> it fires on Google's maintenance schedule rather than yours. `create cluster` -> does not disable it, so do it yourself on every pool that runs workers: -> -> ```bash -> gcloud container node-pools update "${NODE_POOL}" \ -> --cluster "${CLUSTER_NAME}" --location "${CLUSTER_LOCATION}" \ -> --no-enable-autoupgrade -> ``` -> -> This is a management setting, so it takes effect without recreating nodes and -> is safe to apply to a serving cluster. Node auto-repair, preemption and OOM -> kills reach the same path and cannot be configured away, so treat the setting -> as removing the scheduled risk rather than all of it. Change versions through -> the [rolling upgrade runbook](../../docs/upgrade.md), which has you suspend -> every actor on a node at your own pace before the node moves. - ```bash go run ./tools/setup-gcp create cluster [flags] ``` diff --git a/vendor/golang.org/x/net/dns/dnsmessage/message.go b/vendor/golang.org/x/net/dns/dnsmessage/message.go new file mode 100644 index 0000000000..cbe94a38f5 --- /dev/null +++ b/vendor/golang.org/x/net/dns/dnsmessage/message.go @@ -0,0 +1,2742 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package dnsmessage provides a mostly RFC 1035 compliant implementation of +// DNS message packing and unpacking. +// +// The package also supports messages with Extension Mechanisms for DNS +// (EDNS(0)) as defined in RFC 6891. +// +// This implementation is designed to minimize heap allocations and avoid +// unnecessary packing and unpacking as much as possible. +package dnsmessage + +import ( + "errors" +) + +// Message formats +// +// To add a new Resource Record type: +// 1. Create Resource Record types +// 1.1. Add a Type constant named "Type" +// 1.2. Add the corresponding entry to the typeNames map +// 1.3. Add a [ResourceBody] implementation named "Resource" +// 2. Implement packing +// 2.1. Implement Builder.Resource() +// 3. Implement unpacking +// 3.1. Add the unpacking code to unpackResourceBody() +// 3.2. Implement Parser.Resource() + +// A Type is the type of a DNS Resource Record, as defined in the [IANA registry]. +// +// [IANA registry]: https://www.iana.org/assignments/dns-parameters/dns-parameters.xhtml#dns-parameters-4 +type Type uint16 + +const ( + // ResourceHeader.Type and Question.Type + TypeA Type = 1 + TypeNS Type = 2 + TypeCNAME Type = 5 + TypeSOA Type = 6 + TypePTR Type = 12 + TypeMX Type = 15 + TypeTXT Type = 16 + TypeAAAA Type = 28 + TypeSRV Type = 33 + TypeOPT Type = 41 + TypeSVCB Type = 64 + TypeHTTPS Type = 65 + + // Question.Type + TypeWKS Type = 11 + TypeHINFO Type = 13 + TypeMINFO Type = 14 + TypeAXFR Type = 252 + TypeALL Type = 255 +) + +var typeNames = map[Type]string{ + TypeA: "TypeA", + TypeNS: "TypeNS", + TypeCNAME: "TypeCNAME", + TypeSOA: "TypeSOA", + TypePTR: "TypePTR", + TypeMX: "TypeMX", + TypeTXT: "TypeTXT", + TypeAAAA: "TypeAAAA", + TypeSRV: "TypeSRV", + TypeOPT: "TypeOPT", + TypeSVCB: "TypeSVCB", + TypeHTTPS: "TypeHTTPS", + TypeWKS: "TypeWKS", + TypeHINFO: "TypeHINFO", + TypeMINFO: "TypeMINFO", + TypeAXFR: "TypeAXFR", + TypeALL: "TypeALL", +} + +// String implements fmt.Stringer.String. +func (t Type) String() string { + if n, ok := typeNames[t]; ok { + return n + } + return printUint16(uint16(t)) +} + +// GoString implements fmt.GoStringer.GoString. +func (t Type) GoString() string { + if n, ok := typeNames[t]; ok { + return "dnsmessage." + n + } + return printUint16(uint16(t)) +} + +// A Class is a type of network. +type Class uint16 + +const ( + // ResourceHeader.Class and Question.Class + ClassINET Class = 1 + ClassCSNET Class = 2 + ClassCHAOS Class = 3 + ClassHESIOD Class = 4 + + // Question.Class + ClassANY Class = 255 +) + +var classNames = map[Class]string{ + ClassINET: "ClassINET", + ClassCSNET: "ClassCSNET", + ClassCHAOS: "ClassCHAOS", + ClassHESIOD: "ClassHESIOD", + ClassANY: "ClassANY", +} + +// String implements fmt.Stringer.String. +func (c Class) String() string { + if n, ok := classNames[c]; ok { + return n + } + return printUint16(uint16(c)) +} + +// GoString implements fmt.GoStringer.GoString. +func (c Class) GoString() string { + if n, ok := classNames[c]; ok { + return "dnsmessage." + n + } + return printUint16(uint16(c)) +} + +// An OpCode is a DNS operation code. +type OpCode uint16 + +// GoString implements fmt.GoStringer.GoString. +func (o OpCode) GoString() string { + return printUint16(uint16(o)) +} + +// An RCode is a DNS response status code. +type RCode uint16 + +// Header.RCode values. +const ( + RCodeSuccess RCode = 0 // NoError + RCodeFormatError RCode = 1 // FormErr + RCodeServerFailure RCode = 2 // ServFail + RCodeNameError RCode = 3 // NXDomain + RCodeNotImplemented RCode = 4 // NotImp + RCodeRefused RCode = 5 // Refused +) + +var rCodeNames = map[RCode]string{ + RCodeSuccess: "RCodeSuccess", + RCodeFormatError: "RCodeFormatError", + RCodeServerFailure: "RCodeServerFailure", + RCodeNameError: "RCodeNameError", + RCodeNotImplemented: "RCodeNotImplemented", + RCodeRefused: "RCodeRefused", +} + +// String implements fmt.Stringer.String. +func (r RCode) String() string { + if n, ok := rCodeNames[r]; ok { + return n + } + return printUint16(uint16(r)) +} + +// GoString implements fmt.GoStringer.GoString. +func (r RCode) GoString() string { + if n, ok := rCodeNames[r]; ok { + return "dnsmessage." + n + } + return printUint16(uint16(r)) +} + +func printPaddedUint8(i uint8) string { + b := byte(i) + return string([]byte{ + b/100 + '0', + b/10%10 + '0', + b%10 + '0', + }) +} + +func printUint8Bytes(buf []byte, i uint8) []byte { + b := byte(i) + if i >= 100 { + buf = append(buf, b/100+'0') + } + if i >= 10 { + buf = append(buf, b/10%10+'0') + } + return append(buf, b%10+'0') +} + +func printByteSlice(b []byte) string { + if len(b) == 0 { + return "" + } + buf := make([]byte, 0, 5*len(b)) + buf = printUint8Bytes(buf, uint8(b[0])) + for _, n := range b[1:] { + buf = append(buf, ',', ' ') + buf = printUint8Bytes(buf, uint8(n)) + } + return string(buf) +} + +const hexDigits = "0123456789abcdef" + +func printString(str []byte) string { + buf := make([]byte, 0, len(str)) + for i := 0; i < len(str); i++ { + c := str[i] + if c == '.' || c == '-' || c == ' ' || + 'A' <= c && c <= 'Z' || + 'a' <= c && c <= 'z' || + '0' <= c && c <= '9' { + buf = append(buf, c) + continue + } + + upper := c >> 4 + lower := (c << 4) >> 4 + buf = append( + buf, + '\\', + 'x', + hexDigits[upper], + hexDigits[lower], + ) + } + return string(buf) +} + +func printUint16(i uint16) string { + return printUint32(uint32(i)) +} + +func printUint32(i uint32) string { + // Max value is 4294967295. + buf := make([]byte, 10) + for b, d := buf, uint32(1000000000); d > 0; d /= 10 { + b[0] = byte(i/d%10 + '0') + if b[0] == '0' && len(b) == len(buf) && len(buf) > 1 { + buf = buf[1:] + } + b = b[1:] + i %= d + } + return string(buf) +} + +func printBool(b bool) string { + if b { + return "true" + } + return "false" +} + +var ( + // ErrNotStarted indicates that the prerequisite information isn't + // available yet because the previous records haven't been appropriately + // parsed, skipped or finished. + ErrNotStarted = errors.New("parsing/packing of this type isn't available yet") + + // ErrSectionDone indicated that all records in the section have been + // parsed or finished. + ErrSectionDone = errors.New("parsing/packing of this section has completed") + + errBaseLen = errors.New("insufficient data for base length type") + errCalcLen = errors.New("insufficient data for calculated length type") + errReserved = errors.New("segment prefix is reserved") + errTooManyPtr = errors.New("too many pointers (>10)") + errInvalidPtr = errors.New("invalid pointer") + errInvalidName = errors.New("invalid dns name") + errNilResouceBody = errors.New("nil resource body") + errResourceLen = errors.New("insufficient data for resource body length") + errSegTooLong = errors.New("segment length too long") + errNameTooLong = errors.New("name too long") + errZeroSegLen = errors.New("zero length segment") + errResTooLong = errors.New("resource length too long") + errTooManyQuestions = errors.New("too many Questions to pack (>65535)") + errTooManyAnswers = errors.New("too many Answers to pack (>65535)") + errTooManyAuthorities = errors.New("too many Authorities to pack (>65535)") + errTooManyAdditionals = errors.New("too many Additionals to pack (>65535)") + errNonCanonicalName = errors.New("name is not in canonical format (it must end with a .)") + errStringTooLong = errors.New("character string exceeds maximum length (255)") + errParamOutOfOrder = errors.New("parameter out of order") + errTooLongSVCBValue = errors.New("value too long (>65535 bytes)") +) + +// Internal constants. +const ( + // packStartingCap is the default initial buffer size allocated during + // packing. + // + // The starting capacity doesn't matter too much, but most DNS responses + // Will be <= 512 bytes as it is the limit for DNS over UDP. + packStartingCap = 512 + + // uint16Len is the length (in bytes) of a uint16. + uint16Len = 2 + + // uint32Len is the length (in bytes) of a uint32. + uint32Len = 4 + + // headerLen is the length (in bytes) of a DNS header. + // + // A header is comprised of 6 uint16s and no padding. + headerLen = 6 * uint16Len +) + +type nestedError struct { + // s is the current level's error message. + s string + + // err is the nested error. + err error +} + +// nestedError implements error.Error. +func (e *nestedError) Error() string { + return e.s + ": " + e.err.Error() +} + +// Header is a representation of a DNS message header. +type Header struct { + ID uint16 + Response bool + OpCode OpCode + Authoritative bool + Truncated bool + RecursionDesired bool + RecursionAvailable bool + AuthenticData bool + CheckingDisabled bool + RCode RCode +} + +func (m *Header) pack() (id uint16, bits uint16) { + id = m.ID + bits = uint16(m.OpCode)<<11 | uint16(m.RCode) + if m.RecursionAvailable { + bits |= headerBitRA + } + if m.RecursionDesired { + bits |= headerBitRD + } + if m.Truncated { + bits |= headerBitTC + } + if m.Authoritative { + bits |= headerBitAA + } + if m.Response { + bits |= headerBitQR + } + if m.AuthenticData { + bits |= headerBitAD + } + if m.CheckingDisabled { + bits |= headerBitCD + } + return +} + +// GoString implements fmt.GoStringer.GoString. +func (m *Header) GoString() string { + return "dnsmessage.Header{" + + "ID: " + printUint16(m.ID) + ", " + + "Response: " + printBool(m.Response) + ", " + + "OpCode: " + m.OpCode.GoString() + ", " + + "Authoritative: " + printBool(m.Authoritative) + ", " + + "Truncated: " + printBool(m.Truncated) + ", " + + "RecursionDesired: " + printBool(m.RecursionDesired) + ", " + + "RecursionAvailable: " + printBool(m.RecursionAvailable) + ", " + + "AuthenticData: " + printBool(m.AuthenticData) + ", " + + "CheckingDisabled: " + printBool(m.CheckingDisabled) + ", " + + "RCode: " + m.RCode.GoString() + "}" +} + +// Message is a representation of a DNS message. +type Message struct { + Header + Questions []Question + Answers []Resource + Authorities []Resource + Additionals []Resource +} + +type section uint8 + +const ( + sectionNotStarted section = iota + sectionHeader + sectionQuestions + sectionAnswers + sectionAuthorities + sectionAdditionals + sectionDone + + headerBitQR = 1 << 15 // query/response (response=1) + headerBitAA = 1 << 10 // authoritative + headerBitTC = 1 << 9 // truncated + headerBitRD = 1 << 8 // recursion desired + headerBitRA = 1 << 7 // recursion available + headerBitAD = 1 << 5 // authentic data + headerBitCD = 1 << 4 // checking disabled +) + +var sectionNames = map[section]string{ + sectionHeader: "header", + sectionQuestions: "Question", + sectionAnswers: "Answer", + sectionAuthorities: "Authority", + sectionAdditionals: "Additional", +} + +// header is the wire format for a DNS message header. +type header struct { + id uint16 + bits uint16 + questions uint16 + answers uint16 + authorities uint16 + additionals uint16 +} + +func (h *header) count(sec section) uint16 { + switch sec { + case sectionQuestions: + return h.questions + case sectionAnswers: + return h.answers + case sectionAuthorities: + return h.authorities + case sectionAdditionals: + return h.additionals + } + return 0 +} + +// pack appends the wire format of the header to msg. +func (h *header) pack(msg []byte) []byte { + msg = packUint16(msg, h.id) + msg = packUint16(msg, h.bits) + msg = packUint16(msg, h.questions) + msg = packUint16(msg, h.answers) + msg = packUint16(msg, h.authorities) + return packUint16(msg, h.additionals) +} + +func (h *header) unpack(msg []byte, off int) (int, error) { + newOff := off + var err error + if h.id, newOff, err = unpackUint16(msg, newOff); err != nil { + return off, &nestedError{"id", err} + } + if h.bits, newOff, err = unpackUint16(msg, newOff); err != nil { + return off, &nestedError{"bits", err} + } + if h.questions, newOff, err = unpackUint16(msg, newOff); err != nil { + return off, &nestedError{"questions", err} + } + if h.answers, newOff, err = unpackUint16(msg, newOff); err != nil { + return off, &nestedError{"answers", err} + } + if h.authorities, newOff, err = unpackUint16(msg, newOff); err != nil { + return off, &nestedError{"authorities", err} + } + if h.additionals, newOff, err = unpackUint16(msg, newOff); err != nil { + return off, &nestedError{"additionals", err} + } + return newOff, nil +} + +func (h *header) header() Header { + return Header{ + ID: h.id, + Response: (h.bits & headerBitQR) != 0, + OpCode: OpCode(h.bits>>11) & 0xF, + Authoritative: (h.bits & headerBitAA) != 0, + Truncated: (h.bits & headerBitTC) != 0, + RecursionDesired: (h.bits & headerBitRD) != 0, + RecursionAvailable: (h.bits & headerBitRA) != 0, + AuthenticData: (h.bits & headerBitAD) != 0, + CheckingDisabled: (h.bits & headerBitCD) != 0, + RCode: RCode(h.bits & 0xF), + } +} + +// A Resource is a DNS resource record. +type Resource struct { + Header ResourceHeader + Body ResourceBody +} + +func (r *Resource) GoString() string { + return "dnsmessage.Resource{" + + "Header: " + r.Header.GoString() + + ", Body: &" + r.Body.GoString() + + "}" +} + +// A ResourceBody is a DNS resource record minus the header. +type ResourceBody interface { + // pack packs a Resource except for its header. + pack(msg []byte, compression map[string]uint16, compressionOff int) ([]byte, error) + + // realType returns the actual type of the Resource. This is used to + // fill in the header Type field. + realType() Type + + // GoString implements fmt.GoStringer.GoString. + GoString() string +} + +// pack appends the wire format of the Resource to msg. +func (r *Resource) pack(msg []byte, compression map[string]uint16, compressionOff int) ([]byte, error) { + if r.Body == nil { + return msg, errNilResouceBody + } + oldMsg := msg + r.Header.Type = r.Body.realType() + msg, lenOff, err := r.Header.pack(msg, compression, compressionOff) + if err != nil { + return msg, &nestedError{"ResourceHeader", err} + } + preLen := len(msg) + msg, err = r.Body.pack(msg, compression, compressionOff) + if err != nil { + return msg, &nestedError{"content", err} + } + if err := r.Header.fixLen(msg, lenOff, preLen); err != nil { + return oldMsg, err + } + return msg, nil +} + +// A Parser allows incrementally parsing a DNS message. +// +// When parsing is started, the Header is parsed. Next, each Question can be +// either parsed or skipped. Alternatively, all Questions can be skipped at +// once. When all Questions have been parsed, attempting to parse Questions +// will return the [ErrSectionDone] error. +// After all Questions have been either parsed or skipped, all +// Answers, Authorities and Additionals can be either parsed or skipped in the +// same way, and each type of Resource must be fully parsed or skipped before +// proceeding to the next type of Resource. +// +// Parser is safe to copy to preserve the parsing state. +// +// Note that there is no requirement to fully skip or parse the message. +type Parser struct { + msg []byte + header header + + section section + off int + index int + resHeaderValid bool + resHeaderOffset int + resHeaderType Type + resHeaderLength uint16 +} + +// Start parses the header and enables the parsing of Questions. +func (p *Parser) Start(msg []byte) (Header, error) { + if p.msg != nil { + *p = Parser{} + } + p.msg = msg + var err error + if p.off, err = p.header.unpack(msg, 0); err != nil { + return Header{}, &nestedError{"unpacking header", err} + } + p.section = sectionQuestions + return p.header.header(), nil +} + +func (p *Parser) checkAdvance(sec section) error { + if p.section < sec { + return ErrNotStarted + } + if p.section > sec { + return ErrSectionDone + } + p.resHeaderValid = false + if p.index == int(p.header.count(sec)) { + p.index = 0 + p.section++ + return ErrSectionDone + } + return nil +} + +func (p *Parser) resource(sec section) (Resource, error) { + var r Resource + var err error + r.Header, err = p.resourceHeader(sec) + if err != nil { + return r, err + } + p.resHeaderValid = false + r.Body, p.off, err = unpackResourceBody(p.msg, p.off, r.Header) + if err != nil { + return Resource{}, &nestedError{"unpacking " + sectionNames[sec], err} + } + p.index++ + return r, nil +} + +func (p *Parser) resourceHeader(sec section) (ResourceHeader, error) { + if p.resHeaderValid { + p.off = p.resHeaderOffset + } + + if err := p.checkAdvance(sec); err != nil { + return ResourceHeader{}, err + } + var hdr ResourceHeader + off, err := hdr.unpack(p.msg, p.off) + if err != nil { + return ResourceHeader{}, err + } + p.resHeaderValid = true + p.resHeaderOffset = p.off + p.resHeaderType = hdr.Type + p.resHeaderLength = hdr.Length + p.off = off + return hdr, nil +} + +func (p *Parser) skipResource(sec section) error { + if p.resHeaderValid && p.section == sec { + newOff := p.off + int(p.resHeaderLength) + if newOff > len(p.msg) { + return errResourceLen + } + p.off = newOff + p.resHeaderValid = false + p.index++ + return nil + } + if err := p.checkAdvance(sec); err != nil { + return err + } + var err error + p.off, err = skipResource(p.msg, p.off) + if err != nil { + return &nestedError{"skipping: " + sectionNames[sec], err} + } + p.index++ + return nil +} + +// Question parses a single Question. +func (p *Parser) Question() (Question, error) { + if err := p.checkAdvance(sectionQuestions); err != nil { + return Question{}, err + } + var name Name + off, err := name.unpack(p.msg, p.off) + if err != nil { + return Question{}, &nestedError{"unpacking Question.Name", err} + } + typ, off, err := unpackType(p.msg, off) + if err != nil { + return Question{}, &nestedError{"unpacking Question.Type", err} + } + class, off, err := unpackClass(p.msg, off) + if err != nil { + return Question{}, &nestedError{"unpacking Question.Class", err} + } + p.off = off + p.index++ + return Question{name, typ, class}, nil +} + +// AllQuestions parses all Questions. +func (p *Parser) AllQuestions() ([]Question, error) { + // Multiple questions are valid according to the spec, + // but servers don't actually support them. There will + // be at most one question here. + // + // Do not pre-allocate based on info in p.header, since + // the data is untrusted. + qs := []Question{} + for { + q, err := p.Question() + if err == ErrSectionDone { + return qs, nil + } + if err != nil { + return nil, err + } + qs = append(qs, q) + } +} + +// SkipQuestion skips a single Question. +func (p *Parser) SkipQuestion() error { + if err := p.checkAdvance(sectionQuestions); err != nil { + return err + } + off, err := skipName(p.msg, p.off) + if err != nil { + return &nestedError{"skipping Question Name", err} + } + if off, err = skipType(p.msg, off); err != nil { + return &nestedError{"skipping Question Type", err} + } + if off, err = skipClass(p.msg, off); err != nil { + return &nestedError{"skipping Question Class", err} + } + p.off = off + p.index++ + return nil +} + +// SkipAllQuestions skips all Questions. +func (p *Parser) SkipAllQuestions() error { + for { + if err := p.SkipQuestion(); err == ErrSectionDone { + return nil + } else if err != nil { + return err + } + } +} + +// AnswerHeader parses a single Answer ResourceHeader. +func (p *Parser) AnswerHeader() (ResourceHeader, error) { + return p.resourceHeader(sectionAnswers) +} + +// Answer parses a single Answer Resource. +func (p *Parser) Answer() (Resource, error) { + return p.resource(sectionAnswers) +} + +// AllAnswers parses all Answer Resources. +func (p *Parser) AllAnswers() ([]Resource, error) { + // The most common query is for A/AAAA, which usually returns + // a handful of IPs. + // + // Pre-allocate up to a certain limit, since p.header is + // untrusted data. + n := int(p.header.answers) + if n > 20 { + n = 20 + } + as := make([]Resource, 0, n) + for { + a, err := p.Answer() + if err == ErrSectionDone { + return as, nil + } + if err != nil { + return nil, err + } + as = append(as, a) + } +} + +// SkipAnswer skips a single Answer Resource. +// +// It does not perform a complete validation of the resource header, which means +// it may return a nil error when the [AnswerHeader] would actually return an error. +func (p *Parser) SkipAnswer() error { + return p.skipResource(sectionAnswers) +} + +// SkipAllAnswers skips all Answer Resources. +func (p *Parser) SkipAllAnswers() error { + for { + if err := p.SkipAnswer(); err == ErrSectionDone { + return nil + } else if err != nil { + return err + } + } +} + +// AuthorityHeader parses a single Authority ResourceHeader. +func (p *Parser) AuthorityHeader() (ResourceHeader, error) { + return p.resourceHeader(sectionAuthorities) +} + +// Authority parses a single Authority Resource. +func (p *Parser) Authority() (Resource, error) { + return p.resource(sectionAuthorities) +} + +// AllAuthorities parses all Authority Resources. +func (p *Parser) AllAuthorities() ([]Resource, error) { + // Authorities contains SOA in case of NXDOMAIN and friends, + // otherwise it is empty. + // + // Pre-allocate up to a certain limit, since p.header is + // untrusted data. + n := int(p.header.authorities) + if n > 10 { + n = 10 + } + as := make([]Resource, 0, n) + for { + a, err := p.Authority() + if err == ErrSectionDone { + return as, nil + } + if err != nil { + return nil, err + } + as = append(as, a) + } +} + +// SkipAuthority skips a single Authority Resource. +// +// It does not perform a complete validation of the resource header, which means +// it may return a nil error when the [AuthorityHeader] would actually return an error. +func (p *Parser) SkipAuthority() error { + return p.skipResource(sectionAuthorities) +} + +// SkipAllAuthorities skips all Authority Resources. +func (p *Parser) SkipAllAuthorities() error { + for { + if err := p.SkipAuthority(); err == ErrSectionDone { + return nil + } else if err != nil { + return err + } + } +} + +// AdditionalHeader parses a single Additional ResourceHeader. +func (p *Parser) AdditionalHeader() (ResourceHeader, error) { + return p.resourceHeader(sectionAdditionals) +} + +// Additional parses a single Additional Resource. +func (p *Parser) Additional() (Resource, error) { + return p.resource(sectionAdditionals) +} + +// AllAdditionals parses all Additional Resources. +func (p *Parser) AllAdditionals() ([]Resource, error) { + // Additionals usually contain OPT, and sometimes A/AAAA + // glue records. + // + // Pre-allocate up to a certain limit, since p.header is + // untrusted data. + n := int(p.header.additionals) + if n > 10 { + n = 10 + } + as := make([]Resource, 0, n) + for { + a, err := p.Additional() + if err == ErrSectionDone { + return as, nil + } + if err != nil { + return nil, err + } + as = append(as, a) + } +} + +// SkipAdditional skips a single Additional Resource. +// +// It does not perform a complete validation of the resource header, which means +// it may return a nil error when the [AdditionalHeader] would actually return an error. +func (p *Parser) SkipAdditional() error { + return p.skipResource(sectionAdditionals) +} + +// SkipAllAdditionals skips all Additional Resources. +func (p *Parser) SkipAllAdditionals() error { + for { + if err := p.SkipAdditional(); err == ErrSectionDone { + return nil + } else if err != nil { + return err + } + } +} + +// CNAMEResource parses a single CNAMEResource. +// +// One of the XXXHeader methods must have been called before calling this +// method. +func (p *Parser) CNAMEResource() (CNAMEResource, error) { + if !p.resHeaderValid || p.resHeaderType != TypeCNAME { + return CNAMEResource{}, ErrNotStarted + } + r, err := unpackCNAMEResource(p.msg, p.off) + if err != nil { + return CNAMEResource{}, err + } + p.off += int(p.resHeaderLength) + p.resHeaderValid = false + p.index++ + return r, nil +} + +// MXResource parses a single MXResource. +// +// One of the XXXHeader methods must have been called before calling this +// method. +func (p *Parser) MXResource() (MXResource, error) { + if !p.resHeaderValid || p.resHeaderType != TypeMX { + return MXResource{}, ErrNotStarted + } + r, err := unpackMXResource(p.msg, p.off) + if err != nil { + return MXResource{}, err + } + p.off += int(p.resHeaderLength) + p.resHeaderValid = false + p.index++ + return r, nil +} + +// NSResource parses a single NSResource. +// +// One of the XXXHeader methods must have been called before calling this +// method. +func (p *Parser) NSResource() (NSResource, error) { + if !p.resHeaderValid || p.resHeaderType != TypeNS { + return NSResource{}, ErrNotStarted + } + r, err := unpackNSResource(p.msg, p.off) + if err != nil { + return NSResource{}, err + } + p.off += int(p.resHeaderLength) + p.resHeaderValid = false + p.index++ + return r, nil +} + +// PTRResource parses a single PTRResource. +// +// One of the XXXHeader methods must have been called before calling this +// method. +func (p *Parser) PTRResource() (PTRResource, error) { + if !p.resHeaderValid || p.resHeaderType != TypePTR { + return PTRResource{}, ErrNotStarted + } + r, err := unpackPTRResource(p.msg, p.off) + if err != nil { + return PTRResource{}, err + } + p.off += int(p.resHeaderLength) + p.resHeaderValid = false + p.index++ + return r, nil +} + +// SOAResource parses a single SOAResource. +// +// One of the XXXHeader methods must have been called before calling this +// method. +func (p *Parser) SOAResource() (SOAResource, error) { + if !p.resHeaderValid || p.resHeaderType != TypeSOA { + return SOAResource{}, ErrNotStarted + } + r, err := unpackSOAResource(p.msg, p.off) + if err != nil { + return SOAResource{}, err + } + p.off += int(p.resHeaderLength) + p.resHeaderValid = false + p.index++ + return r, nil +} + +// TXTResource parses a single TXTResource. +// +// One of the XXXHeader methods must have been called before calling this +// method. +func (p *Parser) TXTResource() (TXTResource, error) { + if !p.resHeaderValid || p.resHeaderType != TypeTXT { + return TXTResource{}, ErrNotStarted + } + r, err := unpackTXTResource(p.msg, p.off, p.resHeaderLength) + if err != nil { + return TXTResource{}, err + } + p.off += int(p.resHeaderLength) + p.resHeaderValid = false + p.index++ + return r, nil +} + +// SRVResource parses a single SRVResource. +// +// One of the XXXHeader methods must have been called before calling this +// method. +func (p *Parser) SRVResource() (SRVResource, error) { + if !p.resHeaderValid || p.resHeaderType != TypeSRV { + return SRVResource{}, ErrNotStarted + } + r, err := unpackSRVResource(p.msg, p.off) + if err != nil { + return SRVResource{}, err + } + p.off += int(p.resHeaderLength) + p.resHeaderValid = false + p.index++ + return r, nil +} + +// AResource parses a single AResource. +// +// One of the XXXHeader methods must have been called before calling this +// method. +func (p *Parser) AResource() (AResource, error) { + if !p.resHeaderValid || p.resHeaderType != TypeA { + return AResource{}, ErrNotStarted + } + r, err := unpackAResource(p.msg, p.off) + if err != nil { + return AResource{}, err + } + p.off += int(p.resHeaderLength) + p.resHeaderValid = false + p.index++ + return r, nil +} + +// AAAAResource parses a single AAAAResource. +// +// One of the XXXHeader methods must have been called before calling this +// method. +func (p *Parser) AAAAResource() (AAAAResource, error) { + if !p.resHeaderValid || p.resHeaderType != TypeAAAA { + return AAAAResource{}, ErrNotStarted + } + r, err := unpackAAAAResource(p.msg, p.off) + if err != nil { + return AAAAResource{}, err + } + p.off += int(p.resHeaderLength) + p.resHeaderValid = false + p.index++ + return r, nil +} + +// OPTResource parses a single OPTResource. +// +// One of the XXXHeader methods must have been called before calling this +// method. +func (p *Parser) OPTResource() (OPTResource, error) { + if !p.resHeaderValid || p.resHeaderType != TypeOPT { + return OPTResource{}, ErrNotStarted + } + r, err := unpackOPTResource(p.msg, p.off, p.resHeaderLength) + if err != nil { + return OPTResource{}, err + } + p.off += int(p.resHeaderLength) + p.resHeaderValid = false + p.index++ + return r, nil +} + +// UnknownResource parses a single UnknownResource. +// +// One of the XXXHeader methods must have been called before calling this +// method. +func (p *Parser) UnknownResource() (UnknownResource, error) { + if !p.resHeaderValid { + return UnknownResource{}, ErrNotStarted + } + r, err := unpackUnknownResource(p.resHeaderType, p.msg, p.off, p.resHeaderLength) + if err != nil { + return UnknownResource{}, err + } + p.off += int(p.resHeaderLength) + p.resHeaderValid = false + p.index++ + return r, nil +} + +// Unpack parses a full Message. +func (m *Message) Unpack(msg []byte) error { + var p Parser + var err error + if m.Header, err = p.Start(msg); err != nil { + return err + } + if m.Questions, err = p.AllQuestions(); err != nil { + return err + } + if m.Answers, err = p.AllAnswers(); err != nil { + return err + } + if m.Authorities, err = p.AllAuthorities(); err != nil { + return err + } + if m.Additionals, err = p.AllAdditionals(); err != nil { + return err + } + return nil +} + +// Pack packs a full Message. +func (m *Message) Pack() ([]byte, error) { + return m.AppendPack(make([]byte, 0, packStartingCap)) +} + +// AppendPack is like Pack but appends the full Message to b and returns the +// extended buffer. +func (m *Message) AppendPack(b []byte) ([]byte, error) { + // Validate the lengths. It is very unlikely that anyone will try to + // pack more than 65535 of any particular type, but it is possible and + // we should fail gracefully. + if len(m.Questions) > int(^uint16(0)) { + return nil, errTooManyQuestions + } + if len(m.Answers) > int(^uint16(0)) { + return nil, errTooManyAnswers + } + if len(m.Authorities) > int(^uint16(0)) { + return nil, errTooManyAuthorities + } + if len(m.Additionals) > int(^uint16(0)) { + return nil, errTooManyAdditionals + } + + var h header + h.id, h.bits = m.Header.pack() + + h.questions = uint16(len(m.Questions)) + h.answers = uint16(len(m.Answers)) + h.authorities = uint16(len(m.Authorities)) + h.additionals = uint16(len(m.Additionals)) + + compressionOff := len(b) + msg := h.pack(b) + + // RFC 1035 allows (but does not require) compression for packing. RFC + // 1035 requires unpacking implementations to support compression, so + // unconditionally enabling it is fine. + // + // DNS lookups are typically done over UDP, and RFC 1035 states that UDP + // DNS messages can be a maximum of 512 bytes long. Without compression, + // many DNS response messages are over this limit, so enabling + // compression will help ensure compliance. + compression := map[string]uint16{} + + for i := range m.Questions { + var err error + if msg, err = m.Questions[i].pack(msg, compression, compressionOff); err != nil { + return nil, &nestedError{"packing Question", err} + } + } + for i := range m.Answers { + var err error + if msg, err = m.Answers[i].pack(msg, compression, compressionOff); err != nil { + return nil, &nestedError{"packing Answer", err} + } + } + for i := range m.Authorities { + var err error + if msg, err = m.Authorities[i].pack(msg, compression, compressionOff); err != nil { + return nil, &nestedError{"packing Authority", err} + } + } + for i := range m.Additionals { + var err error + if msg, err = m.Additionals[i].pack(msg, compression, compressionOff); err != nil { + return nil, &nestedError{"packing Additional", err} + } + } + + return msg, nil +} + +// GoString implements fmt.GoStringer.GoString. +func (m *Message) GoString() string { + s := "dnsmessage.Message{Header: " + m.Header.GoString() + ", " + + "Questions: []dnsmessage.Question{" + if len(m.Questions) > 0 { + s += m.Questions[0].GoString() + for _, q := range m.Questions[1:] { + s += ", " + q.GoString() + } + } + s += "}, Answers: []dnsmessage.Resource{" + if len(m.Answers) > 0 { + s += m.Answers[0].GoString() + for _, a := range m.Answers[1:] { + s += ", " + a.GoString() + } + } + s += "}, Authorities: []dnsmessage.Resource{" + if len(m.Authorities) > 0 { + s += m.Authorities[0].GoString() + for _, a := range m.Authorities[1:] { + s += ", " + a.GoString() + } + } + s += "}, Additionals: []dnsmessage.Resource{" + if len(m.Additionals) > 0 { + s += m.Additionals[0].GoString() + for _, a := range m.Additionals[1:] { + s += ", " + a.GoString() + } + } + return s + "}}" +} + +// A Builder allows incrementally packing a DNS message. +// +// Example usage: +// +// buf := make([]byte, 2, 514) +// b := NewBuilder(buf, Header{...}) +// b.EnableCompression() +// // Optionally start a section and add things to that section. +// // Repeat adding sections as necessary. +// buf, err := b.Finish() +// // If err is nil, buf[2:] will contain the built bytes. +type Builder struct { + // msg is the storage for the message being built. + msg []byte + + // section keeps track of the current section being built. + section section + + // header keeps track of what should go in the header when Finish is + // called. + header header + + // start is the starting index of the bytes allocated in msg for header. + start int + + // compression is a mapping from name suffixes to their starting index + // in msg. + compression map[string]uint16 +} + +// NewBuilder creates a new builder with compression disabled. +// +// Note: Most users will want to immediately enable compression with the +// EnableCompression method. See that method's comment for why you may or may +// not want to enable compression. +// +// The DNS message is appended to the provided initial buffer buf (which may be +// nil) as it is built. The final message is returned by the (*Builder).Finish +// method, which includes buf[:len(buf)] and may return the same underlying +// array if there was sufficient capacity in the slice. +func NewBuilder(buf []byte, h Header) Builder { + if buf == nil { + buf = make([]byte, 0, packStartingCap) + } + b := Builder{msg: buf, start: len(buf)} + b.header.id, b.header.bits = h.pack() + var hb [headerLen]byte + b.msg = append(b.msg, hb[:]...) + b.section = sectionHeader + return b +} + +// EnableCompression enables compression in the Builder. +// +// Leaving compression disabled avoids compression related allocations, but can +// result in larger message sizes. Be careful with this mode as it can cause +// messages to exceed the UDP size limit. +// +// According to RFC 1035, section 4.1.4, the use of compression is optional, but +// all implementations must accept both compressed and uncompressed DNS +// messages. +// +// Compression should be enabled before any sections are added for best results. +func (b *Builder) EnableCompression() { + b.compression = map[string]uint16{} +} + +func (b *Builder) startCheck(s section) error { + if b.section <= sectionNotStarted { + return ErrNotStarted + } + if b.section > s { + return ErrSectionDone + } + return nil +} + +// StartQuestions prepares the builder for packing Questions. +func (b *Builder) StartQuestions() error { + if err := b.startCheck(sectionQuestions); err != nil { + return err + } + b.section = sectionQuestions + return nil +} + +// StartAnswers prepares the builder for packing Answers. +func (b *Builder) StartAnswers() error { + if err := b.startCheck(sectionAnswers); err != nil { + return err + } + b.section = sectionAnswers + return nil +} + +// StartAuthorities prepares the builder for packing Authorities. +func (b *Builder) StartAuthorities() error { + if err := b.startCheck(sectionAuthorities); err != nil { + return err + } + b.section = sectionAuthorities + return nil +} + +// StartAdditionals prepares the builder for packing Additionals. +func (b *Builder) StartAdditionals() error { + if err := b.startCheck(sectionAdditionals); err != nil { + return err + } + b.section = sectionAdditionals + return nil +} + +func (b *Builder) incrementSectionCount() error { + var count *uint16 + var err error + switch b.section { + case sectionQuestions: + count = &b.header.questions + err = errTooManyQuestions + case sectionAnswers: + count = &b.header.answers + err = errTooManyAnswers + case sectionAuthorities: + count = &b.header.authorities + err = errTooManyAuthorities + case sectionAdditionals: + count = &b.header.additionals + err = errTooManyAdditionals + } + if *count == ^uint16(0) { + return err + } + *count++ + return nil +} + +// Question adds a single Question. +func (b *Builder) Question(q Question) error { + if b.section < sectionQuestions { + return ErrNotStarted + } + if b.section > sectionQuestions { + return ErrSectionDone + } + msg, err := q.pack(b.msg, b.compression, b.start) + if err != nil { + return err + } + if err := b.incrementSectionCount(); err != nil { + return err + } + b.msg = msg + return nil +} + +func (b *Builder) checkResourceSection() error { + if b.section < sectionAnswers { + return ErrNotStarted + } + if b.section > sectionAdditionals { + return ErrSectionDone + } + return nil +} + +// CNAMEResource adds a single CNAMEResource. +func (b *Builder) CNAMEResource(h ResourceHeader, r CNAMEResource) error { + if err := b.checkResourceSection(); err != nil { + return err + } + h.Type = r.realType() + msg, lenOff, err := h.pack(b.msg, b.compression, b.start) + if err != nil { + return &nestedError{"ResourceHeader", err} + } + preLen := len(msg) + if msg, err = r.pack(msg, b.compression, b.start); err != nil { + return &nestedError{"CNAMEResource body", err} + } + if err := h.fixLen(msg, lenOff, preLen); err != nil { + return err + } + if err := b.incrementSectionCount(); err != nil { + return err + } + b.msg = msg + return nil +} + +// MXResource adds a single MXResource. +func (b *Builder) MXResource(h ResourceHeader, r MXResource) error { + if err := b.checkResourceSection(); err != nil { + return err + } + h.Type = r.realType() + msg, lenOff, err := h.pack(b.msg, b.compression, b.start) + if err != nil { + return &nestedError{"ResourceHeader", err} + } + preLen := len(msg) + if msg, err = r.pack(msg, b.compression, b.start); err != nil { + return &nestedError{"MXResource body", err} + } + if err := h.fixLen(msg, lenOff, preLen); err != nil { + return err + } + if err := b.incrementSectionCount(); err != nil { + return err + } + b.msg = msg + return nil +} + +// NSResource adds a single NSResource. +func (b *Builder) NSResource(h ResourceHeader, r NSResource) error { + if err := b.checkResourceSection(); err != nil { + return err + } + h.Type = r.realType() + msg, lenOff, err := h.pack(b.msg, b.compression, b.start) + if err != nil { + return &nestedError{"ResourceHeader", err} + } + preLen := len(msg) + if msg, err = r.pack(msg, b.compression, b.start); err != nil { + return &nestedError{"NSResource body", err} + } + if err := h.fixLen(msg, lenOff, preLen); err != nil { + return err + } + if err := b.incrementSectionCount(); err != nil { + return err + } + b.msg = msg + return nil +} + +// PTRResource adds a single PTRResource. +func (b *Builder) PTRResource(h ResourceHeader, r PTRResource) error { + if err := b.checkResourceSection(); err != nil { + return err + } + h.Type = r.realType() + msg, lenOff, err := h.pack(b.msg, b.compression, b.start) + if err != nil { + return &nestedError{"ResourceHeader", err} + } + preLen := len(msg) + if msg, err = r.pack(msg, b.compression, b.start); err != nil { + return &nestedError{"PTRResource body", err} + } + if err := h.fixLen(msg, lenOff, preLen); err != nil { + return err + } + if err := b.incrementSectionCount(); err != nil { + return err + } + b.msg = msg + return nil +} + +// SOAResource adds a single SOAResource. +func (b *Builder) SOAResource(h ResourceHeader, r SOAResource) error { + if err := b.checkResourceSection(); err != nil { + return err + } + h.Type = r.realType() + msg, lenOff, err := h.pack(b.msg, b.compression, b.start) + if err != nil { + return &nestedError{"ResourceHeader", err} + } + preLen := len(msg) + if msg, err = r.pack(msg, b.compression, b.start); err != nil { + return &nestedError{"SOAResource body", err} + } + if err := h.fixLen(msg, lenOff, preLen); err != nil { + return err + } + if err := b.incrementSectionCount(); err != nil { + return err + } + b.msg = msg + return nil +} + +// TXTResource adds a single TXTResource. +func (b *Builder) TXTResource(h ResourceHeader, r TXTResource) error { + if err := b.checkResourceSection(); err != nil { + return err + } + h.Type = r.realType() + msg, lenOff, err := h.pack(b.msg, b.compression, b.start) + if err != nil { + return &nestedError{"ResourceHeader", err} + } + preLen := len(msg) + if msg, err = r.pack(msg, b.compression, b.start); err != nil { + return &nestedError{"TXTResource body", err} + } + if err := h.fixLen(msg, lenOff, preLen); err != nil { + return err + } + if err := b.incrementSectionCount(); err != nil { + return err + } + b.msg = msg + return nil +} + +// SRVResource adds a single SRVResource. +func (b *Builder) SRVResource(h ResourceHeader, r SRVResource) error { + if err := b.checkResourceSection(); err != nil { + return err + } + h.Type = r.realType() + msg, lenOff, err := h.pack(b.msg, b.compression, b.start) + if err != nil { + return &nestedError{"ResourceHeader", err} + } + preLen := len(msg) + if msg, err = r.pack(msg, b.compression, b.start); err != nil { + return &nestedError{"SRVResource body", err} + } + if err := h.fixLen(msg, lenOff, preLen); err != nil { + return err + } + if err := b.incrementSectionCount(); err != nil { + return err + } + b.msg = msg + return nil +} + +// AResource adds a single AResource. +func (b *Builder) AResource(h ResourceHeader, r AResource) error { + if err := b.checkResourceSection(); err != nil { + return err + } + h.Type = r.realType() + msg, lenOff, err := h.pack(b.msg, b.compression, b.start) + if err != nil { + return &nestedError{"ResourceHeader", err} + } + preLen := len(msg) + if msg, err = r.pack(msg, b.compression, b.start); err != nil { + return &nestedError{"AResource body", err} + } + if err := h.fixLen(msg, lenOff, preLen); err != nil { + return err + } + if err := b.incrementSectionCount(); err != nil { + return err + } + b.msg = msg + return nil +} + +// AAAAResource adds a single AAAAResource. +func (b *Builder) AAAAResource(h ResourceHeader, r AAAAResource) error { + if err := b.checkResourceSection(); err != nil { + return err + } + h.Type = r.realType() + msg, lenOff, err := h.pack(b.msg, b.compression, b.start) + if err != nil { + return &nestedError{"ResourceHeader", err} + } + preLen := len(msg) + if msg, err = r.pack(msg, b.compression, b.start); err != nil { + return &nestedError{"AAAAResource body", err} + } + if err := h.fixLen(msg, lenOff, preLen); err != nil { + return err + } + if err := b.incrementSectionCount(); err != nil { + return err + } + b.msg = msg + return nil +} + +// OPTResource adds a single OPTResource. +func (b *Builder) OPTResource(h ResourceHeader, r OPTResource) error { + if err := b.checkResourceSection(); err != nil { + return err + } + h.Type = r.realType() + msg, lenOff, err := h.pack(b.msg, b.compression, b.start) + if err != nil { + return &nestedError{"ResourceHeader", err} + } + preLen := len(msg) + if msg, err = r.pack(msg, b.compression, b.start); err != nil { + return &nestedError{"OPTResource body", err} + } + if err := h.fixLen(msg, lenOff, preLen); err != nil { + return err + } + if err := b.incrementSectionCount(); err != nil { + return err + } + b.msg = msg + return nil +} + +// UnknownResource adds a single UnknownResource. +func (b *Builder) UnknownResource(h ResourceHeader, r UnknownResource) error { + if err := b.checkResourceSection(); err != nil { + return err + } + h.Type = r.realType() + msg, lenOff, err := h.pack(b.msg, b.compression, b.start) + if err != nil { + return &nestedError{"ResourceHeader", err} + } + preLen := len(msg) + if msg, err = r.pack(msg, b.compression, b.start); err != nil { + return &nestedError{"UnknownResource body", err} + } + if err := h.fixLen(msg, lenOff, preLen); err != nil { + return err + } + if err := b.incrementSectionCount(); err != nil { + return err + } + b.msg = msg + return nil +} + +// Finish ends message building and generates a binary message. +func (b *Builder) Finish() ([]byte, error) { + if b.section < sectionHeader { + return nil, ErrNotStarted + } + b.section = sectionDone + // Space for the header was allocated in NewBuilder. + b.header.pack(b.msg[b.start:b.start]) + return b.msg, nil +} + +// A ResourceHeader is the header of a DNS resource record. There are +// many types of DNS resource records, but they all share the same header. +type ResourceHeader struct { + // Name is the domain name for which this resource record pertains. + Name Name + + // Type is the type of DNS resource record. + // + // This field will be set automatically during packing. + Type Type + + // Class is the class of network to which this DNS resource record + // pertains. + Class Class + + // TTL is the length of time (measured in seconds) which this resource + // record is valid for (time to live). All Resources in a set should + // have the same TTL (RFC 2181 Section 5.2). + TTL uint32 + + // Length is the length of data in the resource record after the header. + // + // This field will be set automatically during packing. + Length uint16 +} + +// GoString implements fmt.GoStringer.GoString. +func (h *ResourceHeader) GoString() string { + return "dnsmessage.ResourceHeader{" + + "Name: " + h.Name.GoString() + ", " + + "Type: " + h.Type.GoString() + ", " + + "Class: " + h.Class.GoString() + ", " + + "TTL: " + printUint32(h.TTL) + ", " + + "Length: " + printUint16(h.Length) + "}" +} + +// pack appends the wire format of the ResourceHeader to oldMsg. +// +// lenOff is the offset in msg where the Length field was packed. +func (h *ResourceHeader) pack(oldMsg []byte, compression map[string]uint16, compressionOff int) (msg []byte, lenOff int, err error) { + msg = oldMsg + if msg, err = h.Name.pack(msg, compression, compressionOff); err != nil { + return oldMsg, 0, &nestedError{"Name", err} + } + msg = packType(msg, h.Type) + msg = packClass(msg, h.Class) + msg = packUint32(msg, h.TTL) + lenOff = len(msg) + msg = packUint16(msg, h.Length) + return msg, lenOff, nil +} + +func (h *ResourceHeader) unpack(msg []byte, off int) (int, error) { + newOff := off + var err error + if newOff, err = h.Name.unpack(msg, newOff); err != nil { + return off, &nestedError{"Name", err} + } + if h.Type, newOff, err = unpackType(msg, newOff); err != nil { + return off, &nestedError{"Type", err} + } + if h.Class, newOff, err = unpackClass(msg, newOff); err != nil { + return off, &nestedError{"Class", err} + } + if h.TTL, newOff, err = unpackUint32(msg, newOff); err != nil { + return off, &nestedError{"TTL", err} + } + if h.Length, newOff, err = unpackUint16(msg, newOff); err != nil { + return off, &nestedError{"Length", err} + } + return newOff, nil +} + +// fixLen updates a packed ResourceHeader to include the length of the +// ResourceBody. +// +// lenOff is the offset of the ResourceHeader.Length field in msg. +// +// preLen is the length that msg was before the ResourceBody was packed. +func (h *ResourceHeader) fixLen(msg []byte, lenOff int, preLen int) error { + conLen := len(msg) - preLen + if conLen > int(^uint16(0)) { + return errResTooLong + } + + // Fill in the length now that we know how long the content is. + packUint16(msg[lenOff:lenOff], uint16(conLen)) + h.Length = uint16(conLen) + + return nil +} + +// EDNS(0) wire constants. +const ( + edns0Version = 0 + + edns0DNSSECOK = 0x00008000 + ednsVersionMask = 0x00ff0000 + edns0DNSSECOKMask = 0x00ff8000 +) + +// SetEDNS0 configures h for EDNS(0). +// +// The provided extRCode must be an extended RCode. +func (h *ResourceHeader) SetEDNS0(udpPayloadLen int, extRCode RCode, dnssecOK bool) error { + h.Name = Name{Data: [255]byte{'.'}, Length: 1} // RFC 6891 section 6.1.2 + h.Type = TypeOPT + h.Class = Class(udpPayloadLen) + h.TTL = uint32(extRCode) >> 4 << 24 + if dnssecOK { + h.TTL |= edns0DNSSECOK + } + return nil +} + +// DNSSECAllowed reports whether the DNSSEC OK bit is set. +func (h *ResourceHeader) DNSSECAllowed() bool { + return h.TTL&edns0DNSSECOKMask == edns0DNSSECOK // RFC 6891 section 6.1.3 +} + +// ExtendedRCode returns an extended RCode. +// +// The provided rcode must be the RCode in DNS message header. +func (h *ResourceHeader) ExtendedRCode(rcode RCode) RCode { + if h.TTL&ednsVersionMask == edns0Version { // RFC 6891 section 6.1.3 + return RCode(h.TTL>>24<<4) | rcode + } + return rcode +} + +func skipResource(msg []byte, off int) (int, error) { + newOff, err := skipName(msg, off) + if err != nil { + return off, &nestedError{"Name", err} + } + if newOff, err = skipType(msg, newOff); err != nil { + return off, &nestedError{"Type", err} + } + if newOff, err = skipClass(msg, newOff); err != nil { + return off, &nestedError{"Class", err} + } + if newOff, err = skipUint32(msg, newOff); err != nil { + return off, &nestedError{"TTL", err} + } + length, newOff, err := unpackUint16(msg, newOff) + if err != nil { + return off, &nestedError{"Length", err} + } + if newOff += int(length); newOff > len(msg) { + return off, errResourceLen + } + return newOff, nil +} + +// packUint16 appends the wire format of field to msg. +func packUint16(msg []byte, field uint16) []byte { + return append(msg, byte(field>>8), byte(field)) +} + +func unpackUint16(msg []byte, off int) (uint16, int, error) { + if off+uint16Len > len(msg) { + return 0, off, errBaseLen + } + return uint16(msg[off])<<8 | uint16(msg[off+1]), off + uint16Len, nil +} + +func skipUint16(msg []byte, off int) (int, error) { + if off+uint16Len > len(msg) { + return off, errBaseLen + } + return off + uint16Len, nil +} + +// packType appends the wire format of field to msg. +func packType(msg []byte, field Type) []byte { + return packUint16(msg, uint16(field)) +} + +func unpackType(msg []byte, off int) (Type, int, error) { + t, o, err := unpackUint16(msg, off) + return Type(t), o, err +} + +func skipType(msg []byte, off int) (int, error) { + return skipUint16(msg, off) +} + +// packClass appends the wire format of field to msg. +func packClass(msg []byte, field Class) []byte { + return packUint16(msg, uint16(field)) +} + +func unpackClass(msg []byte, off int) (Class, int, error) { + c, o, err := unpackUint16(msg, off) + return Class(c), o, err +} + +func skipClass(msg []byte, off int) (int, error) { + return skipUint16(msg, off) +} + +// packUint32 appends the wire format of field to msg. +func packUint32(msg []byte, field uint32) []byte { + return append( + msg, + byte(field>>24), + byte(field>>16), + byte(field>>8), + byte(field), + ) +} + +func unpackUint32(msg []byte, off int) (uint32, int, error) { + if off+uint32Len > len(msg) { + return 0, off, errBaseLen + } + v := uint32(msg[off])<<24 | uint32(msg[off+1])<<16 | uint32(msg[off+2])<<8 | uint32(msg[off+3]) + return v, off + uint32Len, nil +} + +func skipUint32(msg []byte, off int) (int, error) { + if off+uint32Len > len(msg) { + return off, errBaseLen + } + return off + uint32Len, nil +} + +// packText appends the wire format of field to msg. +func packText(msg []byte, field string) ([]byte, error) { + l := len(field) + if l > 255 { + return nil, errStringTooLong + } + msg = append(msg, byte(l)) + msg = append(msg, field...) + + return msg, nil +} + +func unpackText(msg []byte, off int) (string, int, error) { + if off >= len(msg) { + return "", off, errBaseLen + } + beginOff := off + 1 + endOff := beginOff + int(msg[off]) + if endOff > len(msg) { + return "", off, errCalcLen + } + return string(msg[beginOff:endOff]), endOff, nil +} + +// packBytes appends the wire format of field to msg. +func packBytes(msg []byte, field []byte) []byte { + return append(msg, field...) +} + +func unpackBytes(msg []byte, off int, field []byte) (int, error) { + newOff := off + len(field) + if newOff > len(msg) { + return off, errBaseLen + } + copy(field, msg[off:newOff]) + return newOff, nil +} + +const nonEncodedNameMax = 254 + +// A Name is a non-encoded and non-escaped domain name. It is used instead of strings to avoid +// allocations. +type Name struct { + Data [255]byte + Length uint8 +} + +// NewName creates a new Name from a string. +func NewName(name string) (Name, error) { + n := Name{Length: uint8(len(name))} + if len(name) > len(n.Data) { + return Name{}, errCalcLen + } + copy(n.Data[:], name) + return n, nil +} + +// MustNewName creates a new Name from a string and panics on error. +func MustNewName(name string) Name { + n, err := NewName(name) + if err != nil { + panic("creating name: " + err.Error()) + } + return n +} + +// String implements fmt.Stringer.String. +// +// Note: characters inside the labels are not escaped in any way. +func (n Name) String() string { + return string(n.Data[:n.Length]) +} + +// GoString implements fmt.GoStringer.GoString. +func (n *Name) GoString() string { + return `dnsmessage.MustNewName("` + printString(n.Data[:n.Length]) + `")` +} + +// pack appends the wire format of the Name to msg. +// +// Domain names are a sequence of counted strings split at the dots. They end +// with a zero-length string. Compression can be used to reuse domain suffixes. +// +// The compression map will be updated with new domain suffixes. If compression +// is nil, compression will not be used. +func (n *Name) pack(msg []byte, compression map[string]uint16, compressionOff int) ([]byte, error) { + oldMsg := msg + + if n.Length > nonEncodedNameMax { + return nil, errNameTooLong + } + + // Add a trailing dot to canonicalize name. + if n.Length == 0 || n.Data[n.Length-1] != '.' { + return oldMsg, errNonCanonicalName + } + + // Allow root domain. + if n.Data[0] == '.' && n.Length == 1 { + return append(msg, 0), nil + } + + var nameAsStr string + + // Emit sequence of counted strings, chopping at dots. + for i, begin := 0, 0; i < int(n.Length); i++ { + // Check for the end of the segment. + if n.Data[i] == '.' { + // The two most significant bits have special meaning. + // It isn't allowed for segments to be long enough to + // need them. + if i-begin >= 1<<6 { + return oldMsg, errSegTooLong + } + + // Segments must have a non-zero length. + if i-begin == 0 { + return oldMsg, errZeroSegLen + } + + msg = append(msg, byte(i-begin)) + + for j := begin; j < i; j++ { + msg = append(msg, n.Data[j]) + } + + begin = i + 1 + continue + } + + // We can only compress domain suffixes starting with a new + // segment. A pointer is two bytes with the two most significant + // bits set to 1 to indicate that it is a pointer. + if (i == 0 || n.Data[i-1] == '.') && compression != nil { + if ptr, ok := compression[string(n.Data[i:n.Length])]; ok { + // Hit. Emit a pointer instead of the rest of + // the domain. + return append(msg, byte(ptr>>8|0xC0), byte(ptr)), nil + } + + // Miss. Add the suffix to the compression table if the + // offset can be stored in the available 14 bits. + newPtr := len(msg) - compressionOff + if newPtr <= int(^uint16(0)>>2) { + if nameAsStr == "" { + // allocate n.Data on the heap once, to avoid allocating it + // multiple times (for next labels). + nameAsStr = string(n.Data[:n.Length]) + } + compression[nameAsStr[i:]] = uint16(newPtr) + } + } + } + return append(msg, 0), nil +} + +// unpack unpacks a domain name. +func (n *Name) unpack(msg []byte, off int) (int, error) { + // currOff is the current working offset. + currOff := off + + // newOff is the offset where the next record will start. Pointers lead + // to data that belongs to other names and thus doesn't count towards to + // the usage of this name. + newOff := off + + // ptr is the number of pointers followed. + var ptr int + + // Name is a slice representation of the name data. + name := n.Data[:0] + +Loop: + for { + if currOff >= len(msg) { + return off, errBaseLen + } + c := int(msg[currOff]) + currOff++ + switch c & 0xC0 { + case 0x00: // String segment + if c == 0x00 { + // A zero length signals the end of the name. + break Loop + } + endOff := currOff + c + if endOff > len(msg) { + return off, errCalcLen + } + + // Reject names containing dots. + // See issue golang/go#56246 + for _, v := range msg[currOff:endOff] { + if v == '.' { + return off, errInvalidName + } + } + // Reject names that are too long while unpacking + // See issue golang/go#77540 + if len(name)+(endOff-currOff) >= nonEncodedNameMax { + return off, errNameTooLong + } + name = append(name, msg[currOff:endOff]...) + name = append(name, '.') + currOff = endOff + case 0xC0: // Pointer + if currOff >= len(msg) { + return off, errInvalidPtr + } + c1 := msg[currOff] + currOff++ + if ptr == 0 { + newOff = currOff + } + // Don't follow too many pointers, maybe there's a loop. + if ptr++; ptr > 10 { + return off, errTooManyPtr + } + currOff = (c^0xC0)<<8 | int(c1) + default: + // Prefixes 0x80 and 0x40 are reserved. + return off, errReserved + } + } + if len(name) == 0 { + name = append(name, '.') + } + n.Length = uint8(len(name)) + if ptr == 0 { + newOff = currOff + } + return newOff, nil +} + +func skipName(msg []byte, off int) (int, error) { + // newOff is the offset where the next record will start. Pointers lead + // to data that belongs to other names and thus doesn't count towards to + // the usage of this name. + newOff := off + +Loop: + for { + if newOff >= len(msg) { + return off, errBaseLen + } + c := int(msg[newOff]) + newOff++ + switch c & 0xC0 { + case 0x00: + if c == 0x00 { + // A zero length signals the end of the name. + break Loop + } + // literal string + newOff += c + if newOff > len(msg) { + return off, errCalcLen + } + case 0xC0: + // Pointer to somewhere else in msg. + + // Pointers are two bytes. + newOff++ + + // Don't follow the pointer as the data here has ended. + break Loop + default: + // Prefixes 0x80 and 0x40 are reserved. + return off, errReserved + } + } + + return newOff, nil +} + +// A Question is a DNS query. +type Question struct { + Name Name + Type Type + Class Class +} + +// pack appends the wire format of the Question to msg. +func (q *Question) pack(msg []byte, compression map[string]uint16, compressionOff int) ([]byte, error) { + msg, err := q.Name.pack(msg, compression, compressionOff) + if err != nil { + return msg, &nestedError{"Name", err} + } + msg = packType(msg, q.Type) + return packClass(msg, q.Class), nil +} + +// GoString implements fmt.GoStringer.GoString. +func (q *Question) GoString() string { + return "dnsmessage.Question{" + + "Name: " + q.Name.GoString() + ", " + + "Type: " + q.Type.GoString() + ", " + + "Class: " + q.Class.GoString() + "}" +} + +func unpackResourceBody(msg []byte, off int, hdr ResourceHeader) (ResourceBody, int, error) { + var ( + r ResourceBody + err error + name string + ) + switch hdr.Type { + case TypeA: + var rb AResource + rb, err = unpackAResource(msg, off) + r = &rb + name = "A" + case TypeNS: + var rb NSResource + rb, err = unpackNSResource(msg, off) + r = &rb + name = "NS" + case TypeCNAME: + var rb CNAMEResource + rb, err = unpackCNAMEResource(msg, off) + r = &rb + name = "CNAME" + case TypeSOA: + var rb SOAResource + rb, err = unpackSOAResource(msg, off) + r = &rb + name = "SOA" + case TypePTR: + var rb PTRResource + rb, err = unpackPTRResource(msg, off) + r = &rb + name = "PTR" + case TypeMX: + var rb MXResource + rb, err = unpackMXResource(msg, off) + r = &rb + name = "MX" + case TypeTXT: + var rb TXTResource + rb, err = unpackTXTResource(msg, off, hdr.Length) + r = &rb + name = "TXT" + case TypeAAAA: + var rb AAAAResource + rb, err = unpackAAAAResource(msg, off) + r = &rb + name = "AAAA" + case TypeSRV: + var rb SRVResource + rb, err = unpackSRVResource(msg, off) + r = &rb + name = "SRV" + case TypeSVCB: + var rb SVCBResource + rb, err = unpackSVCBResource(msg, off, hdr.Length) + r = &rb + name = "SVCB" + case TypeHTTPS: + var rb HTTPSResource + rb.SVCBResource, err = unpackSVCBResource(msg, off, hdr.Length) + r = &rb + name = "HTTPS" + case TypeOPT: + var rb OPTResource + rb, err = unpackOPTResource(msg, off, hdr.Length) + r = &rb + name = "OPT" + default: + var rb UnknownResource + rb, err = unpackUnknownResource(hdr.Type, msg, off, hdr.Length) + r = &rb + name = "Unknown" + } + if err != nil { + return nil, off, &nestedError{name + " record", err} + } + return r, off + int(hdr.Length), nil +} + +// A CNAMEResource is a CNAME Resource record. +type CNAMEResource struct { + CNAME Name +} + +func (r *CNAMEResource) realType() Type { + return TypeCNAME +} + +// pack appends the wire format of the CNAMEResource to msg. +func (r *CNAMEResource) pack(msg []byte, compression map[string]uint16, compressionOff int) ([]byte, error) { + return r.CNAME.pack(msg, compression, compressionOff) +} + +// GoString implements fmt.GoStringer.GoString. +func (r *CNAMEResource) GoString() string { + return "dnsmessage.CNAMEResource{CNAME: " + r.CNAME.GoString() + "}" +} + +func unpackCNAMEResource(msg []byte, off int) (CNAMEResource, error) { + var cname Name + if _, err := cname.unpack(msg, off); err != nil { + return CNAMEResource{}, err + } + return CNAMEResource{cname}, nil +} + +// An MXResource is an MX Resource record. +type MXResource struct { + Pref uint16 + MX Name +} + +func (r *MXResource) realType() Type { + return TypeMX +} + +// pack appends the wire format of the MXResource to msg. +func (r *MXResource) pack(msg []byte, compression map[string]uint16, compressionOff int) ([]byte, error) { + oldMsg := msg + msg = packUint16(msg, r.Pref) + msg, err := r.MX.pack(msg, compression, compressionOff) + if err != nil { + return oldMsg, &nestedError{"MXResource.MX", err} + } + return msg, nil +} + +// GoString implements fmt.GoStringer.GoString. +func (r *MXResource) GoString() string { + return "dnsmessage.MXResource{" + + "Pref: " + printUint16(r.Pref) + ", " + + "MX: " + r.MX.GoString() + "}" +} + +func unpackMXResource(msg []byte, off int) (MXResource, error) { + pref, off, err := unpackUint16(msg, off) + if err != nil { + return MXResource{}, &nestedError{"Pref", err} + } + var mx Name + if _, err := mx.unpack(msg, off); err != nil { + return MXResource{}, &nestedError{"MX", err} + } + return MXResource{pref, mx}, nil +} + +// An NSResource is an NS Resource record. +type NSResource struct { + NS Name +} + +func (r *NSResource) realType() Type { + return TypeNS +} + +// pack appends the wire format of the NSResource to msg. +func (r *NSResource) pack(msg []byte, compression map[string]uint16, compressionOff int) ([]byte, error) { + return r.NS.pack(msg, compression, compressionOff) +} + +// GoString implements fmt.GoStringer.GoString. +func (r *NSResource) GoString() string { + return "dnsmessage.NSResource{NS: " + r.NS.GoString() + "}" +} + +func unpackNSResource(msg []byte, off int) (NSResource, error) { + var ns Name + if _, err := ns.unpack(msg, off); err != nil { + return NSResource{}, err + } + return NSResource{ns}, nil +} + +// A PTRResource is a PTR Resource record. +type PTRResource struct { + PTR Name +} + +func (r *PTRResource) realType() Type { + return TypePTR +} + +// pack appends the wire format of the PTRResource to msg. +func (r *PTRResource) pack(msg []byte, compression map[string]uint16, compressionOff int) ([]byte, error) { + return r.PTR.pack(msg, compression, compressionOff) +} + +// GoString implements fmt.GoStringer.GoString. +func (r *PTRResource) GoString() string { + return "dnsmessage.PTRResource{PTR: " + r.PTR.GoString() + "}" +} + +func unpackPTRResource(msg []byte, off int) (PTRResource, error) { + var ptr Name + if _, err := ptr.unpack(msg, off); err != nil { + return PTRResource{}, err + } + return PTRResource{ptr}, nil +} + +// An SOAResource is an SOA Resource record. +type SOAResource struct { + NS Name + MBox Name + Serial uint32 + Refresh uint32 + Retry uint32 + Expire uint32 + + // MinTTL the is the default TTL of Resources records which did not + // contain a TTL value and the TTL of negative responses. (RFC 2308 + // Section 4) + MinTTL uint32 +} + +func (r *SOAResource) realType() Type { + return TypeSOA +} + +// pack appends the wire format of the SOAResource to msg. +func (r *SOAResource) pack(msg []byte, compression map[string]uint16, compressionOff int) ([]byte, error) { + oldMsg := msg + msg, err := r.NS.pack(msg, compression, compressionOff) + if err != nil { + return oldMsg, &nestedError{"SOAResource.NS", err} + } + msg, err = r.MBox.pack(msg, compression, compressionOff) + if err != nil { + return oldMsg, &nestedError{"SOAResource.MBox", err} + } + msg = packUint32(msg, r.Serial) + msg = packUint32(msg, r.Refresh) + msg = packUint32(msg, r.Retry) + msg = packUint32(msg, r.Expire) + return packUint32(msg, r.MinTTL), nil +} + +// GoString implements fmt.GoStringer.GoString. +func (r *SOAResource) GoString() string { + return "dnsmessage.SOAResource{" + + "NS: " + r.NS.GoString() + ", " + + "MBox: " + r.MBox.GoString() + ", " + + "Serial: " + printUint32(r.Serial) + ", " + + "Refresh: " + printUint32(r.Refresh) + ", " + + "Retry: " + printUint32(r.Retry) + ", " + + "Expire: " + printUint32(r.Expire) + ", " + + "MinTTL: " + printUint32(r.MinTTL) + "}" +} + +func unpackSOAResource(msg []byte, off int) (SOAResource, error) { + var ns Name + off, err := ns.unpack(msg, off) + if err != nil { + return SOAResource{}, &nestedError{"NS", err} + } + var mbox Name + if off, err = mbox.unpack(msg, off); err != nil { + return SOAResource{}, &nestedError{"MBox", err} + } + serial, off, err := unpackUint32(msg, off) + if err != nil { + return SOAResource{}, &nestedError{"Serial", err} + } + refresh, off, err := unpackUint32(msg, off) + if err != nil { + return SOAResource{}, &nestedError{"Refresh", err} + } + retry, off, err := unpackUint32(msg, off) + if err != nil { + return SOAResource{}, &nestedError{"Retry", err} + } + expire, off, err := unpackUint32(msg, off) + if err != nil { + return SOAResource{}, &nestedError{"Expire", err} + } + minTTL, _, err := unpackUint32(msg, off) + if err != nil { + return SOAResource{}, &nestedError{"MinTTL", err} + } + return SOAResource{ns, mbox, serial, refresh, retry, expire, minTTL}, nil +} + +// A TXTResource is a TXT Resource record. +type TXTResource struct { + TXT []string +} + +func (r *TXTResource) realType() Type { + return TypeTXT +} + +// pack appends the wire format of the TXTResource to msg. +func (r *TXTResource) pack(msg []byte, compression map[string]uint16, compressionOff int) ([]byte, error) { + oldMsg := msg + for _, s := range r.TXT { + var err error + msg, err = packText(msg, s) + if err != nil { + return oldMsg, err + } + } + return msg, nil +} + +// GoString implements fmt.GoStringer.GoString. +func (r *TXTResource) GoString() string { + s := "dnsmessage.TXTResource{TXT: []string{" + if len(r.TXT) == 0 { + return s + "}}" + } + s += `"` + printString([]byte(r.TXT[0])) + for _, t := range r.TXT[1:] { + s += `", "` + printString([]byte(t)) + } + return s + `"}}` +} + +func unpackTXTResource(msg []byte, off int, length uint16) (TXTResource, error) { + txts := make([]string, 0, 1) + for n := uint16(0); n < length; { + var t string + var err error + if t, off, err = unpackText(msg, off); err != nil { + return TXTResource{}, &nestedError{"text", err} + } + // Check if we got too many bytes. + if length-n < uint16(len(t))+1 { + return TXTResource{}, errCalcLen + } + n += uint16(len(t)) + 1 + txts = append(txts, t) + } + return TXTResource{txts}, nil +} + +// An SRVResource is an SRV Resource record. +type SRVResource struct { + Priority uint16 + Weight uint16 + Port uint16 + Target Name // Not compressed as per RFC 2782. +} + +func (r *SRVResource) realType() Type { + return TypeSRV +} + +// pack appends the wire format of the SRVResource to msg. +func (r *SRVResource) pack(msg []byte, compression map[string]uint16, compressionOff int) ([]byte, error) { + oldMsg := msg + msg = packUint16(msg, r.Priority) + msg = packUint16(msg, r.Weight) + msg = packUint16(msg, r.Port) + msg, err := r.Target.pack(msg, nil, compressionOff) + if err != nil { + return oldMsg, &nestedError{"SRVResource.Target", err} + } + return msg, nil +} + +// GoString implements fmt.GoStringer.GoString. +func (r *SRVResource) GoString() string { + return "dnsmessage.SRVResource{" + + "Priority: " + printUint16(r.Priority) + ", " + + "Weight: " + printUint16(r.Weight) + ", " + + "Port: " + printUint16(r.Port) + ", " + + "Target: " + r.Target.GoString() + "}" +} + +func unpackSRVResource(msg []byte, off int) (SRVResource, error) { + priority, off, err := unpackUint16(msg, off) + if err != nil { + return SRVResource{}, &nestedError{"Priority", err} + } + weight, off, err := unpackUint16(msg, off) + if err != nil { + return SRVResource{}, &nestedError{"Weight", err} + } + port, off, err := unpackUint16(msg, off) + if err != nil { + return SRVResource{}, &nestedError{"Port", err} + } + var target Name + if _, err := target.unpack(msg, off); err != nil { + return SRVResource{}, &nestedError{"Target", err} + } + return SRVResource{priority, weight, port, target}, nil +} + +// An AResource is an A Resource record. +type AResource struct { + A [4]byte +} + +func (r *AResource) realType() Type { + return TypeA +} + +// pack appends the wire format of the AResource to msg. +func (r *AResource) pack(msg []byte, compression map[string]uint16, compressionOff int) ([]byte, error) { + return packBytes(msg, r.A[:]), nil +} + +// GoString implements fmt.GoStringer.GoString. +func (r *AResource) GoString() string { + return "dnsmessage.AResource{" + + "A: [4]byte{" + printByteSlice(r.A[:]) + "}}" +} + +func unpackAResource(msg []byte, off int) (AResource, error) { + var a [4]byte + if _, err := unpackBytes(msg, off, a[:]); err != nil { + return AResource{}, err + } + return AResource{a}, nil +} + +// An AAAAResource is an AAAA Resource record. +type AAAAResource struct { + AAAA [16]byte +} + +func (r *AAAAResource) realType() Type { + return TypeAAAA +} + +// GoString implements fmt.GoStringer.GoString. +func (r *AAAAResource) GoString() string { + return "dnsmessage.AAAAResource{" + + "AAAA: [16]byte{" + printByteSlice(r.AAAA[:]) + "}}" +} + +// pack appends the wire format of the AAAAResource to msg. +func (r *AAAAResource) pack(msg []byte, compression map[string]uint16, compressionOff int) ([]byte, error) { + return packBytes(msg, r.AAAA[:]), nil +} + +func unpackAAAAResource(msg []byte, off int) (AAAAResource, error) { + var aaaa [16]byte + if _, err := unpackBytes(msg, off, aaaa[:]); err != nil { + return AAAAResource{}, err + } + return AAAAResource{aaaa}, nil +} + +// An OPTResource is an OPT pseudo Resource record. +// +// The pseudo resource record is part of the extension mechanisms for DNS +// as defined in RFC 6891. +type OPTResource struct { + Options []Option +} + +// An Option represents a DNS message option within OPTResource. +// +// The message option is part of the extension mechanisms for DNS as +// defined in RFC 6891. +type Option struct { + Code uint16 // option code + Data []byte +} + +// GoString implements fmt.GoStringer.GoString. +func (o *Option) GoString() string { + return "dnsmessage.Option{" + + "Code: " + printUint16(o.Code) + ", " + + "Data: []byte{" + printByteSlice(o.Data) + "}}" +} + +func (r *OPTResource) realType() Type { + return TypeOPT +} + +func (r *OPTResource) pack(msg []byte, compression map[string]uint16, compressionOff int) ([]byte, error) { + for _, opt := range r.Options { + msg = packUint16(msg, opt.Code) + l := uint16(len(opt.Data)) + msg = packUint16(msg, l) + msg = packBytes(msg, opt.Data) + } + return msg, nil +} + +// GoString implements fmt.GoStringer.GoString. +func (r *OPTResource) GoString() string { + s := "dnsmessage.OPTResource{Options: []dnsmessage.Option{" + if len(r.Options) == 0 { + return s + "}}" + } + s += r.Options[0].GoString() + for _, o := range r.Options[1:] { + s += ", " + o.GoString() + } + return s + "}}" +} + +func unpackOPTResource(msg []byte, off int, length uint16) (OPTResource, error) { + var opts []Option + for oldOff := off; off < oldOff+int(length); { + var err error + var o Option + o.Code, off, err = unpackUint16(msg, off) + if err != nil { + return OPTResource{}, &nestedError{"Code", err} + } + var l uint16 + l, off, err = unpackUint16(msg, off) + if err != nil { + return OPTResource{}, &nestedError{"Data", err} + } + o.Data = make([]byte, l) + if copy(o.Data, msg[off:]) != int(l) { + return OPTResource{}, &nestedError{"Data", errCalcLen} + } + off += int(l) + opts = append(opts, o) + } + return OPTResource{opts}, nil +} + +// An UnknownResource is a catch-all container for unknown record types. +type UnknownResource struct { + Type Type + Data []byte +} + +func (r *UnknownResource) realType() Type { + return r.Type +} + +// pack appends the wire format of the UnknownResource to msg. +func (r *UnknownResource) pack(msg []byte, compression map[string]uint16, compressionOff int) ([]byte, error) { + return packBytes(msg, r.Data[:]), nil +} + +// GoString implements fmt.GoStringer.GoString. +func (r *UnknownResource) GoString() string { + return "dnsmessage.UnknownResource{" + + "Type: " + r.Type.GoString() + ", " + + "Data: []byte{" + printByteSlice(r.Data) + "}}" +} + +func unpackUnknownResource(recordType Type, msg []byte, off int, length uint16) (UnknownResource, error) { + parsed := UnknownResource{ + Type: recordType, + Data: make([]byte, length), + } + if _, err := unpackBytes(msg, off, parsed.Data); err != nil { + return UnknownResource{}, err + } + return parsed, nil +} diff --git a/vendor/golang.org/x/net/dns/dnsmessage/svcb.go b/vendor/golang.org/x/net/dns/dnsmessage/svcb.go new file mode 100644 index 0000000000..977d91eadc --- /dev/null +++ b/vendor/golang.org/x/net/dns/dnsmessage/svcb.go @@ -0,0 +1,333 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package dnsmessage + +import ( + "slices" +) + +// An SVCBResource is an SVCB Resource record. +type SVCBResource struct { + Priority uint16 + Target Name + Params []SVCParam // Must be in strict increasing order by Key. +} + +func (r *SVCBResource) realType() Type { + return TypeSVCB +} + +// GoString implements fmt.GoStringer.GoString. +func (r *SVCBResource) GoString() string { + b := []byte("dnsmessage.SVCBResource{" + + "Priority: " + printUint16(r.Priority) + ", " + + "Target: " + r.Target.GoString() + ", " + + "Params: []dnsmessage.SVCParam{") + if len(r.Params) > 0 { + b = append(b, r.Params[0].GoString()...) + for _, p := range r.Params[1:] { + b = append(b, ", "+p.GoString()...) + } + } + b = append(b, "}}"...) + return string(b) +} + +// An HTTPSResource is an HTTPS Resource record. +// It has the same format as the SVCB record. +type HTTPSResource struct { + // Alias for SVCB resource record. + SVCBResource +} + +func (r *HTTPSResource) realType() Type { + return TypeHTTPS +} + +// GoString implements fmt.GoStringer.GoString. +func (r *HTTPSResource) GoString() string { + return "dnsmessage.HTTPSResource{SVCBResource: " + r.SVCBResource.GoString() + "}" +} + +// GetParam returns a parameter value by key. +func (r *SVCBResource) GetParam(key SVCParamKey) (value []byte, ok bool) { + for i := range r.Params { + if r.Params[i].Key == key { + return r.Params[i].Value, true + } + if r.Params[i].Key > key { + break + } + } + return nil, false +} + +// SetParam sets a parameter value by key. +// The Params list is kept sorted by key. +func (r *SVCBResource) SetParam(key SVCParamKey, value []byte) { + i := 0 + for i < len(r.Params) { + if r.Params[i].Key >= key { + break + } + i++ + } + + if i < len(r.Params) && r.Params[i].Key == key { + r.Params[i].Value = value + return + } + + r.Params = slices.Insert(r.Params, i, SVCParam{Key: key, Value: value}) +} + +// DeleteParam deletes a parameter by key. +// It returns true if the parameter was present. +func (r *SVCBResource) DeleteParam(key SVCParamKey) bool { + for i := range r.Params { + if r.Params[i].Key == key { + r.Params = slices.Delete(r.Params, i, i+1) + return true + } + if r.Params[i].Key > key { + break + } + } + return false +} + +// A SVCParam is a service parameter. +type SVCParam struct { + Key SVCParamKey + Value []byte +} + +// GoString implements fmt.GoStringer.GoString. +func (p SVCParam) GoString() string { + return "dnsmessage.SVCParam{" + + "Key: " + p.Key.GoString() + ", " + + "Value: []byte{" + printByteSlice(p.Value) + "}}" +} + +// A SVCParamKey is a key for a service parameter. +type SVCParamKey uint16 + +// Values defined at https://www.iana.org/assignments/dns-svcb/dns-svcb.xhtml#dns-svcparamkeys. +const ( + SVCParamMandatory SVCParamKey = 0 + SVCParamALPN SVCParamKey = 1 + SVCParamNoDefaultALPN SVCParamKey = 2 + SVCParamPort SVCParamKey = 3 + SVCParamIPv4Hint SVCParamKey = 4 + SVCParamECH SVCParamKey = 5 + SVCParamIPv6Hint SVCParamKey = 6 + SVCParamDOHPath SVCParamKey = 7 + SVCParamOHTTP SVCParamKey = 8 + SVCParamTLSSupportedGroups SVCParamKey = 9 +) + +var svcParamKeyNames = map[SVCParamKey]string{ + SVCParamMandatory: "Mandatory", + SVCParamALPN: "ALPN", + SVCParamNoDefaultALPN: "NoDefaultALPN", + SVCParamPort: "Port", + SVCParamIPv4Hint: "IPv4Hint", + SVCParamECH: "ECH", + SVCParamIPv6Hint: "IPv6Hint", + SVCParamDOHPath: "DOHPath", + SVCParamOHTTP: "OHTTP", + SVCParamTLSSupportedGroups: "TLSSupportedGroups", +} + +// String implements fmt.Stringer.String. +func (k SVCParamKey) String() string { + if n, ok := svcParamKeyNames[k]; ok { + return n + } + return printUint16(uint16(k)) +} + +// GoString implements fmt.GoStringer.GoString. +func (k SVCParamKey) GoString() string { + if n, ok := svcParamKeyNames[k]; ok { + return "dnsmessage.SVCParam" + n + } + return printUint16(uint16(k)) +} + +func (r *SVCBResource) pack(msg []byte, _ map[string]uint16, _ int) ([]byte, error) { + oldMsg := msg + msg = packUint16(msg, r.Priority) + // https://datatracker.ietf.org/doc/html/rfc3597#section-4 prohibits name + // compression for RR types that are not "well-known". + // https://datatracker.ietf.org/doc/html/rfc9460#section-2.2 explicitly states that + // compression of the Target is prohibited, following RFC 3597. + msg, err := r.Target.pack(msg, nil, 0) + if err != nil { + return oldMsg, &nestedError{"SVCBResource.Target", err} + } + for i, param := range r.Params { + if i > 0 && param.Key <= r.Params[i-1].Key { + return oldMsg, &nestedError{"SVCBResource.Params", errParamOutOfOrder} + } + if len(param.Value) > (1<<16)-1 { + return oldMsg, &nestedError{"SVCBResource.Params", errTooLongSVCBValue} + } + msg = packUint16(msg, uint16(param.Key)) + msg = packUint16(msg, uint16(len(param.Value))) + msg = append(msg, param.Value...) + } + return msg, nil +} + +func unpackSVCBResource(msg []byte, off int, length uint16) (SVCBResource, error) { + // Wire format reference: https://www.rfc-editor.org/rfc/rfc9460.html#section-2.2. + r := SVCBResource{} + paramsOff := off + bodyEnd := off + int(length) + + if bodyEnd > len(msg) { + return SVCBResource{}, errResourceLen + } + + var err error + if r.Priority, paramsOff, err = unpackUint16(msg, paramsOff); err != nil { + return SVCBResource{}, &nestedError{"Priority", err} + } + + if paramsOff, err = r.Target.unpack(msg, paramsOff); err != nil { + return SVCBResource{}, &nestedError{"Target", err} + } + + // Two-pass parsing to avoid allocations. + // First, count the number of params. + n := 0 + var totalValueLen uint16 + off = paramsOff + var previousKey uint16 + for off < bodyEnd { + var key, size uint16 + if key, off, err = unpackUint16(msg, off); err != nil { + return SVCBResource{}, &nestedError{"Params key", err} + } + if n > 0 && key <= previousKey { + // As per https://www.rfc-editor.org/rfc/rfc9460.html#section-2.2, clients MUST + // consider the RR malformed if the SvcParamKeys are not in strictly increasing numeric order + return SVCBResource{}, &nestedError{"Params", errParamOutOfOrder} + } + if size, off, err = unpackUint16(msg, off); err != nil { + return SVCBResource{}, &nestedError{"Params value length", err} + } + if off+int(size) > bodyEnd { + return SVCBResource{}, errResourceLen + } + previousKey = key + totalValueLen += size + off += int(size) + n++ + } + if off != bodyEnd { + return SVCBResource{}, errResourceLen + } + + // Second, fill in the params. + r.Params = make([]SVCParam, n) + // valuesBuf is used to hold all param values to reduce allocations. + // Each param's Value slice will point into this buffer. + valuesBuf := make([]byte, totalValueLen) + off = paramsOff + for i := 0; i < n; i++ { + p := &r.Params[i] + var key, size uint16 + if key, off, err = unpackUint16(msg, off); err != nil { + return SVCBResource{}, &nestedError{"param key", err} + } + p.Key = SVCParamKey(key) + if size, off, err = unpackUint16(msg, off); err != nil { + return SVCBResource{}, &nestedError{"param length", err} + } + if len(msg[off:]) < int(size) { + return SVCBResource{}, &nestedError{"param value", errCalcLen} + } + if copy(valuesBuf, msg[off:][:int(size)]) != int(size) { + return SVCBResource{}, &nestedError{"param value", errCalcLen} + } + p.Value = valuesBuf[:size:size] + valuesBuf = valuesBuf[size:] + off += int(size) + } + + return r, nil +} + +// genericSVCBResource parses a single Resource Record compatible with SVCB. +func (p *Parser) genericSVCBResource(svcbType Type) (SVCBResource, error) { + if !p.resHeaderValid || p.resHeaderType != svcbType { + return SVCBResource{}, ErrNotStarted + } + r, err := unpackSVCBResource(p.msg, p.off, p.resHeaderLength) + if err != nil { + return SVCBResource{}, err + } + p.off += int(p.resHeaderLength) + p.resHeaderValid = false + p.index++ + return r, nil +} + +// SVCBResource parses a single SVCBResource. +// +// One of the XXXHeader methods must have been called before calling this +// method. +func (p *Parser) SVCBResource() (SVCBResource, error) { + return p.genericSVCBResource(TypeSVCB) +} + +// HTTPSResource parses a single HTTPSResource. +// +// One of the XXXHeader methods must have been called before calling this +// method. +func (p *Parser) HTTPSResource() (HTTPSResource, error) { + svcb, err := p.genericSVCBResource(TypeHTTPS) + if err != nil { + return HTTPSResource{}, err + } + return HTTPSResource{svcb}, nil +} + +// genericSVCBResource is the generic implementation for adding SVCB-like resources. +func (b *Builder) genericSVCBResource(h ResourceHeader, r SVCBResource) error { + if err := b.checkResourceSection(); err != nil { + return err + } + msg, lenOff, err := h.pack(b.msg, b.compression, b.start) + if err != nil { + return &nestedError{"ResourceHeader", err} + } + preLen := len(msg) + if msg, err = r.pack(msg, b.compression, b.start); err != nil { + return &nestedError{"ResourceBody", err} + } + if err := h.fixLen(msg, lenOff, preLen); err != nil { + return err + } + if err := b.incrementSectionCount(); err != nil { + return err + } + b.msg = msg + return nil +} + +// SVCBResource adds a single SVCBResource. +func (b *Builder) SVCBResource(h ResourceHeader, r SVCBResource) error { + h.Type = r.realType() + return b.genericSVCBResource(h, r) +} + +// HTTPSResource adds a single HTTPSResource. +func (b *Builder) HTTPSResource(h ResourceHeader, r HTTPSResource) error { + h.Type = r.realType() + return b.genericSVCBResource(h, r.SVCBResource) +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 0a9821bfd8..2af767a13b 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1271,6 +1271,7 @@ golang.org/x/exp/slices # golang.org/x/net v0.58.0 ## explicit; go 1.25.0 golang.org/x/net/bpf +golang.org/x/net/dns/dnsmessage golang.org/x/net/http/httpguts golang.org/x/net/http2 golang.org/x/net/http2/hpack