From 301fcaaeb04e89b7c42f2ef128b854c28c5a39c9 Mon Sep 17 00:00:00 2001 From: Haven Xia Date: Wed, 23 Sep 2026 17:34:59 -0700 Subject: [PATCH 01/15] docs: rewrite the upgrade runbook for an in-place roll --- README.md | 2 +- .../internal/controllers/workerpool_apply.go | 16 +- .../controllers/workerpool_apply_test.go | 11 +- cmd/atenet/README.md | 7 +- docs/upgrade.md | 749 ++++++------------ 5 files changed, 239 insertions(+), 546 deletions(-) diff --git a/README.md b/README.md index 3f6d4b021b..1f3c448a3c 100644 --- a/README.md +++ b/README.md @@ -250,7 +250,7 @@ We provide several sample applications demonstrating Agent Substrate's capabilit * [Egress Traffic](docs/egress-traffic.md): Which protocols an Actor may reach the outside world with and which are blocked. * [Enabling man-in-the-middle (MITM) interception for Actor Egress policy](docs/egress-trust-bundle.md): Egress policies such as header injection depend on MITM interception of Actor traffic. This guide explains how an Actor should be configured to enable interception. * [Request Parking](docs/request-parking.md): How the router parks requests through transient worker-pool saturation. -* [Rolling Upgrade Runbook](docs/upgrade.md): Upgrade a running substrate node by node without losing actor state. +* [Upgrade Runbook](docs/upgrade.md): Upgrade a running install to a newer release in the same release window. * [Security Guide](docs/security.md): Security overview, security principles, and threat modeling. * [Roadmap](docs/roadmap.md): Current limitations and what is planned next. * [Benchmarking Guide](benchmarking/README.md): Locust-based load tests, monitoring stack, and the orchestrated benchmark harness. diff --git a/cmd/atecontroller/internal/controllers/workerpool_apply.go b/cmd/atecontroller/internal/controllers/workerpool_apply.go index b357902247..1b990ef157 100644 --- a/cmd/atecontroller/internal/controllers/workerpool_apply.go +++ b/cmd/atecontroller/internal/controllers/workerpool_apply.go @@ -127,14 +127,14 @@ func buildDeploymentApplyConfig(wp *atev1alpha1.WorkerPool, otel ateomOTelSettin } // --atunnel-broker-identity is newer than the oldest ateom a rolling - // upgrade still has running. docs/upgrade.md keeps the outgoing worker pool - // serving alongside the new one, and that pool's Deployment is reconciled - // by this controller while still pinned to its old image, which exits on an - // unrecognized flag. An ateom without the flag hardcodes the canonical - // identity, and an ateom with it defaults to the same, so omitting the flag - // when it carries that value is equivalent for both and keeps the upgrade - // intact. A relocated or renamed install passes something else and needs an - // image new enough to accept it, which it necessarily has. + // upgrade still has running. docs/upgrade.md upgrades this controller + // before any pool's workerImage, so it reconciles Deployments still pinned + // to the old image, which exits on an unrecognized flag. An ateom without + // the flag hardcodes the canonical identity, and an ateom with it defaults + // to the same, so omitting the flag when it carries that value is + // equivalent for both and keeps the upgrade intact. A relocated or renamed + // install passes something else and needs an image new enough to accept + // it, which it necessarily has. if brokerIdentity := installdefaults.SPIFFEID(systemNamespace, ateletServiceAccount); brokerIdentity != installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace) { args = append(args, "--atunnel-broker-identity="+brokerIdentity) } diff --git a/cmd/atecontroller/internal/controllers/workerpool_apply_test.go b/cmd/atecontroller/internal/controllers/workerpool_apply_test.go index 75dd12db1d..51033e3d43 100644 --- a/cmd/atecontroller/internal/controllers/workerpool_apply_test.go +++ b/cmd/atecontroller/internal/controllers/workerpool_apply_test.go @@ -999,12 +999,11 @@ func TestBuildDeploymentAtunnelIdentitiesPrefixedServiceAccounts(t *testing.T) { } // TestBuildDeploymentOmitsBrokerIdentityForCanonicalInstall pins the flag's -// absence, which is what keeps a rolling upgrade working. docs/upgrade.md runs -// the outgoing worker pool alongside the new one, and this controller -// reconciles that pool's Deployment while it is still pinned to its old image. -// An ateom from before --atunnel-broker-identity existed exits on the -// unrecognized flag, so passing it would crashloop every old worker the moment -// the control plane rolled out. +// absence, which is what keeps a rolling upgrade working. docs/upgrade.md +// upgrades this controller before any pool's workerImage, so it reconciles +// Deployments still pinned to the old image. An ateom from before +// --atunnel-broker-identity existed exits on the unrecognized flag, so passing +// it would crashloop every old worker the moment the control plane rolled out. func TestBuildDeploymentOmitsBrokerIdentityForCanonicalInstall(t *testing.T) { c := buildDeploymentApplyConfig(testWorkerPoolApplyConfig(nil), ateomOTelSettings{}, installdefaults.SystemNamespace, installdefaults.AteletServiceAccount, installdefaults.RouterServiceAccount, egresspolicy.DNSEgressPolicyEnforceInAtunnel). diff --git a/cmd/atenet/README.md b/cmd/atenet/README.md index 43fa5a4482..9cba1d9d14 100644 --- a/cmd/atenet/README.md +++ b/cmd/atenet/README.md @@ -37,9 +37,10 @@ likely be split in the future for better scalability.) Envoy container's `preStop` hook waits for the router's drain-complete marker on a pod-shared emptyDir — so established connections and parked requests finish instead of resetting. The whole sequence must fit within - `terminationGracePeriodSeconds` (see the manifest comments). Upgrades are - whole-system swaps (#473) rather than per-Deployment rolling updates; the - drain is what makes the old system's termination lossless. + `terminationGracePeriodSeconds` (see the manifest comments). An upgrade + rolls the router as an ordinary Deployment update (see + [docs/upgrade.md](../../docs/upgrade.md)); the drain is what makes each old + pod's termination lossless. RBAC permissions: * get, list, watch on ate-system EndpointSlices diff --git a/docs/upgrade.md b/docs/upgrade.md index 10bcb912bd..9b1b1bb194 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -1,652 +1,345 @@ -# Rolling upgrade runbook - -This runbook upgrades a running Agent Substrate install to a new build -version, one node at a time. No actor loses state. On a cluster with -more than one node, at most one node's worth of capacity is out of -service while the rest of the fleet keeps serving; a single-node cluster -stops fully during step 5. It needs `kubectl`, `kubectl ate`, `go run -./cmd/ate-setup`, `jq`, and `grpcurl`. The numbered steps are the same -on every Kubernetes provider; the provider-specific parts sit in their -own sections, [before](#on-gke) and [after](#after-the-roll-on-gke) -the roll. All of the roll's state lives in cluster objects, so you can -stop at any point and pick up again. [Check your -progress](#check-your-progress) tells you where you left off. - -The order is `ate-controller` first, then the dataplane, then the rest -of the control plane. The controller goes first because it manages -the worker pools the roll creates. The dataplane goes before the rest -of the control plane so that, by the time ate-api-server and atenet -change, every atelet and worker already understands requests from -either version of the control plane. - -## Check your progress - -Come back here after a break, and before you touch anything when -something looks wrong. Three commands say where the roll stands: +# Upgrade runbook -```bash -# One DaemonSet: only the old dataplane is installed, and its label -# value is the $OLD_VERSION you are upgrading from. Two: step 3 is done, -# and the other label value is the version you are upgrading to. -kubectl get ds -n ate-system -l app=atelet -L ate.dev/substrate-version - -# Nodes per version. All on the old one: the per-node roll has not -# started. A mix: it is under way. All on the new one: it is done. -kubectl get nodes -L ate.dev/substrate-version --no-headers \ - | awk '{print $NF}' | sort | uniq -c - -# Worker pools. A clone next to each serving pool: step 4 is done. -kubectl get workerpools -A -``` - -The commands above do not show whether steps 1, 2 and 6 have run. -Those steps are idempotent, so run them again if you are not sure. +This runbook upgrades a running Agent Substrate install to a newer release in the same release window, meaning the same `v0.x`: for example, from v0.2.0 to v0.2.1. Between windows, for example from v0.1.x to v0.2.x, reinstall instead. -## Before you start +> [TODO] Add the version skew policy. -### Confirm before you begin -Every item has to hold. Nothing in the roll stops you if one does not. +| Step | What changes | What running actors see | How long | +|---|---|---|---| +| 1 | CRDs | Nothing | Seconds | +| 2 | ate-controller | Usually nothing; see step 2 | A minute, or as long as step 4 if it replaces workers | +| 3 | atelet, one node at a time | While a node's atelet restarts, suspend, pause and resume requests for actors on that node can fail | Seconds to about 6 minutes per node | +| 4 | workers | `SIGTERM`, then a 30-minute window to be suspended before `SIGKILL` | Minutes to hours per pool | +| 5 | ate-api-server, then atenet | API calls and long HTTP requests can be cut once | A few minutes | -- [ ] Every node carries the `ate.dev/substrate-version` label, all - with the same value. That value is `$OLD_VERSION`. +ate-controller manages the WorkerPools, so it goes first. atelet and the workers go before ate-api-server and atenet, so that when the API server changes, every node already understands requests from either version. Rollback is the same list in reverse, from the old release. An actor that crashes along the way goes back to its last snapshot with one revert call. - ```bash - kubectl get nodes -L ate.dev/substrate-version - ``` +The system upgrade does not change sandbox runtimes, the database engine, Kubernetes version or the node OS version. -- [ ] The atelet DaemonSet name carries a version suffix. If it does - not, the cluster was installed from an older build and needs a fresh - install instead; its DaemonSet selector cannot be changed in place. +## Before you start - ```bash - kubectl get ds -n ate-system -l app=atelet - ``` +The commands are for bash. Check that `kubectl config current-context` prints the cluster you are upgrading. -- [ ] If you are upgrading across the release that added the ate-system - NetworkPolicies, every namespace whose pods call the Substrate API - directly (not through `kubectl ate`) is labeled - `ate.dev/api-access=true`. The new manifests deny the API to every - other namespace as soon as they are applied. Worker pods are never - admitted, labeled or not. See - [`ate-system-networkpolicies.yaml`](../manifests/ate-install/ate-system-networkpolicies.yaml). +Check out both releases. ate-setup runs from the new checkout. The old checkout is for rollback, and its `kubectl ate` is the one to use until step 5 is done. - ```bash - kubectl label namespace ate.dev/api-access=true - ``` +```bash +OLD_RELEASE= +NEW_RELEASE= +mkdir ~/ate-upgrade && cd ~/ate-upgrade +git clone --branch $OLD_RELEASE https://github.com/agent-substrate/substrate.git old +git clone --branch $NEW_RELEASE https://github.com/agent-substrate/substrate.git new +(cd old && go install ./cmd/kubectl-ate) +``` -- [ ] Every serving pool is pinned to `$OLD_VERSION`. An unpinned pool - cannot take part in the roll. +Prebuilt images show the installed release in their tag, in `kubectl -n ate-system get deploy,ds -o wide`. For images built from source, `kubectl -n ate-system exec deploy/ate-api-server -c ate-api-server -- /ko-app/ateapi --version` prints it. - ```bash - kubectl get workerpools -A \ - -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,PIN:.spec.template.nodeSelector.ate\.dev/substrate-version' - ``` +### Use the install's settings - Pin a pool whose `PIN` column is `` while no actor is assigned - to its workers, because the edit re-renders the pool's Deployment - (see [the warnings](#three-things-that-break-an-upgrade)): +ate-setup does not store the settings you installed with. A command run without one of them puts that part back to its default, such as a different database or atenet proxy. Write the same `export` lines you installed with into `~/ate-upgrade/settings.sh`, including any from a `.ate-dev-env.sh`, and add: - ```bash - kubectl -n $NS patch workerpool $OLD_WORKERPOOL --type merge \ - -p "spec: {template: {nodeSelector: {ate.dev/substrate-version: '$OLD_VERSION'}}}" - ``` +- **Images:** the new release's `ATE_IMAGE_REPO` and `ATE_IMAGE_TAG`, or `KO_DOCKER_REPO` to build from source. +- **kind:** `ATE_INSTALL_KIND=true`. +- **Wait time:** `ATE_INSTALL_ROLLOUT_TIMEOUT=1h`. The 60-second default is too short for step 3. -- [ ] Every serving pool is healthy: `READY` equals `DESIRED`. +Leave out `ATE_OTLP_ENDPOINT` until the end of step 5. ate-setup does not keep a custom endpoint, and with it set, the later steps restart other components and replace every worker again. Also leave out `ATE_API_POSTGRES_CLOUDSQL_INSTANCE`: ate-setup keeps Cloud SQL by itself, and an empty value removes it. - ```bash - kubectl get workerpools -A - ``` +Run this now, and in every new shell: -- [ ] Nothing drains worker nodes on its own: node auto-upgrade is off - on every pool that runs workers and none of them is spot or - preemptible, as the - [Create Cluster warning](../tools/setup-gcp/README.md#2-create-cluster) - requires. +```bash +cd ~/ate-upgrade/new +source ~/ate-upgrade/settings.sh +``` -- [ ] If the install uses `--cordon-control-plane` with the bundled - PostgreSQL, at least one node carries - `ate.dev/workloadType=ate-postgres`. Releases before the dedicated - postgres pool ran postgres in the shared `ate-control-plane` pool; see - [Cordoned control plane: postgres pool](#cordoned-control-plane-postgres-pool). - `ate-setup` refuses to apply postgres without one. +Then save each pool's image for rollback, and the actors that are already CRASHED: - ```bash - kubectl get nodes -l ate.dev/workloadType=ate-postgres - ``` +```bash +kubectl get workerpools -A -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,IMAGE:.spec.workerImage' > ~/ate-upgrade/before-workerpools.txt +kubectl ate get actors -A | awk '$4=="ACTOR_STATE_CRASHED" {print $1, $2}' > ~/ate-upgrade/before-crashed.txt +``` -- [ ] The installed ate-api-server serves `DrainWorker`. Draining a - worker is one `DrainWorker` RPC, and step 5 calls it with - [grpcurl](https://github.com/fullstorydev/grpcurl). Open access to - the Control API now and keep the port-forward running; step 5 uses - the same `/tmp/ate-ca.pem` and `$TOKEN`. +> [!WARNING] +> Do not run `ate-setup deploy ate-system`, `deploy demo ...`, `deploy benchmarks`, `delete ate-system` or `delete all` during the upgrade. They update components out of order, move pools to the checkout's worker image, or delete every WorkerPool. - ```bash - kubectl -n ate-system port-forward svc/api 8443:443 >/dev/null 2>&1 & +## Upgrade - kubectl get clustertrustbundles -l podcert.ate.dev/canarying=live \ - -o jsonpath='{range .items[?(@.spec.signerName=="servicedns.podcert.ate.dev/identity")]}{.spec.trustBundle}{end}' \ - > /tmp/ate-ca.pem +### Step 1. Apply the new CRDs - TOKEN=$(kubectl -n ate-system create token ate-client \ - --audience=api.ate-system.svc --duration=48h) +```bash +kubectl apply --server-side --field-manager=ate-setup --force-conflicts -f manifests/ate-install/generated +``` - # Must print ateapi.Control.DrainWorker. If it does not, the cluster - # predates the versioned builds and needs a fresh install. - grpcurl -cacert /tmp/ate-ca.pem -authority api.ate-system.svc \ - -H "authorization: Bearer ${TOKEN}" 127.0.0.1:8443 list ateapi.Control \ - | grep DrainWorker - ``` +This updates the three CRDs and ate-controller's RBAC. The flags make kubectl apply the files the way ate-setup does, so step 2 does not conflict with this change. -### Names used throughout +- **Done:** five lines ending in `serverside-applied`. +- **Actors:** nothing changes. The step is safe to rerun. Go on to step 2 right away. -| name | what it is | how to get it | -|---|---|---| -| `$OLD_VERSION` | the version label value you are upgrading from | the one atelet DaemonSet's label, see [Check your progress](#check-your-progress) | -| `$NEW_VERSION` | the version label value you are upgrading to | the second DaemonSet's label once step 3 has run | -| `$NS`, `$OLD_WORKERPOOL` | namespace and name of a serving WorkerPool | `kubectl get workerpools -A` | -| `$NEW_WORKERPOOL` | the clone's name | you pick it in step 4, for example `counter-v2` | -| `$NODE` | the node being rolled | picked per iteration in step 5 from `kubectl get nodes` | - -A cluster usually serves more than one WorkerPool, and every serving -pool moves in the same upgrade. Step 4 clones each of them, the -per-node roll in step 5 covers all pools on a node together, and the -retire at the end deletes each old pool. Where the runbook says -`$OLD_WORKERPOOL`, read "each serving pool". - -### Checkout and environment - -Every `go run ./cmd/ate-setup` command below runs from a checkout of -the new release, with the environment and flags the install used: -`PROJECT_ID`, `CLUSTER_NAME` and `CLUSTER_LOCATION` (or `--context`), -and either `KO_DOCKER_REPO` for a build from source or -`--image-repo`/`--image-tag` for prebuilt images. Keep a checkout of -the old release too as rollback runs the same commands from it. - -`ate-setup` takes the version from `$VERSION` when it is set, else -from `git describe` on the checkout for a build from source, or from -`--image-tag` (`ATE_IMAGE_TAG`) for prebuilt images. That value must come out different -from `$OLD_VERSION`, otherwise dataplane upgrade rolls the running atelet in place -instead of adding a second DaemonSet. A tagged checkout differs by -construction; if the install pinned `VERSION`, pin a new value now -and keep it for every command of this upgrade. - -Install the new `kubectl ate` with `go install ./cmd/kubectl-ate`. - -### On GKE - -GKE needs `gcloud` and three more names: - -| name | what it is | how to get it | -|---|---|---| -| `$CLUSTER`, `$ZONE` | the cluster and its location | `gcloud container clusters list` | -| `$NODEPOOL` | the node pool | `gcloud container node-pools list --cluster $CLUSTER --zone $ZONE` | +### Step 2. Upgrade ate-controller -Park the autoscaler. During the roll, the old pool's pods that lost -their node sit Pending on purpose: they are the rollback reserve. The -autoscaler reads Pending pods as demand and would add nodes for pods -that must never schedule. Save its config first; you restore it -[after the roll](#after-the-roll-on-gke). +> [!WARNING] +> If this release changes the worker pod template, or the install set `ATE_OTLP_ENDPOINT`, step 2 replaces every worker on its old image, like step 4. ```bash -gcloud container node-pools describe $NODEPOOL --cluster $CLUSTER --zone $ZONE \ - --format='json(autoscaling)' # save for after the roll -gcloud container clusters update $CLUSTER --zone $ZONE \ - --node-pool $NODEPOOL --no-enable-autoscaling +go run ./cmd/ate-setup deploy ate-controller ``` -If a GKE maintenance window falls inside the roll, add a maintenance -exclusion for it too: a node GKE recreates mid-roll comes back at the -pool's label, which is still `$OLD_VERSION`. +It applies the CRDs again, deploys the new ate-controller and waits for it. + +- **Done:** the command exits 0, and `kubectl -n ate-system get pods -l app=ate-controller` shows one pod, Running, with RESTARTS 0. Ready only means the container started, so also run `kubectl -n ate-system logs deploy/ate-controller | grep -E 'atecontroller starting|Starting workers|Reconciler error'`: it should show the first two and not the third. +- **Stuck:** the new pod is Pending or in ImagePullBackOff: the old controller still runs. The new pod is in CrashLoopBackOff or its RESTARTS keeps rising: the old controller is already gone, and nothing manages the WorkerPools. Either way, do not go on; [roll back step 2](#roll-back). -Check that the node pool itself carries -`ate.dev/substrate-version=$OLD_VERSION`, or nodes GKE creates later -arrive unlabeled and nothing schedules to them. If it is missing, stamp -it now with the old value: +A minute after the new pod starts, check whether it is replacing workers: ```bash -gcloud container node-pools describe $NODEPOOL --cluster $CLUSTER --zone $ZONE \ - --format='get(config.labels)' # copy the value -gcloud container node-pools update $NODEPOOL --cluster $CLUSTER --zone $ZONE \ - --node-labels=,ate.dev/substrate-version=$OLD_VERSION +kubectl get deploy -A -l ate.dev/worker-pool -o custom-columns='NAMESPACE:.metadata.namespace,POOL:.metadata.name,DESIRED:.spec.replicas,UPDATED:.status.updatedReplicas,AVAILABLE:.status.availableReplicas,DRAINING:.status.terminatingReplicas,ROLLOUT_PAUSED:.spec.paused' ``` -Other providers have their own equivalents: stop whatever adds or -recreates worker nodes during the roll, and make sure a node created -later arrives with the old label. +This is the pool list. A pool is rolling while UPDATED or AVAILABLE is below DESIRED, or DRAINING is above 0; `` means 0 for UPDATED and AVAILABLE. Wait until no pool is rolling, then go to step 3. This can take hours; step 4 says what is normal. -### Cordoned control plane: postgres pool +### Step 3. Upgrade atelet -Skip this section unless the install uses `--cordon-control-plane` with -the bundled PostgreSQL. Under that flag, postgres runs alone in a -one-node pool labeled and tainted -`ate.dev/workloadType=ate-postgres:NoSchedule`, and every other control -plane workload shares the `ate-control-plane` pool. An install from a -release before that split has no postgres pool. The StatefulSet deletes -its running pod before creating the replacement, so without the pool -step 6 would take postgres down with nothing to schedule it on. +> [!WARNING] +> Do not run step 3 while many actors are being suspended, paused or resumed, including automatic resumes when requests arrive, and do not create ActorTemplates during it. While a node's atelet restarts, those calls on that node can fail and leave the actor CRASHED, or stuck in SUSPENDING, PAUSING or RESUMING. -Create the pool before step 6, in the zone of the postgres volume. The -volume is a zonal disk, and a postgres pod on a node in another zone -stays Pending with a volume node affinity conflict. +> [!NOTE] +> TODO: Shrink this warning if the API server retries these calls instead of crashing the actor. ```bash -# The zone the postgres volume lives in. -PG_ZONE=$(kubectl get pv \ - "$(kubectl -n ate-system get pvc data-postgres-0 -o jsonpath='{.spec.volumeName}')" \ - -o jsonpath='{.spec.nodeAffinity.required.nodeSelectorTerms[*].matchExpressions[*].values[0]}') -echo "$PG_ZONE" - -# A machine type that fits the postgres pod's requests: the size10 -# profile asks for 80 CPUs and 140Gi. -gcloud container node-pools create ate-postgres --cluster $CLUSTER --zone $ZONE \ - --node-locations "$PG_ZONE" --num-nodes 1 --machine-type \ - --node-labels ate.dev/workloadType=ate-postgres \ - --node-taints ate.dev/workloadType=ate-postgres:NoSchedule +go run ./cmd/ate-setup deploy atelet +kubectl -n ate-system rollout status ds/atelet ``` -On other providers, create the equivalent: one node in the volume's -zone, carrying the label and the taint. +If ate-setup fails with `waiting for daemonset/atelet in ate-system: ...`, only its wait timed out, and the second command keeps watching. If it fails with any other error, it did not change atelet: fix the error and run step 3 again. -Step 6 then restarts postgres once onto the new node, which is a short -outage for ate-api-server. The other control plane Deployments roll in -steps 2 and 6 as well. Their old pods still carry the required -anti-affinity of the earlier release, so the new pods land on the -`ate-control-plane` pool's spare node first and stay packed there. Once -`kubectl get pods -n ate-system -o wide` shows no pod older than the -roll, spread them again and shrink the pool, which no longer needs one -node per pod: +- **Progress:** `Waiting for daemon set "atelet" rollout to finish: 2 out of 5 new pods have been updated...`, then `... 4 of 5 updated pods are available...`. +- **Done:** `daemon set "atelet" successfully rolled out`. +- **Stuck:** the `N out of M` count does not move for more than about 6 minutes. Run `kubectl -n ate-system get pods -l app=atelet -o wide`. A pod that is not Running leaves its node without atelet. If it is in ImagePullBackOff or CrashLoopBackOff, [roll back step 3](#roll-back). If it is stuck in ContainerCreating, check podcertificate-controller. +- **Actors:** running actors keep running and keep serving HTTP. A resume that reaches a restarting node can return HTTP 500 `error resuming actor /`; a retry usually works. -```bash -kubectl -n ate-system rollout restart deployment \ - ate-api-server ate-controller atenet-router atenet-egress podcertificate-controller -gcloud container clusters resize $CLUSTER --zone $ZONE \ - --node-pool --num-nodes -``` +When it is done, [recover CRASHED actors and actors stuck mid-call](#recover-crashed-actors). -## Three things that break an upgrade - -Each warning comes back at the step where the mistake becomes possible. +### Step 4. Move each WorkerPool to the new worker image > [!WARNING] -> **Do not delete a node's old worker pods before flipping its -> version label.** Otherwise the old pool reschedules replacements onto -> the same node, and old workers end up next to the new atelet: exactly -> the version skew the roll exists to prevent. (Step 5) +> Do not start step 4 until step 3 is done: new workers need the new atelet. Do not start it while any actor is PAUSED. This must print nothing: +> +> ```bash +> kubectl ate get actors -A | awk '$4=="ACTOR_STATE_PAUSED" {print $1, $2}' +> ``` +> +> Do not edit the pool's Deployment with `kubectl set image`, `kubectl rollout undo` or `kubectl edit`: ate-controller puts it back. Do not `kubectl apply -f` a whole WorkerPool: that also resets `spec.replicas` on a pool an autoscaler sizes. If a GitOps tool syncs your WorkerPools, turn off its automatic sync for them first, or it moves the pools back to the old image. -> [!WARNING] -> **Do not edit a serving worker pool, and do not scale it down.** The -> controller would roll the pool's Deployment straight through live -> actors. A deleted worker pod does go through the eviction path: -> `SIGTERM` is forwarded into the actor's containers and the control -> plane keeps accepting a suspend for 30 minutes, so an actor suspended -> inside that window saves its state and stays resumable. Handling -> `SIGTERM` by exiting cleanly is not enough on its own; the suspend has -> to reach the control plane and finish. An actor still awake when the -> window closes moves to `ACTOR_STATE_CRASHED`: `resume` and `suspend` -> are both refused, and everything since its last snapshot is lost. Call -> `RevertActor` (`kubectl ate revert`) to discard the crashed run and -> return the actor to `ACTOR_STATE_SUSPENDED` at its last external snapshot -> so it can be resumed. Scaling a serving pool down removes pods the same -> way, without suspending the actors on them. (Step 4 clones the pool; it -> never edits it.) - -> [!WARNING] -> **On GKE, do not touch the node pool's label until every node is -> rolled.** A pool label update applies in place to every node in the -> pool, so the whole fleet flips at once, with no drain and no pacing. -> (After the roll) - -## Upgrade - -### 1. Apply the new CRDs - -From the new release's checkout, apply its CRDs. Nothing running -changes; the new schema is in place for the controller that follows. +List the pools. Each needs the new image for its sandbox CLASS: ```bash -# in a checkout of the new release -kubectl apply -f manifests/ate-install/generated +kubectl get workerpools -A -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,CLASS:.spec.sandboxClass,IMAGE:.spec.workerImage' ``` -### 2. Upgrade ate-controller +If you build from source, build and push both worker images once. This touches nothing in the cluster and ends by printing `ateom-gvisor: ` and `ateom-microvm: `: ```bash -go run ./cmd/ate-setup deploy ate-controller +go run ./cmd/ate-setup publish worker-images | tee ~/ate-upgrade/worker-images.txt ``` -This rolls `ate-controller` (and re-applies the CRDs, which is a -no-op now). +> [!NOTE] +> TODO: Document where release images are published and their tag format. Open source releases do not publish images yet. -By convention, changes to how the controller renders worker -pods sit behind WorkerPool fields, so the new controller keeps -rendering the serving pools as they are. A release that breaks that -convention says so in its notes. Expect the pools' Deployments to -roll once here in that case. Every actor is suspended through the -worker eviction path, loses no state, and resumes on demand. If they -roll, wait for `READY` to equal `DESIRED` again on every serving pool -(`kubectl get workerpools -A`) before step 4. +Then move the pools one at a time. For each pool: -The move of the node state root from `/var/lib/ateom-gvisor` to -`/var/lib/ate` is such a release. A worker that lands on a node whose -atelet still uses the old path reaches it only once step 5 moves that node. -The old directory can be deleted afterwards. +1. Set the pool: -### 3. Prepare the new dataplane + ```bash + NS= + POOL= + CLASS=$(kubectl -n $NS get workerpool $POOL -o jsonpath='{.spec.sandboxClass}') + ``` -The dataplane roll starts with the new atelet DaemonSet, from the -same checkout: +2. Set its new image. From source, copy the ref after `ateom-$CLASS: ` in `~/ate-upgrade/worker-images.txt`: -```bash -go run ./cmd/ate-setup deploy atelet -``` + ```bash + NEW_IMAGE= + ``` -It lands next to the old one with zero pods until step 5 flips a -node. + With prebuilt images, look it up pinned by digest. Without crane, get the digest with `gcloud artifacts docker images describe --format='value(image_summary.digest)'`. -Then read `$NEW_VERSION` off the cluster: + ```bash + NEW_IMAGE=$ATE_IMAGE_REPO/ateom-$CLASS:$ATE_IMAGE_TAG@$(crane digest $ATE_IMAGE_REPO/ateom-$CLASS:$ATE_IMAGE_TAG) + ``` -```bash -kubectl get ds -n ate-system -l app=atelet -L ate.dev/substrate-version -``` +3. Check that `echo "$NS/$POOL $CLASS $NEW_IMAGE"` shows `ateom-` plus the pool's CLASS and a full `@sha256:` digest. A bad image still removes the pool's first old worker. +4. Patch the pool, wait until ate-controller moves its Deployment to the new image, and watch the roll: -Exactly two DaemonSets print. The SUBSTRATE-VERSION that is not `$OLD_VERSION` is -`$NEW_VERSION`, and the new DaemonSet shows 0 `DESIRED` because no node carries -its label yet. `kubectl get nodes -L ate.dev/substrate-version` still -shows every node at `$OLD_VERSION`. If only one DaemonSet prints, the -version did not change (see Checkout and environment) and the command -rolled the running atelet in place. + ```bash + kubectl -n $NS patch workerpool $POOL --type merge -p "{\"spec\":{\"workerImage\":\"$NEW_IMAGE\"}}" + kubectl -n $NS wait deployment/$POOL --for=jsonpath='{.spec.template.spec.containers[0].image}'="$NEW_IMAGE" --timeout=2m + kubectl -n $NS rollout status deployment/$POOL + ``` -Last, build and push the new worker images. The refs print together -at the end. The one for your pool's `sandboxClasses[].name` goes into the clone -in step 4: +5. Go to the next pool once rollout status prints `successfully rolled out`. The times of pools done one after another add up. Pools patched together roll in parallel and interrupt more actors at once. -```bash -go run ./cmd/ate-setup publish worker-images -``` +What to expect: -If you are using prebuilt images there is nothing to publish. The new worker image -is the release's `ateom-` image under the same repo and -tag as the control plane, pinned by digest: +- **Progress:** rollout status prints, in order: -```bash -NEW_IMAGE=$IMAGE_REPO/ateom-gvisor:$IMAGE_TAG@$(crane digest $IMAGE_REPO/ateom-gvisor:$IMAGE_TAG) -``` + ``` + Waiting for deployment "" rollout to finish: 2 out of 10 new replicas have been updated... + Waiting for deployment "" rollout to finish: 1 old replicas are pending termination... + Waiting for deployment "" rollout to finish: 9 of 10 updated replicas are available... + deployment "" successfully rolled out + ``` + + "Updated" includes workers still starting. Ctrl-C stops only the watch, not the roll. +- **Done:** `successfully rolled out` does not count old workers that are still draining, and their actors may still be inside their 30 minutes. Step 4 is done when every pool shows UPDATED and AVAILABLE equal to DESIRED, and DRAINING 0, in the [pool list](#step-2-upgrade-ate-controller). DRAINING `` means your cluster does not report the count (Kubernetes 1.35 and later do); then wait until `kubectl ate get workers` shows no `WORKER_STATE_DRAINING`. +- **Pace:** how many actors are interrupted at once depends on free node room, not on the batch size. A batch is 10% of the pool's workers, rounded down and at least one, so pools under 20 workers replace one worker at a time. At most one batch is not ready at a time. A 1-worker pool has no ready worker while its worker is replaced. + - While a pool rolls, up to one batch of its workers takes no new actors. If fewer of its workers are idle than that, resumes fail with `no free workers available` until the batch is ready. To avoid this, raise the pool's replicas before you patch it. + - If the nodes have room, replacements start at once, and most old workers in the pool get `SIGTERM` within minutes. + - If the nodes are full, each replacement waits for an old worker to exit: up to 30 minutes plus a pod start per batch. That is up to 30 minutes per worker for pools under 20 workers, and 5 to 8 hours for larger pools. A batch ends early once its actors are suspended, and idle workers exit at once. +- **Stuck, or normal?** + - On full nodes, new pods Pending next to old pods Terminating, with rollout status on one line for up to about 30 minutes, are normal. With an autoscaler, pods Pending for a few minutes while a node is added are normal. + - DESIRED minus READY in `kubectl get workerpools -A` stays above one batch for more than a few minutes, or rollout status fails with `error: deployment "" exceeded its progress deadline` (80 minutes without progress; the roll goes on). Look for Pending pods (no capacity) or ImagePullBackOff (bad image) with `kubectl -n $NS get pods -l ate.dev/worker-pool=$POOL` and fix the cause. Until the roll moves again, rollout status repeats the error at once. + - A bad image stops the roll after the first batch. [Roll back that pool](#roll-back). + - DRAINING above 0 for more than 60 minutes, the longest a worker pod takes to exit, means a pod is stuck Terminating, usually on an unreachable node. Find it with `kubectl -n $NS get pods -l ate.dev/worker-pool=$POOL`. +- **Actors:** each gets `SIGTERM` and 30 minutes to be suspended, as described under eviction in the [API guide](api-guide.md). A resume in flight on a worker when its batch starts can crash that actor at once. The actor had not started running there, so a revert loses nothing. The metric `ate.actor.crashes` with `ate.actor.operation.name` `unknown` counts actors that were not suspended in time. -### 4. Create the new pool +When every pool is done, put the new `workerImage` in any files you apply WorkerPools from, turn GitOps sync back on, and [recover CRASHED actors](#recover-crashed-actors). -**Repeat this step for every serving pool the checklist listed**. +### Step 5. Upgrade ate-api-server, then atenet > [!WARNING] -> Clone the old pool. Do not edit it. An edit rolls its Deployment -> straight through the live actors on it (second warning above). +> Do not start step 5 until step 4 is done, and the [pool list](#step-2-upgrade-ate-controller) shows ROLLOUT_PAUSED `` for every pool. Do not use `deploy ate-system` here. `deploy apiserver` rewrites the API server's database settings from your shell every time, so it runs after a fresh `source` below. -Copy the old pool under a new name (`$NEW_WORKERPOOL`, for example -the old name plus `$NEW_VERSION`) and change only `workerImage` and -the version pin. The command below does exactly that; everything -else, including the `metadata.labels` the scheduler matches actors -by, carries over as is. +First the API server. The last command lists your actors through it: ```bash -NEW_IMAGE= -# Only read when the old pool predates spec.sandboxClasses. -SANDBOX_CLASS= - -kubectl -n $NS get workerpool $OLD_WORKERPOOL -o json \ - | jq --arg name "$NEW_WORKERPOOL" --arg image "$NEW_IMAGE" --arg version "$NEW_VERSION" \ - --arg class "$SANDBOX_CLASS" ' - {apiVersion, kind, - metadata: {name: $name, namespace: .metadata.namespace, - labels: .metadata.labels}, - spec: (.spec + {workerImage: $image})} - | .spec.template.nodeSelector["ate.dev/substrate-version"] = $version - | .spec.sandboxClasses //= [{name: $class}] - ' \ - | kubectl apply -f - +cd ~/ate-upgrade/new +source ~/ate-upgrade/settings.sh +go run ./cmd/ate-setup deploy apiserver +kubectl -n ate-system rollout status deploy/ate-api-server +kubectl ate get actors -A ``` -Do not shrink `spec.template.resources.limits` while cloning. A -worker only accepts an actor whose limits fit under them. +- **Done:** rollout status prints `deployment "ate-api-server" successfully rolled out`, and the list shows your actors. +- **Stuck:** an empty list means the API server points at the wrong database: add the install's `ATE_API_POSTGRES_CONNECTION_STRING` and `ATE_API_POSTGRES_SCHEMA` to `settings.sh` and run the block again. A new pod that never becomes ready makes `deploy apiserver` fail after about 10 minutes with `exceeded its progress deadline`, while the old pods keep serving. Check `kubectl -n ate-system get pods -l app=ate-api-server` and the new pod's logs: a database error has the same fix as an empty list, and ImagePullBackOff means wrong image settings. For anything else, [roll back step 5](#roll-back). -Verify: +Then atenet: ```bash -# $NEW_VERSION, then an image ref that contains @sha256: -kubectl -n $NS get workerpool $NEW_WORKERPOOL \ - -o jsonpath='{.spec.template.nodeSelector.ate\.dev/substrate-version}{"\n"}{.spec.workerImage}{"\n"}' - -# One Deployment per pool; new-pool pods all Pending (no node carries -# the new label yet). -kubectl -n $NS get deploy -l ate.dev/worker-pool -kubectl -n $NS get pods -l ate.dev/worker-pool=$NEW_WORKERPOOL +go run ./cmd/ate-setup deploy atenet +kubectl -n ate-system rollout status deploy/atenet-router +kubectl -n ate-system rollout status deploy/atenet-egress ``` -While both pools serve, placement between them is random, and that -is fine: a snapshot written on either version restores on either -version. The roll converges because step 5 takes old workers out of -service node by node, not because the scheduler prefers the new pool. +- **Done:** both print `successfully rolled out`. +- **Stuck:** a new pod stays Pending, in ImagePullBackOff or CrashLoopBackOff; the old pod keeps serving. Check `kubectl -n ate-system get pods` and [roll back step 5](#roll-back) if you cannot fix it. +- **API clients:** a suspend or resume running when its API server pod stops can be cut off and left mid-call, so harnesses should retry API errors during this step. If a `kubectl ate` command fails once, run it again. A `kubectl port-forward` you opened yourself breaks when its pod goes. +- **Actors:** when the router restarts, an HTTP request that takes longer than about 10 seconds can be cut once, and new connections fail for a few seconds. Outbound connections through the egress gateway are reset once. -### 5. Roll each node +Then finish: -Repeat for every node, one at a time. On a single-node cluster this -step is a full stop: every actor is suspended at once. +1. Install the new `kubectl ate` with `go install ./cmd/kubectl-ate`. Actor owners can now use API fields and commands new in this release. +2. If the install set `ATE_OTLP_ENDPOINT`, restore it now. Suspend any PAUSED actors first. Add the `export` line to `settings.sh`, source it, and run `go run ./cmd/ate-setup deploy atenet`. This restarts atelet, so the step 3 warning applies, and replaces every worker again: wait as in step 4, until DRAINING is 0. If you roll back later, remove the line from `settings.sh` first. +3. [Recover CRASHED actors](#recover-crashed-actors). -**a. Drain the node's workers.** Bound actors keep running. Draining -only stops new placements. +The upgrade is done when step 5 is done and no new actor stays CRASHED. Tell actor owners they can pause actors again. Keep `~/ate-upgrade` while you might still roll back, then delete it: `settings.sh` can hold passwords. -```bash -# A worker's name is its pod's UID; that is what DrainWorker takes. -for w in $(kubectl ate get workers -o json \ - | jq -r --arg node "$NODE" '.workers[] | select(.nodeName == $node) | .metadata.name'); do - grpcurl -cacert /tmp/ate-ca.pem -authority api.ate-system.svc \ - -H "authorization: Bearer ${TOKEN}" \ - -d "{\"worker\": {\"name\": \"${w}\"}}" \ - 127.0.0.1:8443 ateapi.Control/DrainWorker -done -``` +> [!NOTE] +> TODO: podcertificate-controller is not part of the five steps. Say how to upgrade it when a release changes it. -**b. See what is still on the node.** Two lists: the node's workers -with the actor each one hosts, and any paused actor whose local -snapshot lives on this node. A paused actor sits on no worker, so it -shows up only in the second list. +## Something's wrong, or I need to roll back -```bash -kubectl ate get workers -o json | jq -r --arg node "$NODE" ' - ["WORKER", "POD", "ASSIGNED ACTOR"], - (.workers[] | select(.nodeName == $node) - | [.metadata.name, .workerPod, - (.status.assignment.actor | if . then .atespace + "/" + .name else "" end)]) - | @tsv' | column -t -s $'\t' - -kubectl ate get actors -A -o json | jq -r --arg node "$NODE" ' - ["PAUSED_ACTOR", "STATE"], - (.actors[] - | select(.status.assignedNode == $node and (.status.workerAssignment == null)) - | [.metadata.atespace + "/" + .metadata.name, .status.state]) - | @tsv' | column -t -s $'\t' -``` +Stopping between steps is safe. -**c. Suspend them at your own pace.** Every actor in either list has to be suspended -before the node moves (`kubectl ate suspend actor -a -`). Suspend releases the worker and uploads the durable -snapshot, so the actor resumes on demand onto any free matching worker -afterwards. Repeat step b until the `ASSIGNED ACTOR` column reads -`` throughout and the paused list is empty. Then rerun the drain -in step a once more right before flipping. +### Symptoms -> [!WARNING] -> Do not run e before d. Deleting the old-pool pods while the node -> still carries `$OLD_VERSION` lets the old pool put replacements right -> back on this node, next to the new atelet (first warning above). +| You see | Meaning | Do | +|---|---|---| +| `waiting for daemonset/atelet in ate-system: ... (last status: 3/9 nodes updated)` | Only ate-setup's wait timed out. | `kubectl -n ate-system rollout status ds/atelet` | +| An atelet pod in ImagePullBackOff or CrashLoopBackOff | That node has no atelet, and step 3 stopped there. | [Roll back step 3](#roll-back). | +| ate-controller pod restarting after step 2 | Nothing manages the WorkerPools. | [Roll back step 2](#roll-back). | +| `exceeded its progress deadline` for a pool, or worker pods Pending or in ImagePullBackOff | Step 4 is slow or blocked. | See **Stuck, or normal?** in [step 4](#step-4-move-each-workerpool-to-the-new-worker-image). | +| HTTP 503 whose body mentions `ACTOR_STATE_CRASHED` | The actor crashed. | [Revert it](#recover-crashed-actors). | +| HTTP 503 `no free workers available` | During step 4 the pool is short by up to one batch. | Wait, or add capacity. If the actor is PAUSED, suspend it, then resume it. | +| HTTP 503 `another operation is in progress for this actor` | A suspend is running. | Wait. | +| HTTP 503 `actor / unavailable`, with no detail | ate-api-server restarting (step 5). | Retry. | +| HTTP 500 `error resuming actor /` | atelet restarting (step 3). | Retry. | +| InvalidArgument or Unimplemented from a `kubectl ate` command or API call | It is new in this release, and the old API server still serves. | Finish step 5 first. | +| After step 5, the atenet proxy changed or credentials are no longer injected | Install settings were missing from `settings.sh`. | Add them, source it, run `deploy atenet` again. | -**d. Flip the label.** The old atelet pod leaves on its own and the -new one starts. +### Recover CRASHED actors -```bash -kubectl label node $NODE ate.dev/substrate-version=$NEW_VERSION --overwrite -``` +A CRASHED actor needs one revert. It returns the actor to SUSPENDED at its last snapshot, and the next call resumes it. Work since that snapshot is lost, and external volumes are not rewound. -Wait for the new `atelet` pod on the node to be Ready before going on. Usually that is seconds; the new `atelet` pod stays Pending only while the old one is still exiting, since it holds the node's host ports until then. +This lists actors that became CRASHED since you started, each as `> `. An actor can show CRASHED for a moment while its suspend finishes, so run it again after a minute, and revert only actors that stay on the list. ```bash -# The pod is named after atelet-. -kubectl get pods -n ate-system -l app=atelet --field-selector spec.nodeName=$NODE +kubectl ate get actors -A | awk '$4=="ACTOR_STATE_CRASHED" {print $1, $2}' | diff ~/ate-upgrade/before-crashed.txt - ``` -**e. Delete the node's old-pool worker pods.** Step c emptied them, -but they are still Ready and hold capacity the new pool needs on this -node. Their Deployment cannot reschedule them here anymore. Repeat -per pool if the node hosts several. +For each one: ```bash -kubectl -n $NS delete pod -l ate.dev/worker-pool=$OLD_WORKERPOOL \ - --field-selector spec.nodeName=$NODE +ATESPACE= +ACTOR= +kubectl ate revert actor $ACTOR -a $ATESPACE ``` -**f. Confirm the node has moved.** Two checks: +A revert while a suspend is still finishing fails with `code = Aborted desc = another operation is in progress for this actor` and changes nothing; the actor turns SUSPENDED by itself. -```bash -# New-pool workers came up here. -kubectl -n $NS get pods -l ate.dev/worker-pool=$NEW_WORKERPOOL --field-selector spec.nodeName=$NODE +After step 3, also list actors that a failed call left halfway, each as ` `. Actors in the middle of a normal call show up too, so run it again after a few minutes: -# No old-pool pod is left here. -kubectl -n $NS get pods -l ate.dev/worker-pool=$OLD_WORKERPOOL --field-selector spec.nodeName=$NODE +```bash +kubectl ate get actors -A | awk '$4 ~ /^ACTOR_STATE_(SUSPENDING|PAUSING|RESUMING)$/ {print $1, $2, $4}' ``` -New-pool pods Pending on other nodes are expected until those nodes -move. - -**g. Take the next node.** Start again at step a once `kubectl ate get -workers` shows at least one FREE worker for the suspended actors to -land on. +For each actor still listed, run its call again: `kubectl ate suspend actor` for SUSPENDING, `pause actor` for PAUSING (then suspend it, because step 4 must not start with a paused actor), or `resume actor` for RESUMING, each with `$ACTOR -a $ATESPACE`. -You are done when `kubectl get nodes -L ate.dev/substrate-version` -shows every node at `$NEW_VERSION` (a node that joined mid-roll still -carries `$OLD_VERSION`; apply step 5 to roll it) and every assigned -actor in `kubectl ate get actors -A` sits on a new-pool pod. +### Stop a worker roll partway -### 6. Upgrade the rest of the control plane - -Every actor is now on the new dataplane, running or suspended, and -ate-controller moved in step 2. From the same checkout, move -`ate-api-server` first, then everything else: +To stop step 4 between pools, stop patching; unpatched pools do not roll. To stop a pool that is rolling, pause its Deployment. This is the one Deployment change ate-controller leaves in place. ```bash -go run ./cmd/ate-setup deploy apiserver -go run ./cmd/ate-setup deploy ate-system --credential-provider='' +kubectl -n $NS rollout pause deployment/$POOL ``` -`deploy ate-system` requires the credential provider selection; pass the one -the install was made with, or carry it in `ATE_CREDENTIAL_PROVIDER`. An install -made with `--experimental-egress-credential-injection` and the bundled provider -is `{"name":"k8s.io"}`; one without injection is `{"enabled":false}`. The -selection replaces `--credential-provider-name` and -`--credential-provider-address`: a provider you deploy yourself is -`{"name":"","address":":"}`, with the name bare -(`vault.example.com`, not `ate-secret://vault.example.com`). +No more old workers are removed. Workers already removed keep shutting down, and their actors still get the rest of their 30 minutes. The pool's size is not frozen: a change to its replicas, by you or an autoscaler, still adds or removes workers. While the pool is paused, rollout status never finishes; press Ctrl-C. To continue, run `kubectl -n $NS rollout resume deployment/$POOL` and watch as in step 4. -The second command rolls atenet and converges the rest of the -install, postgres included: on a cordoned install it moves postgres to -its own pool (see [Cordoned control plane: postgres -pool](#cordoned-control-plane-postgres-pool)); it re-resolves and re-applies everything, so it could take a -while. The checklist's port-forward dies when the API server rolls. -Restart it and mint a fresh token if you still need to drain. +> [!WARNING] +> Do not leave a pool paused, and do not start step 5 while one is. A paused pool ignores every later change to its workers, including the next upgrade. The [pool list](#step-2-upgrade-ate-controller) shows it as ROLLOUT_PAUSED `true`. -**NOTE**: Until -this step is done the old ate-api-server is still serving, so do not -start using API fields new in this release before upgrade finishes. +### Roll back -### After the roll, on GKE +To go back to the old release, roll back every step you began, finished or not, starting with the last. To fix one pool, roll back only that pool. Rollback runs from the old checkout. Run this block now, and in every new shell during the rollback; the last line is for prebuilt images only: -> [!WARNING] -> Only after every node shows `$NEW_VERSION` in [Check your -> progress](#check-your-progress). Relabeling the node pool applies to -> every node in it at once, with no drain and no pacing. +```bash +cd ~/ate-upgrade/old +source ~/ate-upgrade/settings.sh +export ATE_IMAGE_TAG= +``` -Relabel the node pool with `$NEW_VERSION` so nodes created later start -at the new version, then restore the autoscaler from the config you -saved [before the roll](#on-gke): +Do not edit `settings.sh` for the rollback: going forward again needs the new tag. -```bash -# --node-labels REPLACES the pool's full user label set: list the -# current labels first and carry them all over. -gcloud container node-pools describe $NODEPOOL --cluster $CLUSTER --zone $ZONE \ - --format='get(config.labels)' # copy the output +> [!WARNING] +> Do not roll back step 3 before every pool is back on its old image, with DRAINING 0. A new worker with an old atelet is not supported. -gcloud container node-pools update $NODEPOOL --cluster $CLUSTER --zone $ZONE \ - --node-labels=,ate.dev/substrate-version=$NEW_VERSION +- **Step 5:** tell actor owners to stop using API fields new in this release, because the old API server rejects them. Run `go run ./cmd/ate-setup deploy atenet`, then `go run ./cmd/ate-setup deploy apiserver`, then reinstall the old `kubectl ate` with `go install ./cmd/kubectl-ate`. The database schema stays new; the old API server drops fields it does not know when it rewrites a record. +- **Step 4:** suspend any PAUSED actors again. Then set each patched pool back to its IMAGE in `~/ate-upgrade/before-workerpools.txt`, with the exact string; another reference to the same image counts as a new image and replaces every worker. -# restore the autoscaler config -gcloud container clusters update $CLUSTER --zone $ZONE --node-pool $NODEPOOL \ - --enable-autoscaling --min-nodes --max-nodes -``` + ```bash + NS= + POOL= + OLD_IMAGE= + kubectl -n $NS patch workerpool $POOL --type merge -p "{\"spec\":{\"workerImage\":\"$OLD_IMAGE\"}}" + ``` -## Something's wrong, or I need to undo - -First run [Check your progress](#check-your-progress). It tells you -which step you reached, and everything below is keyed on that. - -If a command failed, rerun it: every step is idempotent. The one -exception is drain. There is no undrain; if you drained the wrong -node, check that its workers are empty (suspending as needed), delete -their pods, and let the Deployment's replacements register as fresh -workers. - -To roll back, undo what you did in reverse order: the control plane -first, then the nodes, then ate-controller last. The roll never edits -or deletes the old objects, so the old `WorkerPool` is intact, the pods -that lost their nodes stay Pending, and the old atelet DaemonSet is -still installed. Rolling nodes back is a matter of label flips. Run the -commands from the old release's checkout with the same environment as -the install, `VERSION` included if the install pinned it. - -- Relabeled the GKE node pool: park the autoscaler again as - [before the roll](#on-gke), then move the node pool label back to - `$OLD_VERSION` with the [after the roll](#after-the-roll-on-gke) - command. -- Past step 6: `go run ./cmd/ate-setup deploy apiserver` now, and - `go run ./cmd/ate-setup deploy ate-system` once the nodes are back. -- Past step 5: roll each flipped node back by running step 5 with the - sides swapped: drain, get every actor off the node as in b and c, - flip the label back to `$OLD_VERSION`, and delete the node's - new-pool pods. -- Past step 4: once no actor is assigned to a new-pool worker, delete - each clone: `kubectl -n $NS delete workerpool $NEW_WORKERPOOL`. -- Past step 3: once no node carries `$NEW_VERSION`, delete the new - DaemonSet: `kubectl delete daemonset -n ate-system -l app=atelet,ate.dev/substrate-version=$NEW_VERSION`. -- Past step 2: `go run ./cmd/ate-setup deploy ate-controller`. - -`kubectl get ds -n ate-system -l app=atelet` must not show a third -DaemonSet afterwards. A third means the old checkout produced a -version other than `$OLD_VERSION`: delete it and check `VERSION`. - -Abandoning the upgrade entirely is the whole list, top to bottom. -Retiring the old pool (below) is a separate, deliberate step. As long -as the old objects exist, rollback is one label flip per node. - -## Retire the old pool - -After the new version has soaked, reclaim the reserve. Save the old -pool's spec first: its `workerImage` ref is by digest and stays -pullable, so the saved file is the last-resort way to recreate the -pool. Then check the guards. Deleting the old objects is what ends -the rollback option. + If you paused the pool, resume it now with `kubectl -n $NS rollout resume deployment/$POOL`. Then watch: -```bash -kubectl -n $NS get workerpool $OLD_WORKERPOOL -o yaml > old-pool-backup.yaml - -# Guards: no node still at $OLD_VERSION; no old-pool pod Running (Pending is -# expected); no actor assigned to an old-pool worker. -kubectl get nodes -l ate.dev/substrate-version=$OLD_VERSION -kubectl -n $NS get pods -l ate.dev/worker-pool=$OLD_WORKERPOOL -kubectl ate get workers - -# Retire the old pool (its Deployment and pods go with it) and the -# old atelet DaemonSet. -kubectl -n $NS delete workerpool $OLD_WORKERPOOL -kubectl delete daemonset -n ate-system -l app=atelet,ate.dev/substrate-version=$OLD_VERSION -``` + ```bash + kubectl -n $NS rollout status deployment/$POOL + ``` + + Partway through a roll, only workers already on the new image are replaced. After a finished roll, every worker is replaced again, with another round of `SIGTERM`. Wait as in step 4, until DRAINING is 0. If an actor turns CRASHED again right after each resume, the old workers cannot restore the snapshot a new worker wrote, and a revert does not help: move that pool forward to the new image again. If you put the new image in your WorkerPool files, put the old one back before turning GitOps sync on. +- **Step 3:** once every pool is back, run `go run ./cmd/ate-setup deploy atelet`. The step 3 warning applies. +- **Step 1 or 2:** run `go run ./cmd/ate-setup deploy ate-controller`. It restores the old CRDs, RBAC and controller. If the release changed the worker pod template, every pool rolls again: suspend any PAUSED actors first, and wait as in step 4. Fields the new release added to WorkerPools, SandboxConfigs or CSIDriverConfigs disappear; remove them from your manifests first, or `kubectl apply` fails. A value only the new release accepts, such as a new `sandboxClass`, stays but is not understood; change it first. If the new release added an API version to a CRD, you cannot roll back past step 1; reinstall. -The new pool keeps its name. Names mean nothing to placement, so -`counter-v2` can serve indefinitely, and the next upgrade clones it -to `counter-v3`. +> [!NOTE] +> TODO: The upgrade end-to-end test does not cover rollback. From fe5d3a8c17040d46d83afb294fad2ab123d505a6 Mon Sep 17 00:00:00 2001 From: Haven Xia Date: Thu, 24 Sep 2026 20:15:02 -0700 Subject: [PATCH 02/15] Go through entire runbook with all new designs and refine --- docs/upgrade.md | 311 +++++++++++++++++------------------------------- 1 file changed, 111 insertions(+), 200 deletions(-) diff --git a/docs/upgrade.md b/docs/upgrade.md index 9b1b1bb194..079a5f7661 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -2,26 +2,23 @@ This runbook upgrades a running Agent Substrate install to a newer release in the same release window, meaning the same `v0.x`: for example, from v0.2.0 to v0.2.1. Between windows, for example from v0.1.x to v0.2.x, reinstall instead. -> [TODO] Add the version skew policy. +> [TODO] Link to the Release Policy. | Step | What changes | What running actors see | How long | |---|---|---|---| -| 1 | CRDs | Nothing | Seconds | -| 2 | ate-controller | Usually nothing; see step 2 | A minute, or as long as step 4 if it replaces workers | -| 3 | atelet, one node at a time | While a node's atelet restarts, suspend, pause and resume requests for actors on that node can fail | Seconds to about 6 minutes per node | -| 4 | workers | `SIGTERM`, then a 30-minute window to be suspended before `SIGKILL` | Minutes to hours per pool | -| 5 | ate-api-server, then atenet | API calls and long HTTP requests can be cut once | A few minutes | +| 1 | podcertificate-controller, CRDs and ate-controller | Usually nothing; see step 1 | A minute, or as long as step 3 if it replaces workers | +| 2 | atelet, one node at a time | Nothing | Seconds to about 6 minutes per node | +| 3 | workers | `SIGTERM`, then a 30-minute window to be suspended before `SIGKILL` | Minutes to hours per pool | +| 4 | ate-api-server, atenet, then SandboxConfig | API calls and long HTTP requests can be cut once | A few minutes | -ate-controller manages the WorkerPools, so it goes first. atelet and the workers go before ate-api-server and atenet, so that when the API server changes, every node already understands requests from either version. Rollback is the same list in reverse, from the old release. An actor that crashes along the way goes back to its last snapshot with one revert call. +podcertificate-controller signs the certificates the other components use, and ate-controller manages the WorkerPools, so they go first, together with the CRDs. atelet and the workers go before ate-api-server and atenet, so that when the API server changes, every node already understands requests from either version. The SandboxConfig admission policy and the default SandboxConfig go last, once every worker can run what they allow. Rollback is the same list in reverse, from the old release. An actor that crashes along the way goes back to its last snapshot with one revert call. -The system upgrade does not change sandbox runtimes, the database engine, Kubernetes version or the node OS version. +The system upgrade does not change the database engine, Kubernetes version or the node OS version. ## Before you start -The commands are for bash. Check that `kubectl config current-context` prints the cluster you are upgrading. - -Check out both releases. ate-setup runs from the new checkout. The old checkout is for rollback, and its `kubectl ate` is the one to use until step 5 is done. +Check out both releases. `ate-setup` runs from the new checkout. The old checkout is kept for rollback. ```bash OLD_RELEASE= @@ -32,126 +29,92 @@ git clone --branch $NEW_RELEASE https://github.com/agent-substrate/substrate.git (cd old && go install ./cmd/kubectl-ate) ``` -Prebuilt images show the installed release in their tag, in `kubectl -n ate-system get deploy,ds -o wide`. For images built from source, `kubectl -n ate-system exec deploy/ate-api-server -c ate-api-server -- /ko-app/ateapi --version` prints it. - -### Use the install's settings - -ate-setup does not store the settings you installed with. A command run without one of them puts that part back to its default, such as a different database or atenet proxy. Write the same `export` lines you installed with into `~/ate-upgrade/settings.sh`, including any from a `.ate-dev-env.sh`, and add: - -- **Images:** the new release's `ATE_IMAGE_REPO` and `ATE_IMAGE_TAG`, or `KO_DOCKER_REPO` to build from source. -- **kind:** `ATE_INSTALL_KIND=true`. -- **Wait time:** `ATE_INSTALL_ROLLOUT_TIMEOUT=1h`. The 60-second default is too short for step 3. +Make two copies of the config file. +> TODO: #1887 defines the config file. -Leave out `ATE_OTLP_ENDPOINT` until the end of step 5. ate-setup does not keep a custom endpoint, and with it set, the later steps restart other components and replace every worker again. Also leave out `ATE_API_POSTGRES_CLOUDSQL_INSTANCE`: ate-setup keeps Cloud SQL by itself, and an empty value removes it. - -Run this now, and in every new shell: +Keep `old.yaml` as it is, for rollback. If you build from source, keep `new.yaml` the same: the checkout you run from decides the release. If you install prebuilt images, set the image tag (`ATE_IMAGE_TAG`) in `new.yaml` to the new release. ```bash -cd ~/ate-upgrade/new -source ~/ate-upgrade/settings.sh +cp ~/ate-upgrade/old.yaml +cp ~/ate-upgrade/new.yaml ``` -Then save each pool's image for rollback, and the actors that are already CRASHED: +You will the steps from the new checkout: ```bash -kubectl get workerpools -A -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,IMAGE:.spec.workerImage' > ~/ate-upgrade/before-workerpools.txt -kubectl ate get actors -A | awk '$4=="ACTOR_STATE_CRASHED" {print $1, $2}' > ~/ate-upgrade/before-crashed.txt +cd ~/ate-upgrade/new ``` -> [!WARNING] -> Do not run `ate-setup deploy ate-system`, `deploy demo ...`, `deploy benchmarks`, `delete ate-system` or `delete all` during the upgrade. They update components out of order, move pools to the checkout's worker image, or delete every WorkerPool. +> [!CAUTION] +> Do not run `ate-setup deploy ate-system` or `ate-setup delete` during the upgrade. They change every component out of upgrade order. ## Upgrade -### Step 1. Apply the new CRDs - -```bash -kubectl apply --server-side --field-manager=ate-setup --force-conflicts -f manifests/ate-install/generated -``` - -This updates the three CRDs and ate-controller's RBAC. The flags make kubectl apply the files the way ate-setup does, so step 2 does not conflict with this change. - -- **Done:** five lines ending in `serverside-applied`. -- **Actors:** nothing changes. The step is safe to rerun. Go on to step 2 right away. - -### Step 2. Upgrade ate-controller - -> [!WARNING] -> If this release changes the worker pod template, or the install set `ATE_OTLP_ENDPOINT`, step 2 replaces every worker on its old image, like step 4. +### Step 1. Upgrade podcertificate-controller, the CRDs and ate-controller ```bash -go run ./cmd/ate-setup deploy ate-controller +go run ./cmd/ate-setup deploy podcertificate-controller --config ~/ate-upgrade/new.yaml +go run ./cmd/ate-setup deploy ate-controller --config ~/ate-upgrade/new.yaml ``` -It applies the CRDs again, deploys the new ate-controller and waits for it. +The first command deploys the new podcertificate-controller and waits until it publishes its trust bundles. The second applies the new CRDs (`WorkerPool`, `SandboxConfig` and `CSIDriverConfig`) and ate-controller's RBAC, waits until the CRDs are established, then deploys the new `ate-controller` and waits for it. Rerunning is safe. -- **Done:** the command exits 0, and `kubectl -n ate-system get pods -l app=ate-controller` shows one pod, Running, with RESTARTS 0. Ready only means the container started, so also run `kubectl -n ate-system logs deploy/ate-controller | grep -E 'atecontroller starting|Starting workers|Reconciler error'`: it should show the first two and not the third. -- **Stuck:** the new pod is Pending or in ImagePullBackOff: the old controller still runs. The new pod is in CrashLoopBackOff or its RESTARTS keeps rising: the old controller is already gone, and nothing manages the WorkerPools. Either way, do not go on; [roll back step 2](#roll-back). +- **Done:** both commands exit 0, and a minute later `kubectl -n ate-system get pods -l app=ate-controller` still shows one pod, Running, with RESTARTS 0. +- **Stuck:** a command fails, or the pod keeps restarting. [Roll back step 1](#roll-back-step-1). A minute after the new pod starts, check whether it is replacing workers: ```bash -kubectl get deploy -A -l ate.dev/worker-pool -o custom-columns='NAMESPACE:.metadata.namespace,POOL:.metadata.name,DESIRED:.spec.replicas,UPDATED:.status.updatedReplicas,AVAILABLE:.status.availableReplicas,DRAINING:.status.terminatingReplicas,ROLLOUT_PAUSED:.spec.paused' +kubectl get deploy -A -l ate.dev/worker-pool ``` -This is the pool list. A pool is rolling while UPDATED or AVAILABLE is below DESIRED, or DRAINING is above 0; `` means 0 for UPDATED and AVAILABLE. Wait until no pool is rolling, then go to step 3. This can take hours; step 4 says what is normal. - -### Step 3. Upgrade atelet +A pool is done rolling when READY shows `n/n`, and UP-TO-DATE and AVAILABLE both show `n`. Wait until every pool is done, then go to step 2. This can happen when this release changes the worker pod template. -> [!WARNING] -> Do not run step 3 while many actors are being suspended, paused or resumed, including automatic resumes when requests arrive, and do not create ActorTemplates during it. While a node's atelet restarts, those calls on that node can fail and leave the actor CRASHED, or stuck in SUSPENDING, PAUSING or RESUMING. +> TODO: Define the API compatibility policy for workerPool. +> +> Proposal: within a release window, ate-controller does not change the pod template it renders, so this step replaces no workers. -> [!NOTE] -> TODO: Shrink this warning if the API server retries these calls instead of crashing the actor. +### Step 2. Upgrade atelet ```bash -go run ./cmd/ate-setup deploy atelet +go run ./cmd/ate-setup deploy atelet --config ~/ate-upgrade/new.yaml --rollout-timeout 1h kubectl -n ate-system rollout status ds/atelet ``` -If ate-setup fails with `waiting for daemonset/atelet in ate-system: ...`, only its wait timed out, and the second command keeps watching. If it fails with any other error, it did not change atelet: fix the error and run step 3 again. - - **Progress:** `Waiting for daemon set "atelet" rollout to finish: 2 out of 5 new pods have been updated...`, then `... 4 of 5 updated pods are available...`. - **Done:** `daemon set "atelet" successfully rolled out`. -- **Stuck:** the `N out of M` count does not move for more than about 6 minutes. Run `kubectl -n ate-system get pods -l app=atelet -o wide`. A pod that is not Running leaves its node without atelet. If it is in ImagePullBackOff or CrashLoopBackOff, [roll back step 3](#roll-back). If it is stuck in ContainerCreating, check podcertificate-controller. -- **Actors:** running actors keep running and keep serving HTTP. A resume that reaches a restarting node can return HTTP 500 `error resuming actor /`; a retry usually works. +- **Stuck:** the `N out of M` count does not move for more than about 6 minutes. A new atelet pod that cannot start stops the roll and leaves its node without atelet; if you cannot fix it quickly, [roll back step 2](#roll-back-step-2). A pod stuck in ContainerCreating usually means podcertificate-controller is not running. +- **Actors:** running actors keep running. A node's atelet is down for a few seconds while it restarts. -When it is done, [recover CRASHED actors and actors stuck mid-call](#recover-crashed-actors). +### Step 3. Move each WorkerPool to the new worker image -### Step 4. Move each WorkerPool to the new worker image - -> [!WARNING] -> Do not start step 4 until step 3 is done: new workers need the new atelet. Do not start it while any actor is PAUSED. This must print nothing: -> -> ```bash -> kubectl ate get actors -A | awk '$4=="ACTOR_STATE_PAUSED" {print $1, $2}' -> ``` +> [!NOTE] +> Do not start step 3 until step 2 is done: new workers need the new atelet. > -> Do not edit the pool's Deployment with `kubectl set image`, `kubectl rollout undo` or `kubectl edit`: ate-controller puts it back. Do not `kubectl apply -f` a whole WorkerPool: that also resets `spec.replicas` on a pool an autoscaler sizes. If a GitOps tool syncs your WorkerPools, turn off its automatic sync for them first, or it moves the pools back to the old image. +> Change the WorkerPool, not its Deployment: `ate-controller` overwrites edits to the Deployment. -List the pools. Each needs the new image for its sandbox CLASS: +Before you patch any pool, list the pools and save the list for rollback. Each pool needs the new image for its sandbox CLASS: ```bash -kubectl get workerpools -A -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,CLASS:.spec.sandboxClass,IMAGE:.spec.workerImage' +kubectl get workerpools -A -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,CLASS:.spec.sandboxClass,IMAGE:.spec.workerImage' | tee ~/ate-upgrade/before-workerpools.txt ``` If you build from source, build and push both worker images once. This touches nothing in the cluster and ends by printing `ateom-gvisor: ` and `ateom-microvm: `: ```bash -go run ./cmd/ate-setup publish worker-images | tee ~/ate-upgrade/worker-images.txt +go run ./cmd/ate-setup publish worker-images --config ~/ate-upgrade/new.yaml | tee ~/ate-upgrade/worker-images.txt ``` -> [!NOTE] -> TODO: Document where release images are published and their tag format. Open source releases do not publish images yet. +> TODO: Figure out whether OSS has prebuilt images published. -Then move the pools one at a time. For each pool: +Then move the pools. For each pool: 1. Set the pool: ```bash - NS= - POOL= - CLASS=$(kubectl -n $NS get workerpool $POOL -o jsonpath='{.spec.sandboxClass}') + NS= + WORKERPOOL= + CLASS=$(kubectl -n $NS get workerpool $WORKERPOOL -o jsonpath='{.spec.sandboxClass}') ``` 2. Set its new image. From source, copy the ref after `ateom-$CLASS: ` in `~/ate-upgrade/worker-images.txt`: @@ -160,186 +123,134 @@ Then move the pools one at a time. For each pool: NEW_IMAGE= ``` - With prebuilt images, look it up pinned by digest. Without crane, get the digest with `gcloud artifacts docker images describe --format='value(image_summary.digest)'`. - - ```bash - NEW_IMAGE=$ATE_IMAGE_REPO/ateom-$CLASS:$ATE_IMAGE_TAG@$(crane digest $ATE_IMAGE_REPO/ateom-$CLASS:$ATE_IMAGE_TAG) - ``` + With prebuilt images, set `NEW_IMAGE` to `ateom-$CLASS` from your image repo at the new tag, pinned by digest (`crane digest ` prints it). -3. Check that `echo "$NS/$POOL $CLASS $NEW_IMAGE"` shows `ateom-` plus the pool's CLASS and a full `@sha256:` digest. A bad image still removes the pool's first old worker. +3. Check that `echo "$NS/$WORKERPOOL $CLASS $NEW_IMAGE"` shows `ateom-` plus the pool's CLASS and a full `@sha256:` digest. A bad image still removes the pool's first old worker. 4. Patch the pool, wait until ate-controller moves its Deployment to the new image, and watch the roll: ```bash - kubectl -n $NS patch workerpool $POOL --type merge -p "{\"spec\":{\"workerImage\":\"$NEW_IMAGE\"}}" - kubectl -n $NS wait deployment/$POOL --for=jsonpath='{.spec.template.spec.containers[0].image}'="$NEW_IMAGE" --timeout=2m - kubectl -n $NS rollout status deployment/$POOL + kubectl -n $NS patch workerpool $WORKERPOOL --type merge -p "{\"spec\":{\"workerImage\":\"$NEW_IMAGE\"}}" + kubectl -n $NS wait deployment/$WORKERPOOL --for=jsonpath='{.spec.template.spec.containers[0].image}'="$NEW_IMAGE" --timeout=2m + kubectl -n $NS rollout status deployment/$WORKERPOOL ``` -5. Go to the next pool once rollout status prints `successfully rolled out`. The times of pools done one after another add up. Pools patched together roll in parallel and interrupt more actors at once. +5. Once the first pool of a CLASS reports `1 of N updated replicas are available`, patch the other pools of that CLASS without waiting. Then repeat on other CLASS. What to expect: -- **Progress:** rollout status prints, in order: - - ``` - Waiting for deployment "" rollout to finish: 2 out of 10 new replicas have been updated... - Waiting for deployment "" rollout to finish: 1 old replicas are pending termination... - Waiting for deployment "" rollout to finish: 9 of 10 updated replicas are available... - deployment "" successfully rolled out - ``` - - "Updated" includes workers still starting. Ctrl-C stops only the watch, not the roll. -- **Done:** `successfully rolled out` does not count old workers that are still draining, and their actors may still be inside their 30 minutes. Step 4 is done when every pool shows UPDATED and AVAILABLE equal to DESIRED, and DRAINING 0, in the [pool list](#step-2-upgrade-ate-controller). DRAINING `` means your cluster does not report the count (Kubernetes 1.35 and later do); then wait until `kubectl ate get workers` shows no `WORKER_STATE_DRAINING`. -- **Pace:** how many actors are interrupted at once depends on free node room, not on the batch size. A batch is 10% of the pool's workers, rounded down and at least one, so pools under 20 workers replace one worker at a time. At most one batch is not ready at a time. A 1-worker pool has no ready worker while its worker is replaced. - - While a pool rolls, up to one batch of its workers takes no new actors. If fewer of its workers are idle than that, resumes fail with `no free workers available` until the batch is ready. To avoid this, raise the pool's replicas before you patch it. - - If the nodes have room, replacements start at once, and most old workers in the pool get `SIGTERM` within minutes. - - If the nodes are full, each replacement waits for an old worker to exit: up to 30 minutes plus a pod start per batch. That is up to 30 minutes per worker for pools under 20 workers, and 5 to 8 hours for larger pools. A batch ends early once its actors are suspended, and idle workers exit at once. -- **Stuck, or normal?** - - On full nodes, new pods Pending next to old pods Terminating, with rollout status on one line for up to about 30 minutes, are normal. With an autoscaler, pods Pending for a few minutes while a node is added are normal. - - DESIRED minus READY in `kubectl get workerpools -A` stays above one batch for more than a few minutes, or rollout status fails with `error: deployment "" exceeded its progress deadline` (80 minutes without progress; the roll goes on). Look for Pending pods (no capacity) or ImagePullBackOff (bad image) with `kubectl -n $NS get pods -l ate.dev/worker-pool=$POOL` and fix the cause. Until the roll moves again, rollout status repeats the error at once. - - A bad image stops the roll after the first batch. [Roll back that pool](#roll-back). - - DRAINING above 0 for more than 60 minutes, the longest a worker pod takes to exit, means a pod is stuck Terminating, usually on an unreachable node. Find it with `kubectl -n $NS get pods -l ate.dev/worker-pool=$POOL`. -- **Actors:** each gets `SIGTERM` and 30 minutes to be suspended, as described under eviction in the [API guide](api-guide.md). A resume in flight on a worker when its batch starts can crash that actor at once. The actor had not started running there, so a revert loses nothing. The metric `ate.actor.crashes` with `ate.actor.operation.name` `unknown` counts actors that were not suspended in time. +- **Progress:** `kubectl rollout status` prints how many new workers are updated and available. Ctrl-C to safely stop the watch. +- **Done:** Step 3 is done when every pool is done rolling with `kubectl get deploy -A -l ate.dev/worker-pool`. Go to Step 4 once done. +- **Pace:** a WorkerPool replaces 10% of its workers per batch. New workers start wherever the scheduler finds room. With spare capacity, every batch has its `SIGTERM` within minutes and the old workers drain in parallel; without it, new workers stay Pending until old ones exit, up to 30 minutes per batch, so a large pool can take hours. +- **Stuck:** rollout status fails with `exceeded its progress deadline` (the roll goes on), or a worker pod stays Terminating for more than 60 minutes. Check `kubectl -n $NS get pods -l ate.dev/worker-pool=$WORKERPOOL`. A bad image stops the roll after the first batch: [roll back that pool](#roll-back-step-3). +- **Actors:** each gets `SIGTERM` and 30 minutes to be suspended, as described under eviction in the [API guide](api-guide.md). -When every pool is done, put the new `workerImage` in any files you apply WorkerPools from, turn GitOps sync back on, and [recover CRASHED actors](#recover-crashed-actors). +### Step 4. Upgrade the Substrate control plane -### Step 5. Upgrade ate-api-server, then atenet - -> [!WARNING] -> Do not start step 5 until step 4 is done, and the [pool list](#step-2-upgrade-ate-controller) shows ROLLOUT_PAUSED `` for every pool. Do not use `deploy ate-system` here. `deploy apiserver` rewrites the API server's database settings from your shell every time, so it runs after a fresh `source` below. - -First the API server. The last command lists your actors through it: +First the Substrate API server. The first new API server pod migrates the database schema. ```bash cd ~/ate-upgrade/new -source ~/ate-upgrade/settings.sh -go run ./cmd/ate-setup deploy apiserver +go run ./cmd/ate-setup deploy apiserver --config ~/ate-upgrade/new.yaml kubectl -n ate-system rollout status deploy/ate-api-server kubectl ate get actors -A ``` -- **Done:** rollout status prints `deployment "ate-api-server" successfully rolled out`, and the list shows your actors. -- **Stuck:** an empty list means the API server points at the wrong database: add the install's `ATE_API_POSTGRES_CONNECTION_STRING` and `ATE_API_POSTGRES_SCHEMA` to `settings.sh` and run the block again. A new pod that never becomes ready makes `deploy apiserver` fail after about 10 minutes with `exceeded its progress deadline`, while the old pods keep serving. Check `kubectl -n ate-system get pods -l app=ate-api-server` and the new pod's logs: a database error has the same fix as an empty list, and ImagePullBackOff means wrong image settings. For anything else, [roll back step 5](#roll-back). - -Then atenet: +Then `atenet`: ```bash -go run ./cmd/ate-setup deploy atenet +go run ./cmd/ate-setup deploy atenet --config ~/ate-upgrade/new.yaml kubectl -n ate-system rollout status deploy/atenet-router kubectl -n ate-system rollout status deploy/atenet-egress ``` -- **Done:** both print `successfully rolled out`. -- **Stuck:** a new pod stays Pending, in ImagePullBackOff or CrashLoopBackOff; the old pod keeps serving. Check `kubectl -n ate-system get pods` and [roll back step 5](#roll-back) if you cannot fix it. -- **API clients:** a suspend or resume running when its API server pod stops can be cut off and left mid-call, so harnesses should retry API errors during this step. If a `kubectl ate` command fails once, run it again. A `kubectl port-forward` you opened yourself breaks when its pod goes. -- **Actors:** when the router restarts, an HTTP request that takes longer than about 10 seconds can be cut once, and new connections fail for a few seconds. Outbound connections through the egress gateway are reset once. +Then the `SandboxConfig` admission policy and the default `SandboxConfig`: -Then finish: +```bash +go run ./cmd/ate-setup deploy sandboxconfig --config ~/ate-upgrade/new.yaml +``` -1. Install the new `kubectl ate` with `go install ./cmd/kubectl-ate`. Actor owners can now use API fields and commands new in this release. -2. If the install set `ATE_OTLP_ENDPOINT`, restore it now. Suspend any PAUSED actors first. Add the `export` line to `settings.sh`, source it, and run `go run ./cmd/ate-setup deploy atenet`. This restarts atelet, so the step 3 warning applies, and replaces every worker again: wait as in step 4, until DRAINING is 0. If you roll back later, remove the line from `settings.sh` first. -3. [Recover CRASHED actors](#recover-crashed-actors). +> [!NOTE] +> TODO: The shape of the default SandboxConfig is not settled. Until each release ships it under its own name, this overwrites `gvisor-default` in place. -The upgrade is done when step 5 is done and no new actor stays CRASHED. Tell actor owners they can pause actors again. Keep `~/ate-upgrade` while you might still roll back, then delete it: `settings.sh` can hold passwords. +- **Done:** all rollout status print `successfully rolled out`, and `deploy sandboxconfig` exits 0. +- **Stuck:** the old pod keeps serving. Roll back step 4 if you cannot fix it. +- **API clients:** an API call can fail once with a retriable error while an API server pod restarts. Retry it. +- **Actors:** when the router restarts, an HTTP request that takes longer than about 10 seconds can be cut once. -> [!NOTE] -> TODO: podcertificate-controller is not part of the five steps. Say how to upgrade it when a release changes it. +Then install the new `kubectl ate` with `go install ./cmd/kubectl-ate`. Actor owners can now use API fields and commands new in this release. + +The upgrade is done when step 4 is done. Keep `~/ate-upgrade` while you might still roll back, then delete it. ## Something's wrong, or I need to roll back Stopping between steps is safe. -### Symptoms +### Pause a pool's roll + +If a pool's new workers look wrong but you are not ready to roll it back, or you want to stop interrupting actors for a while, pause its Deployment. ate-controller leaves this one Deployment change in place. -| You see | Meaning | Do | -|---|---|---| -| `waiting for daemonset/atelet in ate-system: ... (last status: 3/9 nodes updated)` | Only ate-setup's wait timed out. | `kubectl -n ate-system rollout status ds/atelet` | -| An atelet pod in ImagePullBackOff or CrashLoopBackOff | That node has no atelet, and step 3 stopped there. | [Roll back step 3](#roll-back). | -| ate-controller pod restarting after step 2 | Nothing manages the WorkerPools. | [Roll back step 2](#roll-back). | -| `exceeded its progress deadline` for a pool, or worker pods Pending or in ImagePullBackOff | Step 4 is slow or blocked. | See **Stuck, or normal?** in [step 4](#step-4-move-each-workerpool-to-the-new-worker-image). | -| HTTP 503 whose body mentions `ACTOR_STATE_CRASHED` | The actor crashed. | [Revert it](#recover-crashed-actors). | -| HTTP 503 `no free workers available` | During step 4 the pool is short by up to one batch. | Wait, or add capacity. If the actor is PAUSED, suspend it, then resume it. | -| HTTP 503 `another operation is in progress for this actor` | A suspend is running. | Wait. | -| HTTP 503 `actor / unavailable`, with no detail | ate-api-server restarting (step 5). | Retry. | -| HTTP 500 `error resuming actor /` | atelet restarting (step 3). | Retry. | -| InvalidArgument or Unimplemented from a `kubectl ate` command or API call | It is new in this release, and the old API server still serves. | Finish step 5 first. | -| After step 5, the atenet proxy changed or credentials are no longer injected | Install settings were missing from `settings.sh`. | Add them, source it, run `deploy atenet` again. | +```bash +kubectl -n $NS rollout pause deployment/$WORKERPOOL +kubectl -n $NS rollout resume deployment/$WORKERPOOL +``` -### Recover CRASHED actors +Workers already removed still get their 30 minutes. Do not leave a pool paused, and do not start step 4 while one is: a paused pool ignores every later change to its workers. -A CRASHED actor needs one revert. It returns the actor to SUSPENDED at its last snapshot, and the next call resumes it. Work since that snapshot is lost, and external volumes are not rewound. +### Roll back -This lists actors that became CRASHED since you started, each as `> `. An actor can show CRASHED for a moment while its suspend finishes, so run it again after a minute, and revert only actors that stay on the list. +To go back to the old release, roll back every step you began, finished or not, starting with the last. To fix one pool, roll back only that pool. Rollback runs from the old checkout, with `old.yaml`: ```bash -kubectl ate get actors -A | awk '$4=="ACTOR_STATE_CRASHED" {print $1, $2}' | diff ~/ate-upgrade/before-crashed.txt - +cd ~/ate-upgrade/old ``` -For each one: +#### Roll back step 4 + +Tell actor owners to stop using API fields new in this release, because the old API server rejects them. Then: ```bash -ATESPACE= -ACTOR= -kubectl ate revert actor $ACTOR -a $ATESPACE +go run ./cmd/ate-setup deploy sandboxconfig --config ~/ate-upgrade/old.yaml +go run ./cmd/ate-setup deploy atenet --config ~/ate-upgrade/old.yaml +go run ./cmd/ate-setup deploy apiserver --config ~/ate-upgrade/old.yaml +go install ./cmd/kubectl-ate ``` -A revert while a suspend is still finishing fails with `code = Aborted desc = another operation is in progress for this actor` and changes nothing; the actor turns SUSPENDED by itself. +The database schema stays new; the old API server drops fields it does not know when it rewrites a record. -After step 3, also list actors that a failed call left halfway, each as ` `. Actors in the middle of a normal call show up too, so run it again after a few minutes: +#### Roll back step 3 +Set each patched pool back to its IMAGE in `~/ate-upgrade/before-workerpools.txt`, with the exact string. ```bash -kubectl ate get actors -A | awk '$4 ~ /^ACTOR_STATE_(SUSPENDING|PAUSING|RESUMING)$/ {print $1, $2, $4}' +NS= +WORKERPOOL= +OLD_IMAGE= +kubectl -n $NS patch workerpool $WORKERPOOL --type merge -p "{\"spec\":{\"workerImage\":\"$OLD_IMAGE\"}}" ``` -For each actor still listed, run its call again: `kubectl ate suspend actor` for SUSPENDING, `pause actor` for PAUSING (then suspend it, because step 4 must not start with a paused actor), or `resume actor` for RESUMING, each with `$ACTOR -a $ATESPACE`. - -### Stop a worker roll partway - -To stop step 4 between pools, stop patching; unpatched pools do not roll. To stop a pool that is rolling, pause its Deployment. This is the one Deployment change ate-controller leaves in place. +If you paused a WorkerPool, resume it now with `kubectl -n $NS rollout resume deployment/$WORKERPOOL`. Then watch: ```bash -kubectl -n $NS rollout pause deployment/$POOL +kubectl -n $NS rollout status deployment/$WORKERPOOL ``` -No more old workers are removed. Workers already removed keep shutting down, and their actors still get the rest of their 30 minutes. The pool's size is not frozen: a change to its replicas, by you or an autoscaler, still adds or removes workers. While the pool is paused, rollout status never finishes; press Ctrl-C. To continue, run `kubectl -n $NS rollout resume deployment/$POOL` and watch as in step 4. +If a WorkerPool was partway through a roll, only workers already on the new image are replaced. Every new version worker is replaced again, with another round of `SIGTERM`. Wait as in [step 3](#step-3-move-each-workerpool-to-the-new-worker-image). -> [!WARNING] -> Do not leave a pool paused, and do not start step 5 while one is. A paused pool ignores every later change to its workers, including the next upgrade. The [pool list](#step-2-upgrade-ate-controller) shows it as ROLLOUT_PAUSED `true`. -### Roll back +#### Roll back step 2 -To go back to the old release, roll back every step you began, finished or not, starting with the last. To fix one pool, roll back only that pool. Rollback runs from the old checkout. Run this block now, and in every new shell during the rollback; the last line is for prebuilt images only: +**DO NOT** roll back step 2 before every pool is back on its old image and `kubectl get pods -A -l ate.dev/worker-pool` shows no Terminating pod. A new worker with an old atelet is not supported. ```bash -cd ~/ate-upgrade/old -source ~/ate-upgrade/settings.sh -export ATE_IMAGE_TAG= +go run ./cmd/ate-setup deploy atelet --config ~/ate-upgrade/old.yaml --rollout-timeout 1h ``` -Do not edit `settings.sh` for the rollback: going forward again needs the new tag. - -> [!WARNING] -> Do not roll back step 3 before every pool is back on its old image, with DRAINING 0. A new worker with an old atelet is not supported. +#### Roll back step 1 -- **Step 5:** tell actor owners to stop using API fields new in this release, because the old API server rejects them. Run `go run ./cmd/ate-setup deploy atenet`, then `go run ./cmd/ate-setup deploy apiserver`, then reinstall the old `kubectl ate` with `go install ./cmd/kubectl-ate`. The database schema stays new; the old API server drops fields it does not know when it rewrites a record. -- **Step 4:** suspend any PAUSED actors again. Then set each patched pool back to its IMAGE in `~/ate-upgrade/before-workerpools.txt`, with the exact string; another reference to the same image counts as a new image and replaces every worker. - - ```bash - NS= - POOL= - OLD_IMAGE= - kubectl -n $NS patch workerpool $POOL --type merge -p "{\"spec\":{\"workerImage\":\"$OLD_IMAGE\"}}" - ``` - - If you paused the pool, resume it now with `kubectl -n $NS rollout resume deployment/$POOL`. Then watch: - - ```bash - kubectl -n $NS rollout status deployment/$POOL - ``` +```bash +go run ./cmd/ate-setup deploy ate-controller --config ~/ate-upgrade/old.yaml +go run ./cmd/ate-setup deploy podcertificate-controller --config ~/ate-upgrade/old.yaml +``` - Partway through a roll, only workers already on the new image are replaced. After a finished roll, every worker is replaced again, with another round of `SIGTERM`. Wait as in step 4, until DRAINING is 0. If an actor turns CRASHED again right after each resume, the old workers cannot restore the snapshot a new worker wrote, and a revert does not help: move that pool forward to the new image again. If you put the new image in your WorkerPool files, put the old one back before turning GitOps sync on. -- **Step 3:** once every pool is back, run `go run ./cmd/ate-setup deploy atelet`. The step 3 warning applies. -- **Step 1 or 2:** run `go run ./cmd/ate-setup deploy ate-controller`. It restores the old CRDs, RBAC and controller. If the release changed the worker pod template, every pool rolls again: suspend any PAUSED actors first, and wait as in step 4. Fields the new release added to WorkerPools, SandboxConfigs or CSIDriverConfigs disappear; remove them from your manifests first, or `kubectl apply` fails. A value only the new release accepts, such as a new `sandboxClass`, stays but is not understood; change it first. If the new release added an API version to a CRD, you cannot roll back past step 1; reinstall. +These commands restore the old CRDs, RBAC, `ate-controller` and `podcertificate-controller`. If the release changed the worker pod template, every pool rolls again: wait as in step 3. Fields the new release added to WorkerPools, -> [!NOTE] -> TODO: The upgrade end-to-end test does not cover rollback. +SandboxConfigs or CSIDriverConfigs are dropped from those objects. Remove them from your YAML files too, or `kubectl apply` rejects the files. From bf064d3effeca47e02502fb54af7d42fc1614b96 Mon Sep 17 00:00:00 2001 From: Haven Xia Date: Tue, 29 Sep 2026 16:21:11 -0700 Subject: [PATCH 03/15] integrate with install setting design --- docs/upgrade.md | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/docs/upgrade.md b/docs/upgrade.md index 079a5f7661..086e1b5644 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -29,17 +29,19 @@ git clone --branch $NEW_RELEASE https://github.com/agent-substrate/substrate.git (cd old && go install ./cmd/kubectl-ate) ``` -Make two copies of the config file. -> TODO: #1887 defines the config file. +Every `ate-setup` run writes the settings it applied to a local settings file, in the format `--config` reads. Make two copies of the one from your install or your last `ate-setup` run: -Keep `old.yaml` as it is, for rollback. If you build from source, keep `new.yaml` the same: the checkout you run from decides the release. If you install prebuilt images, set the image tag (`ATE_IMAGE_TAG`) in `new.yaml` to the new release. +> [!NOTE] +> TODO: #1887 sets where `ate-setup` writes the settings file. ```bash -cp ~/ate-upgrade/old.yaml -cp ~/ate-upgrade/new.yaml +cp ~/ate-upgrade/old.yaml +cp ~/ate-upgrade/new.yaml ``` -You will the steps from the new checkout: +Keep `old.yaml` as it is, for rollback. If you build from source, keep `new.yaml` the same: the checkout you run from decides the release. If you install prebuilt images, set the image tag in `new.yaml` to the new release. + +You will run the steps from the new checkout: ```bash cd ~/ate-upgrade/new From 6fc4679714cebfad9d4538f301d4c5a81ea4b4b6 Mon Sep 17 00:00:00 2001 From: Haven Xia Date: Wed, 30 Sep 2026 12:15:35 -0700 Subject: [PATCH 04/15] update dependency TODOs --- docs/upgrade.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/docs/upgrade.md b/docs/upgrade.md index 086e1b5644..d031b2f2d9 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -2,7 +2,8 @@ This runbook upgrades a running Agent Substrate install to a newer release in the same release window, meaning the same `v0.x`: for example, from v0.2.0 to v0.2.1. Between windows, for example from v0.1.x to v0.2.x, reinstall instead. -> [TODO] Link to the Release Policy. +> [!NOTE] +> TODO: Depends on the policy doc. Link its release policy (which versions this runbook upgrades between) and its version skew policy (which component versions can run together during the upgrade). | Step | What changes | What running actors see | How long | @@ -32,7 +33,7 @@ git clone --branch $NEW_RELEASE https://github.com/agent-substrate/substrate.git Every `ate-setup` run writes the settings it applied to a local settings file, in the format `--config` reads. Make two copies of the one from your install or your last `ate-setup` run: > [!NOTE] -> TODO: #1887 sets where `ate-setup` writes the settings file. +> TODO: Depends on #1887, which sets where `ate-setup` writes the settings file and its format. ```bash cp ~/ate-upgrade/old.yaml @@ -72,9 +73,8 @@ kubectl get deploy -A -l ate.dev/worker-pool A pool is done rolling when READY shows `n/n`, and UP-TO-DATE and AVAILABLE both show `n`. Wait until every pool is done, then go to step 2. This can happen when this release changes the worker pod template. -> TODO: Define the API compatibility policy for workerPool. -> -> Proposal: within a release window, ate-controller does not change the pod template it renders, so this step replaces no workers. +> [!NOTE] +> TODO: Depends on the API compatibility policy in the policy doc: whether ate-controller may change the worker pod template it renders within a release window. Proposal: it does not, so this step replaces no workers. ### Step 2. Upgrade atelet @@ -107,7 +107,8 @@ If you build from source, build and push both worker images once. This touches n go run ./cmd/ate-setup publish worker-images --config ~/ate-upgrade/new.yaml | tee ~/ate-upgrade/worker-images.txt ``` -> TODO: Figure out whether OSS has prebuilt images published. +> [!NOTE] +> TODO: Depends on the release policy in the policy doc: whether open source releases publish prebuilt images, where, and under which tag. Then move the pools. For each pool: @@ -172,7 +173,7 @@ go run ./cmd/ate-setup deploy sandboxconfig --config ~/ate-upgrade/new.yaml ``` > [!NOTE] -> TODO: The shape of the default SandboxConfig is not settled. Until each release ships it under its own name, this overwrites `gvisor-default` in place. +> TODO: Depends on the shape of the default SandboxConfig. Until each release ships it under its own name, this overwrites `gvisor-default` in place. - **Done:** all rollout status print `successfully rolled out`, and `deploy sandboxconfig` exits 0. - **Stuck:** the old pod keeps serving. Roll back step 4 if you cannot fix it. From 4cee717f196a0639c6f0fbbbd70f7615aaf8492b Mon Sep 17 00:00:00 2001 From: Haven Xia Date: Thu, 1 Oct 2026 14:32:03 -0700 Subject: [PATCH 05/15] docs/upgrade: leave CSI drivers out of the upgrade --- docs/upgrade.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/upgrade.md b/docs/upgrade.md index d031b2f2d9..e28cfb47eb 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -15,7 +15,7 @@ This runbook upgrades a running Agent Substrate install to a newer release in th podcertificate-controller signs the certificates the other components use, and ate-controller manages the WorkerPools, so they go first, together with the CRDs. atelet and the workers go before ate-api-server and atenet, so that when the API server changes, every node already understands requests from either version. The SandboxConfig admission policy and the default SandboxConfig go last, once every worker can run what they allow. Rollback is the same list in reverse, from the old release. An actor that crashes along the way goes back to its last snapshot with one revert call. -The system upgrade does not change the database engine, Kubernetes version or the node OS version. +The system upgrade does not change the database engine, the CSI drivers `--setup-csi` installs, the Kubernetes version or the node OS version. ## Before you start @@ -49,7 +49,7 @@ cd ~/ate-upgrade/new ``` > [!CAUTION] -> Do not run `ate-setup deploy ate-system` or `ate-setup delete` during the upgrade. They change every component out of upgrade order. +> Do not run `ate-setup deploy ate-system`, `ate-setup setup csi` or `ate-setup delete` during the upgrade. They change components out of upgrade order. ## Upgrade From 0c216a3c95a2d78fa6d8fd674cce751e691cd020 Mon Sep 17 00:00:00 2001 From: Haven Xia Date: Tue, 6 Oct 2026 11:46:59 -0700 Subject: [PATCH 06/15] docs/upgrade: upgrade ate-controller after ate-api-server Upgrading ate-controller first did not save a worker roll. When a release changes the worker pod template, the controller step rolls every pool on its own. It also had a new ate-controller call an older ate-api-server, which rejects fields it does not know, and had an older ateom start with flags from a newer ate-controller. Move the CRDs and ate-controller into step 4, after ate-api-server. New workers then start with the flags the old ate-controller renders, and ate-controller only calls an ate-api-server at least as new as itself. Rollback runs the same order in reverse. The atunnel broker identity comments no longer refer to the runbook order. --- .../internal/controllers/workerpool_apply.go | 16 +++--- .../controllers/workerpool_apply_test.go | 8 +-- docs/upgrade.md | 57 ++++++++++--------- 3 files changed, 40 insertions(+), 41 deletions(-) diff --git a/cmd/atecontroller/internal/controllers/workerpool_apply.go b/cmd/atecontroller/internal/controllers/workerpool_apply.go index 1b990ef157..c282f71bdb 100644 --- a/cmd/atecontroller/internal/controllers/workerpool_apply.go +++ b/cmd/atecontroller/internal/controllers/workerpool_apply.go @@ -126,15 +126,13 @@ func buildDeploymentApplyConfig(wp *atev1alpha1.WorkerPool, otel ateomOTelSettin "--atunnel-client-identity=" + installdefaults.SPIFFEID(systemNamespace, routerServiceAccount), } - // --atunnel-broker-identity is newer than the oldest ateom a rolling - // upgrade still has running. docs/upgrade.md upgrades this controller - // before any pool's workerImage, so it reconciles Deployments still pinned - // to the old image, which exits on an unrecognized flag. An ateom without - // the flag hardcodes the canonical identity, and an ateom with it defaults - // to the same, so omitting the flag when it carries that value is - // equivalent for both and keeps the upgrade intact. A relocated or renamed - // install passes something else and needs an image new enough to accept - // it, which it necessarily has. + // --atunnel-broker-identity is newer than the oldest ateom a WorkerPool + // may still pin, and an ateom exits on an unrecognized flag. An ateom + // without the flag hardcodes the canonical identity, and an ateom with it + // defaults to the same, so omitting the flag when it carries that value is + // equivalent for both. A relocated or renamed install passes something + // else and needs an image new enough to accept it, which it necessarily + // has. if brokerIdentity := installdefaults.SPIFFEID(systemNamespace, ateletServiceAccount); brokerIdentity != installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace) { args = append(args, "--atunnel-broker-identity="+brokerIdentity) } diff --git a/cmd/atecontroller/internal/controllers/workerpool_apply_test.go b/cmd/atecontroller/internal/controllers/workerpool_apply_test.go index 51033e3d43..94bc08954e 100644 --- a/cmd/atecontroller/internal/controllers/workerpool_apply_test.go +++ b/cmd/atecontroller/internal/controllers/workerpool_apply_test.go @@ -999,11 +999,9 @@ func TestBuildDeploymentAtunnelIdentitiesPrefixedServiceAccounts(t *testing.T) { } // TestBuildDeploymentOmitsBrokerIdentityForCanonicalInstall pins the flag's -// absence, which is what keeps a rolling upgrade working. docs/upgrade.md -// upgrades this controller before any pool's workerImage, so it reconciles -// Deployments still pinned to the old image. An ateom from before -// --atunnel-broker-identity existed exits on the unrecognized flag, so passing -// it would crashloop every old worker the moment the control plane rolled out. +// absence. A WorkerPool may still pin an ateom from before +// --atunnel-broker-identity existed, which exits on the unrecognized flag, so +// passing it would crashloop that pool's workers. func TestBuildDeploymentOmitsBrokerIdentityForCanonicalInstall(t *testing.T) { c := buildDeploymentApplyConfig(testWorkerPoolApplyConfig(nil), ateomOTelSettings{}, installdefaults.SystemNamespace, installdefaults.AteletServiceAccount, installdefaults.RouterServiceAccount, egresspolicy.DNSEgressPolicyEnforceInAtunnel). diff --git a/docs/upgrade.md b/docs/upgrade.md index e28cfb47eb..86d70ea61d 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -8,12 +8,12 @@ This runbook upgrades a running Agent Substrate install to a newer release in th | Step | What changes | What running actors see | How long | |---|---|---|---| -| 1 | podcertificate-controller, CRDs and ate-controller | Usually nothing; see step 1 | A minute, or as long as step 3 if it replaces workers | +| 1 | podcertificate-controller | Nothing | A minute | | 2 | atelet, one node at a time | Nothing | Seconds to about 6 minutes per node | | 3 | workers | `SIGTERM`, then a 30-minute window to be suspended before `SIGKILL` | Minutes to hours per pool | -| 4 | ate-api-server, atenet, then SandboxConfig | API calls and long HTTP requests can be cut once | A few minutes | +| 4 | ate-api-server, the CRDs and ate-controller, atenet, then SandboxConfig | API calls and long HTTP requests can be cut once; see step 4 | A few minutes, or as long as step 3 if workers roll again | -podcertificate-controller signs the certificates the other components use, and ate-controller manages the WorkerPools, so they go first, together with the CRDs. atelet and the workers go before ate-api-server and atenet, so that when the API server changes, every node already understands requests from either version. The SandboxConfig admission policy and the default SandboxConfig go last, once every worker can run what they allow. Rollback is the same list in reverse, from the old release. An actor that crashes along the way goes back to its last snapshot with one revert call. +podcertificate-controller signs the certificates the other components use, so it goes first. atelet and the workers go before ate-api-server and atenet, so that when the API server changes, every node already understands requests from either version. ate-controller and the CRDs follow the API server: ate-controller is a client of the API server, and new workers start from the pod template the old ate-controller renders. The SandboxConfig admission policy and the default SandboxConfig go last, once every worker can run what they allow. Rollback is the same list in reverse, from the old release. An actor that crashes along the way goes back to its last snapshot with one revert call. The system upgrade does not change the database engine, the CSI drivers `--setup-csi` installs, the Kubernetes version or the node OS version. @@ -53,28 +53,16 @@ cd ~/ate-upgrade/new ## Upgrade -### Step 1. Upgrade podcertificate-controller, the CRDs and ate-controller +### Step 1. Upgrade podcertificate-controller ```bash go run ./cmd/ate-setup deploy podcertificate-controller --config ~/ate-upgrade/new.yaml -go run ./cmd/ate-setup deploy ate-controller --config ~/ate-upgrade/new.yaml -``` - -The first command deploys the new podcertificate-controller and waits until it publishes its trust bundles. The second applies the new CRDs (`WorkerPool`, `SandboxConfig` and `CSIDriverConfig`) and ate-controller's RBAC, waits until the CRDs are established, then deploys the new `ate-controller` and waits for it. Rerunning is safe. - -- **Done:** both commands exit 0, and a minute later `kubectl -n ate-system get pods -l app=ate-controller` still shows one pod, Running, with RESTARTS 0. -- **Stuck:** a command fails, or the pod keeps restarting. [Roll back step 1](#roll-back-step-1). - -A minute after the new pod starts, check whether it is replacing workers: - -```bash -kubectl get deploy -A -l ate.dev/worker-pool ``` -A pool is done rolling when READY shows `n/n`, and UP-TO-DATE and AVAILABLE both show `n`. Wait until every pool is done, then go to step 2. This can happen when this release changes the worker pod template. +This deploys the new podcertificate-controller and waits until it publishes its trust bundles. Rerunning is safe. -> [!NOTE] -> TODO: Depends on the API compatibility policy in the policy doc: whether ate-controller may change the worker pod template it renders within a release window. Proposal: it does not, so this step replaces no workers. +- **Done:** the command exits 0. +- **Stuck:** the command fails. [Roll back step 1](#roll-back-step-1). ### Step 2. Upgrade atelet @@ -158,6 +146,23 @@ kubectl -n ate-system rollout status deploy/ate-api-server kubectl ate get actors -A ``` +Then the CRDs and `ate-controller`: + +```bash +go run ./cmd/ate-setup deploy ate-controller --config ~/ate-upgrade/new.yaml +``` + +This applies the new CRDs (`WorkerPool`, `SandboxConfig` and `CSIDriverConfig`) and ate-controller's RBAC, waits until the CRDs are established, then deploys the new `ate-controller` and waits for it. Rerunning is safe. A minute after the new pod starts, check whether it is replacing workers: + +```bash +kubectl get deploy -A -l ate.dev/worker-pool +``` + +This happens when this release changes the worker pod template. A pool is done rolling when READY shows `n/n`, and UP-TO-DATE and AVAILABLE both show `n`. Wait until every pool is done before you go on. + +> [!NOTE] +> TODO: Depends on the version skew policy: whether a release may change the worker pod template within a major. + Then `atenet`: ```bash @@ -175,12 +180,12 @@ go run ./cmd/ate-setup deploy sandboxconfig --config ~/ate-upgrade/new.yaml > [!NOTE] > TODO: Depends on the shape of the default SandboxConfig. Until each release ships it under its own name, this overwrites `gvisor-default` in place. -- **Done:** all rollout status print `successfully rolled out`, and `deploy sandboxconfig` exits 0. -- **Stuck:** the old pod keeps serving. Roll back step 4 if you cannot fix it. +- **Done:** all rollout status print `successfully rolled out`, `deploy ate-controller` and `deploy sandboxconfig` exit 0, and `kubectl -n ate-system get pods -l app=ate-controller` shows one pod, Running, with RESTARTS 0. +- **Stuck:** an old pod keeps serving, or the new ate-controller pod keeps restarting. Roll back step 4 if you cannot fix it. - **API clients:** an API call can fail once with a retriable error while an API server pod restarts. Retry it. - **Actors:** when the router restarts, an HTTP request that takes longer than about 10 seconds can be cut once. -Then install the new `kubectl ate` with `go install ./cmd/kubectl-ate`. Actor owners can now use API fields and commands new in this release. +Then install the new `kubectl ate` with `go install ./cmd/kubectl-ate`. Actor owners can now use API fields and commands new in this release, and you can set WorkerPool, SandboxConfig and CSIDriverConfig fields new in this release. The upgrade is done when step 4 is done. Keep `~/ate-upgrade` while you might still roll back, then delete it. @@ -214,10 +219,13 @@ Tell actor owners to stop using API fields new in this release, because the old ```bash go run ./cmd/ate-setup deploy sandboxconfig --config ~/ate-upgrade/old.yaml go run ./cmd/ate-setup deploy atenet --config ~/ate-upgrade/old.yaml +go run ./cmd/ate-setup deploy ate-controller --config ~/ate-upgrade/old.yaml go run ./cmd/ate-setup deploy apiserver --config ~/ate-upgrade/old.yaml go install ./cmd/kubectl-ate ``` +`deploy ate-controller` restores the old CRDs, RBAC and `ate-controller`. If the release changed the worker pod template, every pool rolls again: wait as in step 3. Fields the new release added to WorkerPools, SandboxConfigs or CSIDriverConfigs are dropped from those objects. Remove them from your YAML files too, or `kubectl apply` rejects the files. + The database schema stays new; the old API server drops fields it does not know when it rewrites a record. #### Roll back step 3 @@ -250,10 +258,5 @@ go run ./cmd/ate-setup deploy atelet --config ~/ate-upgrade/old.yaml --rollout-t #### Roll back step 1 ```bash -go run ./cmd/ate-setup deploy ate-controller --config ~/ate-upgrade/old.yaml go run ./cmd/ate-setup deploy podcertificate-controller --config ~/ate-upgrade/old.yaml ``` - -These commands restore the old CRDs, RBAC, `ate-controller` and `podcertificate-controller`. If the release changed the worker pod template, every pool rolls again: wait as in step 3. Fields the new release added to WorkerPools, - -SandboxConfigs or CSIDriverConfigs are dropped from those objects. Remove them from your YAML files too, or `kubectl apply` rejects the files. From 0e746b1d2d0f8352dba2a7c89cf1d84d0d628aa7 Mon Sep 17 00:00:00 2001 From: Haven Xia Date: Tue, 6 Oct 2026 14:35:43 -0700 Subject: [PATCH 07/15] docs/upgrade: copy the install record ate-setup writes ate-setup now records the settings of each deploy run in its cache directory, in the format --config reads. Before you start says where to find that record, and the TODO that waited for it is gone. A later deploy run replaces the record, and it holds no database connection strings. So the step asks the operator to check the copy and to export the connection strings when the database is external. It also asks them to point kubectl at the cluster first, because every command in the runbook uses the current context. --- docs/upgrade.md | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/docs/upgrade.md b/docs/upgrade.md index 86d70ea61d..b72a36dffb 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -30,17 +30,24 @@ git clone --branch $NEW_RELEASE https://github.com/agent-substrate/substrate.git (cd old && go install ./cmd/kubectl-ate) ``` -Every `ate-setup` run writes the settings it applied to a local settings file, in the format `--config` reads. Make two copies of the one from your install or your last `ate-setup` run: +Point kubectl at the cluster you upgrade, and keep it there until the upgrade is done: -> [!NOTE] -> TODO: Depends on #1887, which sets where `ate-setup` writes the settings file and its format. +```bash +kubectl config use-context +``` + +Every `ate-setup deploy` run records the settings it was given in a file under `installs/` in the record directory `record.dir` names, and prints the path as `recorded this configuration in `. Make two copies of this cluster's record: ```bash -cp ~/ate-upgrade/old.yaml -cp ~/ate-upgrade/new.yaml +cp ~/ate-upgrade/old.yaml +cp ~/ate-upgrade/new.yaml ``` -Keep `old.yaml` as it is, for rollback. If you build from source, keep `new.yaml` the same: the checkout you run from decides the release. If you install prebuilt images, set the image tag in `new.yaml` to the new release. +Check that the copy holds the settings you installed with. If it is missing or wrong, write the file from those settings; [operator-install.md](operator-install.md) lists them. + +A record holds no database connection strings. If you use an external database, export `ATE_API_POSTGRES_READ_WRITE_CONNECTION_STRING`, and `ATE_API_POSTGRES_OWNER_CONNECTION_STRING` if you set it, in the shell you run the steps from. + +Keep `old.yaml` as it is, for rollback. If you build from source, keep `new.yaml` the same: the checkout you run from decides the release. If you install prebuilt images, set `images.tag` in `new.yaml` to the new release. You will run the steps from the new checkout: From 807ad3e8b58c8a417a5765dc104ff0b912700101 Mon Sep 17 00:00:00 2001 From: Haven Xia Date: Wed, 7 Oct 2026 10:30:05 -0700 Subject: [PATCH 08/15] docs/upgrade: drop the prebuilt-image TODO, make the rest precise Releases do not publish prebuilt images for now, so the runbook no longer waits on where they would live or which tag they would carry. The three TODOs left say what each waits for and what changes in its step once decided: the two policy docs to link, whether a compatible release may change the worker pod template, and per-release SandboxConfig names. --- docs/upgrade.md | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/docs/upgrade.md b/docs/upgrade.md index b72a36dffb..b723322452 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -3,14 +3,14 @@ This runbook upgrades a running Agent Substrate install to a newer release in the same release window, meaning the same `v0.x`: for example, from v0.2.0 to v0.2.1. Between windows, for example from v0.1.x to v0.2.x, reinstall instead. > [!NOTE] -> TODO: Depends on the policy doc. Link its release policy (which versions this runbook upgrades between) and its version skew policy (which component versions can run together during the upgrade). +> TODO: Depends on two policy docs that are not in the repository yet. Link the compatibility policy, which says which releases this runbook upgrades between, and the version skew policy, which says which component releases may run together during the upgrade and so sets the step order below. | Step | What changes | What running actors see | How long | |---|---|---|---| | 1 | podcertificate-controller | Nothing | A minute | | 2 | atelet, one node at a time | Nothing | Seconds to about 6 minutes per node | -| 3 | workers | `SIGTERM`, then a 30-minute window to be suspended before `SIGKILL` | Minutes to hours per pool | +| 3 | workers | `SIGTERM`, then a 30-minute window to be suspended before `SIGKILL` | Seconds to hours per pool | | 4 | ate-api-server, the CRDs and ate-controller, atenet, then SandboxConfig | API calls and long HTTP requests can be cut once; see step 4 | A few minutes, or as long as step 3 if workers roll again | podcertificate-controller signs the certificates the other components use, so it goes first. atelet and the workers go before ate-api-server and atenet, so that when the API server changes, every node already understands requests from either version. ate-controller and the CRDs follow the API server: ate-controller is a client of the API server, and new workers start from the pod template the old ate-controller renders. The SandboxConfig admission policy and the default SandboxConfig go last, once every worker can run what they allow. Rollback is the same list in reverse, from the old release. An actor that crashes along the way goes back to its last snapshot with one revert call. @@ -102,9 +102,6 @@ If you build from source, build and push both worker images once. This touches n go run ./cmd/ate-setup publish worker-images --config ~/ate-upgrade/new.yaml | tee ~/ate-upgrade/worker-images.txt ``` -> [!NOTE] -> TODO: Depends on the release policy in the policy doc: whether open source releases publish prebuilt images, where, and under which tag. - Then move the pools. For each pool: 1. Set the pool: @@ -168,7 +165,7 @@ kubectl get deploy -A -l ate.dev/worker-pool This happens when this release changes the worker pod template. A pool is done rolling when READY shows `n/n`, and UP-TO-DATE and AVAILABLE both show `n`. Wait until every pool is done before you go on. > [!NOTE] -> TODO: Depends on the version skew policy: whether a release may change the worker pod template within a major. +> TODO: Depends on the version skew policy: whether a compatible release may change the worker pod template. If it may not, the pools never roll in this step, and this check and the second roll in the rollback go away. Then `atenet`: @@ -185,7 +182,7 @@ go run ./cmd/ate-setup deploy sandboxconfig --config ~/ate-upgrade/new.yaml ``` > [!NOTE] -> TODO: Depends on the shape of the default SandboxConfig. Until each release ships it under its own name, this overwrites `gvisor-default` in place. +> TODO: Depends on per-release SandboxConfig names. Today this overwrites `gvisor-default` in place, so a release with new sandbox binaries breaks the restore of actors suspended under the old ones. Once each release ships its default SandboxConfig under its own name, this step adds the new one and leaves the old one in place. - **Done:** all rollout status print `successfully rolled out`, `deploy ate-controller` and `deploy sandboxconfig` exit 0, and `kubectl -n ate-system get pods -l app=ate-controller` shows one pod, Running, with RESTARTS 0. - **Stuck:** an old pod keeps serving, or the new ate-controller pod keeps restarting. Roll back step 4 if you cannot fix it. From 250d9f62632bf2717d96fd683f48430755b1210c Mon Sep 17 00:00:00 2001 From: Haven Xia Date: Wed, 7 Oct 2026 13:03:38 -0700 Subject: [PATCH 09/15] e2e test feedback A full upgrade on a GKE cluster showed places where the runbook does not match what the commands print or what happens to actors. ate-setup prints nothing while it waits for the atelet roll, so the kubectl rollout status that followed it only ever printed the final line. Run it in a second terminal instead. kubectl rollout status prints "X of N updated replicas are available" only after all N replicas are updated, so it cannot tell the operator that a pool's first new worker is up. Check the newest ReplicaSet's READY count instead. publish worker-images now prints the configuration report before the refs, so read NEW_IMAGE from its output with sed rather than by hand. Nothing suspends actors during the worker roll. A running demo actor that exited on SIGTERM crashed seconds into the window. Say so in step 3, and leave a TODO for the eviction section of the API guide. The cluster had no record of which release was installed. The install record now carries it as cluster.substrateVersion, so Before you start points there for OLD_RELEASE. WorkerPool now names its sandbox class in the first entry of spec.sandboxClasses. Step 3 reads CLASS from there, in the saved pool list and for each pool. --- docs/upgrade.md | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/docs/upgrade.md b/docs/upgrade.md index b723322452..4d40d5f9f8 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -19,7 +19,7 @@ The system upgrade does not change the database engine, the CSI drivers `--setup ## Before you start -Check out both releases. `ate-setup` runs from the new checkout. The old checkout is kept for rollback. +Check out both releases. `ate-setup` runs from the new checkout. The old checkout is kept for rollback. The installed release is `cluster.substrateVersion` in this cluster's install record, described below. ```bash OLD_RELEASE= @@ -75,6 +75,11 @@ This deploys the new podcertificate-controller and waits until it publishes its ```bash go run ./cmd/ate-setup deploy atelet --config ~/ate-upgrade/new.yaml --rollout-timeout 1h +``` + +`ate-setup` prints nothing until the roll is done. To watch it, open a second terminal and run: + +```bash kubectl -n ate-system rollout status ds/atelet ``` @@ -93,7 +98,7 @@ kubectl -n ate-system rollout status ds/atelet Before you patch any pool, list the pools and save the list for rollback. Each pool needs the new image for its sandbox CLASS: ```bash -kubectl get workerpools -A -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,CLASS:.spec.sandboxClass,IMAGE:.spec.workerImage' | tee ~/ate-upgrade/before-workerpools.txt +kubectl get workerpools -A -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,CLASS:.spec.sandboxClasses[0].name,IMAGE:.spec.workerImage' | tee ~/ate-upgrade/before-workerpools.txt ``` If you build from source, build and push both worker images once. This touches nothing in the cluster and ends by printing `ateom-gvisor: ` and `ateom-microvm: `: @@ -109,13 +114,13 @@ Then move the pools. For each pool: ```bash NS= WORKERPOOL= - CLASS=$(kubectl -n $NS get workerpool $WORKERPOOL -o jsonpath='{.spec.sandboxClass}') + CLASS=$(kubectl -n $NS get workerpool $WORKERPOOL -o jsonpath='{.spec.sandboxClasses[0].name}') ``` -2. Set its new image. From source, copy the ref after `ateom-$CLASS: ` in `~/ate-upgrade/worker-images.txt`: +2. Set its new image. From source, take the ref after `ateom-$CLASS: ` in `~/ate-upgrade/worker-images.txt`: ```bash - NEW_IMAGE= + NEW_IMAGE=$(sed -n "s/^ateom-$CLASS: //p" ~/ate-upgrade/worker-images.txt) ``` With prebuilt images, set `NEW_IMAGE` to `ateom-$CLASS` from your image repo at the new tag, pinned by digest (`crane digest ` prints it). @@ -129,7 +134,13 @@ Then move the pools. For each pool: kubectl -n $NS rollout status deployment/$WORKERPOOL ``` -5. Once the first pool of a CLASS reports `1 of N updated replicas are available`, patch the other pools of that CLASS without waiting. Then repeat on other CLASS. +5. Once the first pool of a CLASS has one new worker running, patch the other pools of that CLASS without waiting for the roll to finish. A new worker is running when the newest ReplicaSet of the pool shows READY 1 or more: + + ```bash + kubectl -n $NS get rs -l ate.dev/worker-pool=$WORKERPOOL + ``` + + Then repeat on the other CLASS. What to expect: @@ -137,7 +148,10 @@ What to expect: - **Done:** Step 3 is done when every pool is done rolling with `kubectl get deploy -A -l ate.dev/worker-pool`. Go to Step 4 once done. - **Pace:** a WorkerPool replaces 10% of its workers per batch. New workers start wherever the scheduler finds room. With spare capacity, every batch has its `SIGTERM` within minutes and the old workers drain in parallel; without it, new workers stay Pending until old ones exit, up to 30 minutes per batch, so a large pool can take hours. - **Stuck:** rollout status fails with `exceeded its progress deadline` (the roll goes on), or a worker pod stays Terminating for more than 60 minutes. Check `kubectl -n $NS get pods -l ate.dev/worker-pool=$WORKERPOOL`. A bad image stops the roll after the first batch: [roll back that pool](#roll-back-step-3). -- **Actors:** each gets `SIGTERM` and 30 minutes to be suspended, as described under eviction in the [API guide](api-guide.md). +- **Actors:** each gets `SIGTERM` and 30 minutes to be suspended. Substrate does not suspend them. An actor that is not suspended by then becomes `CRASHED` and loses its work since its last snapshot. + +> [!NOTE] +> TODO: link to the eviction section in the [API guide](api-guide.md), which will say how an actor is expected to get suspended within the 30 minutes. ### Step 4. Upgrade the Substrate control plane From b7734462e4ad6d0779220281dd3e51bf5782f889 Mon Sep 17 00:00:00 2001 From: Haven Xia Date: Fri, 9 Oct 2026 10:28:12 -0700 Subject: [PATCH 10/15] list which upgrades the runbook supports Operators need to know whether their upgrade path is supported before they start. The runbook used to say only that it works within one v0.x window. It now lists the cases: patch releases can be skipped, minor releases go one at a time, a new major needs a reinstall, and before 1.0 only patch releases of the same v0.x upgrade in place. It also says how far a rollback goes. The policies these rules come from are not in the repository yet, so the TODO now only waits on linking them. --- docs/upgrade.md | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/docs/upgrade.md b/docs/upgrade.md index 4d40d5f9f8..b9cbbc21b1 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -1,9 +1,18 @@ # Upgrade runbook -This runbook upgrades a running Agent Substrate install to a newer release in the same release window, meaning the same `v0.x`: for example, from v0.2.0 to v0.2.1. Between windows, for example from v0.1.x to v0.2.x, reinstall instead. +This runbook upgrades a running Agent Substrate install to a newer release. + +| From → to | Supported | +|---|---| +| A later patch release of the same minor, for example 1.2.1 → 1.2.4 | Yes. Patch releases can be skipped | +| The next minor release, for example 1.2.x → 1.3.y | Yes | +| Skipping a minor release, for example 1.2.x → 1.4.y | No. Upgrade to 1.3 first | +| A new major release, for example 1.x → 2.x | No. Reinstall | + +If an upgrade goes wrong, you can roll back to the release you upgraded from, and no further. > [!NOTE] -> TODO: Depends on two policy docs that are not in the repository yet. Link the compatibility policy, which says which releases this runbook upgrades between, and the version skew policy, which says which component releases may run together during the upgrade and so sets the step order below. +> TODO: Link the compatibility policy, which these rules come from, and the version skew policy, which sets the step order below, once both are in the repository. | Step | What changes | What running actors see | How long | From 243e3a873ae181feb48c997696b0ada624bebe61 Mon Sep 17 00:00:00 2001 From: Haven Xia Date: Fri, 9 Oct 2026 10:41:12 -0700 Subject: [PATCH 11/15] docs/upgrade: say how step 4 will handle per-release SandboxConfigs Each release will ship its default SandboxConfig under its own name, with the release version as a suffix, and leave the old ones in place. A WorkerPool without a configRef moves to the newest default. One with a configRef stays where it is. Every actor start resolves the sandbox from its pool's current config, so an actor suspended under binaries its config no longer offers cold boots and keeps only its files. A separate sandbox upgrade guide will cover pinning a pool and what an actor keeps. None of this is in a release yet: deploy sandboxconfig still overwrites gvisor-default. Keep the TODO and say what the step becomes. Say now that the step does not roll workers. That holds either way, because worker pods take nothing from a SandboxConfig. --- docs/upgrade.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/upgrade.md b/docs/upgrade.md index b9cbbc21b1..a794b96896 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -204,8 +204,10 @@ Then the `SandboxConfig` admission policy and the default `SandboxConfig`: go run ./cmd/ate-setup deploy sandboxconfig --config ~/ate-upgrade/new.yaml ``` +This does not roll workers. + > [!NOTE] -> TODO: Depends on per-release SandboxConfig names. Today this overwrites `gvisor-default` in place, so a release with new sandbox binaries breaks the restore of actors suspended under the old ones. Once each release ships its default SandboxConfig under its own name, this step adds the new one and leaves the old one in place. +> TODO: Today this step overwrites `gvisor-default` in place. Once each release ships its own SandboxConfig, it adds the new one, and WorkerPools without a `configRef` switch to it, so some suspended actors may cold boot. Link the sandbox upgrade guide. - **Done:** all rollout status print `successfully rolled out`, `deploy ate-controller` and `deploy sandboxconfig` exit 0, and `kubectl -n ate-system get pods -l app=ate-controller` shows one pod, Running, with RESTARTS 0. - **Stuck:** an old pod keeps serving, or the new ate-controller pod keeps restarting. Roll back step 4 if you cannot fix it. From 0bfc12e4d6b47bbc74eda6464f90cd707209d00e Mon Sep 17 00:00:00 2001 From: Haven Xia Date: Mon, 12 Oct 2026 09:53:01 -0700 Subject: [PATCH 12/15] docs/upgrade: link the API guide's eviction section The API guide now has an Eviction section that says what an actor needs to keep its state through a worker roll. Link it from step 3 and drop the TODO that waited for it. --- docs/upgrade.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/docs/upgrade.md b/docs/upgrade.md index a794b96896..f76231b524 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -157,10 +157,7 @@ What to expect: - **Done:** Step 3 is done when every pool is done rolling with `kubectl get deploy -A -l ate.dev/worker-pool`. Go to Step 4 once done. - **Pace:** a WorkerPool replaces 10% of its workers per batch. New workers start wherever the scheduler finds room. With spare capacity, every batch has its `SIGTERM` within minutes and the old workers drain in parallel; without it, new workers stay Pending until old ones exit, up to 30 minutes per batch, so a large pool can take hours. - **Stuck:** rollout status fails with `exceeded its progress deadline` (the roll goes on), or a worker pod stays Terminating for more than 60 minutes. Check `kubectl -n $NS get pods -l ate.dev/worker-pool=$WORKERPOOL`. A bad image stops the roll after the first batch: [roll back that pool](#roll-back-step-3). -- **Actors:** each gets `SIGTERM` and 30 minutes to be suspended. Substrate does not suspend them. An actor that is not suspended by then becomes `CRASHED` and loses its work since its last snapshot. - -> [!NOTE] -> TODO: link to the eviction section in the [API guide](api-guide.md), which will say how an actor is expected to get suspended within the 30 minutes. +- **Actors:** each gets `SIGTERM` and 30 minutes to be suspended. Substrate does not suspend them. An actor that is not suspended by then becomes `CRASHED` and loses its work since its last snapshot. [Eviction](api-guide.md#eviction) in the API guide says what an actor needs to keep its state. ### Step 4. Upgrade the Substrate control plane From 6eed7ed965dc4934444d7c462a88ca4c75ed70e3 Mon Sep 17 00:00:00 2001 From: Haven Xia Date: Mon, 12 Oct 2026 13:43:44 -0700 Subject: [PATCH 13/15] docs/upgrade: make the runbook easier to paste and to follow A review walked through the runbook the way an operator would. The commands had <...> placeholders that break when pasted into a shell. Put the three values an operator has to supply in one block with example values, read the installed release from the install record, and look up a pool's old image in the saved list instead of copying it out of a table. Say where the record is by default. Step 3 told the operator to watch rollout status, which blocks until the whole pool has rolled, and then to move on before the roll finishes. Drop the blocking watch from the per-pool steps and start with a small pool of each class. Say that a batch is 10% of the pool rounded down, and that a deploy atelet timeout does not stop the roll. Step 4 ran four deploys under one Done and Stuck. Give each its own, add the egress connections the atenet roll cuts, and say in the summary that workers can roll again. Add a Where am I section that lists each component's image, whose tag is its release in a prebuilt install, and remind the operator to tell actor owners before step 3. The atenet README no longer calls the router drain lossless, since a request still open about 15 seconds into the drain is cut. --- cmd/atenet/README.md | 5 +- docs/upgrade.md | 120 +++++++++++++++++++++++-------------------- 2 files changed, 68 insertions(+), 57 deletions(-) diff --git a/cmd/atenet/README.md b/cmd/atenet/README.md index 9cba1d9d14..8f17a3855c 100644 --- a/cmd/atenet/README.md +++ b/cmd/atenet/README.md @@ -39,8 +39,9 @@ likely be split in the future for better scalability.) requests finish instead of resetting. The whole sequence must fit within `terminationGracePeriodSeconds` (see the manifest comments). An upgrade rolls the router as an ordinary Deployment update (see - [docs/upgrade.md](../../docs/upgrade.md)); the drain is what makes each old - pod's termination lossless. + [docs/upgrade.md](../../docs/upgrade.md)). The drain lets each old pod's + requests finish, except one still open about 15 seconds after the drain + starts, which is cut. RBAC permissions: * get, list, watch on ate-system EndpointSlices diff --git a/docs/upgrade.md b/docs/upgrade.md index f76231b524..356a96917f 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -14,56 +14,59 @@ If an upgrade goes wrong, you can roll back to the release you upgraded from, an > [!NOTE] > TODO: Link the compatibility policy, which these rules come from, and the version skew policy, which sets the step order below, once both are in the repository. - | Step | What changes | What running actors see | How long | |---|---|---|---| | 1 | podcertificate-controller | Nothing | A minute | | 2 | atelet, one node at a time | Nothing | Seconds to about 6 minutes per node | | 3 | workers | `SIGTERM`, then a 30-minute window to be suspended before `SIGKILL` | Seconds to hours per pool | -| 4 | ate-api-server, the CRDs and ate-controller, atenet, then SandboxConfig | API calls and long HTTP requests can be cut once; see step 4 | A few minutes, or as long as step 3 if workers roll again | +| 4 | ate-api-server, the CRDs and ate-controller, atenet, then SandboxConfig | API calls, long requests and outbound connections can be cut once; workers may get `SIGTERM` again. See step 4 | A few minutes, or as long as step 3 if workers roll again | + +Why this order: + +- `podcertificate-controller` signs the certificates the other components use, so it goes first. +- `atelet` and the workers go before `ate-api-server` and `atenet`, so that when the API server changes, every node already understands requests from either version. +- `ate-controller` and the CRDs follow the API server: `ate-controller` is a client of the API server, and new workers start from the pod template the old `ate-controller` renders. +- The `SandboxConfig` admission policy and the default `SandboxConfig` go last, once every worker can run what they allow. -podcertificate-controller signs the certificates the other components use, so it goes first. atelet and the workers go before ate-api-server and atenet, so that when the API server changes, every node already understands requests from either version. ate-controller and the CRDs follow the API server: ate-controller is a client of the API server, and new workers start from the pod template the old ate-controller renders. The SandboxConfig admission policy and the default SandboxConfig go last, once every worker can run what they allow. Rollback is the same list in reverse, from the old release. An actor that crashes along the way goes back to its last snapshot with one revert call. +Rollback is the same list in reverse, from the old release. An actor that crashes along the way goes back to its last snapshot with one revert call. The system upgrade does not change the database engine, the CSI drivers `--setup-csi` installs, the Kubernetes version or the node OS version. -## Before you start +## Where am I? -Check out both releases. `ate-setup` runs from the new checkout. The old checkout is kept for rollback. The installed release is `cluster.substrateVersion` in this cluster's install record, described below. +Come back here after a break or a handoff. For a prebuilt install, each image's tag is its release. atelet moves in step 2, the pools in step 3, the rest in step 4: ```bash -OLD_RELEASE= -NEW_RELEASE= -mkdir ~/ate-upgrade && cd ~/ate-upgrade -git clone --branch $OLD_RELEASE https://github.com/agent-substrate/substrate.git old -git clone --branch $NEW_RELEASE https://github.com/agent-substrate/substrate.git new -(cd old && go install ./cmd/kubectl-ate) +kubectl -n ate-system get deploy,ds -o custom-columns='NAME:.metadata.name,IMAGE:.spec.template.spec.containers[0].image' +kubectl get workerpools -A -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,IMAGE:.spec.workerImage' ``` -Point kubectl at the cluster you upgrade, and keep it there until the upgrade is done: +## Before you start -```bash -kubectl config use-context -``` +Every `ate-setup deploy` run records the settings it was given in `installs/` under the record directory: `ATE_RECORD_DIR` if set, by default `~/.cache/ate-setup` on Linux and `~/Library/Caches/ate-setup` on macOS. It prints the path as `recorded this configuration in `. + +The record's `cluster.substrateVersion` is the installed release. If this machine has no record of the cluster, write one from the settings you installed with ([operator-install.md](operator-install.md) lists them) and set `OLD_RELEASE` to the installed release yourself. -Every `ate-setup deploy` run records the settings it was given in a file under `installs/` in the record directory `record.dir` names, and prints the path as `recorded this configuration in `. Make two copies of this cluster's record: +Fill in the first three lines, then run the rest. `ate-setup` runs from the new checkout. The old checkout is kept for rollback. ```bash -cp ~/ate-upgrade/old.yaml -cp ~/ate-upgrade/new.yaml -``` +NEW_RELEASE=v0.2.1 # change me +RECORD=~/.cache/ate-setup/installs/my-cluster.yaml # change me: this cluster's record +kubectl config use-context my-cluster # change me: keep it until the upgrade is done -Check that the copy holds the settings you installed with. If it is missing or wrong, write the file from those settings; [operator-install.md](operator-install.md) lists them. +OLD_RELEASE=$(sed -n 's/^ *substrateVersion: //p' $RECORD) +mkdir ~/ate-upgrade && cd ~/ate-upgrade +cp $RECORD old.yaml && cp $RECORD new.yaml +git clone --branch $OLD_RELEASE https://github.com/agent-substrate/substrate.git old +git clone --branch $NEW_RELEASE https://github.com/agent-substrate/substrate.git new +(cd old && go install ./cmd/kubectl-ate) +cd new +``` A record holds no database connection strings. If you use an external database, export `ATE_API_POSTGRES_READ_WRITE_CONNECTION_STRING`, and `ATE_API_POSTGRES_OWNER_CONNECTION_STRING` if you set it, in the shell you run the steps from. Keep `old.yaml` as it is, for rollback. If you build from source, keep `new.yaml` the same: the checkout you run from decides the release. If you install prebuilt images, set `images.tag` in `new.yaml` to the new release. -You will run the steps from the new checkout: - -```bash -cd ~/ate-upgrade/new -``` - > [!CAUTION] > Do not run `ate-setup deploy ate-system`, `ate-setup setup csi` or `ate-setup delete` during the upgrade. They change components out of upgrade order. @@ -94,7 +97,7 @@ kubectl -n ate-system rollout status ds/atelet - **Progress:** `Waiting for daemon set "atelet" rollout to finish: 2 out of 5 new pods have been updated...`, then `... 4 of 5 updated pods are available...`. - **Done:** `daemon set "atelet" successfully rolled out`. -- **Stuck:** the `N out of M` count does not move for more than about 6 minutes. A new atelet pod that cannot start stops the roll and leaves its node without atelet; if you cannot fix it quickly, [roll back step 2](#roll-back-step-2). A pod stuck in ContainerCreating usually means podcertificate-controller is not running. +- **Stuck:** the `N out of M` count does not move for more than about 6 minutes. A new atelet pod that cannot start stops the roll and leaves its node without atelet; if you cannot fix it quickly, [roll back step 2](#roll-back-step-2). A pod stuck in ContainerCreating usually means podcertificate-controller is not running. If `deploy atelet` times out, the roll still goes on; keep watching. - **Actors:** running actors keep running. A node's atelet is down for a few seconds while it restarts. ### Step 3. Move each WorkerPool to the new worker image @@ -102,6 +105,8 @@ kubectl -n ate-system rollout status ds/atelet > [!NOTE] > Do not start step 3 until step 2 is done: new workers need the new atelet. > +> Tell actor owners before you start: their running actors get `SIGTERM` in this step. +> > Change the WorkerPool, not its Deployment: `ate-controller` overwrites edits to the Deployment. Before you patch any pool, list the pools and save the list for rollback. Each pool needs the new image for its sandbox CLASS: @@ -121,8 +126,8 @@ Then move the pools. For each pool: 1. Set the pool: ```bash - NS= - WORKERPOOL= + NS=my-namespace # change me + WORKERPOOL=my-pool # change me CLASS=$(kubectl -n $NS get workerpool $WORKERPOOL -o jsonpath='{.spec.sandboxClasses[0].name}') ``` @@ -135,15 +140,14 @@ Then move the pools. For each pool: With prebuilt images, set `NEW_IMAGE` to `ateom-$CLASS` from your image repo at the new tag, pinned by digest (`crane digest ` prints it). 3. Check that `echo "$NS/$WORKERPOOL $CLASS $NEW_IMAGE"` shows `ateom-` plus the pool's CLASS and a full `@sha256:` digest. A bad image still removes the pool's first old worker. -4. Patch the pool, wait until ate-controller moves its Deployment to the new image, and watch the roll: +4. Patch the worker pool and wait until ate-controller moves its Deployment to the new image: ```bash kubectl -n $NS patch workerpool $WORKERPOOL --type merge -p "{\"spec\":{\"workerImage\":\"$NEW_IMAGE\"}}" kubectl -n $NS wait deployment/$WORKERPOOL --for=jsonpath='{.spec.template.spec.containers[0].image}'="$NEW_IMAGE" --timeout=2m - kubectl -n $NS rollout status deployment/$WORKERPOOL ``` -5. Once the first pool of a CLASS has one new worker running, patch the other pools of that CLASS without waiting for the roll to finish. A new worker is running when the newest ReplicaSet of the pool shows READY 1 or more: +5. Start with a small pool of each CLASS that you can afford to disrupt. Once it has one new worker running, patch the other pools of that CLASS without waiting for the roll to finish. A new worker is running when the newest ReplicaSet of the pool shows READY 1 or more: ```bash kubectl -n $NS get rs -l ate.dev/worker-pool=$WORKERPOOL @@ -153,15 +157,17 @@ Then move the pools. For each pool: What to expect: -- **Progress:** `kubectl rollout status` prints how many new workers are updated and available. Ctrl-C to safely stop the watch. +- **Progress:** `kubectl -n $NS rollout status deployment/$WORKERPOOL` prints how many new workers are updated and available. Ctrl-C stops only the watch. - **Done:** Step 3 is done when every pool is done rolling with `kubectl get deploy -A -l ate.dev/worker-pool`. Go to Step 4 once done. -- **Pace:** a WorkerPool replaces 10% of its workers per batch. New workers start wherever the scheduler finds room. With spare capacity, every batch has its `SIGTERM` within minutes and the old workers drain in parallel; without it, new workers stay Pending until old ones exit, up to 30 minutes per batch, so a large pool can take hours. +- **Pace:** a WorkerPool replaces 10% of its workers per batch, rounded down, at least one. With spare capacity, every batch has its `SIGTERM` within minutes and the old workers drain in parallel. Without it, each batch waits up to 30 minutes for old workers to exit. - **Stuck:** rollout status fails with `exceeded its progress deadline` (the roll goes on), or a worker pod stays Terminating for more than 60 minutes. Check `kubectl -n $NS get pods -l ate.dev/worker-pool=$WORKERPOOL`. A bad image stops the roll after the first batch: [roll back that pool](#roll-back-step-3). - **Actors:** each gets `SIGTERM` and 30 minutes to be suspended. Substrate does not suspend them. An actor that is not suspended by then becomes `CRASHED` and loses its work since its last snapshot. [Eviction](api-guide.md#eviction) in the API guide says what an actor needs to keep its state. ### Step 4. Upgrade the Substrate control plane -First the Substrate API server. The first new API server pod migrates the database schema. +#### 4a. ate-api-server + +The first new API server pod migrates the database schema. ```bash cd ~/ate-upgrade/new @@ -170,24 +176,26 @@ kubectl -n ate-system rollout status deploy/ate-api-server kubectl ate get actors -A ``` -Then the CRDs and `ate-controller`: - -```bash -go run ./cmd/ate-setup deploy ate-controller --config ~/ate-upgrade/new.yaml -``` +- **Done:** rollout status prints `successfully rolled out` and `kubectl ate get actors -A` lists your actors. +- **Stuck:** rollout status does not finish, or a new pod keeps restarting. [Roll back step 4](#roll-back-step-4). +- **API clients:** an API call can fail once with a retriable error while an API server pod restarts. Retry it. -This applies the new CRDs (`WorkerPool`, `SandboxConfig` and `CSIDriverConfig`) and ate-controller's RBAC, waits until the CRDs are established, then deploys the new `ate-controller` and waits for it. Rerunning is safe. A minute after the new pod starts, check whether it is replacing workers: +#### 4b. CRDs and ate-controller ```bash +go run ./cmd/ate-setup deploy ate-controller --config ~/ate-upgrade/new.yaml kubectl get deploy -A -l ate.dev/worker-pool ``` -This happens when this release changes the worker pod template. A pool is done rolling when READY shows `n/n`, and UP-TO-DATE and AVAILABLE both show `n`. Wait until every pool is done before you go on. +This applies the new CRDs and ate-controller's RBAC, then deploys the new `ate-controller`. Rerunning is safe. If this release changes the worker pod template, the new ate-controller rolls every pool again, with another round of `SIGTERM`: wait as in step 3. > [!NOTE] > TODO: Depends on the version skew policy: whether a compatible release may change the worker pod template. If it may not, the pools never roll in this step, and this check and the second roll in the rollback go away. -Then `atenet`: +- **Done:** the command exits 0, `kubectl -n ate-system get pods -l app=ate-controller` shows one pod, Running, with RESTARTS 0, and every pool shows READY `n/n` with UP-TO-DATE and AVAILABLE `n`. +- **Stuck:** the new ate-controller pod keeps restarting. [Roll back step 4](#roll-back-step-4). + +#### 4c. atenet ```bash go run ./cmd/ate-setup deploy atenet --config ~/ate-upgrade/new.yaml @@ -195,21 +203,23 @@ kubectl -n ate-system rollout status deploy/atenet-router kubectl -n ate-system rollout status deploy/atenet-egress ``` -Then the `SandboxConfig` admission policy and the default `SandboxConfig`: +- **Done:** both print `successfully rolled out`. +- **Stuck:** rollout status does not finish, or a new pod keeps restarting. [Roll back step 4](#roll-back-step-4). +- **Actors:** a request to an actor, or an outbound connection, still open about 15 seconds after its `atenet` pod starts draining can be cut once. + +#### 4d. SandboxConfig ```bash go run ./cmd/ate-setup deploy sandboxconfig --config ~/ate-upgrade/new.yaml ``` -This does not roll workers. +This applies the SandboxConfig admission policy and the default SandboxConfig. It does not roll workers. > [!NOTE] > TODO: Today this step overwrites `gvisor-default` in place. Once each release ships its own SandboxConfig, it adds the new one, and WorkerPools without a `configRef` switch to it, so some suspended actors may cold boot. Link the sandbox upgrade guide. -- **Done:** all rollout status print `successfully rolled out`, `deploy ate-controller` and `deploy sandboxconfig` exit 0, and `kubectl -n ate-system get pods -l app=ate-controller` shows one pod, Running, with RESTARTS 0. -- **Stuck:** an old pod keeps serving, or the new ate-controller pod keeps restarting. Roll back step 4 if you cannot fix it. -- **API clients:** an API call can fail once with a retriable error while an API server pod restarts. Retry it. -- **Actors:** when the router restarts, an HTTP request that takes longer than about 10 seconds can be cut once. +- **Done:** the command exits 0. +- **Stuck:** the command fails. [Roll back step 4](#roll-back-step-4). Then install the new `kubectl ate` with `go install ./cmd/kubectl-ate`. Actor owners can now use API fields and commands new in this release, and you can set WorkerPool, SandboxConfig and CSIDriverConfig fields new in this release. @@ -256,11 +266,12 @@ The database schema stays new; the old API server drops fields it does not know #### Roll back step 3 -Set each patched pool back to its IMAGE in `~/ate-upgrade/before-workerpools.txt`, with the exact string. +Set each patched pool back to its image in `~/ate-upgrade/before-workerpools.txt`: + ```bash -NS= -WORKERPOOL= -OLD_IMAGE= +NS=my-namespace # change me +WORKERPOOL=my-pool # change me +OLD_IMAGE=$(awk -v ns=$NS -v name=$WORKERPOOL '$1 == ns && $2 == name {print $4}' ~/ate-upgrade/before-workerpools.txt) kubectl -n $NS patch workerpool $WORKERPOOL --type merge -p "{\"spec\":{\"workerImage\":\"$OLD_IMAGE\"}}" ``` @@ -270,8 +281,7 @@ If you paused a WorkerPool, resume it now with `kubectl -n $NS rollout resume de kubectl -n $NS rollout status deployment/$WORKERPOOL ``` -If a WorkerPool was partway through a roll, only workers already on the new image are replaced. Every new version worker is replaced again, with another round of `SIGTERM`. Wait as in [step 3](#step-3-move-each-workerpool-to-the-new-worker-image). - +Only workers already on the new image are replaced, each with another round of rolling. Wait as in [step 3](#step-3-move-each-workerpool-to-the-new-worker-image). #### Roll back step 2 From 65937791da94e6db5eccac7ce112642395851788 Mon Sep 17 00:00:00 2001 From: Haven Xia Date: Tue, 13 Oct 2026 09:17:32 -0700 Subject: [PATCH 14/15] docs/upgrade: guard the setup block and fill gaps from review The setup block now keeps an OLD_RELEASE set by hand, prints both releases so an empty one is caught before cloning, and sets images.tag in new.yaml for prebuilt installs. That edit used to be a prose step after the block, and missing it left new workers on old atelets. Also: - Before 1.0, only patch releases of the same minor upgrade in place. - Back up the database before step 4a, since rollback keeps the new schema, and check that an external database's connection string is set in the shell. - Where am I? tells what has moved from ReplicaSet and ControllerRevision ages rather than image tags, which source builds do not carry. It covers podcertificate-controller and each pool's roll progress and paused state, and gives the upgrade a final check. - The prebuilt NEW_IMAGE is a command, the needed tools are listed, and the revert command is spelled out. - A Git or CI sync of WorkerPools can put the old worker image back. - Actor owners are told before the upgrade starts, not at step 3. - The TODOs are HTML comments, so operators do not read them. - "Why this order" is gone. It explains the version skew policy, which is where it belongs once that policy is in the repository. --- README.md | 2 +- docs/upgrade.md | 87 ++++++++++++++++++++++++++----------------------- 2 files changed, 48 insertions(+), 41 deletions(-) diff --git a/README.md b/README.md index 1f3c448a3c..6432c37fb3 100644 --- a/README.md +++ b/README.md @@ -250,7 +250,7 @@ We provide several sample applications demonstrating Agent Substrate's capabilit * [Egress Traffic](docs/egress-traffic.md): Which protocols an Actor may reach the outside world with and which are blocked. * [Enabling man-in-the-middle (MITM) interception for Actor Egress policy](docs/egress-trust-bundle.md): Egress policies such as header injection depend on MITM interception of Actor traffic. This guide explains how an Actor should be configured to enable interception. * [Request Parking](docs/request-parking.md): How the router parks requests through transient worker-pool saturation. -* [Upgrade Runbook](docs/upgrade.md): Upgrade a running install to a newer release in the same release window. +* [Upgrade Runbook](docs/upgrade.md): Upgrade a running install to a newer supported release, and roll back if needed. * [Security Guide](docs/security.md): Security overview, security principles, and threat modeling. * [Roadmap](docs/roadmap.md): Current limitations and what is planned next. * [Benchmarking Guide](benchmarking/README.md): Locust-based load tests, monitoring stack, and the orchestrated benchmark harness. diff --git a/docs/upgrade.md b/docs/upgrade.md index 356a96917f..e9d15a4737 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -8,11 +8,11 @@ This runbook upgrades a running Agent Substrate install to a newer release. | The next minor release, for example 1.2.x → 1.3.y | Yes | | Skipping a minor release, for example 1.2.x → 1.4.y | No. Upgrade to 1.3 first | | A new major release, for example 1.x → 2.x | No. Reinstall | +| A minor release before 1.0, for example 0.3.x → 0.4.y | No. Reinstall. Before 1.0, breakage still exists for minor versions | If an upgrade goes wrong, you can roll back to the release you upgraded from, and no further. -> [!NOTE] -> TODO: Link the compatibility policy, which these rules come from, and the version skew policy, which sets the step order below, once both are in the repository. + | Step | What changes | What running actors see | How long | |---|---|---|---| @@ -21,51 +21,57 @@ If an upgrade goes wrong, you can roll back to the release you upgraded from, an | 3 | workers | `SIGTERM`, then a 30-minute window to be suspended before `SIGKILL` | Seconds to hours per pool | | 4 | ate-api-server, the CRDs and ate-controller, atenet, then SandboxConfig | API calls, long requests and outbound connections can be cut once; workers may get `SIGTERM` again. See step 4 | A few minutes, or as long as step 3 if workers roll again | -Why this order: - -- `podcertificate-controller` signs the certificates the other components use, so it goes first. -- `atelet` and the workers go before `ate-api-server` and `atenet`, so that when the API server changes, every node already understands requests from either version. -- `ate-controller` and the CRDs follow the API server: `ate-controller` is a client of the API server, and new workers start from the pod template the old `ate-controller` renders. -- The `SandboxConfig` admission policy and the default `SandboxConfig` go last, once every worker can run what they allow. - -Rollback is the same list in reverse, from the old release. An actor that crashes along the way goes back to its last snapshot with one revert call. +Rollback is the same list in reverse, from the old release. An actor that crashes along the way goes back to its last snapshot with `kubectl ate revert actor -a `. The system upgrade does not change the database engine, the CSI drivers `--setup-csi` installs, the Kubernetes version or the node OS version. ## Where am I? -Come back here after a break or a handoff. For a prebuilt install, each image's tag is its release. atelet moves in step 2, the pools in step 3, the rest in step 4: +Come back here after a break or a handoff. Each time a Deployment moves to a new image it gets a new ReplicaSet, and the atelet DaemonSet a new ControllerRevision, so their `AGE` tells what has moved. A component has moved when the `AGE` of its ReplicaSet with pods (`DESIRED` above 0), or of atelet's ControllerRevision with the highest `REVISION`, is shorter than the time since the upgrade began. Two ReplicaSets with pods mean a roll is under way. `podcertificate-controller` moves in step 1, `atelet` in step 2, the rest in step 4: + +```bash +kubectl -n podcertificate-controller-system get rs +kubectl -n ate-system get rs,controllerrevision +``` + +The worker pools move in step 3, and their ReplicaSets read the same way. A pool is done rolling when `UP-TO-DATE` and `READY` equal `WANTED`: ```bash -kubectl -n ate-system get deploy,ds -o custom-columns='NAME:.metadata.name,IMAGE:.spec.template.spec.containers[0].image' -kubectl get workerpools -A -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,IMAGE:.spec.workerImage' +kubectl get rs -A -l ate.dev/worker-pool +kubectl get deploy -A -l ate.dev/worker-pool -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,UP-TO-DATE:.status.updatedReplicas,READY:.status.readyReplicas,WANTED:.spec.replicas,PAUSED:.spec.paused' ``` ## Before you start -Every `ate-setup deploy` run records the settings it was given in `installs/` under the record directory: `ATE_RECORD_DIR` if set, by default `~/.cache/ate-setup` on Linux and `~/Library/Caches/ate-setup` on macOS. It prints the path as `recorded this configuration in `. +You need git, go and kubectl, and crane for prebuilt images. Agree on a time with actor owners: their running actors get `SIGTERM` in step 3. -The record's `cluster.substrateVersion` is the installed release. If this machine has no record of the cluster, write one from the settings you installed with ([operator-install.md](operator-install.md) lists them) and set `OLD_RELEASE` to the installed release yourself. +Every `ate-setup deploy` run records the settings it was given in `installs/` under the record directory: `ATE_RECORD_DIR` if set, by default `~/.cache/ate-setup` on Linux and `~/Library/Caches/ate-setup` on macOS. It prints the path as `recorded this configuration in `. -Fill in the first three lines, then run the rest. `ate-setup` runs from the new checkout. The old checkout is kept for rollback. +The record's `cluster.substrateVersion` is the installed release. If this machine has no record of the cluster, write one from the settings you installed with ([operator-install.md](operator-install.md) lists them), and set `OLD_RELEASE` to the installed release before you run the block below. + +If `~/ate-upgrade` is left from an earlier upgrade, delete it first. Fill in the first three lines, then run the rest. `ate-setup` runs from the new checkout. The old checkout is kept for rollback. ```bash -NEW_RELEASE=v0.2.1 # change me +NEW_RELEASE=v0.4.2 # change me RECORD=~/.cache/ate-setup/installs/my-cluster.yaml # change me: this cluster's record kubectl config use-context my-cluster # change me: keep it until the upgrade is done +# export ATE_API_POSTGRES_READ_WRITE_CONNECTION_STRING=... # external database only -OLD_RELEASE=$(sed -n 's/^ *substrateVersion: //p' $RECORD) +OLD_RELEASE=${OLD_RELEASE:-$(sed -n 's/^ *substrateVersion: //p' $RECORD)} +echo "Upgrading '$OLD_RELEASE' -> '$NEW_RELEASE'" # stop if either is empty or not a release tag mkdir ~/ate-upgrade && cd ~/ate-upgrade cp $RECORD old.yaml && cp $RECORD new.yaml +sed -i.bak "s/^ tag: .*/ tag: $NEW_RELEASE/" new.yaml # prebuilt images: the new tag +grep 'tag:' old.yaml new.yaml # prebuilt images: new.yaml has the new tag git clone --branch $OLD_RELEASE https://github.com/agent-substrate/substrate.git old git clone --branch $NEW_RELEASE https://github.com/agent-substrate/substrate.git new (cd old && go install ./cmd/kubectl-ate) cd new ``` -A record holds no database connection strings. If you use an external database, export `ATE_API_POSTGRES_READ_WRITE_CONNECTION_STRING`, and `ATE_API_POSTGRES_OWNER_CONNECTION_STRING` if you set it, in the shell you run the steps from. +Run every step from this shell. A record holds no database connection strings: with an external database, uncomment the `export` line, and export `ATE_API_POSTGRES_OWNER_CONNECTION_STRING` too if you set it. -Keep `old.yaml` as it is, for rollback. If you build from source, keep `new.yaml` the same: the checkout you run from decides the release. If you install prebuilt images, set `images.tag` in `new.yaml` to the new release. +Keep `old.yaml` as it is, for rollback. With prebuilt images, `new.yaml` now names the new tag. From source, the checkout you run from decides the release. > [!CAUTION] > Do not run `ate-setup deploy ate-system`, `ate-setup setup csi` or `ate-setup delete` during the upgrade. They change components out of upgrade order. @@ -102,12 +108,10 @@ kubectl -n ate-system rollout status ds/atelet ### Step 3. Move each WorkerPool to the new worker image -> [!NOTE] +> [!IMPORTANT] > Do not start step 3 until step 2 is done: new workers need the new atelet. -> -> Tell actor owners before you start: their running actors get `SIGTERM` in this step. -> -> Change the WorkerPool, not its Deployment: `ate-controller` overwrites edits to the Deployment. + +Change the WorkerPool, not its Deployment: `ate-controller` overwrites edits to the Deployment. If your WorkerPools live in Git or are applied by CI, change `workerImage` there as well, or the next sync puts the old image back and rolls every worker again. Before you patch any pool, list the pools and save the list for rollback. Each pool needs the new image for its sandbox CLASS: @@ -121,7 +125,7 @@ If you build from source, build and push both worker images once. This touches n go run ./cmd/ate-setup publish worker-images --config ~/ate-upgrade/new.yaml | tee ~/ate-upgrade/worker-images.txt ``` -Then move the pools. For each pool: +Then move the pools, starting with a small pool of each CLASS that you can afford to disrupt. For each pool: 1. Set the pool: @@ -137,7 +141,12 @@ Then move the pools. For each pool: NEW_IMAGE=$(sed -n "s/^ateom-$CLASS: //p" ~/ate-upgrade/worker-images.txt) ``` - With prebuilt images, set `NEW_IMAGE` to `ateom-$CLASS` from your image repo at the new tag, pinned by digest (`crane digest ` prints it). + With prebuilt images, take `ateom-$CLASS` from your image repo at the new tag, pinned by digest: + + ```bash + NEW_IMAGE=$(sed -n 's/^ repo: //p' ~/ate-upgrade/new.yaml)/ateom-$CLASS:$NEW_RELEASE + NEW_IMAGE=$NEW_IMAGE@$(crane digest $NEW_IMAGE) + ``` 3. Check that `echo "$NS/$WORKERPOOL $CLASS $NEW_IMAGE"` shows `ateom-` plus the pool's CLASS and a full `@sha256:` digest. A bad image still removes the pool's first old worker. 4. Patch the worker pool and wait until ate-controller moves its Deployment to the new image: @@ -147,7 +156,7 @@ Then move the pools. For each pool: kubectl -n $NS wait deployment/$WORKERPOOL --for=jsonpath='{.spec.template.spec.containers[0].image}'="$NEW_IMAGE" --timeout=2m ``` -5. Start with a small pool of each CLASS that you can afford to disrupt. Once it has one new worker running, patch the other pools of that CLASS without waiting for the roll to finish. A new worker is running when the newest ReplicaSet of the pool shows READY 1 or more: +5. Once the first pool of a CLASS has one new worker running, patch the other pools of that CLASS without waiting for the roll to finish. A new worker is running when the newest ReplicaSet of the pool shows READY 1 or more: ```bash kubectl -n $NS get rs -l ate.dev/worker-pool=$WORKERPOOL @@ -160,17 +169,16 @@ What to expect: - **Progress:** `kubectl -n $NS rollout status deployment/$WORKERPOOL` prints how many new workers are updated and available. Ctrl-C stops only the watch. - **Done:** Step 3 is done when every pool is done rolling with `kubectl get deploy -A -l ate.dev/worker-pool`. Go to Step 4 once done. - **Pace:** a WorkerPool replaces 10% of its workers per batch, rounded down, at least one. With spare capacity, every batch has its `SIGTERM` within minutes and the old workers drain in parallel. Without it, each batch waits up to 30 minutes for old workers to exit. -- **Stuck:** rollout status fails with `exceeded its progress deadline` (the roll goes on), or a worker pod stays Terminating for more than 60 minutes. Check `kubectl -n $NS get pods -l ate.dev/worker-pool=$WORKERPOOL`. A bad image stops the roll after the first batch: [roll back that pool](#roll-back-step-3). +- **Stuck:** rollout status fails with `exceeded its progress deadline` (the roll goes on), or a worker pod stays Terminating for more than 60 minutes, its grace period (a worker normally exits within about 30). Check `kubectl -n $NS get pods -l ate.dev/worker-pool=$WORKERPOOL`. A bad image stops the roll after the first batch: [roll back that pool](#roll-back-step-3). - **Actors:** each gets `SIGTERM` and 30 minutes to be suspended. Substrate does not suspend them. An actor that is not suspended by then becomes `CRASHED` and loses its work since its last snapshot. [Eviction](api-guide.md#eviction) in the API guide says what an actor needs to keep its state. ### Step 4. Upgrade the Substrate control plane #### 4a. ate-api-server -The first new API server pod migrates the database schema. +The first new API server pod migrates the database schema, and rolling back does not undo that. Back up the database first: with `pg_dump` for the bundled one, or an on-demand backup for Cloud SQL. With an external database, check that `ATE_API_POSTGRES_READ_WRITE_CONNECTION_STRING` is set in this shell: without it, `deploy apiserver` points the API server at the bundled database. ```bash -cd ~/ate-upgrade/new go run ./cmd/ate-setup deploy apiserver --config ~/ate-upgrade/new.yaml kubectl -n ate-system rollout status deploy/ate-api-server kubectl ate get actors -A @@ -189,10 +197,9 @@ kubectl get deploy -A -l ate.dev/worker-pool This applies the new CRDs and ate-controller's RBAC, then deploys the new `ate-controller`. Rerunning is safe. If this release changes the worker pod template, the new ate-controller rolls every pool again, with another round of `SIGTERM`: wait as in step 3. -> [!NOTE] -> TODO: Depends on the version skew policy: whether a compatible release may change the worker pod template. If it may not, the pools never roll in this step, and this check and the second roll in the rollback go away. + -- **Done:** the command exits 0, `kubectl -n ate-system get pods -l app=ate-controller` shows one pod, Running, with RESTARTS 0, and every pool shows READY `n/n` with UP-TO-DATE and AVAILABLE `n`. +- **Done:** the command exits 0, `kubectl -n ate-system get pods -l app=ate-controller` shows one pod, Running, with RESTARTS 0, and every pool shows READY `n/n` with UP-TO-DATE and AVAILABLE `n`. Check the pools a minute after the new pod starts: a second roll begins only once it reconciles them. - **Stuck:** the new ate-controller pod keeps restarting. [Roll back step 4](#roll-back-step-4). #### 4c. atenet @@ -215,19 +222,18 @@ go run ./cmd/ate-setup deploy sandboxconfig --config ~/ate-upgrade/new.yaml This applies the SandboxConfig admission policy and the default SandboxConfig. It does not roll workers. -> [!NOTE] -> TODO: Today this step overwrites `gvisor-default` in place. Once each release ships its own SandboxConfig, it adds the new one, and WorkerPools without a `configRef` switch to it, so some suspended actors may cold boot. Link the sandbox upgrade guide. + - **Done:** the command exits 0. - **Stuck:** the command fails. [Roll back step 4](#roll-back-step-4). Then install the new `kubectl ate` with `go install ./cmd/kubectl-ate`. Actor owners can now use API fields and commands new in this release, and you can set WorkerPool, SandboxConfig and CSIDriverConfig fields new in this release. -The upgrade is done when step 4 is done. Keep `~/ate-upgrade` while you might still roll back, then delete it. +The upgrade is done when step 4 is done and [Where am I?](#where-am-i) shows every component and pool moved, every pool done rolling and none paused. Keep `~/ate-upgrade` while you might still roll back, then delete it. ## Something's wrong, or I need to roll back -Stopping between steps is safe. +Stopping between steps is safe, and so is stopping partway through step 3. A pool you have patched keeps rolling unless you pause it. ### Pause a pool's roll @@ -281,11 +287,12 @@ If you paused a WorkerPool, resume it now with `kubectl -n $NS rollout resume de kubectl -n $NS rollout status deployment/$WORKERPOOL ``` -Only workers already on the new image are replaced, each with another round of rolling. Wait as in [step 3](#step-3-move-each-workerpool-to-the-new-worker-image). +Only workers already on the new image are replaced, and their actors get `SIGTERM` again. Wait as in [step 3](#step-3-move-each-workerpool-to-the-new-worker-image). #### Roll back step 2 -**DO NOT** roll back step 2 before every pool is back on its old image and `kubectl get pods -A -l ate.dev/worker-pool` shows no Terminating pod. A new worker with an old atelet is not supported. +> [!WARNING] +> Do not roll back step 2 before every pool is back on its old image and `kubectl get pods -A -l ate.dev/worker-pool` shows no Terminating pod. A new worker with an old atelet is not supported. ```bash go run ./cmd/ate-setup deploy atelet --config ~/ate-upgrade/old.yaml --rollout-timeout 1h From fb342d1fc0d0ec156cf4e0bd54e4bf58dd35d0a4 Mon Sep 17 00:00:00 2001 From: Haven Xia Date: Wed, 14 Oct 2026 09:49:20 -0700 Subject: [PATCH 15/15] docs/upgrade: finish step 4 with deploy ate-system The component steps leave out what only the system bundle carries, such as the ate-system NetworkPolicies and the bundled PostgreSQL, so an upgrade never applied them. Step 4e runs deploy ate-system once every component is on the new release, with --setup-csi=none so the CSI drivers stay as they are. Rollback does not undo it. --- docs/upgrade.md | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/docs/upgrade.md b/docs/upgrade.md index e9d15a4737..4f9fdda840 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -19,7 +19,7 @@ If an upgrade goes wrong, you can roll back to the release you upgraded from, an | 1 | podcertificate-controller | Nothing | A minute | | 2 | atelet, one node at a time | Nothing | Seconds to about 6 minutes per node | | 3 | workers | `SIGTERM`, then a 30-minute window to be suspended before `SIGKILL` | Seconds to hours per pool | -| 4 | ate-api-server, the CRDs and ate-controller, atenet, then SandboxConfig | API calls, long requests and outbound connections can be cut once; workers may get `SIGTERM` again. See step 4 | A few minutes, or as long as step 3 if workers roll again | +| 4 | ate-api-server, the CRDs and ate-controller, atenet, SandboxConfig, then the rest of the system bundle | API calls, long requests and outbound connections can be cut once; workers may get `SIGTERM` again. See step 4 | A few minutes, or as long as step 3 if workers roll again | Rollback is the same list in reverse, from the old release. An actor that crashes along the way goes back to its last snapshot with `kubectl ate revert actor -a `. @@ -74,7 +74,7 @@ Run every step from this shell. A record holds no database connection strings: w Keep `old.yaml` as it is, for rollback. With prebuilt images, `new.yaml` now names the new tag. From source, the checkout you run from decides the release. > [!CAUTION] -> Do not run `ate-setup deploy ate-system`, `ate-setup setup csi` or `ate-setup delete` during the upgrade. They change components out of upgrade order. +> Do not run `ate-setup setup csi` or `ate-setup delete` during the upgrade, or `ate-setup deploy ate-system` before step 4e. They change components out of upgrade order. ## Upgrade @@ -227,6 +227,18 @@ This applies the SandboxConfig admission policy and the default SandboxConfig. I - **Done:** the command exits 0. - **Stuck:** the command fails. [Roll back step 4](#roll-back-step-4). +#### 4e. The rest of the system bundle + +```bash +go run ./cmd/ate-setup deploy ate-system --config ~/ate-upgrade/new.yaml --setup-csi=none +``` + +This applies what the steps above do not, such as the ate-system NetworkPolicies and the bundled PostgreSQL. The components above are already on the new release and do not roll again. `--setup-csi=none` leaves the CSI drivers as they are. Rerunning is safe. + +- **Done:** the command exits 0. +- **Stuck:** the command fails. [Roll back step 4](#roll-back-step-4). +- **API clients:** if the release changes the bundled PostgreSQL, it restarts once, and API calls fail until it is back. + Then install the new `kubectl ate` with `go install ./cmd/kubectl-ate`. Actor owners can now use API fields and commands new in this release, and you can set WorkerPool, SandboxConfig and CSIDriverConfig fields new in this release. The upgrade is done when step 4 is done and [Where am I?](#where-am-i) shows every component and pool moved, every pool done rolling and none paused. Keep `~/ate-upgrade` while you might still roll back, then delete it. @@ -268,7 +280,7 @@ go install ./cmd/kubectl-ate `deploy ate-controller` restores the old CRDs, RBAC and `ate-controller`. If the release changed the worker pod template, every pool rolls again: wait as in step 3. Fields the new release added to WorkerPools, SandboxConfigs or CSIDriverConfigs are dropped from those objects. Remove them from your YAML files too, or `kubectl apply` rejects the files. -The database schema stays new; the old API server drops fields it does not know when it rewrites a record. +The database schema stays new; the old API server drops fields it does not know when it rewrites a record. Step 4e is not undone either: the NetworkPolicies and the bundled PostgreSQL stay as the new release left them. #### Roll back step 3