From e04534e645a4b6bf63e4d3b31a9fb979fede5285 Mon Sep 17 00:00:00 2001 From: Satya Patel <14907857+saddlepaddle@users.noreply.github.com> Date: Wed, 18 Nov 2026 21:31:12 +0000 Subject: [PATCH 1/3] feat(sandbox): agent CLIs only from the agents drive claude, codex and pdev become drive-only asset rows like opencode and pi; install-agent-clis and the sandbox-agent-tools-drive flag are removed, so every create and wake mounts the drive of the box's region. Drive.get, the create and the wake's mounts update are retried on 429 and 5xx; after that the create or wake fails with "Agents are unavailable in right now; try again" (serverError.cloudWorkspace.agentsUnavailable) and a Sentry event with the region, bundle and drive. superset-boot links every agent the drive's manifest lists over whatever is on PATH, removes the link of one the drive lacks, never reads an older golden's disk copies, and refuses a boot that finds no manifest. SANDBOX_ROOT_DISK_AGENTS names where older bundles put the agents; the release's golden checks, the probe and boot-twice fail when one is on disk. Image (bun run image --local): 4.89 GB -> 4.10 GB. --- docs/cloud-sandbox-mismatches.md | 26 +- packages/i18n/locales/cs/messages.po | 3 + packages/i18n/locales/de/messages.po | 3 + packages/i18n/locales/en/messages.po | 3 + packages/i18n/locales/es/messages.po | 3 + packages/i18n/locales/fr/messages.po | 3 + packages/i18n/locales/id/messages.po | 3 + packages/i18n/locales/it/messages.po | 3 + packages/i18n/locales/ja/messages.po | 3 + packages/i18n/locales/ko/messages.po | 3 + packages/i18n/locales/nl/messages.po | 3 + packages/i18n/locales/pl/messages.po | 3 + packages/i18n/locales/pt-BR/messages.po | 3 + packages/i18n/locales/ru/messages.po | 3 + packages/i18n/locales/tr/messages.po | 3 + packages/i18n/locales/uk/messages.po | 3 + packages/i18n/locales/vi/messages.po | 3 + packages/i18n/locales/zh-CN/messages.po | 3 + packages/i18n/locales/zh-TW/messages.po | 3 + packages/i18n/src/server-errors.ts | 8 + packages/sandbox/README.md | 8 +- packages/sandbox/bundle/assets.json | 3 - .../bundle/rootfs/etc/profile.d/superset.sh | 5 +- .../bundle/rootfs/usr/local/bin/superset-boot | 26 +- packages/sandbox/bundle/steps.json | 7 +- .../bundle/steps/install-agent-clis.sh | 57 ----- packages/sandbox/src/agent-tools.test.ts | 7 +- packages/sandbox/src/agent-tools.ts | 7 +- packages/sandbox/src/assets/produce.ts | 3 - packages/sandbox/src/boot-twice.ts | 132 ++++++---- packages/sandbox/src/environments/probe.ts | 7 + packages/sandbox/src/environments/release.ts | 9 +- packages/sandbox/src/manifest.test.ts | 2 - packages/sandbox/src/real-sandbox.ts | 1 - packages/shared/src/cloud-agent-launch.ts | 6 +- packages/shared/src/constants.ts | 6 - packages/shared/src/sandbox/contract.ts | 15 ++ packages/trpc/src/lib/sandbox/claim.ts | 8 +- packages/trpc/src/lib/sandbox/index.ts | 1 + packages/trpc/src/lib/sandbox/vercel.test.ts | 236 ++++++++++++++---- packages/trpc/src/lib/sandbox/vercel.ts | 199 ++++++++++----- .../router/cloud-workspace/cloud-workspace.ts | 9 + .../src/router/cloud-workspace/provision.ts | 17 +- 43 files changed, 558 insertions(+), 301 deletions(-) delete mode 100755 packages/sandbox/bundle/steps/install-agent-clis.sh diff --git a/docs/cloud-sandbox-mismatches.md b/docs/cloud-sandbox-mismatches.md index 803c699ddca..e74635317ff 100644 --- a/docs/cloud-sandbox-mismatches.md +++ b/docs/cloud-sandbox-mismatches.md @@ -945,16 +945,22 @@ recorded against the API and a runtime failure against the sandbox, and that id is the only thing that joins the two halves of one broken workspace. **A drive mount lasts one session and a fork does not inherit it — open.** -For organizations with the `sandbox-agent-tools-drive` PostHog flag, the agent -CLIs come from the bundle's `agent-tools--` drive, -mounted read-only at `/opt/agents`. A machine someone owns keeps a mount; a -sandbox gets one only from `Sandbox.create`, `Sandbox.fork` or -`sandbox.update({ mounts })` before the session starts, so `provisionSandbox` -and `wakeSandbox` both pass it. A drive lives in one region and mounts only in -that region. A session without the mount leaves links in `/usr/local/bin` -that point nowhere; `superset-boot` re-runs `install-agent-clis` to put the -image's agents back. OpenCode and Pi exist only on the drive, so for them it -deletes the link instead. +The agent CLIs come only from the bundle's `agent-tools--` +drive, mounted read-only at `/opt/agents`; a new box's own disk has none, and +an older golden's disk copies are left alone and never used. A machine someone +owns keeps a mount; a sandbox gets one only from `Sandbox.create`, +`Sandbox.fork` or `sandbox.update({ mounts })` before the session starts, so +`provisionSandbox`, `wakeSandbox` and the restart in +`promoteSandboxToEnvironment` all pass it. A drive lives in one region and +mounts only there, so it is named by the box's own region, and the release +writes one in every region. A drive that cannot be mounted after a few tries +on 429 and 5xx fails the create or the wake with "Agents are unavailable in + right now; try again" and a Sentry event with the region and bundle; +a promote keeps its golden and leaves the source stopped. A boot that finds no +manifest at `/opt/agents` is refused before host-service starts. **Open:** a +running session whose host-service died is booted again without a remount, +since a mount cannot be added to a live session; if that session has no +drive, its boots are refused until the box is stopped and opened again. **An agent on the drive cannot update itself.** On a machine someone owns, OpenCode and Pi update in place, beside their binary. On a box they run from diff --git a/packages/i18n/locales/cs/messages.po b/packages/i18n/locales/cs/messages.po index 09933a46950..e3e29578dae 100644 --- a/packages/i18n/locales/cs/messages.po +++ b/packages/i18n/locales/cs/messages.po @@ -2416,6 +2416,9 @@ msgstr "Náhledy řízené agenty" msgid "Agents" msgstr "Agenti" +msgid "Agents are unavailable in {region} right now; try again" +msgstr "Agenti teď nejsou v {region} k dispozici; zkuste to znovu" + #. placeholder {0}: displayName(name) msgid "Agents cannot reach {0}'s tools" msgstr "Agenti nemají přístup k nástrojům {0}" diff --git a/packages/i18n/locales/de/messages.po b/packages/i18n/locales/de/messages.po index 6a48fd54ac7..fa5ad2a0ab0 100644 --- a/packages/i18n/locales/de/messages.po +++ b/packages/i18n/locales/de/messages.po @@ -2416,6 +2416,9 @@ msgstr "Vom Agenten gesteuerte Vorschauen" msgid "Agents" msgstr "Agenten" +msgid "Agents are unavailable in {region} right now; try again" +msgstr "Agenten sind in {region} gerade nicht verfügbar; versuche es erneut" + #. placeholder {0}: displayName(name) msgid "Agents cannot reach {0}'s tools" msgstr "Agenten können die Tools von {0} nicht erreichen" diff --git a/packages/i18n/locales/en/messages.po b/packages/i18n/locales/en/messages.po index 22835bd432c..2360a832d95 100644 --- a/packages/i18n/locales/en/messages.po +++ b/packages/i18n/locales/en/messages.po @@ -2416,6 +2416,9 @@ msgstr "Agent-driven previews" msgid "Agents" msgstr "Agents" +msgid "Agents are unavailable in {region} right now; try again" +msgstr "Agents are unavailable in {region} right now; try again" + #. placeholder {0}: displayName(name) msgid "Agents cannot reach {0}'s tools" msgstr "Agents cannot reach {0}'s tools" diff --git a/packages/i18n/locales/es/messages.po b/packages/i18n/locales/es/messages.po index 4bb6cf105a8..67b40d70fea 100644 --- a/packages/i18n/locales/es/messages.po +++ b/packages/i18n/locales/es/messages.po @@ -2416,6 +2416,9 @@ msgstr "Vistas previas controladas por el agente" msgid "Agents" msgstr "Agentes" +msgid "Agents are unavailable in {region} right now; try again" +msgstr "Los agentes no están disponibles en {region} en este momento; inténtalo de nuevo" + #. placeholder {0}: displayName(name) msgid "Agents cannot reach {0}'s tools" msgstr "Los agentes no pueden acceder a las herramientas de {0}" diff --git a/packages/i18n/locales/fr/messages.po b/packages/i18n/locales/fr/messages.po index 1f42b4bba60..ac72c2edc3c 100644 --- a/packages/i18n/locales/fr/messages.po +++ b/packages/i18n/locales/fr/messages.po @@ -2416,6 +2416,9 @@ msgstr "Aperçus pilotés par l'agent" msgid "Agents" msgstr "Agents" +msgid "Agents are unavailable in {region} right now; try again" +msgstr "Les agents ne sont pas disponibles dans {region} pour le moment ; réessayez" + #. placeholder {0}: displayName(name) msgid "Agents cannot reach {0}'s tools" msgstr "Les agents ne peuvent pas accéder aux outils de {0}" diff --git a/packages/i18n/locales/id/messages.po b/packages/i18n/locales/id/messages.po index 82c2d66b65d..18a30865de5 100644 --- a/packages/i18n/locales/id/messages.po +++ b/packages/i18n/locales/id/messages.po @@ -2416,6 +2416,9 @@ msgstr "Pratinjau yang digerakkan agen" msgid "Agents" msgstr "Agen" +msgid "Agents are unavailable in {region} right now; try again" +msgstr "Agen tidak tersedia di {region} saat ini; coba lagi" + #. placeholder {0}: displayName(name) msgid "Agents cannot reach {0}'s tools" msgstr "Agen tidak dapat menjangkau alat {0}" diff --git a/packages/i18n/locales/it/messages.po b/packages/i18n/locales/it/messages.po index 629470d66f5..a9af82615af 100644 --- a/packages/i18n/locales/it/messages.po +++ b/packages/i18n/locales/it/messages.po @@ -2416,6 +2416,9 @@ msgstr "Anteprime guidate dall'agente" msgid "Agents" msgstr "Agenti" +msgid "Agents are unavailable in {region} right now; try again" +msgstr "Gli agenti non sono disponibili in {region} al momento; riprova" + #. placeholder {0}: displayName(name) msgid "Agents cannot reach {0}'s tools" msgstr "Gli agenti non possono raggiungere gli strumenti di {0}" diff --git a/packages/i18n/locales/ja/messages.po b/packages/i18n/locales/ja/messages.po index 4b765ee5866..06f328edad8 100644 --- a/packages/i18n/locales/ja/messages.po +++ b/packages/i18n/locales/ja/messages.po @@ -2416,6 +2416,9 @@ msgstr "エージェントが操作するプレビュー" msgid "Agents" msgstr "エージェント" +msgid "Agents are unavailable in {region} right now; try again" +msgstr "現在 {region} ではエージェントを利用できません。もう一度お試しください" + #. placeholder {0}: displayName(name) msgid "Agents cannot reach {0}'s tools" msgstr "エージェントは {0} のツールにアクセスできません" diff --git a/packages/i18n/locales/ko/messages.po b/packages/i18n/locales/ko/messages.po index 062b60ed8a5..166419e7c02 100644 --- a/packages/i18n/locales/ko/messages.po +++ b/packages/i18n/locales/ko/messages.po @@ -2416,6 +2416,9 @@ msgstr "에이전트가 조작하는 미리보기" msgid "Agents" msgstr "에이전트" +msgid "Agents are unavailable in {region} right now; try again" +msgstr "지금은 {region}에서 에이전트를 사용할 수 없습니다. 다시 시도하세요" + #. placeholder {0}: displayName(name) msgid "Agents cannot reach {0}'s tools" msgstr "에이전트가 {0}의 도구에 접근할 수 없습니다" diff --git a/packages/i18n/locales/nl/messages.po b/packages/i18n/locales/nl/messages.po index e551554bd86..25d9a25289a 100644 --- a/packages/i18n/locales/nl/messages.po +++ b/packages/i18n/locales/nl/messages.po @@ -2416,6 +2416,9 @@ msgstr "Previews aangestuurd door agents" msgid "Agents" msgstr "Agents" +msgid "Agents are unavailable in {region} right now; try again" +msgstr "Agents zijn op dit moment niet beschikbaar in {region}; probeer het opnieuw" + #. placeholder {0}: displayName(name) msgid "Agents cannot reach {0}'s tools" msgstr "Agents kunnen de tools van {0} niet bereiken" diff --git a/packages/i18n/locales/pl/messages.po b/packages/i18n/locales/pl/messages.po index 411313bc257..e705e33d874 100644 --- a/packages/i18n/locales/pl/messages.po +++ b/packages/i18n/locales/pl/messages.po @@ -2416,6 +2416,9 @@ msgstr "Podglądy sterowane przez agenta" msgid "Agents" msgstr "Agenci" +msgid "Agents are unavailable in {region} right now; try again" +msgstr "Agenci są teraz niedostępni w {region}; spróbuj ponownie" + #. placeholder {0}: displayName(name) msgid "Agents cannot reach {0}'s tools" msgstr "Agenci nie mają dostępu do narzędzi {0}" diff --git a/packages/i18n/locales/pt-BR/messages.po b/packages/i18n/locales/pt-BR/messages.po index 80174655017..122cab0bdc5 100644 --- a/packages/i18n/locales/pt-BR/messages.po +++ b/packages/i18n/locales/pt-BR/messages.po @@ -2416,6 +2416,9 @@ msgstr "Prévias controladas pelo agente" msgid "Agents" msgstr "Agentes" +msgid "Agents are unavailable in {region} right now; try again" +msgstr "Os agentes não estão disponíveis em {region} no momento; tente novamente" + #. placeholder {0}: displayName(name) msgid "Agents cannot reach {0}'s tools" msgstr "Os agentes não conseguem acessar as ferramentas de {0}" diff --git a/packages/i18n/locales/ru/messages.po b/packages/i18n/locales/ru/messages.po index 814cd29b4b7..d11b86b5f6d 100644 --- a/packages/i18n/locales/ru/messages.po +++ b/packages/i18n/locales/ru/messages.po @@ -2416,6 +2416,9 @@ msgstr "Превью под управлением агента" msgid "Agents" msgstr "Агенты" +msgid "Agents are unavailable in {region} right now; try again" +msgstr "Агенты сейчас недоступны в {region}; попробуйте ещё раз" + #. placeholder {0}: displayName(name) msgid "Agents cannot reach {0}'s tools" msgstr "Агенты не могут получить доступ к инструментам {0}" diff --git a/packages/i18n/locales/tr/messages.po b/packages/i18n/locales/tr/messages.po index b8e7b7af79a..b26aa904bc5 100644 --- a/packages/i18n/locales/tr/messages.po +++ b/packages/i18n/locales/tr/messages.po @@ -2416,6 +2416,9 @@ msgstr "Ajanın yönettiği önizlemeler" msgid "Agents" msgstr "Ajanlar" +msgid "Agents are unavailable in {region} right now; try again" +msgstr "Ajanlar şu anda {region} bölgesinde kullanılamıyor; tekrar deneyin" + #. placeholder {0}: displayName(name) msgid "Agents cannot reach {0}'s tools" msgstr "Ajanlar {0} araçlarına erişemiyor" diff --git a/packages/i18n/locales/uk/messages.po b/packages/i18n/locales/uk/messages.po index e0cd327ba64..0b30037700a 100644 --- a/packages/i18n/locales/uk/messages.po +++ b/packages/i18n/locales/uk/messages.po @@ -2410,6 +2410,9 @@ msgstr "Попередній перегляд під керуванням аге msgid "Agents" msgstr "Агенти" +msgid "Agents are unavailable in {region} right now; try again" +msgstr "Агенти зараз недоступні в {region}; спробуйте ще раз" + #. placeholder {0}: displayName(name) msgid "Agents cannot reach {0}'s tools" msgstr "Агенти не мають доступу до інструментів {0}" diff --git a/packages/i18n/locales/vi/messages.po b/packages/i18n/locales/vi/messages.po index cd39fe440dd..504b561571f 100644 --- a/packages/i18n/locales/vi/messages.po +++ b/packages/i18n/locales/vi/messages.po @@ -2416,6 +2416,9 @@ msgstr "Bản xem trước do agent điều khiển" msgid "Agents" msgstr "Agent" +msgid "Agents are unavailable in {region} right now; try again" +msgstr "Hiện không thể dùng agent tại {region}; hãy thử lại" + #. placeholder {0}: displayName(name) msgid "Agents cannot reach {0}'s tools" msgstr "Agent không thể truy cập các công cụ của {0}" diff --git a/packages/i18n/locales/zh-CN/messages.po b/packages/i18n/locales/zh-CN/messages.po index ff17e6a7592..7179788de6a 100644 --- a/packages/i18n/locales/zh-CN/messages.po +++ b/packages/i18n/locales/zh-CN/messages.po @@ -2416,6 +2416,9 @@ msgstr "由智能体驱动的预览" msgid "Agents" msgstr "智能体" +msgid "Agents are unavailable in {region} right now; try again" +msgstr "{region} 目前无法使用智能体,请重试" + #. placeholder {0}: displayName(name) msgid "Agents cannot reach {0}'s tools" msgstr "智能体无法访问 {0} 的工具" diff --git a/packages/i18n/locales/zh-TW/messages.po b/packages/i18n/locales/zh-TW/messages.po index b750ae9f547..3f516b492fe 100644 --- a/packages/i18n/locales/zh-TW/messages.po +++ b/packages/i18n/locales/zh-TW/messages.po @@ -2416,6 +2416,9 @@ msgstr "由代理程式驅動的預覽" msgid "Agents" msgstr "代理程式" +msgid "Agents are unavailable in {region} right now; try again" +msgstr "{region} 目前無法使用代理程式,請再試一次" + #. placeholder {0}: displayName(name) msgid "Agents cannot reach {0}'s tools" msgstr "代理程式無法存取 {0} 的工具" diff --git a/packages/i18n/src/server-errors.ts b/packages/i18n/src/server-errors.ts index 2ef4684e458..f5b8c1d9b7e 100644 --- a/packages/i18n/src/server-errors.ts +++ b/packages/i18n/src/server-errors.ts @@ -419,6 +419,14 @@ export const serverErrorMessages: Record< message: "Only owners and admins can see usage by person.", }), ), + "serverError.cloudWorkspace.agentsUnavailable": (params) => { + const region = String(params?.region ?? ""); + return i18n._( + msg({ + message: `Agents are unavailable in ${region} right now; try again`, + }), + ); + }, "serverError.cloudWorkspace.cloudSandboxesAreInternalOnly": (params) => i18n._({ id: "serverError.cloudWorkspace.cloudSandboxesAreInternalOnly", diff --git a/packages/sandbox/README.md b/packages/sandbox/README.md index 7785deac5fb..95c8d742b0e 100644 --- a/packages/sandbox/README.md +++ b/packages/sandbox/README.md @@ -20,10 +20,10 @@ checklist is `plans/20260913-sandbox-implementation-checklist.md`. A step lists the archives it unpacks under `discards` in `steps.json`: the runner deletes them once the step succeeds, keeps the `.hash` sidecar so `sync-assets` does not fetch them again, and fetches them again only when - the step has to re-run. A file the box runs in place (`claude`, `superset`) + the step has to re-run. A file the box runs in place (`superset`) is never discarded. -- **An agent only the agents drive carries** (`opencode`, `pi`) is a row - with no `dest`. `assets.tsv` leaves it out, so neither the image nor a +- **An agent CLI** (`claude`, `codex`, `pdev`, `opencode`, `pi`) is a row + with no `dest`: only the agents drive carries it. `assets.tsv` leaves it out, so neither the image nor a boot fetches it; `bun run build --publish` still mirrors it to the bucket, and `bun run agent-tools` fetches it from there by sha. Its pin goes into the bundle's `drive-assets.tsv`, which no box reads, so a bump alone still @@ -55,7 +55,7 @@ The rule of thumb: if you can answer "where is it on the box", put it there. | `bun run image` | Builds the image: OS, apt lists, vendor tools, the `ubuntu` user, then installs the bundle at its sha and runs every pass once, so a fresh box finds every hash sidecar and step marker matching. A push logs Docker into the registry itself with a credential minted from `VERCEL_SANDBOX_TOKEN`, so there is no login to renew. `--local` builds `superset-sandbox:local` for the boot-twice test; `--dry` prints the Dockerfile. | | boot (`superset-boot`) | On the box, as root, run by the control plane on every create and wake, after it has written `sandbox.conf`, with the host secret in its env: clears `/run/superset`, installs the pinned bundle if the hash differs, runs `apply-rootfs` and the steps marked `beforeHost` in `steps.json` (each a hash compare), starts host-service as `ubuntu` with the secret from the boot env, writes `resources.env`, starts `superset-memguard` (ranks every process for the OOM killer and reaps the largest before the kernel has to). When `sandbox.conf` has `SUPERSET_SANDBOX_BOX_LOGS=1`, it also starts `superset-logship`, which sends the boot, memguard, desktop and host-service logs to Sentry Logs from a byte offset saved on disk. Then, detached: `sync-assets` and the other steps, then the desktop, with the checkout beside them. `checkout.ready` waits for both, and the stamp `steps.end` says the steps are done. Once host-service answers, the environment has been pushed and the checkout is ready, the runner asks host-service to run the repo's `start` hook with that environment. | | `bun run release` | Rewrites the host-service and CLI assets from this checkout and the claude, codex, Chrome, opencode and pi rows to their latest upstream release, publishes the bundle, pushes the image under a candidate tag `release-` (unless `--skip-image`), builds the internal golden from it with the environment's `setup` hook, probes a fork of it (`src/environments/probe.ts`) including a stop and wake, and only after the probe passes moves `latest` to the candidate and writes the environment rows. Every image environment resolves `superset-hostsvc` to `latest`, so a failed release changes no box. | -| `bun run agent-tools --bundle ` | Writes that bundle's agent CLIs to `agent-tools--`, a drive boxes mount read-only at `/opt/agents` when their organization has the `sandbox-agent-tools-drive` flag. A drive is written once and never again; it starts as a fork of the region's newest earlier drive, so only changed agents are fetched, and an agent that fails its download or `--version` check keeps the earlier drive's version or is left out. The drive carries claude, codex and pdev (also in the image) and opencode and pi (only on the drive). `bun run release` does this for every region, six at a time, before it records the bundle. It fails when a region with an environment has no drive, or a drive without claude, codex or pdev. Afterwards a `--production` release deletes the drives that neither an environment row (archived ones too) nor one of the 3 newest released bundles names; one Vercel project holds every database's drives, so no other release deletes any. | +| `bun run agent-tools --bundle ` | Writes that bundle's agent CLIs to `agent-tools--`, a drive every box mounts read-only at `/opt/agents`; a box has no other agent CLIs, so a create or wake whose drive cannot be mounted fails, and a boot without the drive's manifest is refused. A drive is written once and never again; it starts as a fork of the region's newest earlier drive, so only changed agents are fetched, and an agent that fails its download or `--version` check keeps the earlier drive's version or is left out. The drive carries claude, codex, pdev, opencode and pi. `bun run release` does this for every region, six at a time, before it records the bundle. It fails when a region with an environment has no drive, or a drive without claude, codex or pdev. Afterwards a `--production` release deletes the drives that neither an environment row (archived ones too) nor one of the 3 newest released bundles names; one Vercel project holds every database's drives, so no other release deletes any. | | `bun run src/real-sandbox.ts` | The Sandbox workflow dispatched with `real-sandbox`: one real sandbox from the registry image on this checkout's bundle and its agents drive in the default region, the same probe, a stop and a wake. | | Sandbox workflow, `production-release` | `bun run release --production` from `main` on a CI runner: the image, the internal golden and production's environment rows. Runs on a merge that touches the bundle and every night; a failed merge or nightly run opens an issue, or comments on the open one. | diff --git a/packages/sandbox/bundle/assets.json b/packages/sandbox/bundle/assets.json index f387dc1eca7..e5ef7eec26a 100644 --- a/packages/sandbox/bundle/assets.json +++ b/packages/sandbox/bundle/assets.json @@ -10,7 +10,6 @@ "claude": { "sha256": "1e08503dbdf3c2cb0d706d32f3408277388d1c76ef108673e8fe42c1b322925b", "suffix": "", - "dest": "/usr/local/share/superset/media/claude-2.1.280", "mode": "0755", "source": "https://storage.googleapis.com/claude-code-dist-86c565f3-f756-42ad-8dfa-d59b1c096819/claude-code-releases/2.1.280/linux-x64/claude", "version": "2.1.280" @@ -25,7 +24,6 @@ "codex": { "sha256": "a676f5722aae0d86cbe3764332f08eab1fdd89dc0a300b3ad4ffbf48611fa3e3", "suffix": ".tar.gz", - "dest": "/usr/local/share/superset/media/codex-0.162.1.tar.gz", "mode": "0644", "source": "https://github.com/openai/codex/releases/download/rust-v0.162.1/codex-package-x86_64-unknown-linux-musl.tar.gz", "version": "0.162.1" @@ -176,7 +174,6 @@ "pdev": { "sha256": "5a51852a4de0a40e20c0137b873f83e1b26255b4c5d01b6b0a2f608e152d8535", "suffix": ".tar.gz", - "dest": "/usr/local/share/superset/media/pdev-0.2.5.tar.gz", "mode": "0644", "source": "https://github.com/ParadevAI/pdev/releases/download/v0.2.5/pdev-0.2.5-linux-x64.tar.gz", "version": "0.2.5" diff --git a/packages/sandbox/bundle/rootfs/etc/profile.d/superset.sh b/packages/sandbox/bundle/rootfs/etc/profile.d/superset.sh index 6eae7dfec23..f6420c912f4 100644 --- a/packages/sandbox/bundle/rootfs/etc/profile.d/superset.sh +++ b/packages/sandbox/bundle/rootfs/etc/profile.d/superset.sh @@ -10,7 +10,6 @@ export DISPLAY="${DISPLAY:-${SUPERSET_DISPLAY:-:1}}" # The box's memory ceiling and CPUs, written by superset-boot from the cgroup. [ -r "${SUPERSET_RUN_DIR:-/run/superset}/resources.env" ] && . "${SUPERSET_RUN_DIR:-/run/superset}/resources.env" case ":$PATH:" in *":/usr/local/go/bin:"*) ;; *) export PATH="$PATH:/usr/local/go/bin" ;; esac -# The agent CLIs are pinned by the bundle (install-agent-clis) or run from the -# read-only agents drive; a self-update would drift from the pin and only -# prints a warning every session. +# The agent CLIs run from the read-only agents drive, pinned by the bundle; a +# self-update would drift from the pin and only prints a warning every session. export DISABLE_AUTOUPDATER=1 OPENCODE_DISABLE_AUTOUPDATE=1 PI_SKIP_VERSION_CHECK=1 diff --git a/packages/sandbox/bundle/rootfs/usr/local/bin/superset-boot b/packages/sandbox/bundle/rootfs/usr/local/bin/superset-boot index 3077db2b102..93ff127fbde 100755 --- a/packages/sandbox/bundle/rootfs/usr/local/bin/superset-boot +++ b/packages/sandbox/bundle/rootfs/usr/local/bin/superset-boot @@ -9,7 +9,7 @@ # install the pinned bundle only when its hash differs from current # apply-rootfs, and the steps each a hash compare per file and step; # marked beforeHost the only steps host-service waits for -# link the drive's agents only when the control plane mounted one +# link the drive's agents refused without the drive's manifest # start host-service as ubuntu first; everything else is a side effect # then, detached: sync-assets and the other steps, then the desktop; # the checkout beside them; checkout.ready and the start hook after both @@ -118,34 +118,32 @@ if [ -z "${SUPERSET_BOOT_REEXEC:-}" ] && [ "$(self_hash)" != "$BOOT_HASH_BEFORE" fi stamp "steps.host.start"; "$BUNDLE/setup" run-steps before-host >> "$SUPERSET_BOOT_LOG" 2>&1; stamp "steps.host.end" -# --- the agent CLIs from the region's agent-tools drive, when the control -# plane mounted one. Without it the image's agents stay on PATH. Mounts are -# per session, so a link into a mount this session lacks points nowhere. +# --- the agent CLIs, only from the region's agent-tools drive; the box has no +# other copy, so a boot without the drive's manifest is refused. Each listed +# agent's link is replaced, so an older bundle's link on PATH cannot win. Mounts +# are per session, so a link into a mount this session lacks points nowhere. link_agents() { local dir="${SUPERSET_AGENTS_DIR:-/opt/agents}" linked=0 name sha bin target local manifest="${dir}/manifest.json" - if find /usr/local/bin -maxdepth 1 -lname "${dir}/*" ! -exec test -e {} \; -print -quit | grep -q .; then - SUPERSET_BUNDLE_DIR="$BUNDLE" bash "$BUNDLE/steps/install-agent-clis.sh" >> "$SUPERSET_BOOT_LOG" 2>&1 || - stamp "agents.restore.failed" - # What is still dangling has no image copy to restore: opencode, pi. - find /usr/local/bin -maxdepth 1 -lname "${dir}/*" ! -exec test -e {} \; -delete - fi + find /usr/local/bin -maxdepth 1 -lname "${dir}/*" ! -exec test -e {} \; -delete if ! jq -e 'type == "array"' "$manifest" > /dev/null 2>&1; then stamp "agents.unavailable no manifest at ${dir}" - return 0 + refuse "no agents drive at ${dir}; the control plane mounts it on every create and wake" fi while IFS=$'\t' read -r name sha bin; do if ! [[ "$name" =~ ^[a-z0-9-]+$ && "$sha" =~ ^[0-9a-f]{64}$ && "$bin" =~ ^[A-Za-z0-9._/-]+$ && "$bin" != *..* ]]; then stamp "agents.skipped invalid entry ${name:0:32}"; continue fi target="${dir}/${name}/${sha}/${bin}" - [ -x "$target" ] || { stamp "agents.missing ${name} ${sha:0:12}"; continue; } + if [ ! -x "$target" ]; then + rm -f "/usr/local/bin/${name}" + stamp "agents.missing ${name} ${sha:0:12}"; continue + fi ln -sfn "$target" "/usr/local/bin/${name}.tmp" && mv -T "/usr/local/bin/${name}.tmp" "/usr/local/bin/${name}" && linked=$((linked + 1)) done < <(jq -r '.[] | [.name, .sha, .bin] | map(tostring) | @tsv' "$manifest") stamp "agents.linked ${linked} from ${dir}" } -drive_links() { find /usr/local/bin -maxdepth 1 -lname "${SUPERSET_AGENTS_DIR:-/opt/agents}/*" -printf '%f %l\n' | sort; } link_agents # --- host-service, as the sandbox user, first. A runtime that cannot start @@ -342,10 +340,8 @@ checkout() { checkout_pid=$! exec 8> /run/superset-steps.lock flock 8 - links_before="$(drive_links)" stamp "assets.start"; "$BUNDLE/setup" sync-assets >> "$SUPERSET_BOOT_LOG" 2>&1 || stamp "assets.failed continuing"; stamp "assets.end" stamp "steps.start"; "$BUNDLE/setup" run-steps after-host >> "$SUPERSET_BOOT_LOG" 2>&1; stamp "steps.end" - [ "$(drive_links)" = "$links_before" ] || link_agents exec 8>&- as_user env "DISPLAY=${SUPERSET_DISPLAY}" "HOME=${SUPERSET_USER_HOME}" "USER=${SUPERSET_USER}" \ "SUPERSET_SANDBOX_WORKSPACE_ID=${SUPERSET_SANDBOX_WORKSPACE_ID}" \ diff --git a/packages/sandbox/bundle/steps.json b/packages/sandbox/bundle/steps.json index 40709229ab4..a0814d24f95 100644 --- a/packages/sandbox/bundle/steps.json +++ b/packages/sandbox/bundle/steps.json @@ -38,10 +38,5 @@ "discards": ["host-service"], "beforeHost": true }, - { "name": "install-cli", "assets": ["cli"] }, - { - "name": "install-agent-clis", - "assets": ["claude", "codex", "pdev"], - "discards": ["codex", "pdev"] - } + { "name": "install-cli", "assets": ["cli"] } ] diff --git a/packages/sandbox/bundle/steps/install-agent-clis.sh b/packages/sandbox/bundle/steps/install-agent-clis.sh deleted file mode 100755 index aca10a3b16e..00000000000 --- a/packages/sandbox/bundle/steps/install-agent-clis.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env bash -# Puts the pinned `claude`, `codex` and `pdev` on PATH. Each is a row in -# assets.json, so a version bump is a row change and every box picks it up on -# its next boot. Codex unpacks into /opt/codex//: its binary finds the -# bundled rg, bwrap and code-mode host through the codex-package.json beside -# it. pdev unpacks the same way, into /opt/pdev//. -set -euo pipefail -. /etc/superset/contract.sh - -staged() { - local pattern="$1" found="" - for candidate in "${SUPERSET_MEDIA_DIR}"/${pattern}; do - [ -f "${candidate}.hash" ] || continue - if grep -qF "$(tr -d '\n\r' < "${candidate}.hash")" "${SUPERSET_BUNDLE_DIR}/assets.tsv"; then - found="$candidate" - fi - done - [ -n "$found" ] || { echo "no ${pattern} of this bundle under ${SUPERSET_MEDIA_DIR}" >&2; exit 1; } - printf '%s' "$found" -} - -link() { - ln -sfn "$2" "$1.tmp" - mv -T "$1.tmp" "$1" -} - -# Images before this step installed both with npm; its links and packages go. -rm -rf /usr/local/lib/node_modules/@anthropic-ai/claude-code /usr/local/lib/node_modules/@openai/codex - -claude="$(staged 'claude-*')" -chmod 0755 "$claude" -link /usr/local/bin/claude "$claude" - -unpack() { - local name="$1" bin="$2" archive sha target dir - archive="$(staged "${name}-*.tar.gz")" - sha="$(tr -d '\n\r' < "${archive}.hash")" - target="/opt/${name}/${sha}" - if [ ! -x "${target}/${bin}" ]; then - rm -rf "$target" "${target}.partial" - mkdir -p "${target}.partial" - tar -xzf "$archive" -C "${target}.partial" - mv -T "${target}.partial" "$target" - fi - link "/usr/local/bin/${name}" "${target}/${bin}" - for dir in "/opt/${name}"/*/; do - dir="${dir%/}" - [ "$dir" = "$target" ] || rm -rf "$dir" - done -} - -unpack codex bin/codex -unpack pdev pdev/pdev - -claude --version -codex --version -pdev --version diff --git a/packages/sandbox/src/agent-tools.test.ts b/packages/sandbox/src/agent-tools.test.ts index cb8744da80d..2b9f2349b29 100644 --- a/packages/sandbox/src/agent-tools.test.ts +++ b/packages/sandbox/src/agent-tools.test.ts @@ -9,6 +9,7 @@ import { import { tmpdir } from "node:os"; import { join } from "node:path"; import { + AGENT_TOOLS, type AgentToolsManifest, agentToolsDrivesToPrune, agentToolsGaps, @@ -55,7 +56,7 @@ describe("agent-tools manifest", () => { expect(parseAgentToolsManifest(JSON.stringify(manifest))).toEqual(manifest); }); - test("the committed bundle has a row for every agent, and opencode and pi only on the drive", () => { + test("the committed bundle has a row for every agent, each only on the drive", () => { const assets = loadAssets( join(import.meta.dir, "..", "bundle", "assets.json"), ); @@ -66,9 +67,9 @@ describe("agent-tools manifest", () => { "opencode", "pi", ]); - for (const name of ["opencode", "pi"]) { + for (const { name } of AGENT_TOOLS) { expect(assets[name]?.dest).toBeUndefined(); - expect(assets[name]?.source).toStartWith("https://github.com/"); + expect(assets[name]?.source).toStartWith("https://"); } }); diff --git a/packages/sandbox/src/agent-tools.ts b/packages/sandbox/src/agent-tools.ts index f289e3143a0..c7e72a04438 100644 --- a/packages/sandbox/src/agent-tools.ts +++ b/packages/sandbox/src/agent-tools.ts @@ -13,8 +13,7 @@ * * A new drive starts as a fork of the region's newest earlier one, so the * builder fetches only the agents whose sha changed. Each agent fails alone: - * it keeps the earlier drive's version, or is left out and boxes use the - * image's copy. + * it keeps the earlier drive's version, or is left out and boxes have none. * * bun run agent-tools --bundle [--region sfo1 ...] * @@ -411,7 +410,7 @@ export function describeAgentToolsPublish( return results.flatMap((entry) => { if ("error" in entry) { return [ - `::warning::agent-tools: ${entry.region} not published; its boxes keep the image's agents: ${String(entry.error).slice(0, 300)}`, + `::warning::agent-tools: ${entry.region} not published; its boxes have no agents: ${String(entry.error).slice(0, 300)}`, ]; } const { drive, outcome, forkedFrom, agents } = entry.result; @@ -427,7 +426,7 @@ export function describeAgentToolsPublish( ] : agent.outcome === "dropped" ? [ - `::warning::agent-tools: ${drive}: ${agent.name} left out; its boxes keep the image's ${agent.name}, if the image has one`, + `::warning::agent-tools: ${drive}: ${agent.name} left out; its boxes have no ${agent.name}`, ] : [], ), diff --git a/packages/sandbox/src/assets/produce.ts b/packages/sandbox/src/assets/produce.ts index fc17f99335f..7a3e65b85a0 100644 --- a/packages/sandbox/src/assets/produce.ts +++ b/packages/sandbox/src/assets/produce.ts @@ -466,7 +466,6 @@ async function claude(version?: string): Promise { claude: { sha256: hash, suffix: "", - dest: `${SANDBOX_PATHS.media}/claude-${pinned}`, mode: "0755", source, version: pinned, @@ -500,7 +499,6 @@ async function codex(version?: string): Promise { codex: { sha256: hash, suffix: ".tar.gz", - dest: `${SANDBOX_PATHS.media}/codex-${pinned}.tar.gz`, mode: "0644", source: asset.browser_download_url, version: pinned, @@ -534,7 +532,6 @@ async function pdev(version?: string): Promise { pdev: { sha256: hash, suffix: ".tar.gz", - dest: `${SANDBOX_PATHS.media}/pdev-${pinned}.tar.gz`, mode: "0644", source: asset.browser_download_url, version: pinned, diff --git a/packages/sandbox/src/boot-twice.ts b/packages/sandbox/src/boot-twice.ts index 64b37c49a05..a6e21c40d7e 100644 --- a/packages/sandbox/src/boot-twice.ts +++ b/packages/sandbox/src/boot-twice.ts @@ -16,6 +16,7 @@ import { randomUUID } from "node:crypto"; import { readFileSync } from "node:fs"; import { join } from "node:path"; import { + ROOT_DISK_AGENTS_COMMAND, renderSandboxConf, SANDBOX_PATHS, SANDBOX_PORTS, @@ -172,6 +173,19 @@ try { return elapsed; }; + // The agent-tools drive, stubbed as a directory at its mount path: a box + // has no other agent CLIs, so every boot here has one unless it says not. + const stubSha = "f".repeat(64); + const writeDrive = (carried: string[], listed: string[] = carried) => { + const manifest = JSON.stringify( + listed.map((name) => ({ name, sha: stubSha, bin: name })), + ); + exec( + `rm -rf ${SANDBOX_PATHS.agents}; for name in ${carried.join(" ")}; do mkdir -p ${SANDBOX_PATHS.agents}/$name/${stubSha} && printf '#!/bin/sh\\necho drive-%s\\n' $name > ${SANDBOX_PATHS.agents}/$name/${stubSha}/$name && chmod 755 ${SANDBOX_PATHS.agents}/$name/${stubSha}/$name; done && printf '%s' '${manifest}' > ${SANDBOX_PATHS.agents}/manifest.json`, + ); + }; + const allAgents = ["claude", "codex", "pdev", "opencode", "pi"]; + writeDrive(allAgents); const first = boot(1); const log1 = exec(`cat ${SANDBOX_PATHS.bootLog}`); // The image may have been built from an older bundle than the one the @@ -352,15 +366,35 @@ try { ); expect("second boot: nothing fetched", !/fetched \//.test(log2)); expect("second boot: every step skipped", !/step \S+ running/.test(log2)); - const binaries = exec( - "claude --version && superset --version && echo binaries-ok", - { user: "ubuntu", check: false }, - ); + const binaries = exec("superset --version && echo binaries-ok", { + user: "ubuntu", + check: false, + }); expect( - "second boot: claude and superset still run from the media dir", + "second boot: superset still runs from the media dir", /binaries-ok/.test(binaries), binaries.trim().replace(/\n/g, " | "), ); + const rootDiskAgents = exec( + `${ROOT_DISK_AGENTS_COMMAND}; ls -d ${SANDBOX_PATHS.media}/codex-* ${SANDBOX_PATHS.media}/pdev-* 2>/dev/null || true`, + ).trim(); + expect( + "image: no agent CLI on the root disk", + rootDiskAgents === "", + rootDiskAgents.replaceAll("\n", " "), + ); + const resolved = exec( + `for name in ${allAgents.join(" ")}; do readlink -f "$(command -v $name)"; done`, + { user: "ubuntu", check: false }, + ).trim(); + expect( + "every agent on a login shell's PATH runs from the drive", + resolved.split("\n").length === allAgents.length && + resolved + .split("\n") + .every((line) => line.startsWith(`${SANDBOX_PATHS.agents}/`)), + resolved.replaceAll("\n", " "), + ); expect( "second boot: host-service ready", /host\.ready/.test(log2), @@ -379,62 +413,72 @@ try { ); console.log(`\nboot log (second boot):\n${log2}`); - // The agent-tools drive, stubbed as a directory at its mount path: the - // listed agent is linked from it, and a session without it falls back to - // the image's agents rather than to links that point nowhere. + // A box from an older golden still holds that bundle's disk copies and + // their links: the drive's agents win, the copies are left as they are, + // and an agent the drive lacks is missing alone rather than falling back. const stop = () => exec( `pkill -u ubuntu -TERM || true; sleep 1; pkill -u ubuntu -KILL || true; pkill -x Xvnc || true; rm -rf ${SANDBOX_PATHS.run}`, { check: false }, ); - const stubSha = "f".repeat(64); + const leftovers = { + claude: `${SANDBOX_PATHS.media}/claude-old`, + codex: "/opt/codex/old/codex", + pdev: "/opt/pdev/old/pdev", + }; + const leftoverFiles = Object.values(leftovers).join(" "); stop(); - exec( - `: > ${SANDBOX_PATHS.bootLog}; for name in claude pi; do mkdir -p ${SANDBOX_PATHS.agents}/$name/${stubSha} && printf '#!/bin/sh\\necho drive-%s\\n' $name > ${SANDBOX_PATHS.agents}/$name/${stubSha}/$name && chmod 755 ${SANDBOX_PATHS.agents}/$name/${stubSha}/$name; done && printf '%s' '${JSON.stringify( - [ - { name: "claude", sha: stubSha, bin: "claude" }, - { name: "pi", sha: stubSha, bin: "pi" }, - ], - )}' > ${SANDBOX_PATHS.agents}/manifest.json`, - ); + writeDrive(["claude", "codex", "pi"], ["claude", "codex", "pdev", "pi"]); + exec(`: > ${SANDBOX_PATHS.bootLog}`); + for (const [name, file] of Object.entries(leftovers)) { + exec( + `mkdir -p "$(dirname ${file})" && printf '#!/bin/sh\\necho disk-${name}\\n' > ${file} && chmod 755 ${file} && ln -sfn ${file} /usr/local/bin/${name}`, + ); + } boot(3); const logDrive = exec(`cat ${SANDBOX_PATHS.bootLog}`); + const run = (name: string) => + exec(`${name} 2>&1`, { check: false, user: "ubuntu" }); expect( "drive: listed agents linked", - /agents\.linked 2/.test(logDrive) && - /drive-claude/.test(exec("claude", { check: false, user: "ubuntu" })) && - /drive-pi/.test(exec("pi", { check: false, user: "ubuntu" })), - logDrive.match(/agents\..*/)?.[0], - ); - stop(); - exec(`: > ${SANDBOX_PATHS.bootLog}; rm -rf ${SANDBOX_PATHS.agents}`); - const noDriveExit = bootAsUser(); - exec( - `for i in $(seq 1 600); do [ -f ${SANDBOX_PATHS.run}/host-service.ready ] && exit 0; sleep 0.1; done; exit 1`, - { check: false }, + /agents\.linked 3/.test(logDrive) && + /drive-claude/.test(run("claude")) && + /drive-codex/.test(run("codex")) && + /drive-pi/.test(run("pi")), + logDrive.match(/agents\.linked.*/)?.[0], ); - const logNoDrive = exec(`cat ${SANDBOX_PATHS.bootLog}`); expect( - "no drive: boot exits 0 and host-service ready", - /exit=0/.test(noDriveExit) && /host\.ready/.test(logNoDrive), + "drive: boot ignores a leftover disk copy", + exec(`ls ${leftoverFiles} 2>&1`).trim().split("\n").length === 3 && + !/disk-/.test(`${run("claude")}${run("codex")}`), + exec(`ls ${leftoverFiles} 2>&1`).trim().replaceAll("\n", " "), ); expect( - "no drive: stamped unavailable", - /agents\.unavailable/.test(logNoDrive), - ); - expect( - "no drive: image claude back on PATH", - exec("readlink -f /usr/local/bin/claude", { check: false }).startsWith( - SANDBOX_PATHS.media, - ), - exec("readlink -f /usr/local/bin/claude", { check: false }).trim(), + "drive: an agent the drive lacks is missing alone, with no fallback to a disk copy", + /agents\.missing pdev/.test(logDrive) && + exec("command -v pdev || echo none", { + user: "ubuntu", + check: false, + }).trim() === "none", + logDrive.match(/agents\.missing.*/)?.[0], ); + stop(); + exec(`: > ${SANDBOX_PATHS.bootLog}; rm -rf ${SANDBOX_PATHS.agents}`); + const noDriveExit = bootAsUser(); + const logNoDrive = exec(`cat ${SANDBOX_PATHS.bootLog}`); expect( - "no drive: a drive-only agent's link is gone", - exec("test -L /usr/local/bin/pi && echo dangling", { - check: false, - }).trim() === "", + "no manifest: boot refused before host-service starts, so the open fails", + /exit=1/.test(noDriveExit) && + /agents\.unavailable/.test(logNoDrive) && + /boot\.refused/.test(logNoDrive) && + !/host\.(exec|ready)/.test(logNoDrive) && + exec( + `test -e ${SANDBOX_PATHS.run}/host-service.pid && echo up || echo down`, + ).trim() === "down", + logNoDrive.match(/boot\.refused.*/)?.[0], ); + exec(`rm -rf /opt/codex /opt/pdev ${leftovers.claude}`); + writeDrive(allAgents); // Third boot, with host-service deliberately broken: the box must stay // alive, the desktop must still come up, and the log must say so. diff --git a/packages/sandbox/src/environments/probe.ts b/packages/sandbox/src/environments/probe.ts index e1b198bd79d..2162e7da6de 100644 --- a/packages/sandbox/src/environments/probe.ts +++ b/packages/sandbox/src/environments/probe.ts @@ -8,6 +8,7 @@ import { isCloudAgentId } from "@superset/shared/cloud-agent-launch"; import { SANDBOX_CREDENTIAL_PLACEHOLDER } from "@superset/shared/constants"; import { HOST_AGENT_PRESETS } from "@superset/shared/host-agent-presets"; import { + ROOT_DISK_AGENTS_COMMAND, SANDBOX_PATHS, SANDBOX_PORTS, sandboxCheckoutDir, @@ -149,6 +150,12 @@ export async function probeBox(args: ProbeArgs): Promise { out.trim().split("\n").slice(0, 2).join(" "), ); } + const rootDiskAgents = (await run(ROOT_DISK_AGENTS_COMMAND)).trim(); + check( + "no agent CLI on the box's root disk, so none in its golden", + rootDiskAgents === "", + rootDiskAgents.replaceAll("\n", " "), + ); const cliVersion = await run("superset --version"); check("superset runs", /^\d+\.\d+\.\d+/m.test(cliVersion), cliVersion.trim()); if ( diff --git a/packages/sandbox/src/environments/release.ts b/packages/sandbox/src/environments/release.ts index 95dd3faf6c6..661f2617b01 100644 --- a/packages/sandbox/src/environments/release.ts +++ b/packages/sandbox/src/environments/release.ts @@ -39,6 +39,7 @@ import { join } from "node:path"; import { GOLDEN_BOX_VCPUS } from "@superset/shared/billing"; import { SANDBOX_IMAGE_NAME } from "@superset/shared/constants"; import { + ROOT_DISK_AGENTS_COMMAND, SANDBOX_CONTRACT_VERSION, SANDBOX_PATHS, SANDBOX_ROOT_CHECKOUT, @@ -261,7 +262,6 @@ for (;;) { managedEnv: {}, networkPolicy: "allow-all", vcpus: GOLDEN_BOX_VCPUS, - agentTools: true, }, }); break; @@ -285,7 +285,6 @@ await wakeSandbox({ managedEnv: {}, networkPolicy: "allow-all", vcpus: GOLDEN_BOX_VCPUS, - agentTools: true, }, }); log("golden: host-service up; waiting for the checkout"); @@ -369,6 +368,11 @@ const checks: Array<[label: string, command: string, expect: RegExp]> = [ ["ntn", "ntn --version", /\d+\.\d+\.\d+/], ["agent-device", "agent-device --version", /\d+\.\d+\.\d+/], ["stripe", "stripe --version", /\d+\.\d+\.\d+/], + [ + "no agent CLI on the golden's root disk", + `[ -z "$(${ROOT_DISK_AGENTS_COMMAND})" ] && echo clean`, + /^clean$/m, + ], ]; let failed = 0; for (const [label, command, expect] of checks) { @@ -447,7 +451,6 @@ const probeClaim = { managedEnv, networkPolicy, vcpus: GOLDEN_BOX_VCPUS, - agentTools: true, }; log(`probe: provisioning ${probe} as a fork of ${golden}`); await provisionSandbox({ diff --git a/packages/sandbox/src/manifest.test.ts b/packages/sandbox/src/manifest.test.ts index f88f77bd6c7..480595a7669 100644 --- a/packages/sandbox/src/manifest.test.ts +++ b/packages/sandbox/src/manifest.test.ts @@ -357,9 +357,7 @@ describe("renderAssetsTsv", () => { const owners = discardingSteps( loadSteps(join(import.meta.dir, "..", "bundle", "steps.json")), ); - expect(owners.has("claude")).toBe(false); expect(owners.has("cli")).toBe(false); - expect(owners.get("codex")).toBe("install-agent-clis"); expect(owners.get("chrome")).toBe("install-chrome"); expect(owners.get("host-service")).toBe("install-host"); }); diff --git a/packages/sandbox/src/real-sandbox.ts b/packages/sandbox/src/real-sandbox.ts index 1ce5208b71a..60d59024696 100644 --- a/packages/sandbox/src/real-sandbox.ts +++ b/packages/sandbox/src/real-sandbox.ts @@ -108,7 +108,6 @@ const claim = { managedEnv: { SUPERSET_REAL_CHECK: "1" }, networkPolicy: "allow-all" as const, vcpus: GOLDEN_BOX_VCPUS, - agentTools: true, }; log( diff --git a/packages/shared/src/cloud-agent-launch.ts b/packages/shared/src/cloud-agent-launch.ts index af4e496f597..8afe3863ee1 100644 --- a/packages/shared/src/cloud-agent-launch.ts +++ b/packages/shared/src/cloud-agent-launch.ts @@ -3,9 +3,9 @@ import { MAX_ATTACHMENTS } from "./attachment-limits"; import { HOST_AGENT_PRESETS } from "./host-agent-presets"; /** - * The presets a sandbox can actually run: the CLIs the bundle installs - * (`packages/sandbox/bundle/steps.json`, install-agent-clis). Adding one there - * is what makes it launchable here. + * The presets a sandbox can launch. Their CLIs come from the agents drive + * (`AGENT_TOOLS` in `packages/sandbox/src/agent-tools.ts`), which also + * carries opencode and pi; carrying a CLI does not make it launchable here. */ const INSTALLED_IN_SANDBOX = new Set(["claude", "codex", "pdev"]); diff --git a/packages/shared/src/constants.ts b/packages/shared/src/constants.ts index 76cdb18e648..ca3db2fd8f2 100644 --- a/packages/shared/src/constants.ts +++ b/packages/shared/src/constants.ts @@ -196,12 +196,6 @@ export const FEATURE_FLAGS = { CLOUD_WORKSPACES: "cloud-workspaces", /** Per organization: shows Settings, Cloud usage, and gates the `cloud.*` procedures server-side. Unloaded or offline is off. */ CLOUD_USAGE: "cloud-usage", - /** - * Mounts the bundle's agent-tools drive on an organization's boxes, so the - * agent CLIs come from it instead of the image. Evaluated per organization - * on every create and wake. Unloaded or offline means no drive. - */ - SANDBOX_AGENT_TOOLS_DRIVE: "sandbox-agent-tools-drive", /** Sends an organization's box logs to Sentry; needs `SANDBOX_BOX_LOGS_FLEET` too. */ SANDBOX_BOX_LOGS: "sandbox-box-logs", /** One answer for every box (asked for the subject `sandbox-fleet`, no group): off stops box logs at each box's next wake. */ diff --git a/packages/shared/src/sandbox/contract.ts b/packages/shared/src/sandbox/contract.ts index a383b7579a5..f78e5ce2069 100644 --- a/packages/shared/src/sandbox/contract.ts +++ b/packages/shared/src/sandbox/contract.ts @@ -39,6 +39,21 @@ export const SANDBOX_PATHS = { agents: "/opt/agents", } as const; +/** + * Where bundles before the agents drive put the agent CLIs on the box's own + * disk. A golden that holds one of these is from before the drive and every + * fork of it carries the bytes, so it is rebuilt from the image, not forked. + */ +export const SANDBOX_ROOT_DISK_AGENTS = [ + "/opt/codex", + "/opt/pdev", + `${SANDBOX_PATHS.media}/claude-*`, + `${SANDBOX_PATHS.steps}/install-agent-clis.version`, +] as const; + +/** Prints each root-disk agent path that exists; prints nothing on a clean disk. */ +export const ROOT_DISK_AGENTS_COMMAND = `ls -d ${SANDBOX_ROOT_DISK_AGENTS.join(" ")} 2>/dev/null || true`; + /** * The agent CLIs of one bundle in one region: written once by the release, * then only ever mounted read-only. A drive only mounts in its own region. diff --git a/packages/trpc/src/lib/sandbox/claim.ts b/packages/trpc/src/lib/sandbox/claim.ts index c89ecfe46b6..c2a16f87112 100644 --- a/packages/trpc/src/lib/sandbox/claim.ts +++ b/packages/trpc/src/lib/sandbox/claim.ts @@ -16,7 +16,6 @@ import { type CloudAgentLaunch, cloudAgentLaunchToEnv, } from "@superset/shared/cloud-agent-launch"; -import { FEATURE_FLAGS } from "@superset/shared/constants"; import { SANDBOX_CONTRACT_VERSION, type SandboxIdentity, @@ -31,7 +30,7 @@ import { import { resolveEnvironment } from "../../router/environment/resolve-environment"; import { posthog } from "../analytics"; import { resolveMachineSize } from "../billing/plan"; -import { boxLogsEnabled, organizationFlagEnabled } from "../feature-flags"; +import { boxLogsEnabled } from "../feature-flags"; import { githubUserConnectionFor, githubUserTokenFor } from "../github-user"; import { sandboxHostSecretFor } from "./access"; import { deriveSandboxCredentials, gitAuthorFor } from "./credentials"; @@ -184,11 +183,6 @@ export async function buildSandboxClaim(args: { files, ports: repoHooks?.ports, vcpus: await boxVcpus(args.row), - agentTools: await organizationFlagEnabled( - posthog, - FEATURE_FLAGS.SANDBOX_AGENT_TOOLS_DRIVE, - args.row.organizationId, - ), }, environment: { sourceKind: environment.sourceKind, diff --git a/packages/trpc/src/lib/sandbox/index.ts b/packages/trpc/src/lib/sandbox/index.ts index ec376ff4943..af94d059c5e 100644 --- a/packages/trpc/src/lib/sandbox/index.ts +++ b/packages/trpc/src/lib/sandbox/index.ts @@ -54,6 +54,7 @@ export { } from "./repositories"; export { type ActiveWorkspaceSandbox, + AgentToolsUnavailableError, type BoxSession, deleteSandbox, describeSandbox, diff --git a/packages/trpc/src/lib/sandbox/vercel.test.ts b/packages/trpc/src/lib/sandbox/vercel.test.ts index 2141b68005c..af6fb895a56 100644 --- a/packages/trpc/src/lib/sandbox/vercel.test.ts +++ b/packages/trpc/src/lib/sandbox/vercel.test.ts @@ -6,9 +6,12 @@ import { setSystemTime, test, } from "bun:test"; +import * as Sentry from "@sentry/core"; import { APIError, Drive, Sandbox } from "@vercel/sandbox"; import { stub } from "../../../test/stub"; import { + AGENT_TOOLS_RETRY, + AgentToolsUnavailableError, agentToolsMounts, applySandboxPolicy, deleteSandbox, @@ -17,6 +20,7 @@ import { listActiveWorkspaceSandboxes, listSandboxSessions, promoteSandboxToEnvironment, + provisionSandbox, type SandboxClaim, SandboxUnavailableError, SESSION_TIMEOUT_MS, @@ -515,19 +519,48 @@ describe("topUpSandboxSession", () => { }); }); +const failure = (status: number) => + Object.assign(Object.create(APIError.prototype), { + response: { status }, + }) as Error; +const reports: { message: string; tags: Record }[] = []; +stub(Sentry, { + captureMessage: ( + message: string, + context: { tags: Record }, + ) => { + reports.push({ message, tags: context.tags }); + return ""; + }, +}); +stub(AGENT_TOOLS_RETRY, { delayMs: 0 }); + +const BUNDLE = (prefix: string) => + prefix.padEnd(12, "0").repeat(6).slice(0, 64); +const IDENTITY = (bundle: string) => ({ + SUPERSET_SANDBOX_CONTRACT: "1", + SUPERSET_BUNDLE_SHA: bundle, + SUPERSET_API_URL: "https://api.example", + SUPERSET_SANDBOX_WORKSPACE_ID: "6f1c2b8e-1d2a-4c55-9a3e-0b7f5d2c9e11", + SUPERSET_SANDBOX_ORGANIZATION_ID: "6f1c2b8e-1d2a-4c55-9a3e-0b7f5d2c9e12", + SUPERSET_SANDBOX_REPOSITORIES: "[]", + SUPERSET_SANDBOX_IMAGE_TAG: "img", + SUPERSET_SANDBOX_PROVIDER: "vercel", +}); +const claimFor = (bundle: string) => + ({ + vcpus: 2, + hostSecret: "s", + managedEnv: {}, + networkPolicy: "allow-all", + identity: IDENTITY(bundle), + }) as unknown as SandboxClaim; +const unavailableIn = (region: string) => + `Agents are unavailable in ${region} right now; try again`; + describe("agentToolsMounts", () => { const detached = () => Promise.resolve({ currentSandboxName: undefined }); - test("mounts nothing without a drive name", async () => { - let asked = 0; - const getDrive = () => { - asked++; - return detached(); - }; - expect(await agentToolsMounts(null, getDrive)).toBeUndefined(); - expect(asked).toBe(0); - }); - test("mounts a detached drive read-only, and asks only once", async () => { let asked = 0; const getDrive = () => { @@ -549,30 +582,61 @@ describe("agentToolsMounts", () => { expect(asked).toBe(1); }); - test("skips a missing drive, or one a writer still holds", async () => { - expect( - await agentToolsMounts("agent-tools-sfo1-aaaaaaaaaaaa", () => - Promise.reject(new Error("404")), - ), - ).toBeUndefined(); + test("retries a drive lookup that failed with 429 or 5xx", async () => { + const answers = [failure(503), failure(429)]; let asked = 0; - const attached = () => { - asked++; - return Promise.resolve({ currentSandboxName: "builder" }); - }; - expect( - await agentToolsMounts("agent-tools-sfo1-bbbbbbbbbbbb", attached), - ).toBeUndefined(); - expect( - await agentToolsMounts("agent-tools-sfo1-bbbbbbbbbbbb", attached), - ).toBeUndefined(); - expect(asked).toBe(2); + const mounts = await agentToolsMounts( + "agent-tools-iad1-a1a1a1a1a1a1", + () => { + asked++; + const next = answers.shift(); + return next ? Promise.reject(next) : detached(); + }, + ); + expect(asked).toBe(3); + expect(mounts).toBeDefined(); }); + + test("a drive that stays unavailable, is missing or a writer holds fails with the region, and reports region and bundle", async () => { + reports.length = 0; + let asked = 0; + await expect( + agentToolsMounts("agent-tools-sfo1-aaaaaaaaaaaa", () => { + asked++; + return Promise.reject(failure(503)); + }), + ).rejects.toThrow(unavailableIn("sfo1")); + expect(asked).toBe(3); + await expect( + agentToolsMounts("agent-tools-sfo1-bbbbbbbbbbbb", () => + Promise.reject(failure(404)), + ), + ).rejects.toBeInstanceOf(AgentToolsUnavailableError); + await expect( + agentToolsMounts("agent-tools-sfo1-cccccccccccc", () => + Promise.resolve({ currentSandboxName: "builder" }), + ), + ).rejects.toBeInstanceOf(AgentToolsUnavailableError); + expect(reports.map((report) => report.tags)).toEqual( + ["aaaaaaaaaaaa", "bbbbbbbbbbbb", "cccccccccccc"].map((bundle) => ({ + "sandbox.agent_tools_drive": `agent-tools-sfo1-${bundle}`, + "sandbox.region": "sfo1", + "sandbox.bundle": bundle, + })), + ); + }); +}); + +const drives = new Map Promise>(); +stub(Drive, { + get: ({ name }: { name: string }) => + drives.get(name)?.() ?? Promise.resolve({ currentSandboxName: undefined }), }); describe("wakeSandbox", () => { - const asked: string[] = []; + let updateAnswers: Error[] = []; const updates: unknown[] = []; + const resumes: string[] = []; stub(Sandbox, { get: async () => ({ status: "stopped", @@ -582,39 +646,107 @@ describe("wakeSandbox", () => { domain: () => "https://host.example", update: async (patch: unknown) => { updates.push(patch); + const next = updateAnswers.shift(); + if (next) throw next; }, }), }); - stub(Drive, { - get: async ({ name }: { name: string }) => { - asked.push(name); - return { currentSandboxName: undefined }; - }, + const resumed = new Error("resumed"); + const wake = (bundle: string) => + wakeSandbox({ + providerSandboxId: "sbx", + claim: claimFor(bundle), + beforeResume: () => { + resumes.push(bundle); + return Promise.reject(resumed); + }, + }); + const mountOf = (drive: string) => ({ + mounts: { "/opt/agents": { drive, mode: "snapshot" } }, }); test("mounts the drive of the box's own region, not its environment's", async () => { - const stop = new Error("stop before resume"); - await expect( - wakeSandbox({ - providerSandboxId: "sbx", - claim: { - vcpus: 2, - agentTools: true, - identity: { SUPERSET_BUNDLE_SHA: "abcdef012345".repeat(5) }, - } as unknown as SandboxClaim, - beforeResume: () => Promise.reject(stop), - }), - ).rejects.toBe(stop); - expect(asked).toEqual(["agent-tools-iad1-abcdef012345"]); - expect(updates).toEqual([ + updates.length = 0; + await expect(wake(BUNDLE("c1"))).rejects.toBe(resumed); + expect(updates).toEqual([mountOf("agent-tools-iad1-c10000000000")]); + }); + + test("a drive that cannot be mounted fails the wake before the box resumes", async () => { + updates.length = 0; + resumes.length = 0; + drives.set("agent-tools-iad1-c20000000000", () => + Promise.reject(failure(404)), + ); + await expect(wake(BUNDLE("c2"))).rejects.toThrow(unavailableIn("iad1")); + expect(updates).toEqual([]); + expect(resumes).toEqual([]); + }); + + test("a mounts update is retried on 5xx, and one that keeps failing fails the wake", async () => { + updates.length = 0; + resumes.length = 0; + updateAnswers = [failure(502)]; + await expect(wake(BUNDLE("c3"))).rejects.toBe(resumed); + expect(updates).toHaveLength(2); + updates.length = 0; + updateAnswers = [failure(503), failure(503), failure(503)]; + await expect(wake(BUNDLE("c4"))).rejects.toThrow(unavailableIn("iad1")); + expect(updates).toHaveLength(3); + expect(resumes).toEqual([BUNDLE("c3")]); + }); +}); + +describe("provisionSandbox", () => { + let createAnswers: Error[] = []; + const creates: { mounts?: unknown }[] = []; + stub(Sandbox, { + get: () => Promise.reject(failure(404)), + create: async (config: { mounts?: unknown }) => { + creates.push(config); + const next = createAnswers.shift(); + if (next) throw next; + return { + domain: () => "https://host.example", + writeFiles: async () => {}, + runCommand: async () => {}, + }; + }, + }); + const provision = (bundle: string) => + provisionSandbox({ + name: "ws", + environment: { sourceKind: "image", sourceRef: "img", region: "sfo1" }, + claim: claimFor(bundle), + }); + + test("a create that failed with 5xx is retried with the drive mounted", async () => { + creates.length = 0; + createAnswers = [failure(503)]; + expect((await provision(BUNDLE("d1"))).providerSandboxId).toBe("ws"); + expect(creates.map((config) => config.mounts)).toEqual([ + { + "/opt/agents": { + drive: "agent-tools-sfo1-d10000000000", + mode: "snapshot", + }, + }, { - mounts: { - "/opt/agents": { - drive: "agent-tools-iad1-abcdef012345", - mode: "snapshot", - }, + "/opt/agents": { + drive: "agent-tools-sfo1-d10000000000", + mode: "snapshot", }, }, ]); }); + + test("a drive that cannot be mounted fails the create, and no box is made", async () => { + creates.length = 0; + drives.set("agent-tools-sfo1-d20000000000", () => + Promise.reject(failure(404)), + ); + await expect(provision(BUNDLE("d2"))).rejects.toThrow( + unavailableIn("sfo1"), + ); + expect(creates).toEqual([]); + }); }); diff --git a/packages/trpc/src/lib/sandbox/vercel.ts b/packages/trpc/src/lib/sandbox/vercel.ts index b79ff3809e9..e2e588eaae0 100644 --- a/packages/trpc/src/lib/sandbox/vercel.ts +++ b/packages/trpc/src/lib/sandbox/vercel.ts @@ -172,8 +172,6 @@ export interface SandboxClaim { /** Ports the workspace's repository asks to publish, beside the platform's. */ ports?: readonly number[]; vcpus: number; - /** Mount the bundle's agent-tools drive read-only; absent keeps the image's agents. */ - agentTools?: boolean; } function environmentTags(organizationId: string): Record { @@ -199,33 +197,94 @@ function sandboxTags( const publishedAgentToolsDrives = new Set(); +/** A box has no agents without its drive, so a transient refusal gets a few quick tries. */ +export const AGENT_TOOLS_RETRY = { attempts: 3, delayMs: 500 }; + +/** The open fails: a box without its agents drive has no agent CLIs at all. */ +export class AgentToolsUnavailableError extends Error { + readonly region: string; + readonly bundle: string; + constructor( + readonly driveName: string, + cause?: unknown, + ) { + const [, region = "", bundle = ""] = + /^agent-tools-([a-z]+[0-9]+)-([0-9a-f]{12})$/.exec(driveName) ?? []; + super(`Agents are unavailable in ${region} right now; try again`, { + cause, + }); + this.name = "AgentToolsUnavailableError"; + this.region = region; + this.bundle = bundle; + } +} + +function agentToolsUnavailable( + driveName: string, + reason: string, + cause?: unknown, +): AgentToolsUnavailableError { + const error = new AgentToolsUnavailableError(driveName, cause); + console.error( + `[sandbox] agent-tools drive ${driveName} ${reason}`, + cause ?? "", + ); + Sentry.captureMessage(`agent-tools drive not mounted: ${reason}`, { + level: "error", + tags: { + "sandbox.agent_tools_drive": driveName, + "sandbox.region": error.region, + "sandbox.bundle": error.bundle, + }, + extra: { error: cause ? String(cause) : undefined }, + }); + return error; +} + +function isTransient(error: unknown): boolean { + return ( + error instanceof APIError && + (error.response.status === 429 || error.response.status >= 500) + ); +} + +async function withTransientRetry(run: () => Promise): Promise { + for (let attempt = 1; ; attempt++) { + try { + return await run(); + } catch (error) { + if (attempt >= AGENT_TOOLS_RETRY.attempts || !isTransient(error)) + throw error; + await new Promise((resolve) => + setTimeout(resolve, AGENT_TOOLS_RETRY.delayMs * attempt), + ); + } + } +} + /** * A mount of a drive that does not exist fails the create, and a drive a * writer still holds read-write mounts corrupt. Only a drive that exists and * is detached is mounted; that answer is final for a drive, so it is cached. */ export async function agentToolsMounts( - driveName: string | null | undefined, + driveName: string, getDrive: (name: string) => Promise> = ( name, ) => Drive.get({ ...credentials(), name }), -): Promise { - if (!driveName) return undefined; +): Promise { if (!publishedAgentToolsDrives.has(driveName)) { + let drive: Pick; try { - const drive = await getDrive(driveName); - if (drive.currentSandboxName) { - console.warn( - `[sandbox] agent-tools drive ${driveName} is attached to ${drive.currentSandboxName}; not mounting`, - ); - return undefined; - } + drive = await withTransientRetry(() => getDrive(driveName)); } catch (error) { - console.warn( - `[sandbox] agent-tools drive ${driveName} unavailable; not mounting`, - error, + throw agentToolsUnavailable(driveName, "is unavailable", error); + } + if (drive.currentSandboxName) { + throw agentToolsUnavailable( + driveName, + `is attached to ${drive.currentSandboxName}`, ); - return undefined; } publishedAgentToolsDrives.add(driveName); } @@ -234,12 +293,27 @@ export async function agentToolsMounts( /** The box's own region names the drive: a workspace stays where it was created when its environment moves. */ export function agentToolsDriveName( - claim: Pick, + claim: Pick, region: string, -): string | null { - return claim.agentTools - ? sandboxAgentToolsDrive(region, claim.identity.SUPERSET_BUNDLE_SHA) - : null; +): string { + return sandboxAgentToolsDrive(region, claim.identity.SUPERSET_BUNDLE_SHA); +} + +/** Mounts last one session, so a stopped box is handed the drive again before it resumes. */ +async function remountAgentTools( + sandbox: Sandbox, + claim: SandboxClaim, +): Promise { + const { [SANDBOX_PATHS.agents]: _previous, ...others } = sandbox.mounts ?? {}; + const driveName = agentToolsDriveName(claim, sandbox.region); + const agents = await agentToolsMounts(driveName); + try { + await withTransientRetry(() => + sandbox.update({ mounts: { ...others, ...agents } }), + ); + } catch (error) { + throw agentToolsUnavailable(driveName, "mount update failed", error); + } } function publishedPorts(extra: readonly number[] = []): number[] { @@ -330,38 +404,41 @@ export async function provisionSandbox(args: { const mounts = await agentToolsMounts( agentToolsDriveName(args.claim, args.environment.region), ); - const sandbox = - (await getSandboxToWake(args.name)) ?? - (await Sentry.startSpan( - { - name: - args.environment.sourceKind === "fork" ? "fork" : "create from image", - op: "sandbox.provider", - attributes: { - "sandbox.name": args.name, - "sandbox.source": args.environment.sourceRef, - "sandbox.region": args.environment.region, - "sandbox.vcpus": args.claim.vcpus, - }, + const sandbox = await Sentry.startSpan( + { + name: + args.environment.sourceKind === "fork" ? "fork" : "create from image", + op: "sandbox.provider", + attributes: { + "sandbox.name": args.name, + "sandbox.source": args.environment.sourceRef, + "sandbox.region": args.environment.region, + "sandbox.vcpus": args.claim.vcpus, }, - () => - args.environment.sourceKind === "fork" - ? // A fork copies the golden's region; a snapshot only exists where - // it was taken. - Sandbox.fork({ - ...config, - ...(mounts ? { mounts } : {}), - sourceSandbox: args.environment.sourceRef, - resources: { vcpus: args.claim.vcpus }, - }) - : Sandbox.create({ - ...config, - ...(mounts ? { mounts } : {}), - image: args.environment.sourceRef, - region: args.environment.region as SandboxRegion, - resources: { vcpus: args.claim.vcpus }, - }), - )); + }, + () => + // Idempotent on the name: a retry after a create that did land finds it. + withTransientRetry( + async () => + (await getSandboxToWake(args.name)) ?? + (args.environment.sourceKind === "fork" + ? // A fork copies the golden's region; a snapshot only exists where + // it was taken. + Sandbox.fork({ + ...config, + mounts, + sourceSandbox: args.environment.sourceRef, + resources: { vcpus: args.claim.vcpus }, + }) + : Sandbox.create({ + ...config, + mounts, + image: args.environment.sourceRef, + region: args.environment.region as SandboxRegion, + resources: { vcpus: args.claim.vcpus }, + })), + ), + ); await Sentry.startSpan({ name: "start boot", op: "sandbox" }, async () => { await writeIdentity(sandbox, args.claim.identity); await writeSignInFiles(sandbox, args.claim.files); @@ -604,23 +681,7 @@ export async function wakeSandbox(args: { ), ); } - if (!wasRunning) { - const { [SANDBOX_PATHS.agents]: previous, ...others } = - sandbox.mounts ?? {}; - const agents = await agentToolsMounts( - agentToolsDriveName(args.claim, sandbox.region), - ); - if (agents || previous) { - await sandbox - .update({ mounts: { ...others, ...agents } }) - .catch((error) => - console.warn( - `[sandbox] mount update failed for ${args.providerSandboxId}`, - error, - ), - ); - } - } + if (!wasRunning) await remountAgentTools(sandbox, args.claim); if (!wasRunning && args.beforeResume) { await args.beforeResume({ memoryMb: vcpus * MEMORY_MB_PER_VCPU, vcpus }); } diff --git a/packages/trpc/src/router/cloud-workspace/cloud-workspace.ts b/packages/trpc/src/router/cloud-workspace/cloud-workspace.ts index c9bd47c194d..e433c83b2b6 100644 --- a/packages/trpc/src/router/cloud-workspace/cloud-workspace.ts +++ b/packages/trpc/src/router/cloud-workspace/cloud-workspace.ts @@ -37,6 +37,7 @@ import { logControl } from "../../lib/control-log"; import { callMoves } from "../../lib/moves"; import { nudge, readWorkspaceForwards } from "../../lib/realtime"; import { + AgentToolsUnavailableError, deleteSandbox, describeSandbox, HOST_SERVICE_PORT, @@ -172,6 +173,14 @@ async function addressSandbox( : await wakeCloudWorkspace(row); return { ...woken, running: true }; } catch (error) { + if (error instanceof AgentToolsUnavailableError) { + throw userError({ + code: "SERVICE_UNAVAILABLE", + message: error.message, + i18nKey: "serverError.cloudWorkspace.agentsUnavailable", + params: { region: error.region }, + }); + } if (error instanceof SandboxNotReadyError) { throw new TRPCError({ code: "TIMEOUT", diff --git a/packages/trpc/src/router/cloud-workspace/provision.ts b/packages/trpc/src/router/cloud-workspace/provision.ts index 8bc3b4c365c..d8858436ed3 100644 --- a/packages/trpc/src/router/cloud-workspace/provision.ts +++ b/packages/trpc/src/router/cloud-workspace/provision.ts @@ -10,6 +10,7 @@ import { eq } from "drizzle-orm"; import { controlTags } from "../../lib/control-log"; import { nudge } from "../../lib/realtime"; import { + AgentToolsUnavailableError, bootPhases, buildSandboxClaim, deleteSandbox, @@ -263,12 +264,16 @@ async function provision( reason: provisionFailureReason(error), error, }); - await failChatMovesTo(row.id, "The cloud workspace could not start").catch( - (moveError) => - console.error( - `[cloud-workspace] could not fail the chat moves into ${row.id}`, - moveError, - ), + await failChatMovesTo( + row.id, + error instanceof AgentToolsUnavailableError + ? error.message + : "The cloud workspace could not start", + ).catch((moveError) => + console.error( + `[cloud-workspace] could not fail the chat moves into ${row.id}`, + moveError, + ), ); console.error(`[cloud-workspace] provisioning failed for ${row.id}`, error); return "failed"; From ced887cc1b60863a131f7733ecd28650c1e07788 Mon Sep 17 00:00:00 2001 From: Satya Patel <14907857+saddlepaddle@users.noreply.github.com> Date: Wed, 18 Nov 2026 22:06:42 +0000 Subject: [PATCH 2/3] fix(sandbox): show a missing agents drive as a box that will not start A customer never sees the drive. AgentToolsUnavailableError carries the existing "The cloud workspace could not start" as its message, a wake maps it to the existing "Cloud workspace is still starting" like SandboxNotReadyError, and an automation records cloud_not_ready. The drive, region and bundle stay in Sentry and the server log. The agentsUnavailable message and its translations are removed, and the box's refusal no longer names the drive. --- docs/cloud-sandbox-mismatches.md | 15 ++++++----- packages/i18n/locales/cs/messages.po | 3 --- packages/i18n/locales/de/messages.po | 3 --- packages/i18n/locales/en/messages.po | 3 --- packages/i18n/locales/es/messages.po | 3 --- packages/i18n/locales/fr/messages.po | 3 --- packages/i18n/locales/id/messages.po | 3 --- packages/i18n/locales/it/messages.po | 3 --- packages/i18n/locales/ja/messages.po | 3 --- packages/i18n/locales/ko/messages.po | 3 --- packages/i18n/locales/nl/messages.po | 3 --- packages/i18n/locales/pl/messages.po | 3 --- packages/i18n/locales/pt-BR/messages.po | 3 --- packages/i18n/locales/ru/messages.po | 3 --- packages/i18n/locales/tr/messages.po | 3 --- packages/i18n/locales/uk/messages.po | 3 --- packages/i18n/locales/vi/messages.po | 3 --- packages/i18n/locales/zh-CN/messages.po | 3 --- packages/i18n/locales/zh-TW/messages.po | 3 --- packages/i18n/src/server-errors.ts | 8 ------ .../bundle/rootfs/usr/local/bin/superset-boot | 2 +- packages/trpc/src/lib/sandbox/index.ts | 1 + packages/trpc/src/lib/sandbox/vercel.test.ts | 27 ++++++++++++------- packages/trpc/src/lib/sandbox/vercel.ts | 13 ++++++--- .../trpc/src/router/automation/run.test.ts | 9 +++++++ packages/trpc/src/router/automation/run.ts | 11 ++++++-- .../router/cloud-workspace/cloud-workspace.ts | 13 +++------ .../src/router/cloud-workspace/provision.ts | 18 +++++-------- 28 files changed, 65 insertions(+), 106 deletions(-) diff --git a/docs/cloud-sandbox-mismatches.md b/docs/cloud-sandbox-mismatches.md index e74635317ff..916f2f611cc 100644 --- a/docs/cloud-sandbox-mismatches.md +++ b/docs/cloud-sandbox-mismatches.md @@ -950,13 +950,14 @@ drive, mounted read-only at `/opt/agents`; a new box's own disk has none, and an older golden's disk copies are left alone and never used. A machine someone owns keeps a mount; a sandbox gets one only from `Sandbox.create`, `Sandbox.fork` or `sandbox.update({ mounts })` before the session starts, so -`provisionSandbox`, `wakeSandbox` and the restart in -`promoteSandboxToEnvironment` all pass it. A drive lives in one region and -mounts only there, so it is named by the box's own region, and the release -writes one in every region. A drive that cannot be mounted after a few tries -on 429 and 5xx fails the create or the wake with "Agents are unavailable in - right now; try again" and a Sentry event with the region and bundle; -a promote keeps its golden and leaves the source stopped. A boot that finds no +`provisionSandbox` and `wakeSandbox` pass it, and a promote's restart is a +wake. A drive lives in one region and mounts only there, so it is named by +the box's own region, and the release writes one in every region. A drive +that cannot be mounted after a few tries on 429 and 5xx fails the create or +the wake the way any box that will not start does ("The cloud workspace could +not start"; a wake answers "Cloud workspace is still starting"): a customer +never sees the drive. The drive, region and bundle go only to Sentry and the +server log. A promote keeps its golden. A boot that finds no manifest at `/opt/agents` is refused before host-service starts. **Open:** a running session whose host-service died is booted again without a remount, since a mount cannot be added to a live session; if that session has no diff --git a/packages/i18n/locales/cs/messages.po b/packages/i18n/locales/cs/messages.po index e3e29578dae..09933a46950 100644 --- a/packages/i18n/locales/cs/messages.po +++ b/packages/i18n/locales/cs/messages.po @@ -2416,9 +2416,6 @@ msgstr "Náhledy řízené agenty" msgid "Agents" msgstr "Agenti" -msgid "Agents are unavailable in {region} right now; try again" -msgstr "Agenti teď nejsou v {region} k dispozici; zkuste to znovu" - #. placeholder {0}: displayName(name) msgid "Agents cannot reach {0}'s tools" msgstr "Agenti nemají přístup k nástrojům {0}" diff --git a/packages/i18n/locales/de/messages.po b/packages/i18n/locales/de/messages.po index fa5ad2a0ab0..6a48fd54ac7 100644 --- a/packages/i18n/locales/de/messages.po +++ b/packages/i18n/locales/de/messages.po @@ -2416,9 +2416,6 @@ msgstr "Vom Agenten gesteuerte Vorschauen" msgid "Agents" msgstr "Agenten" -msgid "Agents are unavailable in {region} right now; try again" -msgstr "Agenten sind in {region} gerade nicht verfügbar; versuche es erneut" - #. placeholder {0}: displayName(name) msgid "Agents cannot reach {0}'s tools" msgstr "Agenten können die Tools von {0} nicht erreichen" diff --git a/packages/i18n/locales/en/messages.po b/packages/i18n/locales/en/messages.po index 2360a832d95..22835bd432c 100644 --- a/packages/i18n/locales/en/messages.po +++ b/packages/i18n/locales/en/messages.po @@ -2416,9 +2416,6 @@ msgstr "Agent-driven previews" msgid "Agents" msgstr "Agents" -msgid "Agents are unavailable in {region} right now; try again" -msgstr "Agents are unavailable in {region} right now; try again" - #. placeholder {0}: displayName(name) msgid "Agents cannot reach {0}'s tools" msgstr "Agents cannot reach {0}'s tools" diff --git a/packages/i18n/locales/es/messages.po b/packages/i18n/locales/es/messages.po index 67b40d70fea..4bb6cf105a8 100644 --- a/packages/i18n/locales/es/messages.po +++ b/packages/i18n/locales/es/messages.po @@ -2416,9 +2416,6 @@ msgstr "Vistas previas controladas por el agente" msgid "Agents" msgstr "Agentes" -msgid "Agents are unavailable in {region} right now; try again" -msgstr "Los agentes no están disponibles en {region} en este momento; inténtalo de nuevo" - #. placeholder {0}: displayName(name) msgid "Agents cannot reach {0}'s tools" msgstr "Los agentes no pueden acceder a las herramientas de {0}" diff --git a/packages/i18n/locales/fr/messages.po b/packages/i18n/locales/fr/messages.po index ac72c2edc3c..1f42b4bba60 100644 --- a/packages/i18n/locales/fr/messages.po +++ b/packages/i18n/locales/fr/messages.po @@ -2416,9 +2416,6 @@ msgstr "Aperçus pilotés par l'agent" msgid "Agents" msgstr "Agents" -msgid "Agents are unavailable in {region} right now; try again" -msgstr "Les agents ne sont pas disponibles dans {region} pour le moment ; réessayez" - #. placeholder {0}: displayName(name) msgid "Agents cannot reach {0}'s tools" msgstr "Les agents ne peuvent pas accéder aux outils de {0}" diff --git a/packages/i18n/locales/id/messages.po b/packages/i18n/locales/id/messages.po index 18a30865de5..82c2d66b65d 100644 --- a/packages/i18n/locales/id/messages.po +++ b/packages/i18n/locales/id/messages.po @@ -2416,9 +2416,6 @@ msgstr "Pratinjau yang digerakkan agen" msgid "Agents" msgstr "Agen" -msgid "Agents are unavailable in {region} right now; try again" -msgstr "Agen tidak tersedia di {region} saat ini; coba lagi" - #. placeholder {0}: displayName(name) msgid "Agents cannot reach {0}'s tools" msgstr "Agen tidak dapat menjangkau alat {0}" diff --git a/packages/i18n/locales/it/messages.po b/packages/i18n/locales/it/messages.po index a9af82615af..629470d66f5 100644 --- a/packages/i18n/locales/it/messages.po +++ b/packages/i18n/locales/it/messages.po @@ -2416,9 +2416,6 @@ msgstr "Anteprime guidate dall'agente" msgid "Agents" msgstr "Agenti" -msgid "Agents are unavailable in {region} right now; try again" -msgstr "Gli agenti non sono disponibili in {region} al momento; riprova" - #. placeholder {0}: displayName(name) msgid "Agents cannot reach {0}'s tools" msgstr "Gli agenti non possono raggiungere gli strumenti di {0}" diff --git a/packages/i18n/locales/ja/messages.po b/packages/i18n/locales/ja/messages.po index 06f328edad8..4b765ee5866 100644 --- a/packages/i18n/locales/ja/messages.po +++ b/packages/i18n/locales/ja/messages.po @@ -2416,9 +2416,6 @@ msgstr "エージェントが操作するプレビュー" msgid "Agents" msgstr "エージェント" -msgid "Agents are unavailable in {region} right now; try again" -msgstr "現在 {region} ではエージェントを利用できません。もう一度お試しください" - #. placeholder {0}: displayName(name) msgid "Agents cannot reach {0}'s tools" msgstr "エージェントは {0} のツールにアクセスできません" diff --git a/packages/i18n/locales/ko/messages.po b/packages/i18n/locales/ko/messages.po index 166419e7c02..062b60ed8a5 100644 --- a/packages/i18n/locales/ko/messages.po +++ b/packages/i18n/locales/ko/messages.po @@ -2416,9 +2416,6 @@ msgstr "에이전트가 조작하는 미리보기" msgid "Agents" msgstr "에이전트" -msgid "Agents are unavailable in {region} right now; try again" -msgstr "지금은 {region}에서 에이전트를 사용할 수 없습니다. 다시 시도하세요" - #. placeholder {0}: displayName(name) msgid "Agents cannot reach {0}'s tools" msgstr "에이전트가 {0}의 도구에 접근할 수 없습니다" diff --git a/packages/i18n/locales/nl/messages.po b/packages/i18n/locales/nl/messages.po index 25d9a25289a..e551554bd86 100644 --- a/packages/i18n/locales/nl/messages.po +++ b/packages/i18n/locales/nl/messages.po @@ -2416,9 +2416,6 @@ msgstr "Previews aangestuurd door agents" msgid "Agents" msgstr "Agents" -msgid "Agents are unavailable in {region} right now; try again" -msgstr "Agents zijn op dit moment niet beschikbaar in {region}; probeer het opnieuw" - #. placeholder {0}: displayName(name) msgid "Agents cannot reach {0}'s tools" msgstr "Agents kunnen de tools van {0} niet bereiken" diff --git a/packages/i18n/locales/pl/messages.po b/packages/i18n/locales/pl/messages.po index e705e33d874..411313bc257 100644 --- a/packages/i18n/locales/pl/messages.po +++ b/packages/i18n/locales/pl/messages.po @@ -2416,9 +2416,6 @@ msgstr "Podglądy sterowane przez agenta" msgid "Agents" msgstr "Agenci" -msgid "Agents are unavailable in {region} right now; try again" -msgstr "Agenci są teraz niedostępni w {region}; spróbuj ponownie" - #. placeholder {0}: displayName(name) msgid "Agents cannot reach {0}'s tools" msgstr "Agenci nie mają dostępu do narzędzi {0}" diff --git a/packages/i18n/locales/pt-BR/messages.po b/packages/i18n/locales/pt-BR/messages.po index 122cab0bdc5..80174655017 100644 --- a/packages/i18n/locales/pt-BR/messages.po +++ b/packages/i18n/locales/pt-BR/messages.po @@ -2416,9 +2416,6 @@ msgstr "Prévias controladas pelo agente" msgid "Agents" msgstr "Agentes" -msgid "Agents are unavailable in {region} right now; try again" -msgstr "Os agentes não estão disponíveis em {region} no momento; tente novamente" - #. placeholder {0}: displayName(name) msgid "Agents cannot reach {0}'s tools" msgstr "Os agentes não conseguem acessar as ferramentas de {0}" diff --git a/packages/i18n/locales/ru/messages.po b/packages/i18n/locales/ru/messages.po index d11b86b5f6d..814cd29b4b7 100644 --- a/packages/i18n/locales/ru/messages.po +++ b/packages/i18n/locales/ru/messages.po @@ -2416,9 +2416,6 @@ msgstr "Превью под управлением агента" msgid "Agents" msgstr "Агенты" -msgid "Agents are unavailable in {region} right now; try again" -msgstr "Агенты сейчас недоступны в {region}; попробуйте ещё раз" - #. placeholder {0}: displayName(name) msgid "Agents cannot reach {0}'s tools" msgstr "Агенты не могут получить доступ к инструментам {0}" diff --git a/packages/i18n/locales/tr/messages.po b/packages/i18n/locales/tr/messages.po index b26aa904bc5..b8e7b7af79a 100644 --- a/packages/i18n/locales/tr/messages.po +++ b/packages/i18n/locales/tr/messages.po @@ -2416,9 +2416,6 @@ msgstr "Ajanın yönettiği önizlemeler" msgid "Agents" msgstr "Ajanlar" -msgid "Agents are unavailable in {region} right now; try again" -msgstr "Ajanlar şu anda {region} bölgesinde kullanılamıyor; tekrar deneyin" - #. placeholder {0}: displayName(name) msgid "Agents cannot reach {0}'s tools" msgstr "Ajanlar {0} araçlarına erişemiyor" diff --git a/packages/i18n/locales/uk/messages.po b/packages/i18n/locales/uk/messages.po index 0b30037700a..e0cd327ba64 100644 --- a/packages/i18n/locales/uk/messages.po +++ b/packages/i18n/locales/uk/messages.po @@ -2410,9 +2410,6 @@ msgstr "Попередній перегляд під керуванням аге msgid "Agents" msgstr "Агенти" -msgid "Agents are unavailable in {region} right now; try again" -msgstr "Агенти зараз недоступні в {region}; спробуйте ще раз" - #. placeholder {0}: displayName(name) msgid "Agents cannot reach {0}'s tools" msgstr "Агенти не мають доступу до інструментів {0}" diff --git a/packages/i18n/locales/vi/messages.po b/packages/i18n/locales/vi/messages.po index 504b561571f..cd39fe440dd 100644 --- a/packages/i18n/locales/vi/messages.po +++ b/packages/i18n/locales/vi/messages.po @@ -2416,9 +2416,6 @@ msgstr "Bản xem trước do agent điều khiển" msgid "Agents" msgstr "Agent" -msgid "Agents are unavailable in {region} right now; try again" -msgstr "Hiện không thể dùng agent tại {region}; hãy thử lại" - #. placeholder {0}: displayName(name) msgid "Agents cannot reach {0}'s tools" msgstr "Agent không thể truy cập các công cụ của {0}" diff --git a/packages/i18n/locales/zh-CN/messages.po b/packages/i18n/locales/zh-CN/messages.po index 7179788de6a..ff17e6a7592 100644 --- a/packages/i18n/locales/zh-CN/messages.po +++ b/packages/i18n/locales/zh-CN/messages.po @@ -2416,9 +2416,6 @@ msgstr "由智能体驱动的预览" msgid "Agents" msgstr "智能体" -msgid "Agents are unavailable in {region} right now; try again" -msgstr "{region} 目前无法使用智能体,请重试" - #. placeholder {0}: displayName(name) msgid "Agents cannot reach {0}'s tools" msgstr "智能体无法访问 {0} 的工具" diff --git a/packages/i18n/locales/zh-TW/messages.po b/packages/i18n/locales/zh-TW/messages.po index 3f516b492fe..b750ae9f547 100644 --- a/packages/i18n/locales/zh-TW/messages.po +++ b/packages/i18n/locales/zh-TW/messages.po @@ -2416,9 +2416,6 @@ msgstr "由代理程式驅動的預覽" msgid "Agents" msgstr "代理程式" -msgid "Agents are unavailable in {region} right now; try again" -msgstr "{region} 目前無法使用代理程式,請再試一次" - #. placeholder {0}: displayName(name) msgid "Agents cannot reach {0}'s tools" msgstr "代理程式無法存取 {0} 的工具" diff --git a/packages/i18n/src/server-errors.ts b/packages/i18n/src/server-errors.ts index f5b8c1d9b7e..2ef4684e458 100644 --- a/packages/i18n/src/server-errors.ts +++ b/packages/i18n/src/server-errors.ts @@ -419,14 +419,6 @@ export const serverErrorMessages: Record< message: "Only owners and admins can see usage by person.", }), ), - "serverError.cloudWorkspace.agentsUnavailable": (params) => { - const region = String(params?.region ?? ""); - return i18n._( - msg({ - message: `Agents are unavailable in ${region} right now; try again`, - }), - ); - }, "serverError.cloudWorkspace.cloudSandboxesAreInternalOnly": (params) => i18n._({ id: "serverError.cloudWorkspace.cloudSandboxesAreInternalOnly", diff --git a/packages/sandbox/bundle/rootfs/usr/local/bin/superset-boot b/packages/sandbox/bundle/rootfs/usr/local/bin/superset-boot index 93ff127fbde..efde50ece95 100755 --- a/packages/sandbox/bundle/rootfs/usr/local/bin/superset-boot +++ b/packages/sandbox/bundle/rootfs/usr/local/bin/superset-boot @@ -128,7 +128,7 @@ link_agents() { find /usr/local/bin -maxdepth 1 -lname "${dir}/*" ! -exec test -e {} \; -delete if ! jq -e 'type == "array"' "$manifest" > /dev/null 2>&1; then stamp "agents.unavailable no manifest at ${dir}" - refuse "no agents drive at ${dir}; the control plane mounts it on every create and wake" + refuse "no agent CLIs at ${dir}" fi while IFS=$'\t' read -r name sha bin; do if ! [[ "$name" =~ ^[a-z0-9-]+$ && "$sha" =~ ^[0-9a-f]{64}$ && "$bin" =~ ^[A-Za-z0-9._/-]+$ && "$bin" != *..* ]]; then diff --git a/packages/trpc/src/lib/sandbox/index.ts b/packages/trpc/src/lib/sandbox/index.ts index af94d059c5e..e11aa0df798 100644 --- a/packages/trpc/src/lib/sandbox/index.ts +++ b/packages/trpc/src/lib/sandbox/index.ts @@ -56,6 +56,7 @@ export { type ActiveWorkspaceSandbox, AgentToolsUnavailableError, type BoxSession, + CLOUD_WORKSPACE_NOT_STARTED, deleteSandbox, describeSandbox, findSandboxSession, diff --git a/packages/trpc/src/lib/sandbox/vercel.test.ts b/packages/trpc/src/lib/sandbox/vercel.test.ts index af6fb895a56..a3e1c31bacc 100644 --- a/packages/trpc/src/lib/sandbox/vercel.test.ts +++ b/packages/trpc/src/lib/sandbox/vercel.test.ts @@ -14,6 +14,7 @@ import { AgentToolsUnavailableError, agentToolsMounts, applySandboxPolicy, + CLOUD_WORKSPACE_NOT_STARTED, deleteSandbox, describeSandbox, findSandboxSession, @@ -555,8 +556,6 @@ const claimFor = (bundle: string) => networkPolicy: "allow-all", identity: IDENTITY(bundle), }) as unknown as SandboxClaim; -const unavailableIn = (region: string) => - `Agents are unavailable in ${region} right now; try again`; describe("agentToolsMounts", () => { const detached = () => Promise.resolve({ currentSandboxName: undefined }); @@ -597,15 +596,19 @@ describe("agentToolsMounts", () => { expect(mounts).toBeDefined(); }); - test("a drive that stays unavailable, is missing or a writer holds fails with the region, and reports region and bundle", async () => { + test("a drive that stays unavailable, is missing or a writer holds fails like any box that will not start; region, bundle and drive go only to Sentry", async () => { reports.length = 0; let asked = 0; - await expect( - agentToolsMounts("agent-tools-sfo1-aaaaaaaaaaaa", () => { + const error = await agentToolsMounts( + "agent-tools-sfo1-aaaaaaaaaaaa", + () => { asked++; return Promise.reject(failure(503)); - }), - ).rejects.toThrow(unavailableIn("sfo1")); + }, + ).catch((caught: unknown) => caught); + expect(error).toBeInstanceOf(AgentToolsUnavailableError); + expect((error as Error).message).toBe(CLOUD_WORKSPACE_NOT_STARTED); + expect((error as Error).message).not.toMatch(/drive|agent|sfo1/i); expect(asked).toBe(3); await expect( agentToolsMounts("agent-tools-sfo1-bbbbbbbbbbbb", () => @@ -677,7 +680,9 @@ describe("wakeSandbox", () => { drives.set("agent-tools-iad1-c20000000000", () => Promise.reject(failure(404)), ); - await expect(wake(BUNDLE("c2"))).rejects.toThrow(unavailableIn("iad1")); + await expect(wake(BUNDLE("c2"))).rejects.toThrow( + CLOUD_WORKSPACE_NOT_STARTED, + ); expect(updates).toEqual([]); expect(resumes).toEqual([]); }); @@ -690,7 +695,9 @@ describe("wakeSandbox", () => { expect(updates).toHaveLength(2); updates.length = 0; updateAnswers = [failure(503), failure(503), failure(503)]; - await expect(wake(BUNDLE("c4"))).rejects.toThrow(unavailableIn("iad1")); + await expect(wake(BUNDLE("c4"))).rejects.toThrow( + CLOUD_WORKSPACE_NOT_STARTED, + ); expect(updates).toHaveLength(3); expect(resumes).toEqual([BUNDLE("c3")]); }); @@ -745,7 +752,7 @@ describe("provisionSandbox", () => { Promise.reject(failure(404)), ); await expect(provision(BUNDLE("d2"))).rejects.toThrow( - unavailableIn("sfo1"), + CLOUD_WORKSPACE_NOT_STARTED, ); expect(creates).toEqual([]); }); diff --git a/packages/trpc/src/lib/sandbox/vercel.ts b/packages/trpc/src/lib/sandbox/vercel.ts index e2e588eaae0..fad989eff93 100644 --- a/packages/trpc/src/lib/sandbox/vercel.ts +++ b/packages/trpc/src/lib/sandbox/vercel.ts @@ -200,7 +200,14 @@ const publishedAgentToolsDrives = new Set(); /** A box has no agents without its drive, so a transient refusal gets a few quick tries. */ export const AGENT_TOOLS_RETRY = { attempts: 3, delayMs: 500 }; -/** The open fails: a box without its agents drive has no agent CLIs at all. */ +export const CLOUD_WORKSPACE_NOT_STARTED = + "The cloud workspace could not start"; + +/** + * The open fails: a box without its agents drive has no agent CLIs at all. + * The message is what a customer sees for any box that will not start; the + * drive, region and bundle go only to logs and Sentry. + */ export class AgentToolsUnavailableError extends Error { readonly region: string; readonly bundle: string; @@ -210,9 +217,7 @@ export class AgentToolsUnavailableError extends Error { ) { const [, region = "", bundle = ""] = /^agent-tools-([a-z]+[0-9]+)-([0-9a-f]{12})$/.exec(driveName) ?? []; - super(`Agents are unavailable in ${region} right now; try again`, { - cause, - }); + super(CLOUD_WORKSPACE_NOT_STARTED, { cause }); this.name = "AgentToolsUnavailableError"; this.region = region; this.bundle = bundle; diff --git a/packages/trpc/src/router/automation/run.test.ts b/packages/trpc/src/router/automation/run.test.ts index 7b99f29d4b6..f52037cd67e 100644 --- a/packages/trpc/src/router/automation/run.test.ts +++ b/packages/trpc/src/router/automation/run.test.ts @@ -1,5 +1,6 @@ import { describe, expect, test } from "bun:test"; import { userError } from "../../i18n-error"; +import { AgentToolsUnavailableError } from "../../lib/sandbox/vercel"; import { classifyDispatchError } from "./run"; describe("classifyDispatchError", () => { @@ -16,4 +17,12 @@ describe("classifyDispatchError", () => { ).toBe("cloud_usage_limit_reached"); } }); + + test("a box whose agents drive cannot be mounted is a cloud box that is not ready", () => { + expect( + classifyDispatchError( + new AgentToolsUnavailableError("agent-tools-sfo1-0123456789ab"), + ), + ).toBe("cloud_not_ready"); + }); }); diff --git a/packages/trpc/src/router/automation/run.ts b/packages/trpc/src/router/automation/run.ts index 781e10f8b61..1ddbbbe9183 100644 --- a/packages/trpc/src/router/automation/run.ts +++ b/packages/trpc/src/router/automation/run.ts @@ -4,7 +4,10 @@ import { automationRuns, automations } from "@superset/db/schema"; import { TRPCError } from "@trpc/server"; import { and, desc, eq, inArray, isNotNull, isNull } from "drizzle-orm"; import { isI18nErrorCause } from "../../i18n-error"; -import { SandboxNotReadyError } from "../../lib/sandbox"; +import { + AgentToolsUnavailableError, + SandboxNotReadyError, +} from "../../lib/sandbox"; import { RelayDispatchError } from "./relay-client"; /** @@ -184,7 +187,11 @@ export function classifyDispatchError( err: unknown, ): AutomationRunErrorCode | null { if (err instanceof RunFailure) return err.code; - if (err instanceof SandboxNotReadyError) return "cloud_not_ready"; + if ( + err instanceof SandboxNotReadyError || + err instanceof AgentToolsUnavailableError + ) + return "cloud_not_ready"; if (err instanceof TRPCError && isI18nErrorCause(err.cause)) { return CLOUD_ERROR_CODES[err.cause.i18nKey] ?? null; } diff --git a/packages/trpc/src/router/cloud-workspace/cloud-workspace.ts b/packages/trpc/src/router/cloud-workspace/cloud-workspace.ts index e433c83b2b6..5f669be0dc1 100644 --- a/packages/trpc/src/router/cloud-workspace/cloud-workspace.ts +++ b/packages/trpc/src/router/cloud-workspace/cloud-workspace.ts @@ -173,15 +173,10 @@ async function addressSandbox( : await wakeCloudWorkspace(row); return { ...woken, running: true }; } catch (error) { - if (error instanceof AgentToolsUnavailableError) { - throw userError({ - code: "SERVICE_UNAVAILABLE", - message: error.message, - i18nKey: "serverError.cloudWorkspace.agentsUnavailable", - params: { region: error.region }, - }); - } - if (error instanceof SandboxNotReadyError) { + if ( + error instanceof SandboxNotReadyError || + error instanceof AgentToolsUnavailableError + ) { throw new TRPCError({ code: "TIMEOUT", message: "Cloud workspace is still starting", diff --git a/packages/trpc/src/router/cloud-workspace/provision.ts b/packages/trpc/src/router/cloud-workspace/provision.ts index d8858436ed3..4889cdec400 100644 --- a/packages/trpc/src/router/cloud-workspace/provision.ts +++ b/packages/trpc/src/router/cloud-workspace/provision.ts @@ -10,9 +10,9 @@ import { eq } from "drizzle-orm"; import { controlTags } from "../../lib/control-log"; import { nudge } from "../../lib/realtime"; import { - AgentToolsUnavailableError, bootPhases, buildSandboxClaim, + CLOUD_WORKSPACE_NOT_STARTED, deleteSandbox, provisionSandbox, SandboxNotReadyError, @@ -264,16 +264,12 @@ async function provision( reason: provisionFailureReason(error), error, }); - await failChatMovesTo( - row.id, - error instanceof AgentToolsUnavailableError - ? error.message - : "The cloud workspace could not start", - ).catch((moveError) => - console.error( - `[cloud-workspace] could not fail the chat moves into ${row.id}`, - moveError, - ), + await failChatMovesTo(row.id, CLOUD_WORKSPACE_NOT_STARTED).catch( + (moveError) => + console.error( + `[cloud-workspace] could not fail the chat moves into ${row.id}`, + moveError, + ), ); console.error(`[cloud-workspace] provisioning failed for ${row.id}`, error); return "failed"; From 359163d2174eb5b7d5f3085cf11e7f59e1c78779 Mon Sep 17 00:00:00 2001 From: Satya Patel <14907857+saddlepaddle@users.noreply.github.com> Date: Wed, 18 Nov 2026 22:14:49 +0000 Subject: [PATCH 3/3] fix(sandbox): check the root disk only where this checkout built the image real-sandbox boots the registry's latest image, which a release of main built with the agents on its disk, so a clean root disk is not this checkout's to promise there. The probe then checks that the bundle runs no agent-installing step and logs the image's leftovers; the strict check stays for a release that pushed its own image (not --skip-image). --- packages/sandbox/src/environments/probe.ts | 32 +++++++++++++++++--- packages/sandbox/src/environments/release.ts | 9 ++++-- 2 files changed, 33 insertions(+), 8 deletions(-) diff --git a/packages/sandbox/src/environments/probe.ts b/packages/sandbox/src/environments/probe.ts index 2162e7da6de..7dc3cc9bb7f 100644 --- a/packages/sandbox/src/environments/probe.ts +++ b/packages/sandbox/src/environments/probe.ts @@ -29,6 +29,12 @@ export interface ProbeArgs { expectDependencies?: boolean; /** Set when a brokered Anthropic rule is on the box's firewall. */ expectAnthropicRule?: boolean; + /** + * True when the box's image was built from this checkout, so its root disk + * must hold no agent CLI. An older image still has its own copies; then only + * the bundle is checked, which must install none. + */ + imageFromThisCheckout?: boolean; /** A gate to mint a ticket at; skipped when the secret is absent. */ gate?: { workspaceId: string; userId: string } | null; log: (line: string) => void; @@ -151,11 +157,27 @@ export async function probeBox(args: ProbeArgs): Promise { ); } const rootDiskAgents = (await run(ROOT_DISK_AGENTS_COMMAND)).trim(); - check( - "no agent CLI on the box's root disk, so none in its golden", - rootDiskAgents === "", - rootDiskAgents.replaceAll("\n", " "), - ); + if (args.imageFromThisCheckout) { + check( + "no agent CLI on the box's root disk, so none in its golden", + rootDiskAgents === "", + rootDiskAgents.replaceAll("\n", " "), + ); + } else { + const steps = await run( + `cut -f1 ${SANDBOX_PATHS.bundleRoot}/current/steps.tsv`, + ); + check( + "the bundle installs no agent CLI on the root disk", + /\binstall-host\b/.test(steps) && !/install-agent-clis/.test(steps), + steps.trim().replaceAll("\n", " "), + ); + if (rootDiskAgents) { + log( + `info the image predates this checkout and still holds: ${rootDiskAgents.replaceAll("\n", " ")}`, + ); + } + } const cliVersion = await run("superset --version"); check("superset runs", /^\d+\.\d+\.\d+/m.test(cliVersion), cliVersion.trim()); if ( diff --git a/packages/sandbox/src/environments/release.ts b/packages/sandbox/src/environments/release.ts index 661f2617b01..c82f6840b25 100644 --- a/packages/sandbox/src/environments/release.ts +++ b/packages/sandbox/src/environments/release.ts @@ -368,12 +368,14 @@ const checks: Array<[label: string, command: string, expect: RegExp]> = [ ["ntn", "ntn --version", /\d+\.\d+\.\d+/], ["agent-device", "agent-device --version", /\d+\.\d+\.\d+/], ["stripe", "stripe --version", /\d+\.\d+\.\d+/], - [ +]; +if (!SKIP_IMAGE) { + checks.push([ "no agent CLI on the golden's root disk", `[ -z "$(${ROOT_DISK_AGENTS_COMMAND})" ] && echo clean`, /^clean$/m, - ], -]; + ]); +} let failed = 0; for (const [label, command, expect] of checks) { const { logs } = await run(goldenBox, command); @@ -461,6 +463,7 @@ await provisionSandbox({ await wakeSandbox({ providerSandboxId: probe, claim: probeClaim }); let probeFailed = await probeBox({ name: probe, + imageFromThisCheckout: !SKIP_IMAGE, credentials, hostSecret: probeSecret, bundleSha: bundle.sha256,