From 13a86236ea7a0bd84595c37c28912579c31874bc Mon Sep 17 00:00:00 2001 From: Satya Patel <14907857+saddlepaddle@users.noreply.github.com> Date: Mon, 16 Nov 2026 22:30:48 +0000 Subject: [PATCH 1/2] feat(sandbox): mount agent CLIs from a shared read-only drive (draft, flag off) The agent CLIs (claude, codex, pdev; ~1.1 GB with staged archives) are in the image and re-downloaded into every workspace's own disk at each agent release. Snapshots are full block-level copies, so those bytes are paid per workspace. - src/agent-tools.ts publishes each agent into a per-region Vercel Drive by sha, writes manifest.json last, keeps the previous sha, prunes the rest. - provisionSandbox and wakeSandbox mount agent-tools- read-only at /opt/agents when SANDBOX_AGENT_TOOLS_DRIVE=true. Default off. - superset-boot links the manifest's agents onto /usr/local/bin; without a manifest it stamps agents.unavailable and keeps the image's agents, re-running install-agent-clis if a link points into a missing mount. The image still installs the agents; removing them is the follow-up. --- .env.example | 2 + .env.local.example | 1 + docs/cloud-sandbox-mismatches.md | 10 + .../bundle/rootfs/usr/local/bin/superset-boot | 28 ++ packages/sandbox/package.json | 1 + packages/sandbox/src/agent-tools.test.ts | 165 ++++++++++++ packages/sandbox/src/agent-tools.ts | 242 ++++++++++++++++++ packages/sandbox/src/boot-twice.ts | 45 ++++ packages/sandbox/src/boot.test.ts | 10 + packages/shared/src/sandbox/contract.ts | 8 + packages/trpc/src/env.ts | 2 + packages/trpc/src/lib/sandbox/vercel.test.ts | 15 ++ packages/trpc/src/lib/sandbox/vercel.ts | 34 +++ 13 files changed, 563 insertions(+) create mode 100644 packages/sandbox/src/agent-tools.test.ts create mode 100644 packages/sandbox/src/agent-tools.ts diff --git a/.env.example b/.env.example index 6f5e5312b16..2688d1eb7f7 100644 --- a/.env.example +++ b/.env.example @@ -286,6 +286,8 @@ REALTIME_NUDGE_SECRET= VERCEL_SANDBOX_TOKEN= VERCEL_SANDBOX_TEAM_ID= VERCEL_SANDBOX_PROJECT_ID= +# `true` mounts the agent-tools drive on every box; anything else keeps the image agents. +SANDBOX_AGENT_TOOLS_DRIVE= AUTUMN_SECRET_KEY= AUTUMN_WEBHOOK_SECRET= BILLING_PROVIDER= diff --git a/.env.local.example b/.env.local.example index bbc38b0569a..24843280cea 100644 --- a/.env.local.example +++ b/.env.local.example @@ -267,6 +267,7 @@ REALTIME_NUDGE_SECRET=fake-realtime-nudge-secret VERCEL_SANDBOX_TOKEN=fake-vercel-sandbox-token VERCEL_SANDBOX_TEAM_ID=team_fake VERCEL_SANDBOX_PROJECT_ID=prj_fake +SANDBOX_AGENT_TOOLS_DRIVE=false AUTUMN_SECRET_KEY=am_sk_test_fake_local_dev AUTUMN_WEBHOOK_SECRET=whsec_ZmFrZS1sb2NhbC1kZXYtc2VjcmV0 BILLING_PROVIDER= diff --git a/docs/cloud-sandbox-mismatches.md b/docs/cloud-sandbox-mismatches.md index a9ab604c59e..eaf525649b9 100644 --- a/docs/cloud-sandbox-mismatches.md +++ b/docs/cloud-sandbox-mismatches.md @@ -903,6 +903,16 @@ and provider. Keep the workspace id on both sides: a provisioning failure is recorded against the API and a runtime failure against the sandbox, and that id is the only thing that joins the two halves of one broken workspace. +**A drive mount lasts one session and a fork does not inherit it — open.** +Behind `SANDBOX_AGENT_TOOLS_DRIVE`, the agent CLIs come from the region's +`agent-tools-` drive, mounted read-only at `/opt/agents`. A machine +someone owns keeps a mount; a sandbox gets one only from `Sandbox.create`, +`Sandbox.fork` or `sandbox.update({ mounts })` before the session starts, so +`provisionSandbox` and `wakeSandbox` both pass it. A drive lives in one region +and mounts only in that region. A session without the mount leaves links in +`/usr/local/bin` that point nowhere; `superset-boot` re-runs +`install-agent-clis` to put the image's agents back. Untested on Vercel. + ## Running this repo's own dev stack inside a sandbox Reproducing an app bug end to end from a cloud workspace means bringing up diff --git a/packages/sandbox/bundle/rootfs/usr/local/bin/superset-boot b/packages/sandbox/bundle/rootfs/usr/local/bin/superset-boot index a85fc465b59..66dba948078 100755 --- a/packages/sandbox/bundle/rootfs/usr/local/bin/superset-boot +++ b/packages/sandbox/bundle/rootfs/usr/local/bin/superset-boot @@ -9,6 +9,7 @@ # install the pinned bundle only when its hash differs from current # apply-rootfs, sync-assets, each a hash compare per file, asset, step # run-steps +# link the drive's agents only when the control plane mounted one # start host-service as ubuntu first; everything else is a side effect # start the desktop and the checkout, detached # @@ -117,6 +118,33 @@ fi stamp "assets.start"; "$BUNDLE/setup" sync-assets >> "$SUPERSET_BOOT_LOG" 2>&1 || stamp "assets.failed continuing"; stamp "assets.end" stamp "steps.start"; "$BUNDLE/setup" run-steps >> "$SUPERSET_BOOT_LOG" 2>&1; stamp "steps.end" +# --- the agent CLIs from the region's agent-tools drive, when the control +# plane mounted one. Without it the image's agents stay on PATH. Mounts are +# per session, so a link into a mount this session lacks points nowhere. +link_agents() { + local dir="${SUPERSET_AGENTS_DIR:-/opt/agents}" linked=0 name sha bin target + local manifest="${dir}/manifest.json" + if find /usr/local/bin -maxdepth 1 -lname "${dir}/*" ! -exec test -e {} \; -print | grep -q .; then + SUPERSET_BUNDLE_DIR="$BUNDLE" bash "$BUNDLE/steps/install-agent-clis.sh" >> "$SUPERSET_BOOT_LOG" 2>&1 || + stamp "agents.restore.failed" + fi + if ! jq -e 'type == "array"' "$manifest" > /dev/null 2>&1; then + stamp "agents.unavailable no manifest at ${dir}" + return 0 + fi + while IFS=$'\t' read -r name sha bin; do + if ! [[ "$name" =~ ^[a-z0-9-]+$ && "$sha" =~ ^[0-9a-f]{64}$ && "$bin" =~ ^[A-Za-z0-9._/-]+$ && "$bin" != *..* ]]; then + stamp "agents.skipped invalid entry ${name:0:32}"; continue + fi + target="${dir}/${name}/${sha}/${bin}" + [ -x "$target" ] || { stamp "agents.missing ${name} ${sha:0:12}"; continue; } + ln -sfn "$target" "/usr/local/bin/${name}.tmp" && mv -T "/usr/local/bin/${name}.tmp" "/usr/local/bin/${name}" && + linked=$((linked + 1)) + done < <(jq -r '.[] | [.name, .sha, .bin] | map(tostring) | @tsv' "$manifest") + stamp "agents.linked ${linked} from ${dir}" +} +link_agents + # --- host-service, as the sandbox user, first. A runtime that cannot start # is logged, not fatal: the desktop and the checkout still come up so the # box can be looked at, and the control plane marks the workspace failed. diff --git a/packages/sandbox/package.json b/packages/sandbox/package.json index ad490f36232..cc0208accc2 100644 --- a/packages/sandbox/package.json +++ b/packages/sandbox/package.json @@ -9,6 +9,7 @@ "pin": "bun run src/pin.ts", "assets": "bun run src/assets/produce.ts", "release": "bun run src/environments/release.ts", + "agent-tools": "bun run src/agent-tools.ts", "record-bundle": "bun run src/environments/record-bundle.ts", "test": "bun test", "typecheck": "tsc --noEmit --emitDeclarationOnly false" diff --git a/packages/sandbox/src/agent-tools.test.ts b/packages/sandbox/src/agent-tools.test.ts new file mode 100644 index 00000000000..94d8288214a --- /dev/null +++ b/packages/sandbox/src/agent-tools.test.ts @@ -0,0 +1,165 @@ +import { describe, expect, test } from "bun:test"; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + type AgentToolsManifest, + agentToolsManifest, + parseAgentToolsManifest, + renderBuilderScript, + shasToKeep, +} from "./agent-tools"; +import { type Assets, sha256 } from "./manifest"; + +const sha = (c: string) => c.repeat(64); + +function row(sha256: string, suffix: string): Assets[string] { + return { sha256, suffix, dest: "/usr/local/share/x", mode: "0644" }; +} + +describe("agent-tools manifest", () => { + test("lists every agent at its asset sha, bin relative to its dir", () => { + const assets: Assets = { + claude: row(sha("a"), ""), + codex: row(sha("b"), ".tar.gz"), + pdev: row(sha("c"), ".tar.gz"), + chrome: row(sha("d"), ".deb"), + }; + const manifest = agentToolsManifest(assets); + expect(manifest).toEqual([ + { name: "claude", sha: sha("a"), bin: "claude" }, + { name: "codex", sha: sha("b"), bin: "bin/codex" }, + { name: "pdev", sha: sha("c"), bin: "pdev/pdev" }, + ]); + expect(parseAgentToolsManifest(JSON.stringify(manifest))).toEqual(manifest); + }); + + test("a missing or bad manifest reads as empty", () => { + expect(parseAgentToolsManifest(null)).toEqual([]); + expect(parseAgentToolsManifest("{not json")).toEqual([]); + expect( + parseAgentToolsManifest( + JSON.stringify([{ name: "claude", sha: sha("a"), bin: "../x" }]), + ), + ).toEqual([]); + }); + + test("keeps the new and the previous sha of each agent", () => { + expect( + shasToKeep( + [{ name: "claude", sha: sha("a"), bin: "claude" }], + [ + { name: "claude", sha: sha("b"), bin: "claude" }, + { name: "codex", sha: sha("c"), bin: "bin/codex" }, + ], + ), + ).toEqual({ claude: [sha("a"), sha("b")], codex: [sha("c")] }); + }); +}); + +async function runScript(script: string) { + const proc = Bun.spawn(["bash", "-c", script], { + stdout: "pipe", + stderr: "pipe", + }); + const stderr = await new Response(proc.stderr).text(); + return { exitCode: await proc.exited, stderr }; +} + +describe.skipIf(process.platform !== "linux")("builder script", () => { + test("publishes verified agents, writes the manifest, prunes all but two shas", async () => { + const work = mkdtempSync(join(tmpdir(), "agent-tools-")); + const src = join(work, "src"); + mkdirSync(join(src, "bin"), { recursive: true }); + writeFileSync(join(src, "bin", "codex"), "#!/bin/sh\necho codex\n", { + mode: 0o755, + }); + Bun.spawnSync(["tar", "-czf", join(work, "codex.tar.gz"), "-C", src, "."]); + const claude = new TextEncoder().encode("#!/bin/sh\necho claude\n"); + const codex = readFileSync(join(work, "codex.tar.gz")); + const objects = new Map([ + [sha256(claude), claude], + [`${sha256(codex)}.tar.gz`, codex], + ]); + const server = Bun.serve({ + port: 0, + fetch: (request) => { + const body = objects.get(new URL(request.url).pathname.slice(1)); + return body ? new Response(body) : new Response("", { status: 404 }); + }, + }); + const root = join(work, "drive"); + mkdirSync(join(root, "claude", sha("1")), { recursive: true }); + mkdirSync(join(root, "claude", sha("2")), { recursive: true }); + const assets: Assets = { + claude: row(sha256(claude), ""), + codex: row(sha256(codex), ".tar.gz"), + }; + const previous: AgentToolsManifest = [ + { name: "claude", sha: sha("1"), bin: "claude" }, + ]; + const next: AgentToolsManifest = [ + { name: "claude", sha: sha256(claude), bin: "claude" }, + { name: "codex", sha: sha256(codex), bin: "bin/codex" }, + ]; + try { + const run = await runScript( + renderBuilderScript({ + root, + baseUrl: `http://127.0.0.1:${server.port}`, + assets, + previous, + next, + }), + ); + expect(run.exitCode, run.stderr).toBe(0); + expect( + parseAgentToolsManifest( + readFileSync(join(root, "manifest.json"), "utf8"), + ), + ).toEqual(next); + expect( + Bun.spawnSync([join(root, "codex", sha256(codex), "bin", "codex")]) + .stdout.toString() + .trim(), + ).toBe("codex"); + expect(existsSync(join(root, "claude", sha256(claude), "claude"))).toBe( + true, + ); + expect(existsSync(join(root, "claude", sha("1")))).toBe(true); + expect(existsSync(join(root, "claude", sha("2")))).toBe(false); + expect(existsSync(join(root, ".manifest.json.partial"))).toBe(false); + } finally { + server.stop(true); + } + }); + + test("a sha mismatch fails before the manifest is written", async () => { + const root = mkdtempSync(join(tmpdir(), "agent-tools-")); + const server = Bun.serve({ + port: 0, + fetch: () => new Response("not the agent"), + }); + try { + const run = await runScript( + renderBuilderScript({ + root, + baseUrl: `http://127.0.0.1:${server.port}`, + assets: { claude: row(sha("a"), "") }, + previous: [], + next: [{ name: "claude", sha: sha("a"), bin: "claude" }], + }), + ); + expect(run.exitCode).not.toBe(0); + expect(existsSync(join(root, "manifest.json"))).toBe(false); + } finally { + server.stop(true); + } + }); +}); diff --git a/packages/sandbox/src/agent-tools.ts b/packages/sandbox/src/agent-tools.ts new file mode 100644 index 00000000000..82a97b8c4ae --- /dev/null +++ b/packages/sandbox/src/agent-tools.ts @@ -0,0 +1,242 @@ +/** + * Publishes the agent CLIs to each region's agent-tools drive, which boxes + * mount read-only at `SANDBOX_PATHS.agents` instead of carrying the agents + * on their own disks. Per region: a builder sandbox mounts the drive + * read-write, writes each agent into `///` (fetched by sha from + * the asset bucket and verified, unpacked beside its target and renamed in), + * then `/manifest.json` last, then prunes every sha that is neither in the + * new manifest nor in the one it replaced. + * + * bun run agent-tools [--region sfo1 ...] + * + * Needs VERCEL_SANDBOX_TOKEN, VERCEL_SANDBOX_TEAM_ID, VERCEL_SANDBOX_PROJECT_ID. + * Reads the agent rows from bundle/assets.json, so run it after the release + * that moved them. + */ +import { join } from "node:path"; +import { + SANDBOX_ASSET_BASE_URL, + SANDBOX_PATHS, + sandboxAgentToolsDrive, +} from "@superset/shared/sandbox/contract"; +import { + DEFAULT_SANDBOX_REGION, + isSandboxRegionId, + type SandboxRegionId, +} from "@superset/shared/sandbox/regions"; +import { z } from "zod"; +import { type Assets, assetObjectKey, loadAssets, SHA256 } from "./manifest"; + +/** Each agent's asset row, and its executable relative to the unpacked dir. */ +export const AGENT_TOOLS = [ + { name: "claude", bin: "claude" }, + { name: "codex", bin: "bin/codex" }, + { name: "pdev", bin: "pdev/pdev" }, +] as const; + +export const AGENT_TOOLS_MANIFEST = "manifest.json"; + +const agentToolSchema = z.object({ + name: z.string().regex(/^[a-z0-9-]+$/), + sha: z.string().regex(SHA256), + bin: z + .string() + .regex(/^[A-Za-z0-9._-]+(\/[A-Za-z0-9._-]+)*$/) + .refine((bin) => !bin.split("/").includes(".."), "no .. in bin"), +}); +export const agentToolsManifestSchema = z.array(agentToolSchema); +export type AgentToolsManifest = z.infer; + +export function agentToolsManifest(assets: Assets): AgentToolsManifest { + return agentToolsManifestSchema.parse( + AGENT_TOOLS.map(({ name, bin }) => { + const asset = assets[name]; + if (!asset) throw new Error(`assets.json has no ${name} row`); + return { name, sha: asset.sha256, bin }; + }), + ); +} + +/** A drive that has never been published to, or holds a bad manifest, has none. */ +export function parseAgentToolsManifest( + text: string | null, +): AgentToolsManifest { + if (text === null) return []; + try { + return agentToolsManifestSchema.parse(JSON.parse(text)); + } catch { + return []; + } +} + +/** A box may hold a snapshot taken under the previous manifest, so its shas stay too. */ +export function shasToKeep( + previous: AgentToolsManifest, + next: AgentToolsManifest, +): Record { + const keep: Record> = {}; + for (const { name, sha } of [...next, ...previous]) { + keep[name] ??= new Set(); + keep[name].add(sha); + } + return Object.fromEntries( + Object.entries(keep).map(([name, shas]) => [name, [...shas].sort()]), + ); +} + +const quote = (value: string) => `'${value.replaceAll("'", "'\\''")}'`; + +/** The script the builder runs as root against the drive mounted read-write at `root`. */ +export function renderBuilderScript(args: { + root: string; + baseUrl: string; + assets: Assets; + previous: AgentToolsManifest; + next: AgentToolsManifest; +}): string { + const fetches = args.next.map(({ name, sha, bin }) => { + const asset = args.assets[name]; + if (!asset || asset.sha256 !== sha) { + throw new Error(`no asset row for ${name} at ${sha}`); + } + const archive = asset.suffix === ".tar.gz" ? "tar" : "file"; + return `publish ${[name, sha, bin, assetObjectKey(asset), archive].map(quote).join(" ")}`; + }); + const prunes = Object.entries(shasToKeep(args.previous, args.next)).map( + ([name, shas]) => `prune ${quote(name)} ${shas.map(quote).join(" ")}`, + ); + return `set -euo pipefail +root=${quote(args.root)} +base=${quote(args.baseUrl)} + +publish() { + local name="$1" sha="$2" bin="$3" key="$4" kind="$5" + local target="\${root}/\${name}/\${sha}" + if [ -x "\${target}/\${bin}" ]; then echo "\${name} \${sha:0:12} already published"; return 0; fi + local tmp + tmp="$(mktemp)" + curl -fsSL --retry 5 --retry-delay 2 --retry-all-errors -o "$tmp" "\${base}/\${key}" || + curl -fsSL --retry 2 --retry-delay 2 --retry-all-errors -o "$tmp" "\${base}/\${key}?fresh=$(date +%s)" + local got + got="$(sha256sum "$tmp" | cut -d' ' -f1)" + if [ "$got" != "$sha" ]; then echo "\${name}: expected \${sha}, got \${got}" >&2; rm -f "$tmp"; return 1; fi + rm -rf "\${target}.partial" + mkdir -p "\${target}.partial" + if [ "$kind" = tar ]; then + tar -xzf "$tmp" -C "\${target}.partial" + else + install -D -m 0755 "$tmp" "\${target}.partial/\${bin}" + fi + rm -f "$tmp" + [ -x "\${target}.partial/\${bin}" ] || { echo "\${name}: no executable at \${bin}" >&2; return 1; } + rm -rf "$target" + mv -T "\${target}.partial" "$target" + echo "\${name} \${sha:0:12} published" +} + +prune() { + local name="$1" dir sha keep + shift + for dir in "\${root}/\${name}"/*/; do + [ -d "$dir" ] || continue + dir="\${dir%/}" + sha="\${dir##*/}" + keep=0 + for want in "$@"; do [ "$sha" = "$want" ] && keep=1; done + [ "$keep" = 1 ] || { rm -rf "$dir"; echo "\${name} \${sha:0:12} pruned"; } + done +} + +${fetches.join("\n")} +printf '%s\\n' ${quote(JSON.stringify(args.next, null, "\t"))} > "\${root}/.${AGENT_TOOLS_MANIFEST}.partial" +sync +mv -T "\${root}/.${AGENT_TOOLS_MANIFEST}.partial" "\${root}/${AGENT_TOOLS_MANIFEST}" +sync +${prunes.join("\n")} +sync +`; +} + +function regionsFromArgv(argv: string[]): SandboxRegionId[] { + const regions: SandboxRegionId[] = []; + for (let i = 0; i < argv.length; i++) { + if (argv[i] !== "--region") continue; + const region = argv[i + 1] ?? ""; + if (!isSandboxRegionId(region)) throw new Error(`unknown region ${region}`); + regions.push(region); + } + return regions.length ? regions : [DEFAULT_SANDBOX_REGION]; +} + +async function publishRegion( + region: SandboxRegionId, + assets: Assets, + credentials: { token: string; teamId: string; projectId: string }, +): Promise { + const { Drive, Sandbox } = await import("@vercel/sandbox"); + const root = SANDBOX_PATHS.agents; + const drive = await Drive.getOrCreate({ + ...credentials, + name: sandboxAgentToolsDrive(region), + region, + }); + const builder = await Sandbox.create({ + ...credentials, + name: `agent-tools-builder-${region}-${Date.now().toString(36)}`, + region, + persistent: false, + timeout: 20 * 60_000, + mounts: { [root]: drive }, + }); + try { + const previous = parseAgentToolsManifest( + ( + await builder.readFileToBuffer({ + path: `${root}/${AGENT_TOOLS_MANIFEST}`, + }) + )?.toString("utf8") ?? null, + ); + const next = agentToolsManifest(assets); + const result = await builder.runCommand({ + cmd: "bash", + args: [ + "-c", + renderBuilderScript({ + root, + baseUrl: SANDBOX_ASSET_BASE_URL, + assets, + previous, + next, + }), + ], + sudo: true, + }); + console.log(`${region}: ${await result.stdout()}${await result.stderr()}`); + if (result.exitCode !== 0) { + throw new Error(`${region}: builder exited ${result.exitCode}`); + } + } finally { + await builder.stop(); + await builder.delete(); + } +} + +if (import.meta.main) { + const credentials = { + token: process.env.VERCEL_SANDBOX_TOKEN ?? "", + teamId: process.env.VERCEL_SANDBOX_TEAM_ID ?? "", + projectId: process.env.VERCEL_SANDBOX_PROJECT_ID ?? "", + }; + if (!credentials.token || !credentials.teamId || !credentials.projectId) { + console.error( + "VERCEL_SANDBOX_TOKEN, VERCEL_SANDBOX_TEAM_ID and VERCEL_SANDBOX_PROJECT_ID are required", + ); + process.exit(1); + } + const assets = loadAssets( + join(import.meta.dir, "..", "bundle", "assets.json"), + ); + for (const region of regionsFromArgv(process.argv.slice(2))) { + await publishRegion(region, assets, credentials); + } +} diff --git a/packages/sandbox/src/boot-twice.ts b/packages/sandbox/src/boot-twice.ts index bf0974f8c06..bef133de099 100644 --- a/packages/sandbox/src/boot-twice.ts +++ b/packages/sandbox/src/boot-twice.ts @@ -313,6 +313,51 @@ try { ); console.log(`\nboot log (second boot):\n${log2}`); + // The agent-tools drive, stubbed as a directory at its mount path: the + // listed agent is linked from it, and a session without it falls back to + // the image's agents rather than to links that point nowhere. + const stop = () => + exec( + `pkill -u ubuntu -TERM || true; sleep 1; pkill -u ubuntu -KILL || true; pkill -x Xvnc || true; rm -rf ${SANDBOX_PATHS.run}`, + { check: false }, + ); + const stubSha = "f".repeat(64); + stop(); + exec( + `: > ${SANDBOX_PATHS.bootLog}; mkdir -p ${SANDBOX_PATHS.agents}/claude/${stubSha} && printf '#!/bin/sh\\necho drive-claude\\n' > ${SANDBOX_PATHS.agents}/claude/${stubSha}/claude && chmod 755 ${SANDBOX_PATHS.agents}/claude/${stubSha}/claude && printf '%s' '${JSON.stringify([{ name: "claude", sha: stubSha, bin: "claude" }])}' > ${SANDBOX_PATHS.agents}/manifest.json`, + ); + boot(3); + const logDrive = exec(`cat ${SANDBOX_PATHS.bootLog}`); + expect( + "drive: listed agent linked", + /agents\.linked 1/.test(logDrive) && + /drive-claude/.test(exec("claude", { check: false, user: "ubuntu" })), + logDrive.match(/agents\..*/)?.[0], + ); + stop(); + exec(`: > ${SANDBOX_PATHS.bootLog}; rm -rf ${SANDBOX_PATHS.agents}`); + const noDriveExit = bootAsUser(); + exec( + `for i in $(seq 1 600); do [ -f ${SANDBOX_PATHS.run}/host-service.ready ] && exit 0; sleep 0.1; done; exit 1`, + { check: false }, + ); + const logNoDrive = exec(`cat ${SANDBOX_PATHS.bootLog}`); + expect( + "no drive: boot exits 0 and host-service ready", + /exit=0/.test(noDriveExit) && /host\.ready/.test(logNoDrive), + ); + expect( + "no drive: stamped unavailable", + /agents\.unavailable/.test(logNoDrive), + ); + expect( + "no drive: image claude back on PATH", + exec("readlink -f /usr/local/bin/claude", { check: false }).startsWith( + SANDBOX_PATHS.media, + ), + exec("readlink -f /usr/local/bin/claude", { check: false }).trim(), + ); + // Third boot, with host-service deliberately broken: the box must stay // alive, the desktop must still come up, and the log must say so. exec( diff --git a/packages/sandbox/src/boot.test.ts b/packages/sandbox/src/boot.test.ts index cf49be58f6a..620714412b7 100644 --- a/packages/sandbox/src/boot.test.ts +++ b/packages/sandbox/src/boot.test.ts @@ -45,6 +45,16 @@ describe("superset-boot", () => { expect(memguard).toMatch(/swapoff "\$f"/); }); + test("links the drive's agents after the steps, before host-service starts", () => { + const steps = boot.indexOf('"$BUNDLE/setup" run-steps'); + const link = boot.indexOf("\nlink_agents\n"); + const host = boot.indexOf("\nstart_host_service\n"); + expect(steps).toBeGreaterThan(0); + expect(link).toBeGreaterThan(steps); + expect(host).toBeGreaterThan(link); + expect(boot).toMatch(/stamp "agents\.unavailable/); + }); + test("writes the cgroup's memory ceiling and CPU count for login shells", () => { expect(boot).toMatch(/SUPERSET_SANDBOX_MEMORY_MB/); expect(boot).toMatch(/SUPERSET_SANDBOX_VCPUS/); diff --git a/packages/shared/src/sandbox/contract.ts b/packages/shared/src/sandbox/contract.ts index 1f2be836df1..0dff47566d7 100644 --- a/packages/shared/src/sandbox/contract.ts +++ b/packages/shared/src/sandbox/contract.ts @@ -35,8 +35,15 @@ export const SANDBOX_PATHS = { media: "/usr/local/share/superset/media", desktopTemplates: "/usr/local/share/superset/desktop", backgrounds: "/usr/share/backgrounds/superset", + /** A read-only snapshot of the region's agent-tools drive, when one is mounted. */ + agents: "/opt/agents", } as const; +/** The drive the agent CLIs are published to; a drive only mounts in its own region. */ +export function sandboxAgentToolsDrive(region: string): string { + return `agent-tools-${region}`; +} + /** Every port a workspace publishes, and the loopback ones behind them. */ export const SANDBOX_PORTS = { hostService: 4879, @@ -220,6 +227,7 @@ export function renderContractShell(): string { SUPERSET_MEDIA_DIR: SANDBOX_PATHS.media, SUPERSET_DESKTOP_TEMPLATES: SANDBOX_PATHS.desktopTemplates, SUPERSET_BACKGROUNDS_DIR: SANDBOX_PATHS.backgrounds, + SUPERSET_AGENTS_DIR: SANDBOX_PATHS.agents, SUPERSET_HOST_SERVICE_PORT: SANDBOX_PORTS.hostService, SUPERSET_DESKTOP_PORT: SANDBOX_PORTS.desktop, SUPERSET_VNC_PORT: SANDBOX_PORTS.vnc, diff --git a/packages/trpc/src/env.ts b/packages/trpc/src/env.ts index 1ca9332765c..24e09a4dd45 100644 --- a/packages/trpc/src/env.ts +++ b/packages/trpc/src/env.ts @@ -69,6 +69,8 @@ export const env = createEnv({ VERCEL_SANDBOX_TOKEN: z.string().min(1), VERCEL_SANDBOX_TEAM_ID: z.string().min(1), VERCEL_SANDBOX_PROJECT_ID: z.string().min(1), + // Only literal `true` mounts the region's agent-tools drive on every box. + SANDBOX_AGENT_TOOLS_DRIVE: z.string().optional(), // Shared with the gate Worker: signs the tickets clients present // there and derives the secret each sandbox's host-service is booted with. SANDBOX_GATE_SECRET: z.string().min(32), diff --git a/packages/trpc/src/lib/sandbox/vercel.test.ts b/packages/trpc/src/lib/sandbox/vercel.test.ts index 9e50fede634..3971b897b5b 100644 --- a/packages/trpc/src/lib/sandbox/vercel.test.ts +++ b/packages/trpc/src/lib/sandbox/vercel.test.ts @@ -9,6 +9,7 @@ import { import { Sandbox } from "@vercel/sandbox"; import { stub } from "../../../test/stub"; import { + agentToolsMounts, findSandboxSession, listActiveWorkspaceSandboxes, SESSION_TIMEOUT_MS, @@ -160,3 +161,17 @@ describe("topUpSandboxSession", () => { expect(extended).toEqual([SESSION_TIMEOUT_MS / 2]); }); }); + +describe("agentToolsMounts", () => { + test("mounts nothing while the flag is off", () => { + expect(agentToolsMounts("sfo1", false)).toEqual({}); + }); + + test("mounts the region's drive read-only at the agents path", () => { + expect(agentToolsMounts("iad1", true)).toEqual({ + mounts: { + "/opt/agents": { drive: "agent-tools-iad1", mode: "snapshot" }, + }, + }); + }); +}); diff --git a/packages/trpc/src/lib/sandbox/vercel.ts b/packages/trpc/src/lib/sandbox/vercel.ts index d742e6e4c76..4d4cbebd43b 100644 --- a/packages/trpc/src/lib/sandbox/vercel.ts +++ b/packages/trpc/src/lib/sandbox/vercel.ts @@ -17,11 +17,13 @@ import { SANDBOX_PORTS, SANDBOX_PUBLISHED_PORTS, type SandboxIdentity, + sandboxAgentToolsDrive, } from "@superset/shared/sandbox/contract"; import { APIError, type NetworkPolicy, Sandbox, + type SandboxMounts, type SandboxRegion, } from "@vercel/sandbox"; import { env } from "../../env"; @@ -137,6 +139,26 @@ function sandboxTags( }; } +/** + * The region's agent-tools drive, read-only, when the flag is on. Mounts are + * per session and a fork does not inherit them, so every create and every + * wake passes them. + */ +export function agentToolsMounts( + region: string, + enabled = env.SANDBOX_AGENT_TOOLS_DRIVE === "true", +): { mounts: SandboxMounts } | Record { + if (!enabled) return {}; + return { + mounts: { + [SANDBOX_PATHS.agents]: { + drive: sandboxAgentToolsDrive(region), + mode: "snapshot", + }, + }, + }; +} + function publishedPorts(extra: readonly number[] = []): number[] { return [...new Set([...SANDBOX_PUBLISHED_PORTS, ...extra])]; } @@ -221,6 +243,7 @@ export async function provisionSandbox(args: { kind === "environment" ? 0 : WORKSPACE_SNAPSHOT_EXPIRATION_MS, keepLastSnapshots: { count: 1 }, tags: sandboxTags(kind, args.claim.identity), + ...agentToolsMounts(args.environment.region), }; const sandbox = (await getSandbox(args.name)) ?? @@ -477,6 +500,17 @@ export async function wakeSandbox(args: { ), ); } + const { mounts } = agentToolsMounts(sandbox.region); + if (!wasRunning && mounts) { + await sandbox + .update({ mounts }) + .catch((error) => + console.warn( + `[sandbox] mount update failed for ${args.providerSandboxId}`, + error, + ), + ); + } if (!wasRunning && args.beforeResume) { await args.beforeResume({ memoryMb: vcpus * MEMORY_MB_PER_VCPU, vcpus }); } From 1bfe8d6637f9106184c00fae5e41d6184200edb7 Mon Sep 17 00:00:00 2001 From: Satya Patel <14907857+saddlepaddle@users.noreply.github.com> Date: Tue, 17 Nov 2026 00:45:22 +0000 Subject: [PATCH 2/2] feat(sandbox): one immutable agent-tools drive per bundle, behind a per-org flag A box that takes its snapshot mount while a writer holds the drive read-write gets a corrupt filesystem (measured 2026-11-17), so a drive is never written after it is published. - agent-tools--: the release writes it with a builder, stops the builder and waits for the drive to detach, before any environment row names the bundle. Idempotent: a drive with a manifest is left alone, one without is deleted and rewritten. A failed publish only warns. - After the rows, the release deletes agent-tools drives no environment names; a drive a sandbox holds is retried next release. - The claim carries the drive of the environment's bundle and region when the organization has the sandbox-agent-tools-drive PostHog flag (fails closed). - provisionSandbox and wakeSandbox mount it only once Drive.get finds it detached (cached per name); otherwise they warn and boot without it. Wake keeps the sandbox's other mounts and drops a stale agents mount. - SANDBOX_AGENT_TOOLS_DRIVE is gone. --- .env.example | 2 - .env.local.example | 1 - docs/cloud-sandbox-mismatches.md | 26 +- packages/sandbox/README.md | 1 + packages/sandbox/src/agent-tools.test.ts | 176 +++++++-- packages/sandbox/src/agent-tools.ts | 363 ++++++++++++------ packages/sandbox/src/environments/release.ts | 60 ++- packages/shared/src/constants.ts | 6 + packages/shared/src/sandbox/contract.ts | 15 +- packages/trpc/src/env.ts | 2 - .../trpc/src/lib/sandbox/agent-tools.test.ts | 50 +++ packages/trpc/src/lib/sandbox/agent-tools.ts | 26 ++ packages/trpc/src/lib/sandbox/claim.ts | 6 + packages/trpc/src/lib/sandbox/vercel.test.ts | 55 ++- packages/trpc/src/lib/sandbox/vercel.ts | 79 ++-- 15 files changed, 674 insertions(+), 194 deletions(-) create mode 100644 packages/trpc/src/lib/sandbox/agent-tools.test.ts create mode 100644 packages/trpc/src/lib/sandbox/agent-tools.ts diff --git a/.env.example b/.env.example index 2688d1eb7f7..6f5e5312b16 100644 --- a/.env.example +++ b/.env.example @@ -286,8 +286,6 @@ REALTIME_NUDGE_SECRET= VERCEL_SANDBOX_TOKEN= VERCEL_SANDBOX_TEAM_ID= VERCEL_SANDBOX_PROJECT_ID= -# `true` mounts the agent-tools drive on every box; anything else keeps the image agents. -SANDBOX_AGENT_TOOLS_DRIVE= AUTUMN_SECRET_KEY= AUTUMN_WEBHOOK_SECRET= BILLING_PROVIDER= diff --git a/.env.local.example b/.env.local.example index 24843280cea..bbc38b0569a 100644 --- a/.env.local.example +++ b/.env.local.example @@ -267,7 +267,6 @@ REALTIME_NUDGE_SECRET=fake-realtime-nudge-secret VERCEL_SANDBOX_TOKEN=fake-vercel-sandbox-token VERCEL_SANDBOX_TEAM_ID=team_fake VERCEL_SANDBOX_PROJECT_ID=prj_fake -SANDBOX_AGENT_TOOLS_DRIVE=false AUTUMN_SECRET_KEY=am_sk_test_fake_local_dev AUTUMN_WEBHOOK_SECRET=whsec_ZmFrZS1sb2NhbC1kZXYtc2VjcmV0 BILLING_PROVIDER= diff --git a/docs/cloud-sandbox-mismatches.md b/docs/cloud-sandbox-mismatches.md index eaf525649b9..cf9fdb29488 100644 --- a/docs/cloud-sandbox-mismatches.md +++ b/docs/cloud-sandbox-mismatches.md @@ -904,14 +904,24 @@ recorded against the API and a runtime failure against the sandbox, and that id is the only thing that joins the two halves of one broken workspace. **A drive mount lasts one session and a fork does not inherit it — open.** -Behind `SANDBOX_AGENT_TOOLS_DRIVE`, the agent CLIs come from the region's -`agent-tools-` drive, mounted read-only at `/opt/agents`. A machine -someone owns keeps a mount; a sandbox gets one only from `Sandbox.create`, -`Sandbox.fork` or `sandbox.update({ mounts })` before the session starts, so -`provisionSandbox` and `wakeSandbox` both pass it. A drive lives in one region -and mounts only in that region. A session without the mount leaves links in -`/usr/local/bin` that point nowhere; `superset-boot` re-runs -`install-agent-clis` to put the image's agents back. Untested on Vercel. +For organizations with the `sandbox-agent-tools-drive` PostHog flag, the agent +CLIs come from the bundle's `agent-tools--` drive, +mounted read-only at `/opt/agents`. A machine someone owns keeps a mount; a +sandbox gets one only from `Sandbox.create`, `Sandbox.fork` or +`sandbox.update({ mounts })` before the session starts, so `provisionSandbox` +and `wakeSandbox` both pass it. A drive lives in one region and mounts only in +that region. A session without the mount leaves links in `/usr/local/bin` +that point nowhere; `superset-boot` re-runs `install-agent-clis` to put the +image's agents back. + +**A read-only snapshot mount taken while a writer holds the drive is corrupt.** +Measured 2026-11-17: a box that mounted the snapshot while the builder still +had the drive read-write got `ro,norecovery`, `ls` answered "Bad message" and +codex segfaulted. On a disk someone owns, a second reader of a live +filesystem is just slow. Here a drive is never written after it is +published: one drive per bundle, written by a builder that is stopped before +any environment row names the bundle, and the control plane mounts only a +drive that exists and is detached. ## Running this repo's own dev stack inside a sandbox diff --git a/packages/sandbox/README.md b/packages/sandbox/README.md index 160c85a4db6..3b7c39dc9d1 100644 --- a/packages/sandbox/README.md +++ b/packages/sandbox/README.md @@ -42,6 +42,7 @@ The rule of thumb: if you can answer "where is it on the box", put it there. | `bun run image` | Builds the image: OS, apt lists, vendor tools, the `ubuntu` user, then installs the bundle at its sha and runs every pass once, so a fresh box finds every hash sidecar and step marker matching. A push logs Docker into the registry itself with a credential minted from `VERCEL_SANDBOX_TOKEN`, so there is no login to renew. `--local` builds `superset-sandbox:local` for the boot-twice test; `--dry` prints the Dockerfile. | | boot (`superset-boot`) | On the box, as root, run by the control plane on every create and wake, after it has written `sandbox.conf`, with the host secret in its env: clears `/run/superset`, installs the pinned bundle if the hash differs, runs the three passes (each a hash compare), starts host-service as `ubuntu` with the secret from the boot env, writes `resources.env`, starts `superset-memguard` (ranks every process for the OOM killer and reaps the largest before the kernel has to), then the desktop and the checkout. Once host-service answers, the environment has been pushed and the checkout is in, the runner asks host-service to run the repo's `start` hook with that environment. | | `bun run release` | Rewrites the host-service and CLI assets from this checkout and the claude, codex and Chrome rows to their latest upstream release, publishes the bundle, pushes the image under a candidate tag `release-` (unless `--skip-image`), builds the internal golden from it with the environment's `setup` hook, probes a fork of it (`src/environments/probe.ts`) including a stop and wake, and only after the probe passes moves `latest` to the candidate and writes the environment rows. Every image environment resolves `superset-hostsvc` to `latest`, so a failed release changes no box. | +| `bun run agent-tools --bundle ` | Writes that bundle's agent CLIs to `agent-tools--`, a drive boxes mount read-only at `/opt/agents` when their organization has the `sandbox-agent-tools-drive` flag. A drive is written once and never again. `bun run release` does this for every region with an environment before it records the bundle, and afterwards deletes the drives no environment names. | | `bun run src/real-sandbox.ts` | The Sandbox workflow dispatched with `real-sandbox`: one real sandbox from the registry image on this checkout's bundle, the same probe, a stop and a wake. | | Sandbox workflow, `production-release` | `bun run release --production` from `main` on a CI runner: the image, the internal golden and production's environment rows. Runs on a merge that touches the bundle and every night; a failed merge or nightly run opens an issue, or comments on the open one. | diff --git a/packages/sandbox/src/agent-tools.test.ts b/packages/sandbox/src/agent-tools.test.ts index 94d8288214a..30f29379de7 100644 --- a/packages/sandbox/src/agent-tools.test.ts +++ b/packages/sandbox/src/agent-tools.test.ts @@ -10,28 +10,33 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { type AgentToolsManifest, + agentToolsDrivesToPrune, agentToolsManifest, + type DriveHost, parseAgentToolsManifest, + pruneAgentToolsDrives, + publishAgentToolsDrive, renderBuilderScript, - shasToKeep, } from "./agent-tools"; import { type Assets, sha256 } from "./manifest"; const sha = (c: string) => c.repeat(64); +const BUNDLE = "0123456789abcdef".repeat(4); function row(sha256: string, suffix: string): Assets[string] { return { sha256, suffix, dest: "/usr/local/share/x", mode: "0644" }; } +const agentAssets: Assets = { + claude: row(sha("a"), ""), + codex: row(sha("b"), ".tar.gz"), + pdev: row(sha("c"), ".tar.gz"), + chrome: row(sha("d"), ".deb"), +}; + describe("agent-tools manifest", () => { test("lists every agent at its asset sha, bin relative to its dir", () => { - const assets: Assets = { - claude: row(sha("a"), ""), - codex: row(sha("b"), ".tar.gz"), - pdev: row(sha("c"), ".tar.gz"), - chrome: row(sha("d"), ".deb"), - }; - const manifest = agentToolsManifest(assets); + const manifest = agentToolsManifest(agentAssets); expect(manifest).toEqual([ { name: "claude", sha: sha("a"), bin: "claude" }, { name: "codex", sha: sha("b"), bin: "bin/codex" }, @@ -49,17 +54,131 @@ describe("agent-tools manifest", () => { ), ).toEqual([]); }); +}); + +function fakeHost( + drives: Record, +) { + const calls: string[] = []; + const host: DriveHost = { + find: async (name) => { + const drive = drives[name]; + return drive ? { attachedTo: drive.attachedTo } : null; + }, + list: async (prefix) => + Object.keys(drives).filter((name) => name.startsWith(prefix)), + create: async (name) => { + calls.push(`create ${name}`); + drives[name] = {}; + }, + remove: async (name) => { + calls.push(`remove ${name}`); + if (drives[name]?.attachedTo) throw new Error("attached"); + delete drives[name]; + }, + readManifest: async (name) => drives[name]?.manifest ?? null, + write: async (name) => { + calls.push(`write ${name}`); + const drive = drives[name]; + if (drive) + drive.manifest = JSON.stringify(agentToolsManifest(agentAssets)); + }, + }; + return { host, calls, drives }; +} + +describe("publishAgentToolsDrive", () => { + const drive = "agent-tools-sfo1-0123456789ab"; + + test("creates and writes the bundle's drive in the region", async () => { + const { host, calls } = fakeHost({}); + expect( + await publishAgentToolsDrive({ + host, + region: "sfo1", + bundleSha: BUNDLE, + assets: agentAssets, + }), + ).toEqual({ drive, outcome: "published" }); + expect(calls).toEqual([`create ${drive}`, `write ${drive}`]); + }); + + test("leaves a drive that has a manifest alone", async () => { + const { host, calls } = fakeHost({ + [drive]: { manifest: JSON.stringify(agentToolsManifest(agentAssets)) }, + }); + expect( + ( + await publishAgentToolsDrive({ + host, + region: "sfo1", + bundleSha: BUNDLE, + assets: agentAssets, + }) + ).outcome, + ).toBe("exists"); + expect(calls).toEqual([]); + }); + + test("recreates a drive an earlier run left without a manifest", async () => { + const { host, calls } = fakeHost({ [drive]: {} }); + await publishAgentToolsDrive({ + host, + region: "sfo1", + bundleSha: BUNDLE, + assets: agentAssets, + }); + expect(calls).toEqual([ + `remove ${drive}`, + `create ${drive}`, + `write ${drive}`, + ]); + }); - test("keeps the new and the previous sha of each agent", () => { + test("refuses a drive a writer still holds", async () => { + const { host, calls } = fakeHost({ [drive]: { attachedTo: "builder" } }); + await expect( + publishAgentToolsDrive({ + host, + region: "sfo1", + bundleSha: BUNDLE, + assets: agentAssets, + }), + ).rejects.toThrow(/attached to builder/); + expect(calls).toEqual([]); + }); +}); + +describe("pruning", () => { + test("deletes only agent-tools drives no environment references", () => { expect( - shasToKeep( - [{ name: "claude", sha: sha("a"), bin: "claude" }], + agentToolsDrivesToPrune( [ - { name: "claude", sha: sha("b"), bin: "claude" }, - { name: "codex", sha: sha("c"), bin: "bin/codex" }, + "agent-tools-sfo1-0123456789ab", + "agent-tools-sfo1-aaaaaaaaaaaa", + "agent-tools-iad1-0123456789ab", + "agent-tools-sfo1", + "agent-tools-builder-sfo1-xyz", ], + new Set(["agent-tools-sfo1-0123456789ab"]), ), - ).toEqual({ claude: [sha("a"), sha("b")], codex: [sha("c")] }); + ).toEqual([ + "agent-tools-sfo1-aaaaaaaaaaaa", + "agent-tools-iad1-0123456789ab", + ]); + }); + + test("keeps a drive a sandbox holds for the next release", async () => { + const { host } = fakeHost({ + "agent-tools-sfo1-aaaaaaaaaaaa": {}, + "agent-tools-sfo1-bbbbbbbbbbbb": { attachedTo: "box" }, + }); + expect( + await pruneAgentToolsDrives({ host, referenced: new Set() }), + ).toEqual({ + deleted: ["agent-tools-sfo1-aaaaaaaaaaaa"], + kept: ["agent-tools-sfo1-bbbbbbbbbbbb"], + }); }); }); @@ -73,7 +192,7 @@ async function runScript(script: string) { } describe.skipIf(process.platform !== "linux")("builder script", () => { - test("publishes verified agents, writes the manifest, prunes all but two shas", async () => { + test("writes verified agents, then the manifest last", async () => { const work = mkdtempSync(join(tmpdir(), "agent-tools-")); const src = join(work, "src"); mkdirSync(join(src, "bin"), { recursive: true }); @@ -95,16 +214,8 @@ describe.skipIf(process.platform !== "linux")("builder script", () => { }, }); const root = join(work, "drive"); - mkdirSync(join(root, "claude", sha("1")), { recursive: true }); - mkdirSync(join(root, "claude", sha("2")), { recursive: true }); - const assets: Assets = { - claude: row(sha256(claude), ""), - codex: row(sha256(codex), ".tar.gz"), - }; - const previous: AgentToolsManifest = [ - { name: "claude", sha: sha("1"), bin: "claude" }, - ]; - const next: AgentToolsManifest = [ + mkdirSync(root); + const manifest: AgentToolsManifest = [ { name: "claude", sha: sha256(claude), bin: "claude" }, { name: "codex", sha: sha256(codex), bin: "bin/codex" }, ]; @@ -113,9 +224,11 @@ describe.skipIf(process.platform !== "linux")("builder script", () => { renderBuilderScript({ root, baseUrl: `http://127.0.0.1:${server.port}`, - assets, - previous, - next, + assets: { + claude: row(sha256(claude), ""), + codex: row(sha256(codex), ".tar.gz"), + }, + manifest, }), ); expect(run.exitCode, run.stderr).toBe(0); @@ -123,7 +236,7 @@ describe.skipIf(process.platform !== "linux")("builder script", () => { parseAgentToolsManifest( readFileSync(join(root, "manifest.json"), "utf8"), ), - ).toEqual(next); + ).toEqual(manifest); expect( Bun.spawnSync([join(root, "codex", sha256(codex), "bin", "codex")]) .stdout.toString() @@ -132,8 +245,6 @@ describe.skipIf(process.platform !== "linux")("builder script", () => { expect(existsSync(join(root, "claude", sha256(claude), "claude"))).toBe( true, ); - expect(existsSync(join(root, "claude", sha("1")))).toBe(true); - expect(existsSync(join(root, "claude", sha("2")))).toBe(false); expect(existsSync(join(root, ".manifest.json.partial"))).toBe(false); } finally { server.stop(true); @@ -152,8 +263,7 @@ describe.skipIf(process.platform !== "linux")("builder script", () => { root, baseUrl: `http://127.0.0.1:${server.port}`, assets: { claude: row(sha("a"), "") }, - previous: [], - next: [{ name: "claude", sha: sha("a"), bin: "claude" }], + manifest: [{ name: "claude", sha: sha("a"), bin: "claude" }], }), ); expect(run.exitCode).not.toBe(0); diff --git a/packages/sandbox/src/agent-tools.ts b/packages/sandbox/src/agent-tools.ts index 82a97b8c4ae..31413c88bb8 100644 --- a/packages/sandbox/src/agent-tools.ts +++ b/packages/sandbox/src/agent-tools.ts @@ -1,20 +1,24 @@ /** - * Publishes the agent CLIs to each region's agent-tools drive, which boxes - * mount read-only at `SANDBOX_PATHS.agents` instead of carrying the agents - * on their own disks. Per region: a builder sandbox mounts the drive - * read-write, writes each agent into `///` (fetched by sha from - * the asset bucket and verified, unpacked beside its target and renamed in), - * then `/manifest.json` last, then prunes every sha that is neither in the - * new manifest nor in the one it replaced. + * Publishes the agent CLIs of one bundle to a drive per region, + * `agent-tools--`, which boxes mount read-only at + * `SANDBOX_PATHS.agents` instead of carrying the agents on their own disks. * - * bun run agent-tools [--region sfo1 ...] + * A drive is written once and never again: a box that takes its snapshot + * mount while a writer still holds the drive read-write gets a corrupt + * filesystem. So a builder sandbox writes the drive (each agent fetched by + * sha from the asset bucket and verified, `/manifest.json` last), is + * stopped, and the drive counts as published only once the builder has + * stopped and the drive is detached. The release publishes before any + * environment row points at the bundle, so no box mounts a drive mid-write. + * + * bun run agent-tools --bundle [--region sfo1 ...] * * Needs VERCEL_SANDBOX_TOKEN, VERCEL_SANDBOX_TEAM_ID, VERCEL_SANDBOX_PROJECT_ID. - * Reads the agent rows from bundle/assets.json, so run it after the release - * that moved them. + * The agent rows come from bundle/assets.json, so they must be that bundle's. */ import { join } from "node:path"; import { + SANDBOX_AGENT_TOOLS_DRIVE_PATTERN, SANDBOX_ASSET_BASE_URL, SANDBOX_PATHS, sandboxAgentToolsDrive, @@ -57,7 +61,7 @@ export function agentToolsManifest(assets: Assets): AgentToolsManifest { ); } -/** A drive that has never been published to, or holds a bad manifest, has none. */ +/** A drive that was never finished, or holds a bad manifest, has none. */ export function parseAgentToolsManifest( text: string | null, ): AgentToolsManifest { @@ -69,32 +73,16 @@ export function parseAgentToolsManifest( } } -/** A box may hold a snapshot taken under the previous manifest, so its shas stay too. */ -export function shasToKeep( - previous: AgentToolsManifest, - next: AgentToolsManifest, -): Record { - const keep: Record> = {}; - for (const { name, sha } of [...next, ...previous]) { - keep[name] ??= new Set(); - keep[name].add(sha); - } - return Object.fromEntries( - Object.entries(keep).map(([name, shas]) => [name, [...shas].sort()]), - ); -} - const quote = (value: string) => `'${value.replaceAll("'", "'\\''")}'`; -/** The script the builder runs as root against the drive mounted read-write at `root`. */ +/** The script the builder runs as root against the empty drive mounted read-write at `root`. */ export function renderBuilderScript(args: { root: string; baseUrl: string; assets: Assets; - previous: AgentToolsManifest; - next: AgentToolsManifest; + manifest: AgentToolsManifest; }): string { - const fetches = args.next.map(({ name, sha, bin }) => { + const fetches = args.manifest.map(({ name, sha, bin }) => { const asset = args.assets[name]; if (!asset || asset.sha256 !== sha) { throw new Error(`no asset row for ${name} at ${sha}`); @@ -102,9 +90,6 @@ export function renderBuilderScript(args: { const archive = asset.suffix === ".tar.gz" ? "tar" : "file"; return `publish ${[name, sha, bin, assetObjectKey(asset), archive].map(quote).join(" ")}`; }); - const prunes = Object.entries(shasToKeep(args.previous, args.next)).map( - ([name, shas]) => `prune ${quote(name)} ${shas.map(quote).join(" ")}`, - ); return `set -euo pipefail root=${quote(args.root)} base=${quote(args.baseUrl)} @@ -112,7 +97,6 @@ base=${quote(args.baseUrl)} publish() { local name="$1" sha="$2" bin="$3" key="$4" kind="$5" local target="\${root}/\${name}/\${sha}" - if [ -x "\${target}/\${bin}" ]; then echo "\${name} \${sha:0:12} already published"; return 0; fi local tmp tmp="$(mktemp)" curl -fsSL --retry 5 --retry-delay 2 --retry-all-errors -o "$tmp" "\${base}/\${key}" || @@ -131,112 +115,273 @@ publish() { [ -x "\${target}.partial/\${bin}" ] || { echo "\${name}: no executable at \${bin}" >&2; return 1; } rm -rf "$target" mv -T "\${target}.partial" "$target" - echo "\${name} \${sha:0:12} published" -} - -prune() { - local name="$1" dir sha keep - shift - for dir in "\${root}/\${name}"/*/; do - [ -d "$dir" ] || continue - dir="\${dir%/}" - sha="\${dir##*/}" - keep=0 - for want in "$@"; do [ "$sha" = "$want" ] && keep=1; done - [ "$keep" = 1 ] || { rm -rf "$dir"; echo "\${name} \${sha:0:12} pruned"; } - done + echo "\${name} \${sha:0:12} written" } ${fetches.join("\n")} -printf '%s\\n' ${quote(JSON.stringify(args.next, null, "\t"))} > "\${root}/.${AGENT_TOOLS_MANIFEST}.partial" +printf '%s\\n' ${quote(JSON.stringify(args.manifest, null, "\t"))} > "\${root}/.${AGENT_TOOLS_MANIFEST}.partial" sync mv -T "\${root}/.${AGENT_TOOLS_MANIFEST}.partial" "\${root}/${AGENT_TOOLS_MANIFEST}" sync -${prunes.join("\n")} -sync `; } -function regionsFromArgv(argv: string[]): SandboxRegionId[] { - const regions: SandboxRegionId[] = []; - for (let i = 0; i < argv.length; i++) { - if (argv[i] !== "--region") continue; - const region = argv[i + 1] ?? ""; - if (!isSandboxRegionId(region)) throw new Error(`unknown region ${region}`); - regions.push(region); - } - return regions.length ? regions : [DEFAULT_SANDBOX_REGION]; +/** What the publish needs from the provider; `vercelDriveHost` is the real one. */ +export interface DriveHost { + /** Null when no drive has the name. */ + find(name: string): Promise<{ attachedTo?: string } | null>; + list(prefix: string): Promise; + create(name: string, region: SandboxRegionId): Promise; + /** Throws while the drive is attached to a sandbox. */ + remove(name: string): Promise; + /** Through a read-only snapshot mount; null when the drive has no manifest. */ + readManifest(name: string, region: SandboxRegionId): Promise; + /** Resolves only once the builder has stopped and the drive is detached. */ + write(name: string, region: SandboxRegionId, script: string): Promise; } -async function publishRegion( - region: SandboxRegionId, - assets: Assets, - credentials: { token: string; teamId: string; projectId: string }, -): Promise { - const { Drive, Sandbox } = await import("@vercel/sandbox"); - const root = SANDBOX_PATHS.agents; - const drive = await Drive.getOrCreate({ - ...credentials, - name: sandboxAgentToolsDrive(region), - region, - }); - const builder = await Sandbox.create({ - ...credentials, - name: `agent-tools-builder-${region}-${Date.now().toString(36)}`, - region, - persistent: false, - timeout: 20 * 60_000, - mounts: { [root]: drive }, - }); - try { - const previous = parseAgentToolsManifest( - ( - await builder.readFileToBuffer({ - path: `${root}/${AGENT_TOOLS_MANIFEST}`, - }) - )?.toString("utf8") ?? null, +/** + * Idempotent on the bundle: a drive that has a manifest is published and is + * left alone; one without (an earlier run that failed) is deleted and + * written again. + */ +export async function publishAgentToolsDrive(args: { + host: DriveHost; + region: SandboxRegionId; + bundleSha: string; + assets: Assets; + baseUrl?: string; +}): Promise<{ drive: string; outcome: "exists" | "published" }> { + const drive = sandboxAgentToolsDrive(args.region, args.bundleSha); + const manifest = agentToolsManifest(args.assets); + const existing = await args.host.find(drive); + if (existing?.attachedTo) { + throw new Error( + `${drive} is attached to ${existing.attachedTo}; an earlier publish may still be writing it`, ); - const next = agentToolsManifest(assets); - const result = await builder.runCommand({ - cmd: "bash", - args: [ - "-c", - renderBuilderScript({ - root, - baseUrl: SANDBOX_ASSET_BASE_URL, - assets, - previous, - next, - }), - ], - sudo: true, - }); - console.log(`${region}: ${await result.stdout()}${await result.stderr()}`); - if (result.exitCode !== 0) { - throw new Error(`${region}: builder exited ${result.exitCode}`); + } + if (existing) { + const found = parseAgentToolsManifest( + await args.host.readManifest(drive, args.region), + ); + if (found.length > 0) return { drive, outcome: "exists" }; + await args.host.remove(drive); + } + await args.host.create(drive, args.region); + await args.host.write( + drive, + args.region, + renderBuilderScript({ + root: SANDBOX_PATHS.agents, + baseUrl: args.baseUrl ?? SANDBOX_ASSET_BASE_URL, + assets: args.assets, + manifest, + }), + ); + return { drive, outcome: "published" }; +} + +/** Agent-tools drives no environment row's bundle and region name any more. */ +export function agentToolsDrivesToPrune( + drives: readonly string[], + referenced: ReadonlySet, +): string[] { + return drives.filter( + (name) => + SANDBOX_AGENT_TOOLS_DRIVE_PATTERN.test(name) && !referenced.has(name), + ); +} + +/** Returns the drives it could not delete; a drive a sandbox holds is retried next release. */ +export async function pruneAgentToolsDrives(args: { + host: DriveHost; + referenced: ReadonlySet; +}): Promise<{ deleted: string[]; kept: string[] }> { + const deleted: string[] = []; + const kept: string[] = []; + for (const name of agentToolsDrivesToPrune( + await args.host.list("agent-tools-"), + args.referenced, + )) { + try { + await args.host.remove(name); + deleted.push(name); + } catch { + kept.push(name); + } + } + return { deleted, kept }; +} + +interface VercelCredentials { + token: string; + teamId: string; + projectId: string; +} + +export function vercelDriveHost(credentials: VercelCredentials): DriveHost { + const sdk = () => import("@vercel/sandbox"); + const isNotFound = async (error: unknown) => { + const { APIError } = await sdk(); + return error instanceof APIError && error.response.status === 404; + }; + const getDrive = async (name: string) => { + const { Drive } = await sdk(); + try { + return await Drive.get({ ...credentials, name }); + } catch (error) { + if (await isNotFound(error)) return null; + throw error; } - } finally { + }; + const builderName = (region: string) => + `agent-tools-builder-${region}-${Date.now().toString(36)}`; + const stopAndDelete = async ( + builder: import("@vercel/sandbox").Sandbox, + drive: string, + ) => { + const { Sandbox } = await sdk(); await builder.stop(); + const deadline = Date.now() + 3 * 60_000; + for (;;) { + const box = await Sandbox.get({ + ...credentials, + name: builder.name, + resume: false, + }); + const attached = (await getDrive(drive))?.currentSandboxName; + if (box.status === "stopped" && !attached) break; + if (Date.now() > deadline) { + throw new Error(`${builder.name} did not stop and release ${drive}`); + } + await new Promise((resolve) => setTimeout(resolve, 2_000)); + } await builder.delete(); - } + }; + return { + async find(name) { + const drive = await getDrive(name); + return drive ? { attachedTo: drive.currentSandboxName } : null; + }, + async list(prefix) { + const { Drive } = await sdk(); + const names: string[] = []; + for await (const drive of await Drive.list({ + ...credentials, + namePrefix: prefix, + })) { + names.push(drive.name); + } + return names; + }, + async create(name, region) { + const { Drive } = await sdk(); + await Drive.getOrCreate({ ...credentials, name, region }); + }, + async remove(name) { + await (await getDrive(name))?.delete(); + }, + async readManifest(name, region) { + const { Sandbox } = await sdk(); + const drive = await getDrive(name); + if (!drive) return null; + const reader = await Sandbox.create({ + ...credentials, + name: builderName(region), + region, + persistent: false, + timeout: 5 * 60_000, + mounts: { [SANDBOX_PATHS.agents]: drive.snapshot() }, + }); + try { + return ( + ( + await reader.readFileToBuffer({ + path: `${SANDBOX_PATHS.agents}/${AGENT_TOOLS_MANIFEST}`, + }) + )?.toString("utf8") ?? null + ); + } finally { + await reader.stop(); + await reader.delete(); + } + }, + async write(name, region, script) { + const { Sandbox } = await sdk(); + const drive = await getDrive(name); + if (!drive) throw new Error(`${name} does not exist`); + const builder = await Sandbox.create({ + ...credentials, + name: builderName(region), + region, + persistent: false, + timeout: 20 * 60_000, + mounts: { [SANDBOX_PATHS.agents]: drive }, + }); + let failure: Error | null = null; + try { + const result = await builder.runCommand({ + cmd: "bash", + args: ["-c", script], + sudo: true, + }); + console.log( + `${name}: ${await result.stdout()}${await result.stderr()}`, + ); + if (result.exitCode !== 0) { + failure = new Error(`${name}: builder exited ${result.exitCode}`); + } + } finally { + await stopAndDelete(builder, name); + } + if (failure) throw failure; + }, + }; +} + +function argValues(argv: string[], flag: string): string[] { + return argv.flatMap((arg, i) => + arg === flag && argv[i + 1] ? [argv[i + 1] as string] : [], + ); } if (import.meta.main) { + const argv = process.argv.slice(2); const credentials = { token: process.env.VERCEL_SANDBOX_TOKEN ?? "", teamId: process.env.VERCEL_SANDBOX_TEAM_ID ?? "", projectId: process.env.VERCEL_SANDBOX_PROJECT_ID ?? "", }; + const [bundleSha] = argValues(argv, "--bundle"); if (!credentials.token || !credentials.teamId || !credentials.projectId) { console.error( "VERCEL_SANDBOX_TOKEN, VERCEL_SANDBOX_TEAM_ID and VERCEL_SANDBOX_PROJECT_ID are required", ); process.exit(1); } + if (!bundleSha || !SHA256.test(bundleSha)) { + console.error("--bundle is required"); + process.exit(1); + } + const regions = argValues(argv, "--region"); + const unknown = regions.filter((region) => !isSandboxRegionId(region)); + if (unknown.length) { + console.error(`unknown region ${unknown.join(", ")}`); + process.exit(1); + } const assets = loadAssets( join(import.meta.dir, "..", "bundle", "assets.json"), ); - for (const region of regionsFromArgv(process.argv.slice(2))) { - await publishRegion(region, assets, credentials); + const host = vercelDriveHost(credentials); + for (const region of regions.length + ? (regions as SandboxRegionId[]) + : [DEFAULT_SANDBOX_REGION]) { + const { drive, outcome } = await publishAgentToolsDrive({ + host, + region, + bundleSha, + assets, + }); + console.log(`${drive}: ${outcome}`); } } diff --git a/packages/sandbox/src/environments/release.ts b/packages/sandbox/src/environments/release.ts index d3b5cc7a6e5..2d049be17aa 100644 --- a/packages/sandbox/src/environments/release.ts +++ b/packages/sandbox/src/environments/release.ts @@ -43,9 +43,14 @@ import { SANDBOX_PATHS, SANDBOX_ROOT_CHECKOUT, type SandboxIdentity, + sandboxAgentToolsDrive, sandboxCheckoutDir, } from "@superset/shared/sandbox/contract"; -import { DEFAULT_SANDBOX_REGION } from "@superset/shared/sandbox/regions"; +import { + DEFAULT_SANDBOX_REGION, + isSandboxRegionId, + type SandboxRegionId, +} from "@superset/shared/sandbox/regions"; import { Sandbox } from "@vercel/sandbox"; // The provisioning code imports the API env schema; an operator running this @@ -147,6 +152,39 @@ log( await publishBundle(bundle, { dry: false }); log(`bundle: ${bundle.sha256} published`); +// 1b. the bundle's agent-tools drive in every region with an environment, +// before any row points at the bundle so no box mounts one mid-write. Boxes +// fail open to the image's agents, so a failed publish only warns. +const agentTools = await import("../agent-tools"); +const driveHost = agentTools.vercelDriveHost(credentials); +async function environmentRegions(): Promise { + const { db } = await import("@superset/db/client"); + const rows = await db.query.environments.findMany({ + columns: { region: true }, + where: (row, { isNull: unset }) => unset(row.archivedAt), + }); + return [ + ...new Set( + [REGION, ...rows.map((row) => row.region)].filter(isSandboxRegionId), + ), + ]; +} +for (const region of await environmentRegions()) { + try { + const { drive, outcome } = await agentTools.publishAgentToolsDrive({ + host: driveHost, + region, + bundleSha: bundle.sha256, + assets: bundle.assets, + }); + log(`agent-tools: ${drive} ${outcome}`); + } catch (error) { + console.log( + `::warning::agent-tools: ${region} not published; its boxes keep the image's agents: ${String(error).slice(0, 300)}`, + ); + } +} + // 2. image const candidateTag = `release-${bundle.sha256.slice(0, 12)}`; const goldenImage = SKIP_IMAGE @@ -564,6 +602,26 @@ if (!previous) `rows: created; set SUPERSET_INTERNAL_ENVIRONMENT_ID=${internal.id} so the next release updates it`, ); +try { + const live = await db.query.environments.findMany({ + columns: { region: true, bundleSha: true }, + where: (row, { isNull: unset }) => unset(row.archivedAt), + }); + const { deleted, kept } = await agentTools.pruneAgentToolsDrives({ + host: driveHost, + referenced: new Set( + live.map((row) => sandboxAgentToolsDrive(row.region, row.bundleSha)), + ), + }); + log( + `agent-tools: pruned ${deleted.length}${kept.length ? `; ${kept.join(", ")} still attached, next release retries` : ""}`, + ); +} catch (error) { + console.log( + `::warning::agent-tools: prune failed: ${String(error).slice(0, 300)}`, + ); +} + if ( previous && previous.sourceKind === "fork" && diff --git a/packages/shared/src/constants.ts b/packages/shared/src/constants.ts index 313f79dbe10..5f69d086fb6 100644 --- a/packages/shared/src/constants.ts +++ b/packages/shared/src/constants.ts @@ -220,6 +220,12 @@ export const FEATURE_FLAGS = { * conditions are the allowlist, so access changes without a deploy. */ CLOUD_WORKSPACES: "cloud-workspaces", + /** + * Mounts the bundle's agent-tools drive on an organization's boxes, so the + * agent CLIs come from it instead of the image. Evaluated per organization + * on every create and wake. Unloaded or offline means no drive. + */ + SANDBOX_AGENT_TOOLS_DRIVE: "sandbox-agent-tools-drive", /** * Moving a workspace and its chats to the cloud or to another computer: * shows the Move items, and gates diff --git a/packages/shared/src/sandbox/contract.ts b/packages/shared/src/sandbox/contract.ts index 0dff47566d7..9fd3d568b03 100644 --- a/packages/shared/src/sandbox/contract.ts +++ b/packages/shared/src/sandbox/contract.ts @@ -39,11 +39,20 @@ export const SANDBOX_PATHS = { agents: "/opt/agents", } as const; -/** The drive the agent CLIs are published to; a drive only mounts in its own region. */ -export function sandboxAgentToolsDrive(region: string): string { - return `agent-tools-${region}`; +/** + * The agent CLIs of one bundle in one region: written once by the release, + * then only ever mounted read-only. A drive only mounts in its own region. + */ +export function sandboxAgentToolsDrive( + region: string, + bundleSha: string, +): string { + return `agent-tools-${region}-${bundleSha.slice(0, 12)}`; } +export const SANDBOX_AGENT_TOOLS_DRIVE_PATTERN = + /^agent-tools-[a-z]+[0-9]+-[0-9a-f]{12}$/; + /** Every port a workspace publishes, and the loopback ones behind them. */ export const SANDBOX_PORTS = { hostService: 4879, diff --git a/packages/trpc/src/env.ts b/packages/trpc/src/env.ts index 24e09a4dd45..1ca9332765c 100644 --- a/packages/trpc/src/env.ts +++ b/packages/trpc/src/env.ts @@ -69,8 +69,6 @@ export const env = createEnv({ VERCEL_SANDBOX_TOKEN: z.string().min(1), VERCEL_SANDBOX_TEAM_ID: z.string().min(1), VERCEL_SANDBOX_PROJECT_ID: z.string().min(1), - // Only literal `true` mounts the region's agent-tools drive on every box. - SANDBOX_AGENT_TOOLS_DRIVE: z.string().optional(), // Shared with the gate Worker: signs the tickets clients present // there and derives the secret each sandbox's host-service is booted with. SANDBOX_GATE_SECRET: z.string().min(32), diff --git a/packages/trpc/src/lib/sandbox/agent-tools.test.ts b/packages/trpc/src/lib/sandbox/agent-tools.test.ts new file mode 100644 index 00000000000..00fe6117571 --- /dev/null +++ b/packages/trpc/src/lib/sandbox/agent-tools.test.ts @@ -0,0 +1,50 @@ +import { describe, expect, test } from "bun:test"; +import { stub } from "../../../test/stub"; +import { posthog } from "../analytics"; +import { agentToolsDriveFor } from "./agent-tools"; + +let answer: () => Promise = () => Promise.resolve(true); +const asked: unknown[][] = []; +stub(posthog, { + isFeatureEnabled: (...args: unknown[]) => { + asked.push(args); + return answer(); + }, +}); + +const BUNDLE = "0123456789abcdef".repeat(4); +const ORG = "6f1c2b8e-1d2a-4c55-9a3e-0b7f5d2c9e11"; + +describe("agentToolsDriveFor", () => { + test("names the bundle's drive in the box's region when the org's flag is on", async () => { + answer = () => Promise.resolve(true); + expect( + await agentToolsDriveFor({ + organizationId: ORG, + region: "sfo1", + bundleSha: BUNDLE, + }), + ).toBe("agent-tools-sfo1-0123456789ab"); + expect(asked.at(-1)?.[0]).toBe("sandbox-agent-tools-drive"); + expect(asked.at(-1)?.[2]).toMatchObject({ + groups: { organization: ORG }, + }); + }); + + test("fails closed when the flag is off, unloaded or PostHog is down", async () => { + for (const next of [ + () => Promise.resolve(false), + () => Promise.resolve(undefined), + () => Promise.reject(new Error("offline")), + ]) { + answer = next; + expect( + await agentToolsDriveFor({ + organizationId: ORG, + region: "sfo1", + bundleSha: BUNDLE, + }), + ).toBeNull(); + } + }); +}); diff --git a/packages/trpc/src/lib/sandbox/agent-tools.ts b/packages/trpc/src/lib/sandbox/agent-tools.ts new file mode 100644 index 00000000000..92d069b9d8e --- /dev/null +++ b/packages/trpc/src/lib/sandbox/agent-tools.ts @@ -0,0 +1,26 @@ +import { FEATURE_FLAGS } from "@superset/shared/constants"; +import { sandboxAgentToolsDrive } from "@superset/shared/sandbox/contract"; +import { posthog } from "../analytics"; + +/** Fails closed: PostHog unreachable means no drive. */ +export async function agentToolsDriveEnabled( + organizationId: string, +): Promise { + const enabled = await posthog + .isFeatureEnabled(FEATURE_FLAGS.SANDBOX_AGENT_TOOLS_DRIVE, organizationId, { + groups: { organization: organizationId }, + sendFeatureFlagEvents: false, + }) + .catch(() => undefined); + return enabled === true; +} + +/** The drive of the bundle the box boots, or null while the organization's flag is off. */ +export async function agentToolsDriveFor(args: { + organizationId: string; + region: string; + bundleSha: string; +}): Promise { + if (!(await agentToolsDriveEnabled(args.organizationId))) return null; + return sandboxAgentToolsDrive(args.region, args.bundleSha); +} diff --git a/packages/trpc/src/lib/sandbox/claim.ts b/packages/trpc/src/lib/sandbox/claim.ts index bd1e715c445..fc496d4d084 100644 --- a/packages/trpc/src/lib/sandbox/claim.ts +++ b/packages/trpc/src/lib/sandbox/claim.ts @@ -28,6 +28,7 @@ import { resolveEnvironment } from "../../router/environment/resolve-environment import { resolveMachineSize } from "../billing/plan"; import { githubUserConnectionFor, githubUserTokenFor } from "../github-user"; import { sandboxHostSecretFor } from "./access"; +import { agentToolsDriveFor } from "./agent-tools"; import { deriveSandboxCredentials, gitAuthorFor } from "./credentials"; import { creatorPlugins } from "./plugins"; import { readRepoHooks } from "./repo-hooks"; @@ -174,6 +175,11 @@ export async function buildSandboxClaim(args: { files, ports: repoHooks?.ports, vcpus: await boxVcpus(args.row), + agentToolsDrive: await agentToolsDriveFor({ + organizationId: args.row.organizationId, + region: environment.region, + bundleSha: environment.bundleSha, + }), }, environment: { sourceKind: environment.sourceKind, diff --git a/packages/trpc/src/lib/sandbox/vercel.test.ts b/packages/trpc/src/lib/sandbox/vercel.test.ts index 3971b897b5b..4a24e20353a 100644 --- a/packages/trpc/src/lib/sandbox/vercel.test.ts +++ b/packages/trpc/src/lib/sandbox/vercel.test.ts @@ -163,15 +163,56 @@ describe("topUpSandboxSession", () => { }); describe("agentToolsMounts", () => { - test("mounts nothing while the flag is off", () => { - expect(agentToolsMounts("sfo1", false)).toEqual({}); + const detached = () => Promise.resolve({ currentSandboxName: undefined }); + + test("mounts nothing without a drive name", async () => { + let asked = 0; + const getDrive = () => { + asked++; + return detached(); + }; + expect(await agentToolsMounts(null, getDrive)).toBeUndefined(); + expect(asked).toBe(0); }); - test("mounts the region's drive read-only at the agents path", () => { - expect(agentToolsMounts("iad1", true)).toEqual({ - mounts: { - "/opt/agents": { drive: "agent-tools-iad1", mode: "snapshot" }, + test("mounts a detached drive read-only, and asks only once", async () => { + let asked = 0; + const getDrive = () => { + asked++; + return detached(); + }; + const want = { + "/opt/agents": { + drive: "agent-tools-iad1-0123456789ab", + mode: "snapshot", }, - }); + }; + expect( + await agentToolsMounts("agent-tools-iad1-0123456789ab", getDrive), + ).toEqual(want); + expect( + await agentToolsMounts("agent-tools-iad1-0123456789ab", getDrive), + ).toEqual(want); + expect(asked).toBe(1); + }); + + test("skips a missing drive, or one a writer still holds", async () => { + expect( + await agentToolsMounts("agent-tools-sfo1-aaaaaaaaaaaa", () => + Promise.reject(new Error("404")), + ), + ).toBeUndefined(); + let asked = 0; + const attached = () => { + asked++; + return Promise.resolve({ currentSandboxName: "builder" }); + }; + expect( + await agentToolsMounts("agent-tools-sfo1-bbbbbbbbbbbb", attached), + ).toBeUndefined(); + expect( + await agentToolsMounts("agent-tools-sfo1-bbbbbbbbbbbb", attached), + ).toBeUndefined(); + expect(asked).toBe(2); }); }); diff --git a/packages/trpc/src/lib/sandbox/vercel.ts b/packages/trpc/src/lib/sandbox/vercel.ts index 4d4cbebd43b..786002a5c37 100644 --- a/packages/trpc/src/lib/sandbox/vercel.ts +++ b/packages/trpc/src/lib/sandbox/vercel.ts @@ -17,10 +17,10 @@ import { SANDBOX_PORTS, SANDBOX_PUBLISHED_PORTS, type SandboxIdentity, - sandboxAgentToolsDrive, } from "@superset/shared/sandbox/contract"; import { APIError, + Drive, type NetworkPolicy, Sandbox, type SandboxMounts, @@ -121,6 +121,8 @@ export interface SandboxClaim { /** Ports the workspace's repository asks to publish, beside the platform's. */ ports?: readonly number[]; vcpus: number; + /** The bundle's agent-tools drive to mount read-only; absent keeps the image's agents. */ + agentToolsDrive?: string | null; } /** Vercel allows 5 tags per sandbox. */ @@ -139,24 +141,39 @@ function sandboxTags( }; } +const publishedAgentToolsDrives = new Set(); + /** - * The region's agent-tools drive, read-only, when the flag is on. Mounts are - * per session and a fork does not inherit them, so every create and every - * wake passes them. + * A mount of a drive that does not exist fails the create, and a drive a + * writer still holds read-write mounts corrupt. Only a drive that exists and + * is detached is mounted; that answer is final for a drive, so it is cached. */ -export function agentToolsMounts( - region: string, - enabled = env.SANDBOX_AGENT_TOOLS_DRIVE === "true", -): { mounts: SandboxMounts } | Record { - if (!enabled) return {}; - return { - mounts: { - [SANDBOX_PATHS.agents]: { - drive: sandboxAgentToolsDrive(region), - mode: "snapshot", - }, - }, - }; +export async function agentToolsMounts( + driveName: string | null | undefined, + getDrive: (name: string) => Promise> = ( + name, + ) => Drive.get({ ...credentials(), name }), +): Promise { + if (!driveName) return undefined; + if (!publishedAgentToolsDrives.has(driveName)) { + try { + const drive = await getDrive(driveName); + if (drive.currentSandboxName) { + console.warn( + `[sandbox] agent-tools drive ${driveName} is attached to ${drive.currentSandboxName}; not mounting`, + ); + return undefined; + } + } catch (error) { + console.warn( + `[sandbox] agent-tools drive ${driveName} unavailable; not mounting`, + error, + ); + return undefined; + } + publishedAgentToolsDrives.add(driveName); + } + return { [SANDBOX_PATHS.agents]: { drive: driveName, mode: "snapshot" } }; } function publishedPorts(extra: readonly number[] = []): number[] { @@ -243,8 +260,8 @@ export async function provisionSandbox(args: { kind === "environment" ? 0 : WORKSPACE_SNAPSHOT_EXPIRATION_MS, keepLastSnapshots: { count: 1 }, tags: sandboxTags(kind, args.claim.identity), - ...agentToolsMounts(args.environment.region), }; + const mounts = await agentToolsMounts(args.claim.agentToolsDrive); const sandbox = (await getSandbox(args.name)) ?? (await Sentry.startSpan( @@ -265,11 +282,13 @@ export async function provisionSandbox(args: { // it was taken. Sandbox.fork({ ...config, + ...(mounts ? { mounts } : {}), sourceSandbox: args.environment.sourceRef, resources: { vcpus: args.claim.vcpus }, }) : Sandbox.create({ ...config, + ...(mounts ? { mounts } : {}), image: args.environment.sourceRef, region: args.environment.region as SandboxRegion, resources: { vcpus: args.claim.vcpus }, @@ -500,16 +519,20 @@ export async function wakeSandbox(args: { ), ); } - const { mounts } = agentToolsMounts(sandbox.region); - if (!wasRunning && mounts) { - await sandbox - .update({ mounts }) - .catch((error) => - console.warn( - `[sandbox] mount update failed for ${args.providerSandboxId}`, - error, - ), - ); + if (!wasRunning) { + const { [SANDBOX_PATHS.agents]: previous, ...others } = + sandbox.mounts ?? {}; + const agents = await agentToolsMounts(args.claim.agentToolsDrive); + if (agents || previous) { + await sandbox + .update({ mounts: { ...others, ...agents } }) + .catch((error) => + console.warn( + `[sandbox] mount update failed for ${args.providerSandboxId}`, + error, + ), + ); + } } if (!wasRunning && args.beforeResume) { await args.beforeResume({ memoryMb: vcpus * MEMORY_MB_PER_VCPU, vcpus });