From 47e9d4bf7e935c76fb2f1795ed65e600a663dc0c Mon Sep 17 00:00:00 2001 From: Satya Patel <14907857+saddlepaddle@users.noreply.github.com> Date: Tue, 17 Nov 2026 02:28:00 +0000 Subject: [PATCH] feat(sandbox): publish agent-tools drives in parallel, from a fork, per agent Regions publish six at a time and a failed region only warns. Each new drive is a copy-on-write fork of the region's newest free earlier drive, so the builder fetches only agents whose sha changed and removes agent directories the new manifest no longer names. Each agent fails alone: a failed download, checksum, unpack or `--version` check keeps the earlier drive's version, or leaves the agent out so boxes use the image's copy. The manifest keeps its `[{ name, sha, bin }]` shape. --- packages/sandbox/README.md | 2 +- packages/sandbox/src/agent-tools.test.ts | 437 ++++++++++++++++--- packages/sandbox/src/agent-tools.ts | 359 ++++++++++++--- packages/sandbox/src/environments/release.ts | 24 +- 4 files changed, 689 insertions(+), 133 deletions(-) diff --git a/packages/sandbox/README.md b/packages/sandbox/README.md index ab330b9bae6..2fdb4b2c7ea 100644 --- a/packages/sandbox/README.md +++ b/packages/sandbox/README.md @@ -47,7 +47,7 @@ The rule of thumb: if you can answer "where is it on the box", put it there. | `bun run image` | Builds the image: OS, apt lists, vendor tools, the `ubuntu` user, then installs the bundle at its sha and runs every pass once, so a fresh box finds every hash sidecar and step marker matching. A push logs Docker into the registry itself with a credential minted from `VERCEL_SANDBOX_TOKEN`, so there is no login to renew. `--local` builds `superset-sandbox:local` for the boot-twice test; `--dry` prints the Dockerfile. | | boot (`superset-boot`) | On the box, as root, run by the control plane on every create and wake, after it has written `sandbox.conf`, with the host secret in its env: clears `/run/superset`, installs the pinned bundle if the hash differs, runs the three passes (each a hash compare), starts host-service as `ubuntu` with the secret from the boot env, writes `resources.env`, starts `superset-memguard` (ranks every process for the OOM killer and reaps the largest before the kernel has to), then the desktop and the checkout. Once host-service answers, the environment has been pushed and the checkout is in, the runner asks host-service to run the repo's `start` hook with that environment. | | `bun run release` | Rewrites the host-service and CLI assets from this checkout and the claude, codex and Chrome rows to their latest upstream release, publishes the bundle, pushes the image under a candidate tag `release-` (unless `--skip-image`), builds the internal golden from it with the environment's `setup` hook, probes a fork of it (`src/environments/probe.ts`) including a stop and wake, and only after the probe passes moves `latest` to the candidate and writes the environment rows. Every image environment resolves `superset-hostsvc` to `latest`, so a failed release changes no box. | -| `bun run agent-tools --bundle ` | Writes that bundle's agent CLIs to `agent-tools--`, a drive boxes mount read-only at `/opt/agents` when their organization has the `sandbox-agent-tools-drive` flag. A drive is written once and never again. `bun run release` does this for every region with an environment before it records the bundle, and afterwards deletes the drives no environment names. | +| `bun run agent-tools --bundle ` | Writes that bundle's agent CLIs to `agent-tools--`, a drive boxes mount read-only at `/opt/agents` when their organization has the `sandbox-agent-tools-drive` flag. A drive is written once and never again; it starts as a fork of the region's newest earlier drive, so only changed agents are fetched, and an agent that fails its download or `--version` check keeps the earlier drive's version or is left out. `bun run release` does this for every region with an environment, six at a time, before it records the bundle, and afterwards deletes the drives no environment names. | | `bun run src/real-sandbox.ts` | The Sandbox workflow dispatched with `real-sandbox`: one real sandbox from the registry image on this checkout's bundle, the same probe, a stop and a wake. | | Sandbox workflow, `production-release` | `bun run release --production` from `main` on a CI runner: the image, the internal golden and production's environment rows. Runs on a merge that touches the bundle and every night; a failed merge or nightly run opens an issue, or comments on the open one. | diff --git a/packages/sandbox/src/agent-tools.test.ts b/packages/sandbox/src/agent-tools.test.ts index 30f29379de7..7dac023edce 100644 --- a/packages/sandbox/src/agent-tools.test.ts +++ b/packages/sandbox/src/agent-tools.test.ts @@ -13,9 +13,12 @@ import { agentToolsDrivesToPrune, agentToolsManifest, type DriveHost, + describeAgentToolsPublish, parseAgentToolsManifest, + parseBuilderResults, pruneAgentToolsDrives, publishAgentToolsDrive, + publishAgentToolsDrives, renderBuilderScript, } from "./agent-tools"; import { type Assets, sha256 } from "./manifest"; @@ -56,20 +59,49 @@ describe("agent-tools manifest", () => { }); }); +interface FakeDrive { + manifest?: string; + attachedTo?: string; + createdAt?: number; +} + +function written(assets: Assets = agentAssets) { + return agentToolsManifest(assets) + .map(({ name, sha }) => `agent-tools-result ${name} written ${sha}`) + .join("\n"); +} + function fakeHost( - drives: Record, + drives: Record, + options: { + builder?: (name: string) => string; + failFork?: boolean; + } = {}, ) { const calls: string[] = []; + const scripts: Record = {}; + let clock = 100; const host: DriveHost = { find: async (name) => { const drive = drives[name]; return drive ? { attachedTo: drive.attachedTo } : null; }, list: async (prefix) => - Object.keys(drives).filter((name) => name.startsWith(prefix)), + Object.entries(drives) + .filter(([name]) => name.startsWith(prefix)) + .map(([name, drive]) => ({ + name, + createdAt: drive.createdAt ?? 0, + attachedTo: drive.attachedTo, + })), create: async (name) => { calls.push(`create ${name}`); - drives[name] = {}; + drives[name] = { createdAt: clock++ }; + }, + fork: async (parent, name) => { + calls.push(`fork ${parent} ${name}`); + drives[name] = { ...drives[parent], createdAt: clock++ }; + if (options.failFork) throw new Error("fork refused"); }, remove: async (name) => { calls.push(`remove ${name}`); @@ -77,57 +109,59 @@ function fakeHost( delete drives[name]; }, readManifest: async (name) => drives[name]?.manifest ?? null, - write: async (name) => { + write: async (name, _region, script) => { calls.push(`write ${name}`); const drive = drives[name]; + scripts[name] = script(drive?.manifest ?? null); + const stdout = options.builder?.(name) ?? written(); if (drive) drive.manifest = JSON.stringify(agentToolsManifest(agentAssets)); + return stdout; }, }; - return { host, calls, drives }; + return { host, calls, drives, scripts }; } +const agentLine = (script: string | undefined, name: string) => + script?.split("\n").find((line) => line.startsWith(`agent '${name}' `)); + describe("publishAgentToolsDrive", () => { const drive = "agent-tools-sfo1-0123456789ab"; + const publish = (host: DriveHost, warn: (line: string) => void = () => {}) => + publishAgentToolsDrive({ + host, + region: "sfo1", + bundleSha: BUNDLE, + assets: agentAssets, + warn, + }); - test("creates and writes the bundle's drive in the region", async () => { - const { host, calls } = fakeHost({}); - expect( - await publishAgentToolsDrive({ - host, - region: "sfo1", - bundleSha: BUNDLE, - assets: agentAssets, - }), - ).toEqual({ drive, outcome: "published" }); + test("with no earlier drive, creates the drive and writes every agent", async () => { + const { host, calls, scripts } = fakeHost({}); + const result = await publish(host); + expect(result.drive).toBe(drive); + expect(result.outcome).toBe("published"); + expect(result.forkedFrom).toBeUndefined(); + expect(result.agents.map((agent) => agent.outcome)).toEqual([ + "written", + "written", + "written", + ]); expect(calls).toEqual([`create ${drive}`, `write ${drive}`]); + expect(agentLine(scripts[drive], "claude")).toEndWith("'' '' '--version'"); }); test("leaves a drive that has a manifest alone", async () => { const { host, calls } = fakeHost({ [drive]: { manifest: JSON.stringify(agentToolsManifest(agentAssets)) }, }); - expect( - ( - await publishAgentToolsDrive({ - host, - region: "sfo1", - bundleSha: BUNDLE, - assets: agentAssets, - }) - ).outcome, - ).toBe("exists"); + expect((await publish(host)).outcome).toBe("exists"); expect(calls).toEqual([]); }); test("recreates a drive an earlier run left without a manifest", async () => { const { host, calls } = fakeHost({ [drive]: {} }); - await publishAgentToolsDrive({ - host, - region: "sfo1", - bundleSha: BUNDLE, - assets: agentAssets, - }); + await publish(host); expect(calls).toEqual([ `remove ${drive}`, `create ${drive}`, @@ -137,16 +171,155 @@ describe("publishAgentToolsDrive", () => { test("refuses a drive a writer still holds", async () => { const { host, calls } = fakeHost({ [drive]: { attachedTo: "builder" } }); - await expect( - publishAgentToolsDrive({ - host, - region: "sfo1", - bundleSha: BUNDLE, - assets: agentAssets, - }), - ).rejects.toThrow(/attached to builder/); + await expect(publish(host)).rejects.toThrow(/attached to builder/); expect(calls).toEqual([]); }); + + test("forks the region's newest free drive and hands the builder its manifest", async () => { + const earlier = [ + { name: "claude", sha: sha("e"), bin: "claude" }, + { name: "codex", sha: sha("b"), bin: "bin/codex" }, + ]; + const { host, calls, scripts } = fakeHost({ + "agent-tools-sfo1-111111111111": { + createdAt: 1, + manifest: JSON.stringify([ + { name: "claude", sha: sha("1"), bin: "claude" }, + ]), + }, + "agent-tools-sfo1-222222222222": { + createdAt: 2, + manifest: JSON.stringify(earlier), + }, + "agent-tools-sfo1-333333333333": { createdAt: 3, attachedTo: "box" }, + "agent-tools-iad1-444444444444": { createdAt: 4 }, + }); + const result = await publish(host); + expect(result.forkedFrom).toBe("agent-tools-sfo1-222222222222"); + expect(calls).toEqual([ + `fork agent-tools-sfo1-222222222222 ${drive}`, + `write ${drive}`, + ]); + expect(agentLine(scripts[drive], "claude")).toEndWith( + `'${sha("e")}' 'claude' '--version'`, + ); + expect(agentLine(scripts[drive], "pdev")).toEndWith("'' '' '--version'"); + }); + + test("a failed fork warns and falls back to writing every agent", async () => { + const warnings: string[] = []; + const { host, calls, scripts } = fakeHost( + { + "agent-tools-sfo1-222222222222": { + createdAt: 2, + manifest: JSON.stringify(agentToolsManifest(agentAssets)), + }, + }, + { failFork: true }, + ); + const result = await publish(host, (line) => warnings.push(line)); + expect(result.forkedFrom).toBeUndefined(); + expect(calls).toEqual([ + `fork agent-tools-sfo1-222222222222 ${drive}`, + `remove ${drive}`, + `create ${drive}`, + `write ${drive}`, + ]); + expect(warnings.join("\n")).toContain("fork refused"); + expect(agentLine(scripts[drive], "claude")).toEndWith("'' '' '--version'"); + }); + + test("reports each agent's outcome and warns for the ones that missed the bundle", async () => { + const tools = [ + { name: "claude", bin: "claude" }, + { name: "codex", bin: "bin/codex" }, + { name: "pdev", bin: "pdev/pdev" }, + { name: "chrome", bin: "chrome", check: ["--headless", "--version"] }, + ]; + const { host, scripts } = fakeHost( + {}, + { + builder: () => + [ + "noise", + `agent-tools-result claude written ${sha("a")}`, + `agent-tools-result codex reused ${sha("b")}`, + `agent-tools-result pdev kept-previous ${sha("9")}`, + "agent-tools-result chrome dropped ", + ].join("\n"), + }, + ); + const result = await publishAgentToolsDrive({ + host, + region: "sfo1", + bundleSha: BUNDLE, + assets: agentAssets, + tools, + }); + expect(result.agents).toEqual([ + { name: "claude", outcome: "written", sha: sha("a") }, + { name: "codex", outcome: "reused", sha: sha("b") }, + { name: "pdev", outcome: "kept-previous", sha: sha("9") }, + { name: "chrome", outcome: "dropped" }, + ]); + expect(agentLine(scripts[drive], "chrome")).toEndWith( + "'--headless' '--version'", + ); + const lines = describeAgentToolsPublish([{ region: "sfo1", result }]); + expect(lines[0]).toBe( + `agent-tools: ${drive} published: claude written, codex reused, pdev kept-previous, chrome dropped`, + ); + expect(lines.slice(1)).toEqual([ + expect.stringMatching(/^::warning::.*pdev kept 999999999999/), + expect.stringMatching(/^::warning::.*chrome left out/), + ]); + }); +}); + +describe("publishAgentToolsDrives", () => { + test("runs regions in parallel up to the limit, and one failure stays in its region", async () => { + const { host } = fakeHost({}); + let running = 0; + let peak = 0; + const write = host.write; + host.write = async (name, region, script) => { + running++; + peak = Math.max(peak, running); + await new Promise((resolve) => setTimeout(resolve, 1)); + running--; + if (region === "iad1") throw new Error("builder exited 1"); + return write(name, region, script); + }; + const regions = ["sfo1", "iad1", "fra1", "lhr1", "sin1"] as const; + const results = await publishAgentToolsDrives({ + host, + regions, + bundleSha: BUNDLE, + assets: agentAssets, + concurrency: 2, + }); + expect(peak).toBe(2); + expect(results.map((entry) => [entry.region, "error" in entry])).toEqual([ + ["sfo1", false], + ["iad1", true], + ["fra1", false], + ["lhr1", false], + ["sin1", false], + ]); + expect( + describeAgentToolsPublish(results).filter((line) => + line.startsWith("::warning::"), + ), + ).toEqual([expect.stringMatching(/iad1 not published.*builder exited 1/)]); + }); +}); + +test("builder results ignore lines that are not results", () => { + expect( + parseBuilderResults( + `claude: written\nagent-tools-result claude reused ${sha("a")}\nagent-tools-result x bogus\n`, + ), + ).toEqual([{ name: "claude", outcome: "reused", sha: sha("a") }]); }); describe("pruning", () => { @@ -187,8 +360,34 @@ async function runScript(script: string) { stdout: "pipe", stderr: "pipe", }); - const stderr = await new Response(proc.stderr).text(); - return { exitCode: await proc.exited, stderr }; + const [stdout, stderr] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + ]); + return { exitCode: await proc.exited, stdout, stderr }; +} + +function serve(objects: Map) { + const requests: string[] = []; + const server = Bun.serve({ + port: 0, + fetch: (request) => { + const key = new URL(request.url).pathname.slice(1); + requests.push(key); + return new Response(objects.get(key) ?? "not the agent"); + }, + }); + return { server, requests, baseUrl: `http://127.0.0.1:${server.port}` }; +} + +const agentScript = (body: string) => + new TextEncoder().encode(`#!/bin/sh\n${body}\n`); + +function placeAgent(root: string, name: string, body: Uint8Array) { + const dir = join(root, name, sha256(body)); + mkdirSync(dir, { recursive: true }); + writeFileSync(join(dir, name), body, { mode: 0o755 }); + return { name, sha: sha256(body), bin: name }; } describe.skipIf(process.platform !== "linux")("builder script", () => { @@ -200,19 +399,14 @@ describe.skipIf(process.platform !== "linux")("builder script", () => { mode: 0o755, }); Bun.spawnSync(["tar", "-czf", join(work, "codex.tar.gz"), "-C", src, "."]); - const claude = new TextEncoder().encode("#!/bin/sh\necho claude\n"); + const claude = agentScript("echo claude"); const codex = readFileSync(join(work, "codex.tar.gz")); - const objects = new Map([ - [sha256(claude), claude], - [`${sha256(codex)}.tar.gz`, codex], - ]); - const server = Bun.serve({ - port: 0, - fetch: (request) => { - const body = objects.get(new URL(request.url).pathname.slice(1)); - return body ? new Response(body) : new Response("", { status: 404 }); - }, - }); + const { server, baseUrl } = serve( + new Map([ + [sha256(claude), claude], + [`${sha256(codex)}.tar.gz`, codex], + ]), + ); const root = join(work, "drive"); mkdirSync(root); const manifest: AgentToolsManifest = [ @@ -223,7 +417,7 @@ describe.skipIf(process.platform !== "linux")("builder script", () => { const run = await runScript( renderBuilderScript({ root, - baseUrl: `http://127.0.0.1:${server.port}`, + baseUrl, assets: { claude: row(sha256(claude), ""), codex: row(sha256(codex), ".tar.gz"), @@ -232,6 +426,10 @@ describe.skipIf(process.platform !== "linux")("builder script", () => { }), ); expect(run.exitCode, run.stderr).toBe(0); + expect(parseBuilderResults(run.stdout)).toEqual([ + { name: "claude", outcome: "written", sha: sha256(claude) }, + { name: "codex", outcome: "written", sha: sha256(codex) }, + ]); expect( parseAgentToolsManifest( readFileSync(join(root, "manifest.json"), "utf8"), @@ -242,34 +440,147 @@ describe.skipIf(process.platform !== "linux")("builder script", () => { .stdout.toString() .trim(), ).toBe("codex"); - expect(existsSync(join(root, "claude", sha256(claude), "claude"))).toBe( - true, - ); expect(existsSync(join(root, ".manifest.json.partial"))).toBe(false); } finally { server.stop(true); } }); - test("a sha mismatch fails before the manifest is written", async () => { + test("an agent that fails with no earlier copy is left out; with none left there is no manifest", async () => { const root = mkdtempSync(join(tmpdir(), "agent-tools-")); - const server = Bun.serve({ - port: 0, - fetch: () => new Response("not the agent"), - }); + const { server, baseUrl } = serve(new Map()); try { const run = await runScript( renderBuilderScript({ root, - baseUrl: `http://127.0.0.1:${server.port}`, + baseUrl, assets: { claude: row(sha("a"), "") }, manifest: [{ name: "claude", sha: sha("a"), bin: "claude" }], }), ); expect(run.exitCode).not.toBe(0); + expect(parseBuilderResults(run.stdout)).toEqual([ + { name: "claude", outcome: "dropped" }, + ]); expect(existsSync(join(root, "manifest.json"))).toBe(false); } finally { server.stop(true); } }); + + test("on a fork, fetches only changed agents, falls back per agent and removes what the manifest drops", async () => { + const root = mkdtempSync(join(tmpdir(), "agent-tools-")); + const oldClaude = placeAgent(root, "claude", agentScript("echo old")); + const oldCodex = placeAgent(root, "codex", agentScript("echo old")); + const pdev = placeAgent(root, "pdev", agentScript("echo pdev")); + const gone = placeAgent(root, "gone", agentScript("echo gone")); + mkdirSync(join(root, "lost+found")); + mkdirSync(join(root, "pdev", `${pdev.sha}.partial`)); + const previous = [oldClaude, oldCodex, pdev, gone]; + writeFileSync(join(root, "manifest.json"), JSON.stringify(previous)); + + const newClaude = agentScript("echo new"); + const newCodex = sha("b"); + const fresh = sha("f"); + const { server, requests, baseUrl } = serve( + new Map([[sha256(newClaude), newClaude]]), + ); + const manifest: AgentToolsManifest = [ + { name: "claude", sha: sha256(newClaude), bin: "claude" }, + { name: "codex", sha: newCodex, bin: "codex" }, + pdev, + { name: "fresh", sha: fresh, bin: "fresh" }, + ]; + try { + const run = await runScript( + renderBuilderScript({ + root, + baseUrl, + assets: { + claude: row(sha256(newClaude), ""), + codex: row(newCodex, ""), + pdev: row(pdev.sha, ""), + fresh: row(fresh, ""), + }, + manifest, + previous, + }), + ); + expect(run.exitCode, run.stderr).toBe(0); + expect(parseBuilderResults(run.stdout)).toEqual([ + { name: "claude", outcome: "written", sha: sha256(newClaude) }, + { name: "codex", outcome: "kept-previous", sha: oldCodex.sha }, + { name: "pdev", outcome: "reused", sha: pdev.sha }, + { name: "fresh", outcome: "dropped" }, + ]); + expect(requests.sort()).toEqual( + [sha256(newClaude), newCodex, fresh].sort(), + ); + expect( + parseAgentToolsManifest( + readFileSync(join(root, "manifest.json"), "utf8"), + ), + ).toEqual([manifest[0], oldCodex, pdev] as AgentToolsManifest); + expect(existsSync(join(root, "claude", oldClaude.sha))).toBe(false); + expect(existsSync(join(root, "gone"))).toBe(false); + expect(existsSync(join(root, "fresh"))).toBe(false); + expect(existsSync(join(root, "pdev", `${pdev.sha}.partial`))).toBe(false); + expect(existsSync(join(root, "lost+found"))).toBe(true); + } finally { + server.stop(true); + } + }); + + test("an agent that fails its check is not named: the earlier one is kept, or it is left out", async () => { + const root = mkdtempSync(join(tmpdir(), "agent-tools-")); + const good = placeAgent( + root, + "claude", + agentScript('[ "$1" = selftest ] || exit 9'), + ); + const broken = agentScript("exit 3"); + const hangs = agentScript("sleep 30"); + const { server, baseUrl } = serve( + new Map([ + [sha256(broken), broken], + [sha256(hangs), hangs], + ]), + ); + try { + const run = await runScript( + renderBuilderScript({ + root, + baseUrl, + assets: { + claude: row(sha256(broken), ""), + codex: row(sha256(hangs), ""), + }, + manifest: [ + { name: "claude", sha: sha256(broken), bin: "claude" }, + { name: "codex", sha: sha256(hangs), bin: "codex" }, + ], + previous: [good], + tools: [ + { name: "claude", bin: "claude", check: ["selftest"] }, + { name: "codex", bin: "codex" }, + ], + checkTimeoutSeconds: 1, + }), + ); + expect(run.exitCode, run.stderr).toBe(0); + expect(parseBuilderResults(run.stdout)).toEqual([ + { name: "claude", outcome: "kept-previous", sha: good.sha }, + { name: "codex", outcome: "dropped" }, + ]); + expect(run.stderr).toContain("check failed"); + expect( + parseAgentToolsManifest( + readFileSync(join(root, "manifest.json"), "utf8"), + ), + ).toEqual([good]); + expect(existsSync(join(root, "codex"))).toBe(false); + } finally { + server.stop(true); + } + }); }); diff --git a/packages/sandbox/src/agent-tools.ts b/packages/sandbox/src/agent-tools.ts index a9967404976..1c422daa7c8 100644 --- a/packages/sandbox/src/agent-tools.ts +++ b/packages/sandbox/src/agent-tools.ts @@ -11,6 +11,11 @@ * stopped and the drive is detached. The release publishes before any * environment row points at the bundle, so no box mounts a drive mid-write. * + * A new drive starts as a fork of the region's newest earlier one, so the + * builder fetches only the agents whose sha changed. Each agent fails alone: + * it keeps the earlier drive's version, or is left out and boxes use the + * image's copy. + * * bun run agent-tools --bundle [--region sfo1 ...] * * Needs VERCEL_SANDBOX_TOKEN, VERCEL_SANDBOX_TEAM_ID, VERCEL_SANDBOX_PROJECT_ID. @@ -32,11 +37,21 @@ import { z } from "zod"; import { type Assets, assetObjectKey, loadAssets, SHA256 } from "./manifest"; /** Each agent's asset row, and its executable relative to the unpacked dir. */ -export const AGENT_TOOLS = [ +export const AGENT_TOOLS: readonly AgentTool[] = [ { name: "claude", bin: "claude" }, { name: "codex", bin: "bin/codex" }, { name: "pdev", bin: "pdev/pdev" }, -] as const; +]; + +export interface AgentTool { + name: string; + bin: string; + /** Arguments the builder runs the agent with before it names it; the default is `--version`. */ + check?: readonly string[]; +} + +const DEFAULT_CHECK = ["--version"]; +const CHECK_TIMEOUT_SECONDS = 30; export const AGENT_TOOLS_MANIFEST = "manifest.json"; @@ -51,9 +66,12 @@ const agentToolSchema = z.object({ export const agentToolsManifestSchema = z.array(agentToolSchema); export type AgentToolsManifest = z.infer; -export function agentToolsManifest(assets: Assets): AgentToolsManifest { +export function agentToolsManifest( + assets: Assets, + tools: readonly AgentTool[] = AGENT_TOOLS, +): AgentToolsManifest { return agentToolsManifestSchema.parse( - AGENT_TOOLS.map(({ name, bin }) => { + tools.map(({ name, bin }) => { const asset = assets[name]; if (!asset) throw new Error(`assets.json has no ${name} row`); return { name, sha: asset.sha256, bin }; @@ -75,69 +93,198 @@ export function parseAgentToolsManifest( const quote = (value: string) => `'${value.replaceAll("'", "'\\''")}'`; -/** The script the builder runs as root against the empty drive mounted read-write at `root`. */ +export type AgentOutcome = "written" | "reused" | "kept-previous" | "dropped"; + +export interface AgentResult { + name: string; + outcome: AgentOutcome; + /** The sha the new manifest names; absent when the agent was dropped. */ + sha?: string; +} + +const RESULT_LINE = + /^agent-tools-result ([a-z0-9-]+) (written|reused|kept-previous|dropped) ?([0-9a-f]{64})?$/gm; + +export function parseBuilderResults(stdout: string): AgentResult[] { + return [...stdout.matchAll(RESULT_LINE)].map(([, name, outcome, sha]) => ({ + name: name as string, + outcome: outcome as AgentOutcome, + ...(sha ? { sha } : {}), + })); +} + +/** + * The script the builder runs as root against the drive mounted read-write + * at `root`: empty, or a fork of an earlier drive whose manifest is `previous`. + */ export function renderBuilderScript(args: { root: string; baseUrl: string; assets: Assets; manifest: AgentToolsManifest; + previous?: AgentToolsManifest; + tools?: readonly AgentTool[]; + checkTimeoutSeconds?: number; }): string { - const fetches = args.manifest.map(({ name, sha, bin }) => { + const tools = args.tools ?? AGENT_TOOLS; + const agents = args.manifest.map(({ name, sha, bin }) => { const asset = args.assets[name]; if (!asset || asset.sha256 !== sha) { throw new Error(`no asset row for ${name} at ${sha}`); } const archive = asset.suffix === ".tar.gz" ? "tar" : "file"; - return `publish ${[name, sha, bin, assetObjectKey(asset), archive].map(quote).join(" ")}`; + const previous = args.previous?.find((entry) => entry.name === name); + const check = + tools.find((tool) => tool.name === name)?.check ?? DEFAULT_CHECK; + return `agent ${[ + name, + sha, + bin, + assetObjectKey(asset), + archive, + previous?.sha ?? "", + previous?.bin ?? "", + ...check, + ] + .map(quote) + .join(" ")}`; }); return `set -euo pipefail +shopt -s nullglob root=${quote(args.root)} base=${quote(args.baseUrl)} +declare -A keep_sha=() +entries=() + +rm -f "\${root}/${AGENT_TOOLS_MANIFEST}" "\${root}/.${AGENT_TOOLS_MANIFEST}.partial" +sync -publish() { +fetch() { local name="$1" sha="$2" bin="$3" key="$4" kind="$5" - local target="\${root}/\${name}/\${sha}" - local tmp - tmp="$(mktemp)" - curl -fsSL --retry 5 --retry-delay 2 --retry-all-errors -o "$tmp" "\${base}/\${key}" || - curl -fsSL --retry 2 --retry-delay 2 --retry-all-errors -o "$tmp" "\${base}/\${key}?fresh=$(date +%s)" - local got + local target="\${root}/\${name}/\${sha}" tmp got + tmp="$(mktemp)" || return 1 + if ! { curl -fsSL --retry 5 --retry-delay 2 --retry-all-errors -o "$tmp" "\${base}/\${key}" || + curl -fsSL --retry 2 --retry-delay 2 --retry-all-errors -o "$tmp" "\${base}/\${key}?fresh=$(date +%s)"; }; then + echo "\${name}: download failed" >&2; rm -f "$tmp"; return 1 + fi got="$(sha256sum "$tmp" | cut -d' ' -f1)" if [ "$got" != "$sha" ]; then echo "\${name}: expected \${sha}, got \${got}" >&2; rm -f "$tmp"; return 1; fi - rm -rf "\${target}.partial" - mkdir -p "\${target}.partial" + rm -rf "\${target}.partial" && mkdir -p "\${target}.partial" || { rm -f "$tmp"; return 1; } if [ "$kind" = tar ]; then - tar -xzf "$tmp" -C "\${target}.partial" + tar -xzf "$tmp" -C "\${target}.partial" || { echo "\${name}: unpack failed" >&2; rm -f "$tmp"; rm -rf "\${target}.partial"; return 1; } else - install -D -m 0755 "$tmp" "\${target}.partial/\${bin}" + install -D -m 0755 "$tmp" "\${target}.partial/\${bin}" || { rm -f "$tmp"; rm -rf "\${target}.partial"; return 1; } fi rm -f "$tmp" - [ -x "\${target}.partial/\${bin}" ] || { echo "\${name}: no executable at \${bin}" >&2; return 1; } - rm -rf "$target" - mv -T "\${target}.partial" "$target" - echo "\${name} \${sha:0:12} written" + [ -x "\${target}.partial/\${bin}" ] || { echo "\${name}: no executable at \${bin}" >&2; rm -rf "\${target}.partial"; return 1; } + rm -rf "$target" && mv -T "\${target}.partial" "$target" } -${fetches.join("\n")} -printf '%s\\n' ${quote(JSON.stringify(args.manifest, null, "\t"))} > "\${root}/.${AGENT_TOOLS_MANIFEST}.partial" +smoke() { + local dir="$1" bin="$2" home + shift 2 + home="$(mktemp -d)" || return 1 + if (cd "$home" && HOME="$home" timeout -k 5 ${args.checkTimeoutSeconds ?? CHECK_TIMEOUT_SECONDS} "\${dir}/\${bin}" "$@" > "\${home}/.out" 2>&1 < /dev/null); then + rm -rf "$home"; return 0 + fi + echo "\${dir}/\${bin} $*: check failed: $(tail -c 300 "\${home}/.out")" >&2 + rm -rf "$home"; return 1 +} + +keep() { + keep_sha[$1]="$2" + entries+=("$(printf '{"name":"%s","sha":"%s","bin":"%s"}' "$1" "$2" "$3")") +} + +agent() { + local name="$1" sha="$2" bin="$3" key="$4" kind="$5" prev_sha="$6" prev_bin="$7" outcome=failed + shift 7 + if [ -x "\${root}/\${name}/\${sha}/\${bin}" ]; then outcome=reused + elif fetch "$name" "$sha" "$bin" "$key" "$kind"; then outcome=written + fi + if [ "$outcome" != failed ] && smoke "\${root}/\${name}/\${sha}" "$bin" "$@"; then + keep "$name" "$sha" "$bin" + elif [ -n "$prev_sha" ] && [ "$prev_sha" != "$sha" ] && [ -x "\${root}/\${name}/\${prev_sha}/\${prev_bin}" ] && + smoke "\${root}/\${name}/\${prev_sha}" "$prev_bin" "$@"; then + outcome=kept-previous sha="$prev_sha" + keep "$name" "$prev_sha" "$prev_bin" + else + outcome=dropped sha= + fi + echo "agent-tools-result \${name} \${outcome} \${sha}" +} + +${agents.join("\n")} + +for dir in "\${root}"/*/; do + dir="\${dir%/}" name="\${dir##*/}" + [[ "$name" =~ ^[a-z0-9-]+$ ]] || continue + if [ -z "\${keep_sha[$name]:-}" ]; then rm -rf "$dir"; continue; fi + for entry in "\${dir}"/* "\${dir}"/.[!.]*; do + [ "\${entry##*/}" = "\${keep_sha[$name]}" ] || rm -rf "$entry" + done +done + +[ "\${#entries[@]}" -gt 0 ] || { echo "no agent could be published" >&2; exit 1; } +printf '[%s]\\n' "$(IFS=,; echo "\${entries[*]}")" > "\${root}/.${AGENT_TOOLS_MANIFEST}.partial" sync mv -T "\${root}/.${AGENT_TOOLS_MANIFEST}.partial" "\${root}/${AGENT_TOOLS_MANIFEST}" sync `; } +export interface DriveEntry { + name: string; + createdAt: number; + attachedTo?: string; +} + /** What the publish needs from the provider; `vercelDriveHost` is the real one. */ export interface DriveHost { /** Null when no drive has the name. */ find(name: string): Promise<{ attachedTo?: string } | null>; - list(prefix: string): Promise; + list(prefix: string): Promise; create(name: string, region: SandboxRegionId): Promise; + /** Copy-on-write. The parent must have been written, stopped and detached. */ + fork(parent: string, name: string): Promise; /** Throws while the drive is attached to a sandbox. */ remove(name: string): Promise; /** Through a read-only snapshot mount; null when the drive has no manifest. */ readManifest(name: string, region: SandboxRegionId): Promise; - /** Resolves only once the builder has stopped and the drive is detached. */ - write(name: string, region: SandboxRegionId, script: string): Promise; + /** + * Renders the script from the manifest the mounted drive holds, runs it, + * and returns its stdout. Resolves only once the builder has stopped and + * the drive is detached. + */ + write( + name: string, + region: SandboxRegionId, + script: (manifest: string | null) => string, + ): Promise; +} + +export interface AgentToolsPublish { + drive: string; + outcome: "exists" | "published"; + forkedFrom?: string; + agents: AgentResult[]; +} + +/** The region's newest other agent-tools drive that no sandbox holds read-write. */ +export function previousAgentToolsDrive( + drives: readonly DriveEntry[], + region: SandboxRegionId, + drive: string, +): string | undefined { + return drives + .filter( + (entry) => + entry.name !== drive && + entry.name.startsWith(`agent-tools-${region}-`) && + SANDBOX_AGENT_TOOLS_DRIVE_PATTERN.test(entry.name) && + !entry.attachedTo, + ) + .sort((a, b) => b.createdAt - a.createdAt)[0]?.name; } /** @@ -151,9 +298,12 @@ export async function publishAgentToolsDrive(args: { bundleSha: string; assets: Assets; baseUrl?: string; -}): Promise<{ drive: string; outcome: "exists" | "published" }> { + tools?: readonly AgentTool[]; + warn?: (line: string) => void; +}): Promise { + const warn = args.warn ?? ((line) => console.log(`::warning::${line}`)); const drive = sandboxAgentToolsDrive(args.region, args.bundleSha); - const manifest = agentToolsManifest(args.assets); + const manifest = agentToolsManifest(args.assets, args.tools); const existing = await args.host.find(drive); if (existing?.attachedTo) { throw new Error( @@ -164,21 +314,109 @@ export async function publishAgentToolsDrive(args: { const found = parseAgentToolsManifest( await args.host.readManifest(drive, args.region), ); - if (found.length > 0) return { drive, outcome: "exists" }; + if (found.length > 0) return { drive, outcome: "exists", agents: [] }; await args.host.remove(drive); } - await args.host.create(drive, args.region); - await args.host.write( - drive, - args.region, + let forkedFrom: string | undefined; + let parent: string | undefined; + try { + parent = previousAgentToolsDrive( + await args.host.list(`agent-tools-${args.region}-`), + args.region, + drive, + ); + if (parent) { + await args.host.fork(parent, drive); + forkedFrom = parent; + } + } catch (error) { + warn( + `agent-tools: ${drive} not forked from ${parent ?? "an earlier drive"}; writing every agent: ${String(error).slice(0, 300)}`, + ); + if (await args.host.find(drive)) await args.host.remove(drive); + } + if (!forkedFrom) await args.host.create(drive, args.region); + const stdout = await args.host.write(drive, args.region, (current) => renderBuilderScript({ root: SANDBOX_PATHS.agents, baseUrl: args.baseUrl ?? SANDBOX_ASSET_BASE_URL, assets: args.assets, manifest, + previous: forkedFrom ? parseAgentToolsManifest(current) : [], + tools: args.tools, }), ); - return { drive, outcome: "published" }; + return { + drive, + outcome: "published", + ...(forkedFrom ? { forkedFrom } : {}), + agents: parseBuilderResults(stdout), + }; +} + +export type RegionPublish = + | { region: SandboxRegionId; result: AgentToolsPublish } + | { region: SandboxRegionId; error: unknown }; + +/** Publishes `concurrency` regions at a time; a region that fails does not stop the others. */ +export async function publishAgentToolsDrives( + args: Omit[0], "region"> & { + regions: readonly SandboxRegionId[]; + concurrency?: number; + }, +): Promise { + const { regions, concurrency = 6, ...rest } = args; + const results: RegionPublish[] = []; + let next = 0; + const worker = async () => { + while (next < regions.length) { + const index = next++; + const region = regions[index] as SandboxRegionId; + try { + results[index] = { + region, + result: await publishAgentToolsDrive({ ...rest, region }), + }; + } catch (error) { + results[index] = { region, error }; + } + } + }; + await Promise.all( + Array.from({ length: Math.min(concurrency, regions.length) }, worker), + ); + return results; +} + +/** One line per region and per agent that did not get the bundle's version; warnings start `::warning::`. */ +export function describeAgentToolsPublish( + results: readonly RegionPublish[], +): string[] { + return results.flatMap((entry) => { + if ("error" in entry) { + return [ + `::warning::agent-tools: ${entry.region} not published; its boxes keep the image's agents: ${String(entry.error).slice(0, 300)}`, + ]; + } + const { drive, outcome, forkedFrom, agents } = entry.result; + const summary = agents + .map((agent) => `${agent.name} ${agent.outcome}`) + .join(", "); + return [ + `agent-tools: ${drive} ${outcome}${forkedFrom ? ` from ${forkedFrom}` : ""}${summary ? `: ${summary}` : ""}`, + ...agents.flatMap((agent) => + agent.outcome === "kept-previous" + ? [ + `::warning::agent-tools: ${drive}: ${agent.name} kept ${agent.sha?.slice(0, 12)} from the earlier drive; the bundle's version failed`, + ] + : agent.outcome === "dropped" + ? [ + `::warning::agent-tools: ${drive}: ${agent.name} left out; its boxes keep the image's ${agent.name}`, + ] + : [], + ), + ]; + }); } /** Agent-tools drives no environment row's bundle and region name any more. */ @@ -200,7 +438,7 @@ export async function pruneAgentToolsDrives(args: { const deleted: string[] = []; const kept: string[] = []; for (const name of agentToolsDrivesToPrune( - await args.host.list("agent-tools-"), + (await args.host.list("agent-tools-")).map((drive) => drive.name), args.referenced, )) { try { @@ -265,20 +503,29 @@ export function vercelDriveHost(credentials: VercelCredentials): DriveHost { }, async list(prefix) { const { Drive } = await sdk(); - const names: string[] = []; + const drives: DriveEntry[] = []; for await (const drive of await Drive.list({ ...credentials, namePrefix: prefix, sortBy: "name", })) { - names.push(drive.name); + drives.push({ + name: drive.name, + createdAt: drive.createdAt.getTime(), + attachedTo: drive.currentSandboxName, + }); } - return names; + return drives; }, async create(name, region) { const { Drive } = await sdk(); await Drive.getOrCreate({ ...credentials, name, region }); }, + async fork(parent, name) { + const drive = await getDrive(parent); + if (!drive) throw new Error(`${parent} does not exist`); + await drive.fork({ name }); + }, async remove(name) { await (await getDrive(name))?.delete(); }, @@ -320,15 +567,18 @@ export function vercelDriveHost(credentials: VercelCredentials): DriveHost { mounts: { [SANDBOX_PATHS.agents]: drive }, }); let failure: Error | null = null; + let stdout = ""; try { + const current = await builder.readFileToBuffer({ + path: `${SANDBOX_PATHS.agents}/${AGENT_TOOLS_MANIFEST}`, + }); const result = await builder.runCommand({ cmd: "bash", - args: ["-c", script], + args: ["-c", script(current?.toString("utf8") ?? null)], sudo: true, }); - console.log( - `${name}: ${await result.stdout()}${await result.stderr()}`, - ); + stdout = await result.stdout(); + console.log(`${name}: ${stdout}${await result.stderr()}`); if (result.exitCode !== 0) { failure = new Error(`${name}: builder exited ${result.exitCode}`); } @@ -336,6 +586,7 @@ export function vercelDriveHost(credentials: VercelCredentials): DriveHost { await stopAndDelete(builder, name); } if (failure) throw failure; + return stdout; }, }; } @@ -373,16 +624,14 @@ if (import.meta.main) { const assets = loadAssets( join(import.meta.dir, "..", "bundle", "assets.json"), ); - const host = vercelDriveHost(credentials); - for (const region of regions.length - ? (regions as SandboxRegionId[]) - : [DEFAULT_SANDBOX_REGION]) { - const { drive, outcome } = await publishAgentToolsDrive({ - host, - region, - bundleSha, - assets, - }); - console.log(`${drive}: ${outcome}`); - } + const results = await publishAgentToolsDrives({ + host: vercelDriveHost(credentials), + regions: regions.length + ? (regions as SandboxRegionId[]) + : [DEFAULT_SANDBOX_REGION], + bundleSha, + assets, + }); + for (const line of describeAgentToolsPublish(results)) console.log(line); + if (results.some((entry) => "error" in entry)) process.exit(1); } diff --git a/packages/sandbox/src/environments/release.ts b/packages/sandbox/src/environments/release.ts index 2d049be17aa..72836cb16ee 100644 --- a/packages/sandbox/src/environments/release.ts +++ b/packages/sandbox/src/environments/release.ts @@ -169,20 +169,16 @@ async function environmentRegions(): Promise { ), ]; } -for (const region of await environmentRegions()) { - try { - const { drive, outcome } = await agentTools.publishAgentToolsDrive({ - host: driveHost, - region, - bundleSha: bundle.sha256, - assets: bundle.assets, - }); - log(`agent-tools: ${drive} ${outcome}`); - } catch (error) { - console.log( - `::warning::agent-tools: ${region} not published; its boxes keep the image's agents: ${String(error).slice(0, 300)}`, - ); - } +for (const line of agentTools.describeAgentToolsPublish( + await agentTools.publishAgentToolsDrives({ + host: driveHost, + regions: await environmentRegions(), + bundleSha: bundle.sha256, + assets: bundle.assets, + }), +)) { + if (line.startsWith("::warning::")) console.log(line); + else log(line); } // 2. image