diff --git a/packages/sandbox/README.md b/packages/sandbox/README.md index ab330b9bae6..2fdb4b2c7ea 100644 --- a/packages/sandbox/README.md +++ b/packages/sandbox/README.md @@ -47,7 +47,7 @@ The rule of thumb: if you can answer "where is it on the box", put it there. | `bun run image` | Builds the image: OS, apt lists, vendor tools, the `ubuntu` user, then installs the bundle at its sha and runs every pass once, so a fresh box finds every hash sidecar and step marker matching. A push logs Docker into the registry itself with a credential minted from `VERCEL_SANDBOX_TOKEN`, so there is no login to renew. `--local` builds `superset-sandbox:local` for the boot-twice test; `--dry` prints the Dockerfile. | | boot (`superset-boot`) | On the box, as root, run by the control plane on every create and wake, after it has written `sandbox.conf`, with the host secret in its env: clears `/run/superset`, installs the pinned bundle if the hash differs, runs the three passes (each a hash compare), starts host-service as `ubuntu` with the secret from the boot env, writes `resources.env`, starts `superset-memguard` (ranks every process for the OOM killer and reaps the largest before the kernel has to), then the desktop and the checkout. Once host-service answers, the environment has been pushed and the checkout is in, the runner asks host-service to run the repo's `start` hook with that environment. | | `bun run release` | Rewrites the host-service and CLI assets from this checkout and the claude, codex and Chrome rows to their latest upstream release, publishes the bundle, pushes the image under a candidate tag `release-` (unless `--skip-image`), builds the internal golden from it with the environment's `setup` hook, probes a fork of it (`src/environments/probe.ts`) including a stop and wake, and only after the probe passes moves `latest` to the candidate and writes the environment rows. Every image environment resolves `superset-hostsvc` to `latest`, so a failed release changes no box. | -| `bun run agent-tools --bundle ` | Writes that bundle's agent CLIs to `agent-tools--`, a drive boxes mount read-only at `/opt/agents` when their organization has the `sandbox-agent-tools-drive` flag. A drive is written once and never again. `bun run release` does this for every region with an environment before it records the bundle, and afterwards deletes the drives no environment names. | +| `bun run agent-tools --bundle ` | Writes that bundle's agent CLIs to `agent-tools--`, a drive boxes mount read-only at `/opt/agents` when their organization has the `sandbox-agent-tools-drive` flag. A drive is written once and never again; it starts as a fork of the region's newest earlier drive, so only changed agents are fetched, and an agent that fails its download or `--version` check keeps the earlier drive's version or is left out. `bun run release` does this for every region with an environment, six at a time, before it records the bundle, and afterwards deletes the drives no environment names. | | `bun run src/real-sandbox.ts` | The Sandbox workflow dispatched with `real-sandbox`: one real sandbox from the registry image on this checkout's bundle, the same probe, a stop and a wake. | | Sandbox workflow, `production-release` | `bun run release --production` from `main` on a CI runner: the image, the internal golden and production's environment rows. Runs on a merge that touches the bundle and every night; a failed merge or nightly run opens an issue, or comments on the open one. | diff --git a/packages/sandbox/src/agent-tools.test.ts b/packages/sandbox/src/agent-tools.test.ts index 30f29379de7..7dac023edce 100644 --- a/packages/sandbox/src/agent-tools.test.ts +++ b/packages/sandbox/src/agent-tools.test.ts @@ -13,9 +13,12 @@ import { agentToolsDrivesToPrune, agentToolsManifest, type DriveHost, + describeAgentToolsPublish, parseAgentToolsManifest, + parseBuilderResults, pruneAgentToolsDrives, publishAgentToolsDrive, + publishAgentToolsDrives, renderBuilderScript, } from "./agent-tools"; import { type Assets, sha256 } from "./manifest"; @@ -56,20 +59,49 @@ describe("agent-tools manifest", () => { }); }); +interface FakeDrive { + manifest?: string; + attachedTo?: string; + createdAt?: number; +} + +function written(assets: Assets = agentAssets) { + return agentToolsManifest(assets) + .map(({ name, sha }) => `agent-tools-result ${name} written ${sha}`) + .join("\n"); +} + function fakeHost( - drives: Record, + drives: Record, + options: { + builder?: (name: string) => string; + failFork?: boolean; + } = {}, ) { const calls: string[] = []; + const scripts: Record = {}; + let clock = 100; const host: DriveHost = { find: async (name) => { const drive = drives[name]; return drive ? { attachedTo: drive.attachedTo } : null; }, list: async (prefix) => - Object.keys(drives).filter((name) => name.startsWith(prefix)), + Object.entries(drives) + .filter(([name]) => name.startsWith(prefix)) + .map(([name, drive]) => ({ + name, + createdAt: drive.createdAt ?? 0, + attachedTo: drive.attachedTo, + })), create: async (name) => { calls.push(`create ${name}`); - drives[name] = {}; + drives[name] = { createdAt: clock++ }; + }, + fork: async (parent, name) => { + calls.push(`fork ${parent} ${name}`); + drives[name] = { ...drives[parent], createdAt: clock++ }; + if (options.failFork) throw new Error("fork refused"); }, remove: async (name) => { calls.push(`remove ${name}`); @@ -77,57 +109,59 @@ function fakeHost( delete drives[name]; }, readManifest: async (name) => drives[name]?.manifest ?? null, - write: async (name) => { + write: async (name, _region, script) => { calls.push(`write ${name}`); const drive = drives[name]; + scripts[name] = script(drive?.manifest ?? null); + const stdout = options.builder?.(name) ?? written(); if (drive) drive.manifest = JSON.stringify(agentToolsManifest(agentAssets)); + return stdout; }, }; - return { host, calls, drives }; + return { host, calls, drives, scripts }; } +const agentLine = (script: string | undefined, name: string) => + script?.split("\n").find((line) => line.startsWith(`agent '${name}' `)); + describe("publishAgentToolsDrive", () => { const drive = "agent-tools-sfo1-0123456789ab"; + const publish = (host: DriveHost, warn: (line: string) => void = () => {}) => + publishAgentToolsDrive({ + host, + region: "sfo1", + bundleSha: BUNDLE, + assets: agentAssets, + warn, + }); - test("creates and writes the bundle's drive in the region", async () => { - const { host, calls } = fakeHost({}); - expect( - await publishAgentToolsDrive({ - host, - region: "sfo1", - bundleSha: BUNDLE, - assets: agentAssets, - }), - ).toEqual({ drive, outcome: "published" }); + test("with no earlier drive, creates the drive and writes every agent", async () => { + const { host, calls, scripts } = fakeHost({}); + const result = await publish(host); + expect(result.drive).toBe(drive); + expect(result.outcome).toBe("published"); + expect(result.forkedFrom).toBeUndefined(); + expect(result.agents.map((agent) => agent.outcome)).toEqual([ + "written", + "written", + "written", + ]); expect(calls).toEqual([`create ${drive}`, `write ${drive}`]); + expect(agentLine(scripts[drive], "claude")).toEndWith("'' '' '--version'"); }); test("leaves a drive that has a manifest alone", async () => { const { host, calls } = fakeHost({ [drive]: { manifest: JSON.stringify(agentToolsManifest(agentAssets)) }, }); - expect( - ( - await publishAgentToolsDrive({ - host, - region: "sfo1", - bundleSha: BUNDLE, - assets: agentAssets, - }) - ).outcome, - ).toBe("exists"); + expect((await publish(host)).outcome).toBe("exists"); expect(calls).toEqual([]); }); test("recreates a drive an earlier run left without a manifest", async () => { const { host, calls } = fakeHost({ [drive]: {} }); - await publishAgentToolsDrive({ - host, - region: "sfo1", - bundleSha: BUNDLE, - assets: agentAssets, - }); + await publish(host); expect(calls).toEqual([ `remove ${drive}`, `create ${drive}`, @@ -137,16 +171,155 @@ describe("publishAgentToolsDrive", () => { test("refuses a drive a writer still holds", async () => { const { host, calls } = fakeHost({ [drive]: { attachedTo: "builder" } }); - await expect( - publishAgentToolsDrive({ - host, - region: "sfo1", - bundleSha: BUNDLE, - assets: agentAssets, - }), - ).rejects.toThrow(/attached to builder/); + await expect(publish(host)).rejects.toThrow(/attached to builder/); expect(calls).toEqual([]); }); + + test("forks the region's newest free drive and hands the builder its manifest", async () => { + const earlier = [ + { name: "claude", sha: sha("e"), bin: "claude" }, + { name: "codex", sha: sha("b"), bin: "bin/codex" }, + ]; + const { host, calls, scripts } = fakeHost({ + "agent-tools-sfo1-111111111111": { + createdAt: 1, + manifest: JSON.stringify([ + { name: "claude", sha: sha("1"), bin: "claude" }, + ]), + }, + "agent-tools-sfo1-222222222222": { + createdAt: 2, + manifest: JSON.stringify(earlier), + }, + "agent-tools-sfo1-333333333333": { createdAt: 3, attachedTo: "box" }, + "agent-tools-iad1-444444444444": { createdAt: 4 }, + }); + const result = await publish(host); + expect(result.forkedFrom).toBe("agent-tools-sfo1-222222222222"); + expect(calls).toEqual([ + `fork agent-tools-sfo1-222222222222 ${drive}`, + `write ${drive}`, + ]); + expect(agentLine(scripts[drive], "claude")).toEndWith( + `'${sha("e")}' 'claude' '--version'`, + ); + expect(agentLine(scripts[drive], "pdev")).toEndWith("'' '' '--version'"); + }); + + test("a failed fork warns and falls back to writing every agent", async () => { + const warnings: string[] = []; + const { host, calls, scripts } = fakeHost( + { + "agent-tools-sfo1-222222222222": { + createdAt: 2, + manifest: JSON.stringify(agentToolsManifest(agentAssets)), + }, + }, + { failFork: true }, + ); + const result = await publish(host, (line) => warnings.push(line)); + expect(result.forkedFrom).toBeUndefined(); + expect(calls).toEqual([ + `fork agent-tools-sfo1-222222222222 ${drive}`, + `remove ${drive}`, + `create ${drive}`, + `write ${drive}`, + ]); + expect(warnings.join("\n")).toContain("fork refused"); + expect(agentLine(scripts[drive], "claude")).toEndWith("'' '' '--version'"); + }); + + test("reports each agent's outcome and warns for the ones that missed the bundle", async () => { + const tools = [ + { name: "claude", bin: "claude" }, + { name: "codex", bin: "bin/codex" }, + { name: "pdev", bin: "pdev/pdev" }, + { name: "chrome", bin: "chrome", check: ["--headless", "--version"] }, + ]; + const { host, scripts } = fakeHost( + {}, + { + builder: () => + [ + "noise", + `agent-tools-result claude written ${sha("a")}`, + `agent-tools-result codex reused ${sha("b")}`, + `agent-tools-result pdev kept-previous ${sha("9")}`, + "agent-tools-result chrome dropped ", + ].join("\n"), + }, + ); + const result = await publishAgentToolsDrive({ + host, + region: "sfo1", + bundleSha: BUNDLE, + assets: agentAssets, + tools, + }); + expect(result.agents).toEqual([ + { name: "claude", outcome: "written", sha: sha("a") }, + { name: "codex", outcome: "reused", sha: sha("b") }, + { name: "pdev", outcome: "kept-previous", sha: sha("9") }, + { name: "chrome", outcome: "dropped" }, + ]); + expect(agentLine(scripts[drive], "chrome")).toEndWith( + "'--headless' '--version'", + ); + const lines = describeAgentToolsPublish([{ region: "sfo1", result }]); + expect(lines[0]).toBe( + `agent-tools: ${drive} published: claude written, codex reused, pdev kept-previous, chrome dropped`, + ); + expect(lines.slice(1)).toEqual([ + expect.stringMatching(/^::warning::.*pdev kept 999999999999/), + expect.stringMatching(/^::warning::.*chrome left out/), + ]); + }); +}); + +describe("publishAgentToolsDrives", () => { + test("runs regions in parallel up to the limit, and one failure stays in its region", async () => { + const { host } = fakeHost({}); + let running = 0; + let peak = 0; + const write = host.write; + host.write = async (name, region, script) => { + running++; + peak = Math.max(peak, running); + await new Promise((resolve) => setTimeout(resolve, 1)); + running--; + if (region === "iad1") throw new Error("builder exited 1"); + return write(name, region, script); + }; + const regions = ["sfo1", "iad1", "fra1", "lhr1", "sin1"] as const; + const results = await publishAgentToolsDrives({ + host, + regions, + bundleSha: BUNDLE, + assets: agentAssets, + concurrency: 2, + }); + expect(peak).toBe(2); + expect(results.map((entry) => [entry.region, "error" in entry])).toEqual([ + ["sfo1", false], + ["iad1", true], + ["fra1", false], + ["lhr1", false], + ["sin1", false], + ]); + expect( + describeAgentToolsPublish(results).filter((line) => + line.startsWith("::warning::"), + ), + ).toEqual([expect.stringMatching(/iad1 not published.*builder exited 1/)]); + }); +}); + +test("builder results ignore lines that are not results", () => { + expect( + parseBuilderResults( + `claude: written\nagent-tools-result claude reused ${sha("a")}\nagent-tools-result x bogus\n`, + ), + ).toEqual([{ name: "claude", outcome: "reused", sha: sha("a") }]); }); describe("pruning", () => { @@ -187,8 +360,34 @@ async function runScript(script: string) { stdout: "pipe", stderr: "pipe", }); - const stderr = await new Response(proc.stderr).text(); - return { exitCode: await proc.exited, stderr }; + const [stdout, stderr] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + ]); + return { exitCode: await proc.exited, stdout, stderr }; +} + +function serve(objects: Map) { + const requests: string[] = []; + const server = Bun.serve({ + port: 0, + fetch: (request) => { + const key = new URL(request.url).pathname.slice(1); + requests.push(key); + return new Response(objects.get(key) ?? "not the agent"); + }, + }); + return { server, requests, baseUrl: `http://127.0.0.1:${server.port}` }; +} + +const agentScript = (body: string) => + new TextEncoder().encode(`#!/bin/sh\n${body}\n`); + +function placeAgent(root: string, name: string, body: Uint8Array) { + const dir = join(root, name, sha256(body)); + mkdirSync(dir, { recursive: true }); + writeFileSync(join(dir, name), body, { mode: 0o755 }); + return { name, sha: sha256(body), bin: name }; } describe.skipIf(process.platform !== "linux")("builder script", () => { @@ -200,19 +399,14 @@ describe.skipIf(process.platform !== "linux")("builder script", () => { mode: 0o755, }); Bun.spawnSync(["tar", "-czf", join(work, "codex.tar.gz"), "-C", src, "."]); - const claude = new TextEncoder().encode("#!/bin/sh\necho claude\n"); + const claude = agentScript("echo claude"); const codex = readFileSync(join(work, "codex.tar.gz")); - const objects = new Map([ - [sha256(claude), claude], - [`${sha256(codex)}.tar.gz`, codex], - ]); - const server = Bun.serve({ - port: 0, - fetch: (request) => { - const body = objects.get(new URL(request.url).pathname.slice(1)); - return body ? new Response(body) : new Response("", { status: 404 }); - }, - }); + const { server, baseUrl } = serve( + new Map([ + [sha256(claude), claude], + [`${sha256(codex)}.tar.gz`, codex], + ]), + ); const root = join(work, "drive"); mkdirSync(root); const manifest: AgentToolsManifest = [ @@ -223,7 +417,7 @@ describe.skipIf(process.platform !== "linux")("builder script", () => { const run = await runScript( renderBuilderScript({ root, - baseUrl: `http://127.0.0.1:${server.port}`, + baseUrl, assets: { claude: row(sha256(claude), ""), codex: row(sha256(codex), ".tar.gz"), @@ -232,6 +426,10 @@ describe.skipIf(process.platform !== "linux")("builder script", () => { }), ); expect(run.exitCode, run.stderr).toBe(0); + expect(parseBuilderResults(run.stdout)).toEqual([ + { name: "claude", outcome: "written", sha: sha256(claude) }, + { name: "codex", outcome: "written", sha: sha256(codex) }, + ]); expect( parseAgentToolsManifest( readFileSync(join(root, "manifest.json"), "utf8"), @@ -242,34 +440,147 @@ describe.skipIf(process.platform !== "linux")("builder script", () => { .stdout.toString() .trim(), ).toBe("codex"); - expect(existsSync(join(root, "claude", sha256(claude), "claude"))).toBe( - true, - ); expect(existsSync(join(root, ".manifest.json.partial"))).toBe(false); } finally { server.stop(true); } }); - test("a sha mismatch fails before the manifest is written", async () => { + test("an agent that fails with no earlier copy is left out; with none left there is no manifest", async () => { const root = mkdtempSync(join(tmpdir(), "agent-tools-")); - const server = Bun.serve({ - port: 0, - fetch: () => new Response("not the agent"), - }); + const { server, baseUrl } = serve(new Map()); try { const run = await runScript( renderBuilderScript({ root, - baseUrl: `http://127.0.0.1:${server.port}`, + baseUrl, assets: { claude: row(sha("a"), "") }, manifest: [{ name: "claude", sha: sha("a"), bin: "claude" }], }), ); expect(run.exitCode).not.toBe(0); + expect(parseBuilderResults(run.stdout)).toEqual([ + { name: "claude", outcome: "dropped" }, + ]); expect(existsSync(join(root, "manifest.json"))).toBe(false); } finally { server.stop(true); } }); + + test("on a fork, fetches only changed agents, falls back per agent and removes what the manifest drops", async () => { + const root = mkdtempSync(join(tmpdir(), "agent-tools-")); + const oldClaude = placeAgent(root, "claude", agentScript("echo old")); + const oldCodex = placeAgent(root, "codex", agentScript("echo old")); + const pdev = placeAgent(root, "pdev", agentScript("echo pdev")); + const gone = placeAgent(root, "gone", agentScript("echo gone")); + mkdirSync(join(root, "lost+found")); + mkdirSync(join(root, "pdev", `${pdev.sha}.partial`)); + const previous = [oldClaude, oldCodex, pdev, gone]; + writeFileSync(join(root, "manifest.json"), JSON.stringify(previous)); + + const newClaude = agentScript("echo new"); + const newCodex = sha("b"); + const fresh = sha("f"); + const { server, requests, baseUrl } = serve( + new Map([[sha256(newClaude), newClaude]]), + ); + const manifest: AgentToolsManifest = [ + { name: "claude", sha: sha256(newClaude), bin: "claude" }, + { name: "codex", sha: newCodex, bin: "codex" }, + pdev, + { name: "fresh", sha: fresh, bin: "fresh" }, + ]; + try { + const run = await runScript( + renderBuilderScript({ + root, + baseUrl, + assets: { + claude: row(sha256(newClaude), ""), + codex: row(newCodex, ""), + pdev: row(pdev.sha, ""), + fresh: row(fresh, ""), + }, + manifest, + previous, + }), + ); + expect(run.exitCode, run.stderr).toBe(0); + expect(parseBuilderResults(run.stdout)).toEqual([ + { name: "claude", outcome: "written", sha: sha256(newClaude) }, + { name: "codex", outcome: "kept-previous", sha: oldCodex.sha }, + { name: "pdev", outcome: "reused", sha: pdev.sha }, + { name: "fresh", outcome: "dropped" }, + ]); + expect(requests.sort()).toEqual( + [sha256(newClaude), newCodex, fresh].sort(), + ); + expect( + parseAgentToolsManifest( + readFileSync(join(root, "manifest.json"), "utf8"), + ), + ).toEqual([manifest[0], oldCodex, pdev] as AgentToolsManifest); + expect(existsSync(join(root, "claude", oldClaude.sha))).toBe(false); + expect(existsSync(join(root, "gone"))).toBe(false); + expect(existsSync(join(root, "fresh"))).toBe(false); + expect(existsSync(join(root, "pdev", `${pdev.sha}.partial`))).toBe(false); + expect(existsSync(join(root, "lost+found"))).toBe(true); + } finally { + server.stop(true); + } + }); + + test("an agent that fails its check is not named: the earlier one is kept, or it is left out", async () => { + const root = mkdtempSync(join(tmpdir(), "agent-tools-")); + const good = placeAgent( + root, + "claude", + agentScript('[ "$1" = selftest ] || exit 9'), + ); + const broken = agentScript("exit 3"); + const hangs = agentScript("sleep 30"); + const { server, baseUrl } = serve( + new Map([ + [sha256(broken), broken], + [sha256(hangs), hangs], + ]), + ); + try { + const run = await runScript( + renderBuilderScript({ + root, + baseUrl, + assets: { + claude: row(sha256(broken), ""), + codex: row(sha256(hangs), ""), + }, + manifest: [ + { name: "claude", sha: sha256(broken), bin: "claude" }, + { name: "codex", sha: sha256(hangs), bin: "codex" }, + ], + previous: [good], + tools: [ + { name: "claude", bin: "claude", check: ["selftest"] }, + { name: "codex", bin: "codex" }, + ], + checkTimeoutSeconds: 1, + }), + ); + expect(run.exitCode, run.stderr).toBe(0); + expect(parseBuilderResults(run.stdout)).toEqual([ + { name: "claude", outcome: "kept-previous", sha: good.sha }, + { name: "codex", outcome: "dropped" }, + ]); + expect(run.stderr).toContain("check failed"); + expect( + parseAgentToolsManifest( + readFileSync(join(root, "manifest.json"), "utf8"), + ), + ).toEqual([good]); + expect(existsSync(join(root, "codex"))).toBe(false); + } finally { + server.stop(true); + } + }); }); diff --git a/packages/sandbox/src/agent-tools.ts b/packages/sandbox/src/agent-tools.ts index a9967404976..1c422daa7c8 100644 --- a/packages/sandbox/src/agent-tools.ts +++ b/packages/sandbox/src/agent-tools.ts @@ -11,6 +11,11 @@ * stopped and the drive is detached. The release publishes before any * environment row points at the bundle, so no box mounts a drive mid-write. * + * A new drive starts as a fork of the region's newest earlier one, so the + * builder fetches only the agents whose sha changed. Each agent fails alone: + * it keeps the earlier drive's version, or is left out and boxes use the + * image's copy. + * * bun run agent-tools --bundle [--region sfo1 ...] * * Needs VERCEL_SANDBOX_TOKEN, VERCEL_SANDBOX_TEAM_ID, VERCEL_SANDBOX_PROJECT_ID. @@ -32,11 +37,21 @@ import { z } from "zod"; import { type Assets, assetObjectKey, loadAssets, SHA256 } from "./manifest"; /** Each agent's asset row, and its executable relative to the unpacked dir. */ -export const AGENT_TOOLS = [ +export const AGENT_TOOLS: readonly AgentTool[] = [ { name: "claude", bin: "claude" }, { name: "codex", bin: "bin/codex" }, { name: "pdev", bin: "pdev/pdev" }, -] as const; +]; + +export interface AgentTool { + name: string; + bin: string; + /** Arguments the builder runs the agent with before it names it; the default is `--version`. */ + check?: readonly string[]; +} + +const DEFAULT_CHECK = ["--version"]; +const CHECK_TIMEOUT_SECONDS = 30; export const AGENT_TOOLS_MANIFEST = "manifest.json"; @@ -51,9 +66,12 @@ const agentToolSchema = z.object({ export const agentToolsManifestSchema = z.array(agentToolSchema); export type AgentToolsManifest = z.infer; -export function agentToolsManifest(assets: Assets): AgentToolsManifest { +export function agentToolsManifest( + assets: Assets, + tools: readonly AgentTool[] = AGENT_TOOLS, +): AgentToolsManifest { return agentToolsManifestSchema.parse( - AGENT_TOOLS.map(({ name, bin }) => { + tools.map(({ name, bin }) => { const asset = assets[name]; if (!asset) throw new Error(`assets.json has no ${name} row`); return { name, sha: asset.sha256, bin }; @@ -75,69 +93,198 @@ export function parseAgentToolsManifest( const quote = (value: string) => `'${value.replaceAll("'", "'\\''")}'`; -/** The script the builder runs as root against the empty drive mounted read-write at `root`. */ +export type AgentOutcome = "written" | "reused" | "kept-previous" | "dropped"; + +export interface AgentResult { + name: string; + outcome: AgentOutcome; + /** The sha the new manifest names; absent when the agent was dropped. */ + sha?: string; +} + +const RESULT_LINE = + /^agent-tools-result ([a-z0-9-]+) (written|reused|kept-previous|dropped) ?([0-9a-f]{64})?$/gm; + +export function parseBuilderResults(stdout: string): AgentResult[] { + return [...stdout.matchAll(RESULT_LINE)].map(([, name, outcome, sha]) => ({ + name: name as string, + outcome: outcome as AgentOutcome, + ...(sha ? { sha } : {}), + })); +} + +/** + * The script the builder runs as root against the drive mounted read-write + * at `root`: empty, or a fork of an earlier drive whose manifest is `previous`. + */ export function renderBuilderScript(args: { root: string; baseUrl: string; assets: Assets; manifest: AgentToolsManifest; + previous?: AgentToolsManifest; + tools?: readonly AgentTool[]; + checkTimeoutSeconds?: number; }): string { - const fetches = args.manifest.map(({ name, sha, bin }) => { + const tools = args.tools ?? AGENT_TOOLS; + const agents = args.manifest.map(({ name, sha, bin }) => { const asset = args.assets[name]; if (!asset || asset.sha256 !== sha) { throw new Error(`no asset row for ${name} at ${sha}`); } const archive = asset.suffix === ".tar.gz" ? "tar" : "file"; - return `publish ${[name, sha, bin, assetObjectKey(asset), archive].map(quote).join(" ")}`; + const previous = args.previous?.find((entry) => entry.name === name); + const check = + tools.find((tool) => tool.name === name)?.check ?? DEFAULT_CHECK; + return `agent ${[ + name, + sha, + bin, + assetObjectKey(asset), + archive, + previous?.sha ?? "", + previous?.bin ?? "", + ...check, + ] + .map(quote) + .join(" ")}`; }); return `set -euo pipefail +shopt -s nullglob root=${quote(args.root)} base=${quote(args.baseUrl)} +declare -A keep_sha=() +entries=() + +rm -f "\${root}/${AGENT_TOOLS_MANIFEST}" "\${root}/.${AGENT_TOOLS_MANIFEST}.partial" +sync -publish() { +fetch() { local name="$1" sha="$2" bin="$3" key="$4" kind="$5" - local target="\${root}/\${name}/\${sha}" - local tmp - tmp="$(mktemp)" - curl -fsSL --retry 5 --retry-delay 2 --retry-all-errors -o "$tmp" "\${base}/\${key}" || - curl -fsSL --retry 2 --retry-delay 2 --retry-all-errors -o "$tmp" "\${base}/\${key}?fresh=$(date +%s)" - local got + local target="\${root}/\${name}/\${sha}" tmp got + tmp="$(mktemp)" || return 1 + if ! { curl -fsSL --retry 5 --retry-delay 2 --retry-all-errors -o "$tmp" "\${base}/\${key}" || + curl -fsSL --retry 2 --retry-delay 2 --retry-all-errors -o "$tmp" "\${base}/\${key}?fresh=$(date +%s)"; }; then + echo "\${name}: download failed" >&2; rm -f "$tmp"; return 1 + fi got="$(sha256sum "$tmp" | cut -d' ' -f1)" if [ "$got" != "$sha" ]; then echo "\${name}: expected \${sha}, got \${got}" >&2; rm -f "$tmp"; return 1; fi - rm -rf "\${target}.partial" - mkdir -p "\${target}.partial" + rm -rf "\${target}.partial" && mkdir -p "\${target}.partial" || { rm -f "$tmp"; return 1; } if [ "$kind" = tar ]; then - tar -xzf "$tmp" -C "\${target}.partial" + tar -xzf "$tmp" -C "\${target}.partial" || { echo "\${name}: unpack failed" >&2; rm -f "$tmp"; rm -rf "\${target}.partial"; return 1; } else - install -D -m 0755 "$tmp" "\${target}.partial/\${bin}" + install -D -m 0755 "$tmp" "\${target}.partial/\${bin}" || { rm -f "$tmp"; rm -rf "\${target}.partial"; return 1; } fi rm -f "$tmp" - [ -x "\${target}.partial/\${bin}" ] || { echo "\${name}: no executable at \${bin}" >&2; return 1; } - rm -rf "$target" - mv -T "\${target}.partial" "$target" - echo "\${name} \${sha:0:12} written" + [ -x "\${target}.partial/\${bin}" ] || { echo "\${name}: no executable at \${bin}" >&2; rm -rf "\${target}.partial"; return 1; } + rm -rf "$target" && mv -T "\${target}.partial" "$target" } -${fetches.join("\n")} -printf '%s\\n' ${quote(JSON.stringify(args.manifest, null, "\t"))} > "\${root}/.${AGENT_TOOLS_MANIFEST}.partial" +smoke() { + local dir="$1" bin="$2" home + shift 2 + home="$(mktemp -d)" || return 1 + if (cd "$home" && HOME="$home" timeout -k 5 ${args.checkTimeoutSeconds ?? CHECK_TIMEOUT_SECONDS} "\${dir}/\${bin}" "$@" > "\${home}/.out" 2>&1 < /dev/null); then + rm -rf "$home"; return 0 + fi + echo "\${dir}/\${bin} $*: check failed: $(tail -c 300 "\${home}/.out")" >&2 + rm -rf "$home"; return 1 +} + +keep() { + keep_sha[$1]="$2" + entries+=("$(printf '{"name":"%s","sha":"%s","bin":"%s"}' "$1" "$2" "$3")") +} + +agent() { + local name="$1" sha="$2" bin="$3" key="$4" kind="$5" prev_sha="$6" prev_bin="$7" outcome=failed + shift 7 + if [ -x "\${root}/\${name}/\${sha}/\${bin}" ]; then outcome=reused + elif fetch "$name" "$sha" "$bin" "$key" "$kind"; then outcome=written + fi + if [ "$outcome" != failed ] && smoke "\${root}/\${name}/\${sha}" "$bin" "$@"; then + keep "$name" "$sha" "$bin" + elif [ -n "$prev_sha" ] && [ "$prev_sha" != "$sha" ] && [ -x "\${root}/\${name}/\${prev_sha}/\${prev_bin}" ] && + smoke "\${root}/\${name}/\${prev_sha}" "$prev_bin" "$@"; then + outcome=kept-previous sha="$prev_sha" + keep "$name" "$prev_sha" "$prev_bin" + else + outcome=dropped sha= + fi + echo "agent-tools-result \${name} \${outcome} \${sha}" +} + +${agents.join("\n")} + +for dir in "\${root}"/*/; do + dir="\${dir%/}" name="\${dir##*/}" + [[ "$name" =~ ^[a-z0-9-]+$ ]] || continue + if [ -z "\${keep_sha[$name]:-}" ]; then rm -rf "$dir"; continue; fi + for entry in "\${dir}"/* "\${dir}"/.[!.]*; do + [ "\${entry##*/}" = "\${keep_sha[$name]}" ] || rm -rf "$entry" + done +done + +[ "\${#entries[@]}" -gt 0 ] || { echo "no agent could be published" >&2; exit 1; } +printf '[%s]\\n' "$(IFS=,; echo "\${entries[*]}")" > "\${root}/.${AGENT_TOOLS_MANIFEST}.partial" sync mv -T "\${root}/.${AGENT_TOOLS_MANIFEST}.partial" "\${root}/${AGENT_TOOLS_MANIFEST}" sync `; } +export interface DriveEntry { + name: string; + createdAt: number; + attachedTo?: string; +} + /** What the publish needs from the provider; `vercelDriveHost` is the real one. */ export interface DriveHost { /** Null when no drive has the name. */ find(name: string): Promise<{ attachedTo?: string } | null>; - list(prefix: string): Promise; + list(prefix: string): Promise; create(name: string, region: SandboxRegionId): Promise; + /** Copy-on-write. The parent must have been written, stopped and detached. */ + fork(parent: string, name: string): Promise; /** Throws while the drive is attached to a sandbox. */ remove(name: string): Promise; /** Through a read-only snapshot mount; null when the drive has no manifest. */ readManifest(name: string, region: SandboxRegionId): Promise; - /** Resolves only once the builder has stopped and the drive is detached. */ - write(name: string, region: SandboxRegionId, script: string): Promise; + /** + * Renders the script from the manifest the mounted drive holds, runs it, + * and returns its stdout. Resolves only once the builder has stopped and + * the drive is detached. + */ + write( + name: string, + region: SandboxRegionId, + script: (manifest: string | null) => string, + ): Promise; +} + +export interface AgentToolsPublish { + drive: string; + outcome: "exists" | "published"; + forkedFrom?: string; + agents: AgentResult[]; +} + +/** The region's newest other agent-tools drive that no sandbox holds read-write. */ +export function previousAgentToolsDrive( + drives: readonly DriveEntry[], + region: SandboxRegionId, + drive: string, +): string | undefined { + return drives + .filter( + (entry) => + entry.name !== drive && + entry.name.startsWith(`agent-tools-${region}-`) && + SANDBOX_AGENT_TOOLS_DRIVE_PATTERN.test(entry.name) && + !entry.attachedTo, + ) + .sort((a, b) => b.createdAt - a.createdAt)[0]?.name; } /** @@ -151,9 +298,12 @@ export async function publishAgentToolsDrive(args: { bundleSha: string; assets: Assets; baseUrl?: string; -}): Promise<{ drive: string; outcome: "exists" | "published" }> { + tools?: readonly AgentTool[]; + warn?: (line: string) => void; +}): Promise { + const warn = args.warn ?? ((line) => console.log(`::warning::${line}`)); const drive = sandboxAgentToolsDrive(args.region, args.bundleSha); - const manifest = agentToolsManifest(args.assets); + const manifest = agentToolsManifest(args.assets, args.tools); const existing = await args.host.find(drive); if (existing?.attachedTo) { throw new Error( @@ -164,21 +314,109 @@ export async function publishAgentToolsDrive(args: { const found = parseAgentToolsManifest( await args.host.readManifest(drive, args.region), ); - if (found.length > 0) return { drive, outcome: "exists" }; + if (found.length > 0) return { drive, outcome: "exists", agents: [] }; await args.host.remove(drive); } - await args.host.create(drive, args.region); - await args.host.write( - drive, - args.region, + let forkedFrom: string | undefined; + let parent: string | undefined; + try { + parent = previousAgentToolsDrive( + await args.host.list(`agent-tools-${args.region}-`), + args.region, + drive, + ); + if (parent) { + await args.host.fork(parent, drive); + forkedFrom = parent; + } + } catch (error) { + warn( + `agent-tools: ${drive} not forked from ${parent ?? "an earlier drive"}; writing every agent: ${String(error).slice(0, 300)}`, + ); + if (await args.host.find(drive)) await args.host.remove(drive); + } + if (!forkedFrom) await args.host.create(drive, args.region); + const stdout = await args.host.write(drive, args.region, (current) => renderBuilderScript({ root: SANDBOX_PATHS.agents, baseUrl: args.baseUrl ?? SANDBOX_ASSET_BASE_URL, assets: args.assets, manifest, + previous: forkedFrom ? parseAgentToolsManifest(current) : [], + tools: args.tools, }), ); - return { drive, outcome: "published" }; + return { + drive, + outcome: "published", + ...(forkedFrom ? { forkedFrom } : {}), + agents: parseBuilderResults(stdout), + }; +} + +export type RegionPublish = + | { region: SandboxRegionId; result: AgentToolsPublish } + | { region: SandboxRegionId; error: unknown }; + +/** Publishes `concurrency` regions at a time; a region that fails does not stop the others. */ +export async function publishAgentToolsDrives( + args: Omit[0], "region"> & { + regions: readonly SandboxRegionId[]; + concurrency?: number; + }, +): Promise { + const { regions, concurrency = 6, ...rest } = args; + const results: RegionPublish[] = []; + let next = 0; + const worker = async () => { + while (next < regions.length) { + const index = next++; + const region = regions[index] as SandboxRegionId; + try { + results[index] = { + region, + result: await publishAgentToolsDrive({ ...rest, region }), + }; + } catch (error) { + results[index] = { region, error }; + } + } + }; + await Promise.all( + Array.from({ length: Math.min(concurrency, regions.length) }, worker), + ); + return results; +} + +/** One line per region and per agent that did not get the bundle's version; warnings start `::warning::`. */ +export function describeAgentToolsPublish( + results: readonly RegionPublish[], +): string[] { + return results.flatMap((entry) => { + if ("error" in entry) { + return [ + `::warning::agent-tools: ${entry.region} not published; its boxes keep the image's agents: ${String(entry.error).slice(0, 300)}`, + ]; + } + const { drive, outcome, forkedFrom, agents } = entry.result; + const summary = agents + .map((agent) => `${agent.name} ${agent.outcome}`) + .join(", "); + return [ + `agent-tools: ${drive} ${outcome}${forkedFrom ? ` from ${forkedFrom}` : ""}${summary ? `: ${summary}` : ""}`, + ...agents.flatMap((agent) => + agent.outcome === "kept-previous" + ? [ + `::warning::agent-tools: ${drive}: ${agent.name} kept ${agent.sha?.slice(0, 12)} from the earlier drive; the bundle's version failed`, + ] + : agent.outcome === "dropped" + ? [ + `::warning::agent-tools: ${drive}: ${agent.name} left out; its boxes keep the image's ${agent.name}`, + ] + : [], + ), + ]; + }); } /** Agent-tools drives no environment row's bundle and region name any more. */ @@ -200,7 +438,7 @@ export async function pruneAgentToolsDrives(args: { const deleted: string[] = []; const kept: string[] = []; for (const name of agentToolsDrivesToPrune( - await args.host.list("agent-tools-"), + (await args.host.list("agent-tools-")).map((drive) => drive.name), args.referenced, )) { try { @@ -265,20 +503,29 @@ export function vercelDriveHost(credentials: VercelCredentials): DriveHost { }, async list(prefix) { const { Drive } = await sdk(); - const names: string[] = []; + const drives: DriveEntry[] = []; for await (const drive of await Drive.list({ ...credentials, namePrefix: prefix, sortBy: "name", })) { - names.push(drive.name); + drives.push({ + name: drive.name, + createdAt: drive.createdAt.getTime(), + attachedTo: drive.currentSandboxName, + }); } - return names; + return drives; }, async create(name, region) { const { Drive } = await sdk(); await Drive.getOrCreate({ ...credentials, name, region }); }, + async fork(parent, name) { + const drive = await getDrive(parent); + if (!drive) throw new Error(`${parent} does not exist`); + await drive.fork({ name }); + }, async remove(name) { await (await getDrive(name))?.delete(); }, @@ -320,15 +567,18 @@ export function vercelDriveHost(credentials: VercelCredentials): DriveHost { mounts: { [SANDBOX_PATHS.agents]: drive }, }); let failure: Error | null = null; + let stdout = ""; try { + const current = await builder.readFileToBuffer({ + path: `${SANDBOX_PATHS.agents}/${AGENT_TOOLS_MANIFEST}`, + }); const result = await builder.runCommand({ cmd: "bash", - args: ["-c", script], + args: ["-c", script(current?.toString("utf8") ?? null)], sudo: true, }); - console.log( - `${name}: ${await result.stdout()}${await result.stderr()}`, - ); + stdout = await result.stdout(); + console.log(`${name}: ${stdout}${await result.stderr()}`); if (result.exitCode !== 0) { failure = new Error(`${name}: builder exited ${result.exitCode}`); } @@ -336,6 +586,7 @@ export function vercelDriveHost(credentials: VercelCredentials): DriveHost { await stopAndDelete(builder, name); } if (failure) throw failure; + return stdout; }, }; } @@ -373,16 +624,14 @@ if (import.meta.main) { const assets = loadAssets( join(import.meta.dir, "..", "bundle", "assets.json"), ); - const host = vercelDriveHost(credentials); - for (const region of regions.length - ? (regions as SandboxRegionId[]) - : [DEFAULT_SANDBOX_REGION]) { - const { drive, outcome } = await publishAgentToolsDrive({ - host, - region, - bundleSha, - assets, - }); - console.log(`${drive}: ${outcome}`); - } + const results = await publishAgentToolsDrives({ + host: vercelDriveHost(credentials), + regions: regions.length + ? (regions as SandboxRegionId[]) + : [DEFAULT_SANDBOX_REGION], + bundleSha, + assets, + }); + for (const line of describeAgentToolsPublish(results)) console.log(line); + if (results.some((entry) => "error" in entry)) process.exit(1); } diff --git a/packages/sandbox/src/environments/release.ts b/packages/sandbox/src/environments/release.ts index 2d049be17aa..72836cb16ee 100644 --- a/packages/sandbox/src/environments/release.ts +++ b/packages/sandbox/src/environments/release.ts @@ -169,20 +169,16 @@ async function environmentRegions(): Promise { ), ]; } -for (const region of await environmentRegions()) { - try { - const { drive, outcome } = await agentTools.publishAgentToolsDrive({ - host: driveHost, - region, - bundleSha: bundle.sha256, - assets: bundle.assets, - }); - log(`agent-tools: ${drive} ${outcome}`); - } catch (error) { - console.log( - `::warning::agent-tools: ${region} not published; its boxes keep the image's agents: ${String(error).slice(0, 300)}`, - ); - } +for (const line of agentTools.describeAgentToolsPublish( + await agentTools.publishAgentToolsDrives({ + host: driveHost, + regions: await environmentRegions(), + bundleSha: bundle.sha256, + assets: bundle.assets, + }), +)) { + if (line.startsWith("::warning::")) console.log(line); + else log(line); } // 2. image