diff --git a/docs/cloud-sandbox-mismatches.md b/docs/cloud-sandbox-mismatches.md index a9ab604c59e..cf9fdb29488 100644 --- a/docs/cloud-sandbox-mismatches.md +++ b/docs/cloud-sandbox-mismatches.md @@ -903,6 +903,26 @@ and provider. Keep the workspace id on both sides: a provisioning failure is recorded against the API and a runtime failure against the sandbox, and that id is the only thing that joins the two halves of one broken workspace. +**A drive mount lasts one session and a fork does not inherit it — open.** +For organizations with the `sandbox-agent-tools-drive` PostHog flag, the agent +CLIs come from the bundle's `agent-tools--` drive, +mounted read-only at `/opt/agents`. A machine someone owns keeps a mount; a +sandbox gets one only from `Sandbox.create`, `Sandbox.fork` or +`sandbox.update({ mounts })` before the session starts, so `provisionSandbox` +and `wakeSandbox` both pass it. A drive lives in one region and mounts only in +that region. A session without the mount leaves links in `/usr/local/bin` +that point nowhere; `superset-boot` re-runs `install-agent-clis` to put the +image's agents back. + +**A read-only snapshot mount taken while a writer holds the drive is corrupt.** +Measured 2026-11-17: a box that mounted the snapshot while the builder still +had the drive read-write got `ro,norecovery`, `ls` answered "Bad message" and +codex segfaulted. On a disk someone owns, a second reader of a live +filesystem is just slow. Here a drive is never written after it is +published: one drive per bundle, written by a builder that is stopped before +any environment row names the bundle, and the control plane mounts only a +drive that exists and is detached. + ## Running this repo's own dev stack inside a sandbox Reproducing an app bug end to end from a cloud workspace means bringing up diff --git a/packages/sandbox/README.md b/packages/sandbox/README.md index 160c85a4db6..3b7c39dc9d1 100644 --- a/packages/sandbox/README.md +++ b/packages/sandbox/README.md @@ -42,6 +42,7 @@ The rule of thumb: if you can answer "where is it on the box", put it there. | `bun run image` | Builds the image: OS, apt lists, vendor tools, the `ubuntu` user, then installs the bundle at its sha and runs every pass once, so a fresh box finds every hash sidecar and step marker matching. A push logs Docker into the registry itself with a credential minted from `VERCEL_SANDBOX_TOKEN`, so there is no login to renew. `--local` builds `superset-sandbox:local` for the boot-twice test; `--dry` prints the Dockerfile. | | boot (`superset-boot`) | On the box, as root, run by the control plane on every create and wake, after it has written `sandbox.conf`, with the host secret in its env: clears `/run/superset`, installs the pinned bundle if the hash differs, runs the three passes (each a hash compare), starts host-service as `ubuntu` with the secret from the boot env, writes `resources.env`, starts `superset-memguard` (ranks every process for the OOM killer and reaps the largest before the kernel has to), then the desktop and the checkout. Once host-service answers, the environment has been pushed and the checkout is in, the runner asks host-service to run the repo's `start` hook with that environment. | | `bun run release` | Rewrites the host-service and CLI assets from this checkout and the claude, codex and Chrome rows to their latest upstream release, publishes the bundle, pushes the image under a candidate tag `release-` (unless `--skip-image`), builds the internal golden from it with the environment's `setup` hook, probes a fork of it (`src/environments/probe.ts`) including a stop and wake, and only after the probe passes moves `latest` to the candidate and writes the environment rows. Every image environment resolves `superset-hostsvc` to `latest`, so a failed release changes no box. | +| `bun run agent-tools --bundle ` | Writes that bundle's agent CLIs to `agent-tools--`, a drive boxes mount read-only at `/opt/agents` when their organization has the `sandbox-agent-tools-drive` flag. A drive is written once and never again. `bun run release` does this for every region with an environment before it records the bundle, and afterwards deletes the drives no environment names. | | `bun run src/real-sandbox.ts` | The Sandbox workflow dispatched with `real-sandbox`: one real sandbox from the registry image on this checkout's bundle, the same probe, a stop and a wake. | | Sandbox workflow, `production-release` | `bun run release --production` from `main` on a CI runner: the image, the internal golden and production's environment rows. Runs on a merge that touches the bundle and every night; a failed merge or nightly run opens an issue, or comments on the open one. | diff --git a/packages/sandbox/bundle/rootfs/usr/local/bin/superset-boot b/packages/sandbox/bundle/rootfs/usr/local/bin/superset-boot index a85fc465b59..66dba948078 100755 --- a/packages/sandbox/bundle/rootfs/usr/local/bin/superset-boot +++ b/packages/sandbox/bundle/rootfs/usr/local/bin/superset-boot @@ -9,6 +9,7 @@ # install the pinned bundle only when its hash differs from current # apply-rootfs, sync-assets, each a hash compare per file, asset, step # run-steps +# link the drive's agents only when the control plane mounted one # start host-service as ubuntu first; everything else is a side effect # start the desktop and the checkout, detached # @@ -117,6 +118,33 @@ fi stamp "assets.start"; "$BUNDLE/setup" sync-assets >> "$SUPERSET_BOOT_LOG" 2>&1 || stamp "assets.failed continuing"; stamp "assets.end" stamp "steps.start"; "$BUNDLE/setup" run-steps >> "$SUPERSET_BOOT_LOG" 2>&1; stamp "steps.end" +# --- the agent CLIs from the region's agent-tools drive, when the control +# plane mounted one. Without it the image's agents stay on PATH. Mounts are +# per session, so a link into a mount this session lacks points nowhere. +link_agents() { + local dir="${SUPERSET_AGENTS_DIR:-/opt/agents}" linked=0 name sha bin target + local manifest="${dir}/manifest.json" + if find /usr/local/bin -maxdepth 1 -lname "${dir}/*" ! -exec test -e {} \; -print | grep -q .; then + SUPERSET_BUNDLE_DIR="$BUNDLE" bash "$BUNDLE/steps/install-agent-clis.sh" >> "$SUPERSET_BOOT_LOG" 2>&1 || + stamp "agents.restore.failed" + fi + if ! jq -e 'type == "array"' "$manifest" > /dev/null 2>&1; then + stamp "agents.unavailable no manifest at ${dir}" + return 0 + fi + while IFS=$'\t' read -r name sha bin; do + if ! [[ "$name" =~ ^[a-z0-9-]+$ && "$sha" =~ ^[0-9a-f]{64}$ && "$bin" =~ ^[A-Za-z0-9._/-]+$ && "$bin" != *..* ]]; then + stamp "agents.skipped invalid entry ${name:0:32}"; continue + fi + target="${dir}/${name}/${sha}/${bin}" + [ -x "$target" ] || { stamp "agents.missing ${name} ${sha:0:12}"; continue; } + ln -sfn "$target" "/usr/local/bin/${name}.tmp" && mv -T "/usr/local/bin/${name}.tmp" "/usr/local/bin/${name}" && + linked=$((linked + 1)) + done < <(jq -r '.[] | [.name, .sha, .bin] | map(tostring) | @tsv' "$manifest") + stamp "agents.linked ${linked} from ${dir}" +} +link_agents + # --- host-service, as the sandbox user, first. A runtime that cannot start # is logged, not fatal: the desktop and the checkout still come up so the # box can be looked at, and the control plane marks the workspace failed. diff --git a/packages/sandbox/package.json b/packages/sandbox/package.json index ad490f36232..cc0208accc2 100644 --- a/packages/sandbox/package.json +++ b/packages/sandbox/package.json @@ -9,6 +9,7 @@ "pin": "bun run src/pin.ts", "assets": "bun run src/assets/produce.ts", "release": "bun run src/environments/release.ts", + "agent-tools": "bun run src/agent-tools.ts", "record-bundle": "bun run src/environments/record-bundle.ts", "test": "bun test", "typecheck": "tsc --noEmit --emitDeclarationOnly false" diff --git a/packages/sandbox/src/agent-tools.test.ts b/packages/sandbox/src/agent-tools.test.ts new file mode 100644 index 00000000000..30f29379de7 --- /dev/null +++ b/packages/sandbox/src/agent-tools.test.ts @@ -0,0 +1,275 @@ +import { describe, expect, test } from "bun:test"; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + type AgentToolsManifest, + agentToolsDrivesToPrune, + agentToolsManifest, + type DriveHost, + parseAgentToolsManifest, + pruneAgentToolsDrives, + publishAgentToolsDrive, + renderBuilderScript, +} from "./agent-tools"; +import { type Assets, sha256 } from "./manifest"; + +const sha = (c: string) => c.repeat(64); +const BUNDLE = "0123456789abcdef".repeat(4); + +function row(sha256: string, suffix: string): Assets[string] { + return { sha256, suffix, dest: "/usr/local/share/x", mode: "0644" }; +} + +const agentAssets: Assets = { + claude: row(sha("a"), ""), + codex: row(sha("b"), ".tar.gz"), + pdev: row(sha("c"), ".tar.gz"), + chrome: row(sha("d"), ".deb"), +}; + +describe("agent-tools manifest", () => { + test("lists every agent at its asset sha, bin relative to its dir", () => { + const manifest = agentToolsManifest(agentAssets); + expect(manifest).toEqual([ + { name: "claude", sha: sha("a"), bin: "claude" }, + { name: "codex", sha: sha("b"), bin: "bin/codex" }, + { name: "pdev", sha: sha("c"), bin: "pdev/pdev" }, + ]); + expect(parseAgentToolsManifest(JSON.stringify(manifest))).toEqual(manifest); + }); + + test("a missing or bad manifest reads as empty", () => { + expect(parseAgentToolsManifest(null)).toEqual([]); + expect(parseAgentToolsManifest("{not json")).toEqual([]); + expect( + parseAgentToolsManifest( + JSON.stringify([{ name: "claude", sha: sha("a"), bin: "../x" }]), + ), + ).toEqual([]); + }); +}); + +function fakeHost( + drives: Record, +) { + const calls: string[] = []; + const host: DriveHost = { + find: async (name) => { + const drive = drives[name]; + return drive ? { attachedTo: drive.attachedTo } : null; + }, + list: async (prefix) => + Object.keys(drives).filter((name) => name.startsWith(prefix)), + create: async (name) => { + calls.push(`create ${name}`); + drives[name] = {}; + }, + remove: async (name) => { + calls.push(`remove ${name}`); + if (drives[name]?.attachedTo) throw new Error("attached"); + delete drives[name]; + }, + readManifest: async (name) => drives[name]?.manifest ?? null, + write: async (name) => { + calls.push(`write ${name}`); + const drive = drives[name]; + if (drive) + drive.manifest = JSON.stringify(agentToolsManifest(agentAssets)); + }, + }; + return { host, calls, drives }; +} + +describe("publishAgentToolsDrive", () => { + const drive = "agent-tools-sfo1-0123456789ab"; + + test("creates and writes the bundle's drive in the region", async () => { + const { host, calls } = fakeHost({}); + expect( + await publishAgentToolsDrive({ + host, + region: "sfo1", + bundleSha: BUNDLE, + assets: agentAssets, + }), + ).toEqual({ drive, outcome: "published" }); + expect(calls).toEqual([`create ${drive}`, `write ${drive}`]); + }); + + test("leaves a drive that has a manifest alone", async () => { + const { host, calls } = fakeHost({ + [drive]: { manifest: JSON.stringify(agentToolsManifest(agentAssets)) }, + }); + expect( + ( + await publishAgentToolsDrive({ + host, + region: "sfo1", + bundleSha: BUNDLE, + assets: agentAssets, + }) + ).outcome, + ).toBe("exists"); + expect(calls).toEqual([]); + }); + + test("recreates a drive an earlier run left without a manifest", async () => { + const { host, calls } = fakeHost({ [drive]: {} }); + await publishAgentToolsDrive({ + host, + region: "sfo1", + bundleSha: BUNDLE, + assets: agentAssets, + }); + expect(calls).toEqual([ + `remove ${drive}`, + `create ${drive}`, + `write ${drive}`, + ]); + }); + + test("refuses a drive a writer still holds", async () => { + const { host, calls } = fakeHost({ [drive]: { attachedTo: "builder" } }); + await expect( + publishAgentToolsDrive({ + host, + region: "sfo1", + bundleSha: BUNDLE, + assets: agentAssets, + }), + ).rejects.toThrow(/attached to builder/); + expect(calls).toEqual([]); + }); +}); + +describe("pruning", () => { + test("deletes only agent-tools drives no environment references", () => { + expect( + agentToolsDrivesToPrune( + [ + "agent-tools-sfo1-0123456789ab", + "agent-tools-sfo1-aaaaaaaaaaaa", + "agent-tools-iad1-0123456789ab", + "agent-tools-sfo1", + "agent-tools-builder-sfo1-xyz", + ], + new Set(["agent-tools-sfo1-0123456789ab"]), + ), + ).toEqual([ + "agent-tools-sfo1-aaaaaaaaaaaa", + "agent-tools-iad1-0123456789ab", + ]); + }); + + test("keeps a drive a sandbox holds for the next release", async () => { + const { host } = fakeHost({ + "agent-tools-sfo1-aaaaaaaaaaaa": {}, + "agent-tools-sfo1-bbbbbbbbbbbb": { attachedTo: "box" }, + }); + expect( + await pruneAgentToolsDrives({ host, referenced: new Set() }), + ).toEqual({ + deleted: ["agent-tools-sfo1-aaaaaaaaaaaa"], + kept: ["agent-tools-sfo1-bbbbbbbbbbbb"], + }); + }); +}); + +async function runScript(script: string) { + const proc = Bun.spawn(["bash", "-c", script], { + stdout: "pipe", + stderr: "pipe", + }); + const stderr = await new Response(proc.stderr).text(); + return { exitCode: await proc.exited, stderr }; +} + +describe.skipIf(process.platform !== "linux")("builder script", () => { + test("writes verified agents, then the manifest last", async () => { + const work = mkdtempSync(join(tmpdir(), "agent-tools-")); + const src = join(work, "src"); + mkdirSync(join(src, "bin"), { recursive: true }); + writeFileSync(join(src, "bin", "codex"), "#!/bin/sh\necho codex\n", { + mode: 0o755, + }); + Bun.spawnSync(["tar", "-czf", join(work, "codex.tar.gz"), "-C", src, "."]); + const claude = new TextEncoder().encode("#!/bin/sh\necho claude\n"); + const codex = readFileSync(join(work, "codex.tar.gz")); + const objects = new Map([ + [sha256(claude), claude], + [`${sha256(codex)}.tar.gz`, codex], + ]); + const server = Bun.serve({ + port: 0, + fetch: (request) => { + const body = objects.get(new URL(request.url).pathname.slice(1)); + return body ? new Response(body) : new Response("", { status: 404 }); + }, + }); + const root = join(work, "drive"); + mkdirSync(root); + const manifest: AgentToolsManifest = [ + { name: "claude", sha: sha256(claude), bin: "claude" }, + { name: "codex", sha: sha256(codex), bin: "bin/codex" }, + ]; + try { + const run = await runScript( + renderBuilderScript({ + root, + baseUrl: `http://127.0.0.1:${server.port}`, + assets: { + claude: row(sha256(claude), ""), + codex: row(sha256(codex), ".tar.gz"), + }, + manifest, + }), + ); + expect(run.exitCode, run.stderr).toBe(0); + expect( + parseAgentToolsManifest( + readFileSync(join(root, "manifest.json"), "utf8"), + ), + ).toEqual(manifest); + expect( + Bun.spawnSync([join(root, "codex", sha256(codex), "bin", "codex")]) + .stdout.toString() + .trim(), + ).toBe("codex"); + expect(existsSync(join(root, "claude", sha256(claude), "claude"))).toBe( + true, + ); + expect(existsSync(join(root, ".manifest.json.partial"))).toBe(false); + } finally { + server.stop(true); + } + }); + + test("a sha mismatch fails before the manifest is written", async () => { + const root = mkdtempSync(join(tmpdir(), "agent-tools-")); + const server = Bun.serve({ + port: 0, + fetch: () => new Response("not the agent"), + }); + try { + const run = await runScript( + renderBuilderScript({ + root, + baseUrl: `http://127.0.0.1:${server.port}`, + assets: { claude: row(sha("a"), "") }, + manifest: [{ name: "claude", sha: sha("a"), bin: "claude" }], + }), + ); + expect(run.exitCode).not.toBe(0); + expect(existsSync(join(root, "manifest.json"))).toBe(false); + } finally { + server.stop(true); + } + }); +}); diff --git a/packages/sandbox/src/agent-tools.ts b/packages/sandbox/src/agent-tools.ts new file mode 100644 index 00000000000..31413c88bb8 --- /dev/null +++ b/packages/sandbox/src/agent-tools.ts @@ -0,0 +1,387 @@ +/** + * Publishes the agent CLIs of one bundle to a drive per region, + * `agent-tools--`, which boxes mount read-only at + * `SANDBOX_PATHS.agents` instead of carrying the agents on their own disks. + * + * A drive is written once and never again: a box that takes its snapshot + * mount while a writer still holds the drive read-write gets a corrupt + * filesystem. So a builder sandbox writes the drive (each agent fetched by + * sha from the asset bucket and verified, `/manifest.json` last), is + * stopped, and the drive counts as published only once the builder has + * stopped and the drive is detached. The release publishes before any + * environment row points at the bundle, so no box mounts a drive mid-write. + * + * bun run agent-tools --bundle [--region sfo1 ...] + * + * Needs VERCEL_SANDBOX_TOKEN, VERCEL_SANDBOX_TEAM_ID, VERCEL_SANDBOX_PROJECT_ID. + * The agent rows come from bundle/assets.json, so they must be that bundle's. + */ +import { join } from "node:path"; +import { + SANDBOX_AGENT_TOOLS_DRIVE_PATTERN, + SANDBOX_ASSET_BASE_URL, + SANDBOX_PATHS, + sandboxAgentToolsDrive, +} from "@superset/shared/sandbox/contract"; +import { + DEFAULT_SANDBOX_REGION, + isSandboxRegionId, + type SandboxRegionId, +} from "@superset/shared/sandbox/regions"; +import { z } from "zod"; +import { type Assets, assetObjectKey, loadAssets, SHA256 } from "./manifest"; + +/** Each agent's asset row, and its executable relative to the unpacked dir. */ +export const AGENT_TOOLS = [ + { name: "claude", bin: "claude" }, + { name: "codex", bin: "bin/codex" }, + { name: "pdev", bin: "pdev/pdev" }, +] as const; + +export const AGENT_TOOLS_MANIFEST = "manifest.json"; + +const agentToolSchema = z.object({ + name: z.string().regex(/^[a-z0-9-]+$/), + sha: z.string().regex(SHA256), + bin: z + .string() + .regex(/^[A-Za-z0-9._-]+(\/[A-Za-z0-9._-]+)*$/) + .refine((bin) => !bin.split("/").includes(".."), "no .. in bin"), +}); +export const agentToolsManifestSchema = z.array(agentToolSchema); +export type AgentToolsManifest = z.infer; + +export function agentToolsManifest(assets: Assets): AgentToolsManifest { + return agentToolsManifestSchema.parse( + AGENT_TOOLS.map(({ name, bin }) => { + const asset = assets[name]; + if (!asset) throw new Error(`assets.json has no ${name} row`); + return { name, sha: asset.sha256, bin }; + }), + ); +} + +/** A drive that was never finished, or holds a bad manifest, has none. */ +export function parseAgentToolsManifest( + text: string | null, +): AgentToolsManifest { + if (text === null) return []; + try { + return agentToolsManifestSchema.parse(JSON.parse(text)); + } catch { + return []; + } +} + +const quote = (value: string) => `'${value.replaceAll("'", "'\\''")}'`; + +/** The script the builder runs as root against the empty drive mounted read-write at `root`. */ +export function renderBuilderScript(args: { + root: string; + baseUrl: string; + assets: Assets; + manifest: AgentToolsManifest; +}): string { + const fetches = args.manifest.map(({ name, sha, bin }) => { + const asset = args.assets[name]; + if (!asset || asset.sha256 !== sha) { + throw new Error(`no asset row for ${name} at ${sha}`); + } + const archive = asset.suffix === ".tar.gz" ? "tar" : "file"; + return `publish ${[name, sha, bin, assetObjectKey(asset), archive].map(quote).join(" ")}`; + }); + return `set -euo pipefail +root=${quote(args.root)} +base=${quote(args.baseUrl)} + +publish() { + local name="$1" sha="$2" bin="$3" key="$4" kind="$5" + local target="\${root}/\${name}/\${sha}" + local tmp + tmp="$(mktemp)" + curl -fsSL --retry 5 --retry-delay 2 --retry-all-errors -o "$tmp" "\${base}/\${key}" || + curl -fsSL --retry 2 --retry-delay 2 --retry-all-errors -o "$tmp" "\${base}/\${key}?fresh=$(date +%s)" + local got + got="$(sha256sum "$tmp" | cut -d' ' -f1)" + if [ "$got" != "$sha" ]; then echo "\${name}: expected \${sha}, got \${got}" >&2; rm -f "$tmp"; return 1; fi + rm -rf "\${target}.partial" + mkdir -p "\${target}.partial" + if [ "$kind" = tar ]; then + tar -xzf "$tmp" -C "\${target}.partial" + else + install -D -m 0755 "$tmp" "\${target}.partial/\${bin}" + fi + rm -f "$tmp" + [ -x "\${target}.partial/\${bin}" ] || { echo "\${name}: no executable at \${bin}" >&2; return 1; } + rm -rf "$target" + mv -T "\${target}.partial" "$target" + echo "\${name} \${sha:0:12} written" +} + +${fetches.join("\n")} +printf '%s\\n' ${quote(JSON.stringify(args.manifest, null, "\t"))} > "\${root}/.${AGENT_TOOLS_MANIFEST}.partial" +sync +mv -T "\${root}/.${AGENT_TOOLS_MANIFEST}.partial" "\${root}/${AGENT_TOOLS_MANIFEST}" +sync +`; +} + +/** What the publish needs from the provider; `vercelDriveHost` is the real one. */ +export interface DriveHost { + /** Null when no drive has the name. */ + find(name: string): Promise<{ attachedTo?: string } | null>; + list(prefix: string): Promise; + create(name: string, region: SandboxRegionId): Promise; + /** Throws while the drive is attached to a sandbox. */ + remove(name: string): Promise; + /** Through a read-only snapshot mount; null when the drive has no manifest. */ + readManifest(name: string, region: SandboxRegionId): Promise; + /** Resolves only once the builder has stopped and the drive is detached. */ + write(name: string, region: SandboxRegionId, script: string): Promise; +} + +/** + * Idempotent on the bundle: a drive that has a manifest is published and is + * left alone; one without (an earlier run that failed) is deleted and + * written again. + */ +export async function publishAgentToolsDrive(args: { + host: DriveHost; + region: SandboxRegionId; + bundleSha: string; + assets: Assets; + baseUrl?: string; +}): Promise<{ drive: string; outcome: "exists" | "published" }> { + const drive = sandboxAgentToolsDrive(args.region, args.bundleSha); + const manifest = agentToolsManifest(args.assets); + const existing = await args.host.find(drive); + if (existing?.attachedTo) { + throw new Error( + `${drive} is attached to ${existing.attachedTo}; an earlier publish may still be writing it`, + ); + } + if (existing) { + const found = parseAgentToolsManifest( + await args.host.readManifest(drive, args.region), + ); + if (found.length > 0) return { drive, outcome: "exists" }; + await args.host.remove(drive); + } + await args.host.create(drive, args.region); + await args.host.write( + drive, + args.region, + renderBuilderScript({ + root: SANDBOX_PATHS.agents, + baseUrl: args.baseUrl ?? SANDBOX_ASSET_BASE_URL, + assets: args.assets, + manifest, + }), + ); + return { drive, outcome: "published" }; +} + +/** Agent-tools drives no environment row's bundle and region name any more. */ +export function agentToolsDrivesToPrune( + drives: readonly string[], + referenced: ReadonlySet, +): string[] { + return drives.filter( + (name) => + SANDBOX_AGENT_TOOLS_DRIVE_PATTERN.test(name) && !referenced.has(name), + ); +} + +/** Returns the drives it could not delete; a drive a sandbox holds is retried next release. */ +export async function pruneAgentToolsDrives(args: { + host: DriveHost; + referenced: ReadonlySet; +}): Promise<{ deleted: string[]; kept: string[] }> { + const deleted: string[] = []; + const kept: string[] = []; + for (const name of agentToolsDrivesToPrune( + await args.host.list("agent-tools-"), + args.referenced, + )) { + try { + await args.host.remove(name); + deleted.push(name); + } catch { + kept.push(name); + } + } + return { deleted, kept }; +} + +interface VercelCredentials { + token: string; + teamId: string; + projectId: string; +} + +export function vercelDriveHost(credentials: VercelCredentials): DriveHost { + const sdk = () => import("@vercel/sandbox"); + const isNotFound = async (error: unknown) => { + const { APIError } = await sdk(); + return error instanceof APIError && error.response.status === 404; + }; + const getDrive = async (name: string) => { + const { Drive } = await sdk(); + try { + return await Drive.get({ ...credentials, name }); + } catch (error) { + if (await isNotFound(error)) return null; + throw error; + } + }; + const builderName = (region: string) => + `agent-tools-builder-${region}-${Date.now().toString(36)}`; + const stopAndDelete = async ( + builder: import("@vercel/sandbox").Sandbox, + drive: string, + ) => { + const { Sandbox } = await sdk(); + await builder.stop(); + const deadline = Date.now() + 3 * 60_000; + for (;;) { + const box = await Sandbox.get({ + ...credentials, + name: builder.name, + resume: false, + }); + const attached = (await getDrive(drive))?.currentSandboxName; + if (box.status === "stopped" && !attached) break; + if (Date.now() > deadline) { + throw new Error(`${builder.name} did not stop and release ${drive}`); + } + await new Promise((resolve) => setTimeout(resolve, 2_000)); + } + await builder.delete(); + }; + return { + async find(name) { + const drive = await getDrive(name); + return drive ? { attachedTo: drive.currentSandboxName } : null; + }, + async list(prefix) { + const { Drive } = await sdk(); + const names: string[] = []; + for await (const drive of await Drive.list({ + ...credentials, + namePrefix: prefix, + })) { + names.push(drive.name); + } + return names; + }, + async create(name, region) { + const { Drive } = await sdk(); + await Drive.getOrCreate({ ...credentials, name, region }); + }, + async remove(name) { + await (await getDrive(name))?.delete(); + }, + async readManifest(name, region) { + const { Sandbox } = await sdk(); + const drive = await getDrive(name); + if (!drive) return null; + const reader = await Sandbox.create({ + ...credentials, + name: builderName(region), + region, + persistent: false, + timeout: 5 * 60_000, + mounts: { [SANDBOX_PATHS.agents]: drive.snapshot() }, + }); + try { + return ( + ( + await reader.readFileToBuffer({ + path: `${SANDBOX_PATHS.agents}/${AGENT_TOOLS_MANIFEST}`, + }) + )?.toString("utf8") ?? null + ); + } finally { + await reader.stop(); + await reader.delete(); + } + }, + async write(name, region, script) { + const { Sandbox } = await sdk(); + const drive = await getDrive(name); + if (!drive) throw new Error(`${name} does not exist`); + const builder = await Sandbox.create({ + ...credentials, + name: builderName(region), + region, + persistent: false, + timeout: 20 * 60_000, + mounts: { [SANDBOX_PATHS.agents]: drive }, + }); + let failure: Error | null = null; + try { + const result = await builder.runCommand({ + cmd: "bash", + args: ["-c", script], + sudo: true, + }); + console.log( + `${name}: ${await result.stdout()}${await result.stderr()}`, + ); + if (result.exitCode !== 0) { + failure = new Error(`${name}: builder exited ${result.exitCode}`); + } + } finally { + await stopAndDelete(builder, name); + } + if (failure) throw failure; + }, + }; +} + +function argValues(argv: string[], flag: string): string[] { + return argv.flatMap((arg, i) => + arg === flag && argv[i + 1] ? [argv[i + 1] as string] : [], + ); +} + +if (import.meta.main) { + const argv = process.argv.slice(2); + const credentials = { + token: process.env.VERCEL_SANDBOX_TOKEN ?? "", + teamId: process.env.VERCEL_SANDBOX_TEAM_ID ?? "", + projectId: process.env.VERCEL_SANDBOX_PROJECT_ID ?? "", + }; + const [bundleSha] = argValues(argv, "--bundle"); + if (!credentials.token || !credentials.teamId || !credentials.projectId) { + console.error( + "VERCEL_SANDBOX_TOKEN, VERCEL_SANDBOX_TEAM_ID and VERCEL_SANDBOX_PROJECT_ID are required", + ); + process.exit(1); + } + if (!bundleSha || !SHA256.test(bundleSha)) { + console.error("--bundle is required"); + process.exit(1); + } + const regions = argValues(argv, "--region"); + const unknown = regions.filter((region) => !isSandboxRegionId(region)); + if (unknown.length) { + console.error(`unknown region ${unknown.join(", ")}`); + process.exit(1); + } + const assets = loadAssets( + join(import.meta.dir, "..", "bundle", "assets.json"), + ); + const host = vercelDriveHost(credentials); + for (const region of regions.length + ? (regions as SandboxRegionId[]) + : [DEFAULT_SANDBOX_REGION]) { + const { drive, outcome } = await publishAgentToolsDrive({ + host, + region, + bundleSha, + assets, + }); + console.log(`${drive}: ${outcome}`); + } +} diff --git a/packages/sandbox/src/boot-twice.ts b/packages/sandbox/src/boot-twice.ts index bf0974f8c06..bef133de099 100644 --- a/packages/sandbox/src/boot-twice.ts +++ b/packages/sandbox/src/boot-twice.ts @@ -313,6 +313,51 @@ try { ); console.log(`\nboot log (second boot):\n${log2}`); + // The agent-tools drive, stubbed as a directory at its mount path: the + // listed agent is linked from it, and a session without it falls back to + // the image's agents rather than to links that point nowhere. + const stop = () => + exec( + `pkill -u ubuntu -TERM || true; sleep 1; pkill -u ubuntu -KILL || true; pkill -x Xvnc || true; rm -rf ${SANDBOX_PATHS.run}`, + { check: false }, + ); + const stubSha = "f".repeat(64); + stop(); + exec( + `: > ${SANDBOX_PATHS.bootLog}; mkdir -p ${SANDBOX_PATHS.agents}/claude/${stubSha} && printf '#!/bin/sh\\necho drive-claude\\n' > ${SANDBOX_PATHS.agents}/claude/${stubSha}/claude && chmod 755 ${SANDBOX_PATHS.agents}/claude/${stubSha}/claude && printf '%s' '${JSON.stringify([{ name: "claude", sha: stubSha, bin: "claude" }])}' > ${SANDBOX_PATHS.agents}/manifest.json`, + ); + boot(3); + const logDrive = exec(`cat ${SANDBOX_PATHS.bootLog}`); + expect( + "drive: listed agent linked", + /agents\.linked 1/.test(logDrive) && + /drive-claude/.test(exec("claude", { check: false, user: "ubuntu" })), + logDrive.match(/agents\..*/)?.[0], + ); + stop(); + exec(`: > ${SANDBOX_PATHS.bootLog}; rm -rf ${SANDBOX_PATHS.agents}`); + const noDriveExit = bootAsUser(); + exec( + `for i in $(seq 1 600); do [ -f ${SANDBOX_PATHS.run}/host-service.ready ] && exit 0; sleep 0.1; done; exit 1`, + { check: false }, + ); + const logNoDrive = exec(`cat ${SANDBOX_PATHS.bootLog}`); + expect( + "no drive: boot exits 0 and host-service ready", + /exit=0/.test(noDriveExit) && /host\.ready/.test(logNoDrive), + ); + expect( + "no drive: stamped unavailable", + /agents\.unavailable/.test(logNoDrive), + ); + expect( + "no drive: image claude back on PATH", + exec("readlink -f /usr/local/bin/claude", { check: false }).startsWith( + SANDBOX_PATHS.media, + ), + exec("readlink -f /usr/local/bin/claude", { check: false }).trim(), + ); + // Third boot, with host-service deliberately broken: the box must stay // alive, the desktop must still come up, and the log must say so. exec( diff --git a/packages/sandbox/src/boot.test.ts b/packages/sandbox/src/boot.test.ts index cf49be58f6a..620714412b7 100644 --- a/packages/sandbox/src/boot.test.ts +++ b/packages/sandbox/src/boot.test.ts @@ -45,6 +45,16 @@ describe("superset-boot", () => { expect(memguard).toMatch(/swapoff "\$f"/); }); + test("links the drive's agents after the steps, before host-service starts", () => { + const steps = boot.indexOf('"$BUNDLE/setup" run-steps'); + const link = boot.indexOf("\nlink_agents\n"); + const host = boot.indexOf("\nstart_host_service\n"); + expect(steps).toBeGreaterThan(0); + expect(link).toBeGreaterThan(steps); + expect(host).toBeGreaterThan(link); + expect(boot).toMatch(/stamp "agents\.unavailable/); + }); + test("writes the cgroup's memory ceiling and CPU count for login shells", () => { expect(boot).toMatch(/SUPERSET_SANDBOX_MEMORY_MB/); expect(boot).toMatch(/SUPERSET_SANDBOX_VCPUS/); diff --git a/packages/sandbox/src/environments/release.ts b/packages/sandbox/src/environments/release.ts index d3b5cc7a6e5..2d049be17aa 100644 --- a/packages/sandbox/src/environments/release.ts +++ b/packages/sandbox/src/environments/release.ts @@ -43,9 +43,14 @@ import { SANDBOX_PATHS, SANDBOX_ROOT_CHECKOUT, type SandboxIdentity, + sandboxAgentToolsDrive, sandboxCheckoutDir, } from "@superset/shared/sandbox/contract"; -import { DEFAULT_SANDBOX_REGION } from "@superset/shared/sandbox/regions"; +import { + DEFAULT_SANDBOX_REGION, + isSandboxRegionId, + type SandboxRegionId, +} from "@superset/shared/sandbox/regions"; import { Sandbox } from "@vercel/sandbox"; // The provisioning code imports the API env schema; an operator running this @@ -147,6 +152,39 @@ log( await publishBundle(bundle, { dry: false }); log(`bundle: ${bundle.sha256} published`); +// 1b. the bundle's agent-tools drive in every region with an environment, +// before any row points at the bundle so no box mounts one mid-write. Boxes +// fail open to the image's agents, so a failed publish only warns. +const agentTools = await import("../agent-tools"); +const driveHost = agentTools.vercelDriveHost(credentials); +async function environmentRegions(): Promise { + const { db } = await import("@superset/db/client"); + const rows = await db.query.environments.findMany({ + columns: { region: true }, + where: (row, { isNull: unset }) => unset(row.archivedAt), + }); + return [ + ...new Set( + [REGION, ...rows.map((row) => row.region)].filter(isSandboxRegionId), + ), + ]; +} +for (const region of await environmentRegions()) { + try { + const { drive, outcome } = await agentTools.publishAgentToolsDrive({ + host: driveHost, + region, + bundleSha: bundle.sha256, + assets: bundle.assets, + }); + log(`agent-tools: ${drive} ${outcome}`); + } catch (error) { + console.log( + `::warning::agent-tools: ${region} not published; its boxes keep the image's agents: ${String(error).slice(0, 300)}`, + ); + } +} + // 2. image const candidateTag = `release-${bundle.sha256.slice(0, 12)}`; const goldenImage = SKIP_IMAGE @@ -564,6 +602,26 @@ if (!previous) `rows: created; set SUPERSET_INTERNAL_ENVIRONMENT_ID=${internal.id} so the next release updates it`, ); +try { + const live = await db.query.environments.findMany({ + columns: { region: true, bundleSha: true }, + where: (row, { isNull: unset }) => unset(row.archivedAt), + }); + const { deleted, kept } = await agentTools.pruneAgentToolsDrives({ + host: driveHost, + referenced: new Set( + live.map((row) => sandboxAgentToolsDrive(row.region, row.bundleSha)), + ), + }); + log( + `agent-tools: pruned ${deleted.length}${kept.length ? `; ${kept.join(", ")} still attached, next release retries` : ""}`, + ); +} catch (error) { + console.log( + `::warning::agent-tools: prune failed: ${String(error).slice(0, 300)}`, + ); +} + if ( previous && previous.sourceKind === "fork" && diff --git a/packages/shared/src/constants.ts b/packages/shared/src/constants.ts index 313f79dbe10..5f69d086fb6 100644 --- a/packages/shared/src/constants.ts +++ b/packages/shared/src/constants.ts @@ -220,6 +220,12 @@ export const FEATURE_FLAGS = { * conditions are the allowlist, so access changes without a deploy. */ CLOUD_WORKSPACES: "cloud-workspaces", + /** + * Mounts the bundle's agent-tools drive on an organization's boxes, so the + * agent CLIs come from it instead of the image. Evaluated per organization + * on every create and wake. Unloaded or offline means no drive. + */ + SANDBOX_AGENT_TOOLS_DRIVE: "sandbox-agent-tools-drive", /** * Moving a workspace and its chats to the cloud or to another computer: * shows the Move items, and gates diff --git a/packages/shared/src/sandbox/contract.ts b/packages/shared/src/sandbox/contract.ts index 1f2be836df1..9fd3d568b03 100644 --- a/packages/shared/src/sandbox/contract.ts +++ b/packages/shared/src/sandbox/contract.ts @@ -35,8 +35,24 @@ export const SANDBOX_PATHS = { media: "/usr/local/share/superset/media", desktopTemplates: "/usr/local/share/superset/desktop", backgrounds: "/usr/share/backgrounds/superset", + /** A read-only snapshot of the region's agent-tools drive, when one is mounted. */ + agents: "/opt/agents", } as const; +/** + * The agent CLIs of one bundle in one region: written once by the release, + * then only ever mounted read-only. A drive only mounts in its own region. + */ +export function sandboxAgentToolsDrive( + region: string, + bundleSha: string, +): string { + return `agent-tools-${region}-${bundleSha.slice(0, 12)}`; +} + +export const SANDBOX_AGENT_TOOLS_DRIVE_PATTERN = + /^agent-tools-[a-z]+[0-9]+-[0-9a-f]{12}$/; + /** Every port a workspace publishes, and the loopback ones behind them. */ export const SANDBOX_PORTS = { hostService: 4879, @@ -220,6 +236,7 @@ export function renderContractShell(): string { SUPERSET_MEDIA_DIR: SANDBOX_PATHS.media, SUPERSET_DESKTOP_TEMPLATES: SANDBOX_PATHS.desktopTemplates, SUPERSET_BACKGROUNDS_DIR: SANDBOX_PATHS.backgrounds, + SUPERSET_AGENTS_DIR: SANDBOX_PATHS.agents, SUPERSET_HOST_SERVICE_PORT: SANDBOX_PORTS.hostService, SUPERSET_DESKTOP_PORT: SANDBOX_PORTS.desktop, SUPERSET_VNC_PORT: SANDBOX_PORTS.vnc, diff --git a/packages/trpc/src/lib/sandbox/agent-tools.test.ts b/packages/trpc/src/lib/sandbox/agent-tools.test.ts new file mode 100644 index 00000000000..00fe6117571 --- /dev/null +++ b/packages/trpc/src/lib/sandbox/agent-tools.test.ts @@ -0,0 +1,50 @@ +import { describe, expect, test } from "bun:test"; +import { stub } from "../../../test/stub"; +import { posthog } from "../analytics"; +import { agentToolsDriveFor } from "./agent-tools"; + +let answer: () => Promise = () => Promise.resolve(true); +const asked: unknown[][] = []; +stub(posthog, { + isFeatureEnabled: (...args: unknown[]) => { + asked.push(args); + return answer(); + }, +}); + +const BUNDLE = "0123456789abcdef".repeat(4); +const ORG = "6f1c2b8e-1d2a-4c55-9a3e-0b7f5d2c9e11"; + +describe("agentToolsDriveFor", () => { + test("names the bundle's drive in the box's region when the org's flag is on", async () => { + answer = () => Promise.resolve(true); + expect( + await agentToolsDriveFor({ + organizationId: ORG, + region: "sfo1", + bundleSha: BUNDLE, + }), + ).toBe("agent-tools-sfo1-0123456789ab"); + expect(asked.at(-1)?.[0]).toBe("sandbox-agent-tools-drive"); + expect(asked.at(-1)?.[2]).toMatchObject({ + groups: { organization: ORG }, + }); + }); + + test("fails closed when the flag is off, unloaded or PostHog is down", async () => { + for (const next of [ + () => Promise.resolve(false), + () => Promise.resolve(undefined), + () => Promise.reject(new Error("offline")), + ]) { + answer = next; + expect( + await agentToolsDriveFor({ + organizationId: ORG, + region: "sfo1", + bundleSha: BUNDLE, + }), + ).toBeNull(); + } + }); +}); diff --git a/packages/trpc/src/lib/sandbox/agent-tools.ts b/packages/trpc/src/lib/sandbox/agent-tools.ts new file mode 100644 index 00000000000..92d069b9d8e --- /dev/null +++ b/packages/trpc/src/lib/sandbox/agent-tools.ts @@ -0,0 +1,26 @@ +import { FEATURE_FLAGS } from "@superset/shared/constants"; +import { sandboxAgentToolsDrive } from "@superset/shared/sandbox/contract"; +import { posthog } from "../analytics"; + +/** Fails closed: PostHog unreachable means no drive. */ +export async function agentToolsDriveEnabled( + organizationId: string, +): Promise { + const enabled = await posthog + .isFeatureEnabled(FEATURE_FLAGS.SANDBOX_AGENT_TOOLS_DRIVE, organizationId, { + groups: { organization: organizationId }, + sendFeatureFlagEvents: false, + }) + .catch(() => undefined); + return enabled === true; +} + +/** The drive of the bundle the box boots, or null while the organization's flag is off. */ +export async function agentToolsDriveFor(args: { + organizationId: string; + region: string; + bundleSha: string; +}): Promise { + if (!(await agentToolsDriveEnabled(args.organizationId))) return null; + return sandboxAgentToolsDrive(args.region, args.bundleSha); +} diff --git a/packages/trpc/src/lib/sandbox/claim.ts b/packages/trpc/src/lib/sandbox/claim.ts index bd1e715c445..fc496d4d084 100644 --- a/packages/trpc/src/lib/sandbox/claim.ts +++ b/packages/trpc/src/lib/sandbox/claim.ts @@ -28,6 +28,7 @@ import { resolveEnvironment } from "../../router/environment/resolve-environment import { resolveMachineSize } from "../billing/plan"; import { githubUserConnectionFor, githubUserTokenFor } from "../github-user"; import { sandboxHostSecretFor } from "./access"; +import { agentToolsDriveFor } from "./agent-tools"; import { deriveSandboxCredentials, gitAuthorFor } from "./credentials"; import { creatorPlugins } from "./plugins"; import { readRepoHooks } from "./repo-hooks"; @@ -174,6 +175,11 @@ export async function buildSandboxClaim(args: { files, ports: repoHooks?.ports, vcpus: await boxVcpus(args.row), + agentToolsDrive: await agentToolsDriveFor({ + organizationId: args.row.organizationId, + region: environment.region, + bundleSha: environment.bundleSha, + }), }, environment: { sourceKind: environment.sourceKind, diff --git a/packages/trpc/src/lib/sandbox/vercel.test.ts b/packages/trpc/src/lib/sandbox/vercel.test.ts index 9e50fede634..4a24e20353a 100644 --- a/packages/trpc/src/lib/sandbox/vercel.test.ts +++ b/packages/trpc/src/lib/sandbox/vercel.test.ts @@ -9,6 +9,7 @@ import { import { Sandbox } from "@vercel/sandbox"; import { stub } from "../../../test/stub"; import { + agentToolsMounts, findSandboxSession, listActiveWorkspaceSandboxes, SESSION_TIMEOUT_MS, @@ -160,3 +161,58 @@ describe("topUpSandboxSession", () => { expect(extended).toEqual([SESSION_TIMEOUT_MS / 2]); }); }); + +describe("agentToolsMounts", () => { + const detached = () => Promise.resolve({ currentSandboxName: undefined }); + + test("mounts nothing without a drive name", async () => { + let asked = 0; + const getDrive = () => { + asked++; + return detached(); + }; + expect(await agentToolsMounts(null, getDrive)).toBeUndefined(); + expect(asked).toBe(0); + }); + + test("mounts a detached drive read-only, and asks only once", async () => { + let asked = 0; + const getDrive = () => { + asked++; + return detached(); + }; + const want = { + "/opt/agents": { + drive: "agent-tools-iad1-0123456789ab", + mode: "snapshot", + }, + }; + expect( + await agentToolsMounts("agent-tools-iad1-0123456789ab", getDrive), + ).toEqual(want); + expect( + await agentToolsMounts("agent-tools-iad1-0123456789ab", getDrive), + ).toEqual(want); + expect(asked).toBe(1); + }); + + test("skips a missing drive, or one a writer still holds", async () => { + expect( + await agentToolsMounts("agent-tools-sfo1-aaaaaaaaaaaa", () => + Promise.reject(new Error("404")), + ), + ).toBeUndefined(); + let asked = 0; + const attached = () => { + asked++; + return Promise.resolve({ currentSandboxName: "builder" }); + }; + expect( + await agentToolsMounts("agent-tools-sfo1-bbbbbbbbbbbb", attached), + ).toBeUndefined(); + expect( + await agentToolsMounts("agent-tools-sfo1-bbbbbbbbbbbb", attached), + ).toBeUndefined(); + expect(asked).toBe(2); + }); +}); diff --git a/packages/trpc/src/lib/sandbox/vercel.ts b/packages/trpc/src/lib/sandbox/vercel.ts index d742e6e4c76..786002a5c37 100644 --- a/packages/trpc/src/lib/sandbox/vercel.ts +++ b/packages/trpc/src/lib/sandbox/vercel.ts @@ -20,8 +20,10 @@ import { } from "@superset/shared/sandbox/contract"; import { APIError, + Drive, type NetworkPolicy, Sandbox, + type SandboxMounts, type SandboxRegion, } from "@vercel/sandbox"; import { env } from "../../env"; @@ -119,6 +121,8 @@ export interface SandboxClaim { /** Ports the workspace's repository asks to publish, beside the platform's. */ ports?: readonly number[]; vcpus: number; + /** The bundle's agent-tools drive to mount read-only; absent keeps the image's agents. */ + agentToolsDrive?: string | null; } /** Vercel allows 5 tags per sandbox. */ @@ -137,6 +141,41 @@ function sandboxTags( }; } +const publishedAgentToolsDrives = new Set(); + +/** + * A mount of a drive that does not exist fails the create, and a drive a + * writer still holds read-write mounts corrupt. Only a drive that exists and + * is detached is mounted; that answer is final for a drive, so it is cached. + */ +export async function agentToolsMounts( + driveName: string | null | undefined, + getDrive: (name: string) => Promise> = ( + name, + ) => Drive.get({ ...credentials(), name }), +): Promise { + if (!driveName) return undefined; + if (!publishedAgentToolsDrives.has(driveName)) { + try { + const drive = await getDrive(driveName); + if (drive.currentSandboxName) { + console.warn( + `[sandbox] agent-tools drive ${driveName} is attached to ${drive.currentSandboxName}; not mounting`, + ); + return undefined; + } + } catch (error) { + console.warn( + `[sandbox] agent-tools drive ${driveName} unavailable; not mounting`, + error, + ); + return undefined; + } + publishedAgentToolsDrives.add(driveName); + } + return { [SANDBOX_PATHS.agents]: { drive: driveName, mode: "snapshot" } }; +} + function publishedPorts(extra: readonly number[] = []): number[] { return [...new Set([...SANDBOX_PUBLISHED_PORTS, ...extra])]; } @@ -222,6 +261,7 @@ export async function provisionSandbox(args: { keepLastSnapshots: { count: 1 }, tags: sandboxTags(kind, args.claim.identity), }; + const mounts = await agentToolsMounts(args.claim.agentToolsDrive); const sandbox = (await getSandbox(args.name)) ?? (await Sentry.startSpan( @@ -242,11 +282,13 @@ export async function provisionSandbox(args: { // it was taken. Sandbox.fork({ ...config, + ...(mounts ? { mounts } : {}), sourceSandbox: args.environment.sourceRef, resources: { vcpus: args.claim.vcpus }, }) : Sandbox.create({ ...config, + ...(mounts ? { mounts } : {}), image: args.environment.sourceRef, region: args.environment.region as SandboxRegion, resources: { vcpus: args.claim.vcpus }, @@ -477,6 +519,21 @@ export async function wakeSandbox(args: { ), ); } + if (!wasRunning) { + const { [SANDBOX_PATHS.agents]: previous, ...others } = + sandbox.mounts ?? {}; + const agents = await agentToolsMounts(args.claim.agentToolsDrive); + if (agents || previous) { + await sandbox + .update({ mounts: { ...others, ...agents } }) + .catch((error) => + console.warn( + `[sandbox] mount update failed for ${args.providerSandboxId}`, + error, + ), + ); + } + } if (!wasRunning && args.beforeResume) { await args.beforeResume({ memoryMb: vcpus * MEMORY_MB_PER_VCPU, vcpus }); }