From ebea6309f63e6c12ed60254f07a7090a32cbbac9 Mon Sep 17 00:00:00 2001 From: Satya Patel <14907857+saddlepaddle@users.noreply.github.com> Date: Thu, 19 Nov 2026 02:16:14 +0000 Subject: [PATCH] fix(sandbox): an agents drive lives as long as the sandboxes that mount it The production release deletes agent-tools-- only when it is not this release's bundle and no sandbox in the Vercel project, of any status or database, lists it under mounts. This replaces keeping the newest released bundles and reading environment rows, which a release cannot do for preview databases that share the project. The whole sandbox listing is read first; a failed page, a repeated cursor, an empty listing or a missing new golden deletes nothing. Every drive kept is logged with its reason and every delete is logged; a refused delete waits for the next release. Other releases and --prune-dry-run only log. --- .github/workflows/sandbox.yml | 31 +++ packages/sandbox/README.md | 2 +- packages/sandbox/src/agent-tools.test.ts | 211 ++++++++++++---- packages/sandbox/src/agent-tools.ts | 243 ++++++++++++++++--- packages/sandbox/src/environments/release.ts | 49 ++-- 5 files changed, 419 insertions(+), 117 deletions(-) diff --git a/.github/workflows/sandbox.yml b/.github/workflows/sandbox.yml index 31d4d56969c..07d8ccd3dd7 100644 --- a/.github/workflows/sandbox.yml +++ b/.github/workflows/sandbox.yml @@ -24,7 +24,12 @@ on: - real-sandbox - production-release - inventory + - drive-prune-dry-run default: real-sandbox + bundle: + description: "drive-prune-dry-run: the current release's bundle sha256" + type: string + required: false jobs: boot-twice: @@ -173,6 +178,32 @@ jobs: VERCEL_SANDBOX_PROJECT_ID: ${{ secrets.VERCEL_SANDBOX_PROJECT_ID }} run: bun run inventory + drive-prune-dry-run: + name: Agent-tools drive prune, dry run + if: github.event_name == 'workflow_dispatch' && inputs.run == 'drive-prune-dry-run' + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Check out code + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + + - name: Setup Bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version-file: .bun-version + + - name: Install dependencies + run: bun install --frozen --ignore-scripts + + - name: List what the prune would delete + working-directory: packages/sandbox + env: + VERCEL_SANDBOX_TOKEN: ${{ secrets.VERCEL_SANDBOX_TOKEN }} + VERCEL_SANDBOX_TEAM_ID: ${{ secrets.VERCEL_SANDBOX_TEAM_ID }} + VERCEL_SANDBOX_PROJECT_ID: ${{ secrets.VERCEL_SANDBOX_PROJECT_ID }} + BUNDLE: ${{ inputs.bundle }} + run: bun run agent-tools --bundle "$BUNDLE" --prune-dry-run + production-release: name: Production release # A merge that touches the image or the bundle releases it, and the nightly diff --git a/packages/sandbox/README.md b/packages/sandbox/README.md index 7785deac5fb..6f43c244034 100644 --- a/packages/sandbox/README.md +++ b/packages/sandbox/README.md @@ -55,7 +55,7 @@ The rule of thumb: if you can answer "where is it on the box", put it there. | `bun run image` | Builds the image: OS, apt lists, vendor tools, the `ubuntu` user, then installs the bundle at its sha and runs every pass once, so a fresh box finds every hash sidecar and step marker matching. A push logs Docker into the registry itself with a credential minted from `VERCEL_SANDBOX_TOKEN`, so there is no login to renew. `--local` builds `superset-sandbox:local` for the boot-twice test; `--dry` prints the Dockerfile. | | boot (`superset-boot`) | On the box, as root, run by the control plane on every create and wake, after it has written `sandbox.conf`, with the host secret in its env: clears `/run/superset`, installs the pinned bundle if the hash differs, runs `apply-rootfs` and the steps marked `beforeHost` in `steps.json` (each a hash compare), starts host-service as `ubuntu` with the secret from the boot env, writes `resources.env`, starts `superset-memguard` (ranks every process for the OOM killer and reaps the largest before the kernel has to). When `sandbox.conf` has `SUPERSET_SANDBOX_BOX_LOGS=1`, it also starts `superset-logship`, which sends the boot, memguard, desktop and host-service logs to Sentry Logs from a byte offset saved on disk. Then, detached: `sync-assets` and the other steps, then the desktop, with the checkout beside them. `checkout.ready` waits for both, and the stamp `steps.end` says the steps are done. Once host-service answers, the environment has been pushed and the checkout is ready, the runner asks host-service to run the repo's `start` hook with that environment. | | `bun run release` | Rewrites the host-service and CLI assets from this checkout and the claude, codex, Chrome, opencode and pi rows to their latest upstream release, publishes the bundle, pushes the image under a candidate tag `release-` (unless `--skip-image`), builds the internal golden from it with the environment's `setup` hook, probes a fork of it (`src/environments/probe.ts`) including a stop and wake, and only after the probe passes moves `latest` to the candidate and writes the environment rows. Every image environment resolves `superset-hostsvc` to `latest`, so a failed release changes no box. | -| `bun run agent-tools --bundle ` | Writes that bundle's agent CLIs to `agent-tools--`, a drive boxes mount read-only at `/opt/agents` when their organization has the `sandbox-agent-tools-drive` flag. A drive is written once and never again; it starts as a fork of the region's newest earlier drive, so only changed agents are fetched, and an agent that fails its download or `--version` check keeps the earlier drive's version or is left out. The drive carries claude, codex and pdev (also in the image) and opencode and pi (only on the drive). `bun run release` does this for every region, six at a time, before it records the bundle. It fails when a region with an environment has no drive, or a drive without claude, codex or pdev. Afterwards a `--production` release deletes the drives that neither an environment row (archived ones too) nor one of the 3 newest released bundles names; one Vercel project holds every database's drives, so no other release deletes any. | +| `bun run agent-tools --bundle ` | Writes that bundle's agent CLIs to `agent-tools--`, a drive boxes mount read-only at `/opt/agents` when their organization has the `sandbox-agent-tools-drive` flag. A drive is written once and never again; it starts as a fork of the region's newest earlier drive, so only changed agents are fetched, and an agent that fails its download or `--version` check keeps the earlier drive's version or is left out. The drive carries claude, codex and pdev (also in the image) and opencode and pi (only on the drive). `bun run release` does this for every region, six at a time, before it records the bundle. It fails when a region with an environment has no drive, or a drive without claude, codex or pdev. Afterwards a `--production` release deletes a drive once it is not this release's bundle and no sandbox in the Vercel project, of any status or database, lists it under mounts: a drive lives as long as the boxes that mount it. It reads every page of the sandbox listing first and deletes nothing if a page fails, a cursor repeats, or the new golden is missing; it logs each drive it keeps, with the reason, and each it deletes; a delete Vercel refuses (the drive is attached) waits for the next release. Any other release, and `bun run agent-tools --bundle --prune-dry-run` (the Sandbox workflow's `drive-prune-dry-run`), only logs what it would delete. A stopped box's listing names the drive it mounted last, but its wake mounts its database's environment row bundle. Production's rows always name the current release, so this is safe there; a preview database keeps the bundle it was branched with, and once no box lists that drive, the drive is deleted and those preview boxes fail their wake with the generic "could not start" error. `bun run agent-tools --bundle --region ` writes the drive again. | | `bun run src/real-sandbox.ts` | The Sandbox workflow dispatched with `real-sandbox`: one real sandbox from the registry image on this checkout's bundle and its agents drive in the default region, the same probe, a stop and a wake. | | Sandbox workflow, `production-release` | `bun run release --production` from `main` on a CI runner: the image, the internal golden and production's environment rows. Runs on a merge that touches the bundle and every night; a failed merge or nightly run opens an issue, or comments on the open one. | diff --git a/packages/sandbox/src/agent-tools.test.ts b/packages/sandbox/src/agent-tools.test.ts index cb8744da80d..fb637c8946b 100644 --- a/packages/sandbox/src/agent-tools.test.ts +++ b/packages/sandbox/src/agent-tools.test.ts @@ -10,19 +10,22 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { type AgentToolsManifest, - agentToolsDrivesToPrune, agentToolsGaps, agentToolsManifest, + agentToolsPrunePlan, type DriveHost, describeAgentToolsPublish, + type ListedSandbox, parseAgentToolsManifest, parseBuilderResults, pruneAgentToolsDrives, publishAgentToolsDrive, publishAgentToolsDrives, type RegionPublish, - referencedAgentToolsDrives, renderBuilderScript, + type SandboxLister, + type SandboxPage, + vercelSandboxLister, } from "./agent-tools"; import { type Assets, loadAssets, sha256 } from "./manifest"; @@ -387,56 +390,178 @@ describe("agentToolsGaps", () => { }); }); -test("referenced drives: every row's bundle in its region, and the recent bundles in every region", () => { - const older = "1".repeat(64); - expect( - [ - ...referencedAgentToolsDrives({ - rows: [ - { region: "iad1", bundleSha: older }, - { region: "fra1", bundleSha: null }, - ], - recentBundles: [BUNDLE], - regions: ["sfo1", "iad1"], - }), - ].sort(), - ).toEqual([ - "agent-tools-iad1-0123456789ab", - "agent-tools-iad1-111111111111", - "agent-tools-sfo1-0123456789ab", - ]); -}); - describe("pruning", () => { - test("deletes only agent-tools drives no environment references", () => { + const CURRENT = "c".repeat(64); + const drive = (region: string, bundle: string) => + `agent-tools-${region}-${bundle.slice(0, 12)}`; + const pages = + (...list: SandboxPage[]): SandboxLister => + async (cursor) => { + const page = list[cursor === null ? 0 : Number(cursor)]; + if (!page) throw new Error(`no page ${cursor}`); + return page; + }; + const box = (name: string, mounted?: string): ListedSandbox => ({ + name, + ...(mounted ? { mounts: { "/opt/agents": { drive: mounted } } } : {}), + }); + const prune = ( + host: DriveHost, + sandboxes: SandboxLister, + options: { dryRun?: boolean; expected?: string[] } = {}, + ) => { + const lines: string[] = []; + return pruneAgentToolsDrives({ + host, + sandboxes, + currentBundle: CURRENT, + expectedSandboxes: options.expected, + dryRun: options.dryRun ?? false, + log: (line) => lines.push(line), + }).then((result) => ({ result, lines })); + }; + + test("keeps the current release's drive in every region, mounted or not", () => { expect( - agentToolsDrivesToPrune( - [ - "agent-tools-sfo1-0123456789ab", - "agent-tools-sfo1-aaaaaaaaaaaa", - "agent-tools-iad1-0123456789ab", - "agent-tools-sfo1", - "agent-tools-builder-sfo1-xyz", - ], - new Set(["agent-tools-sfo1-0123456789ab"]), - ), + agentToolsPrunePlan({ + drives: [{ name: drive("sfo1", CURRENT), createdAt: 0 }], + mounted: new Map(), + currentBundle: CURRENT, + }), ).toEqual([ - "agent-tools-sfo1-aaaaaaaaaaaa", - "agent-tools-iad1-0123456789ab", + { + drive: drive("sfo1", CURRENT), + action: "keep", + reason: "current release", + }, ]); }); - test("keeps a drive a sandbox holds for the next release", async () => { + test("keeps a drive any listed sandbox mounts, on any page, and deletes one none mounts", async () => { + const { host, calls } = fakeHost({ + [drive("iad1", sha("1"))]: {}, + [drive("fra1", sha("2"))]: {}, + [drive("sfo1", sha("3"))]: {}, + "agent-tools-builder-sfo1-x": {}, + }); + const { result, lines } = await prune( + host, + pages( + { sandboxes: [box("ws-a", drive("iad1", sha("1")))], next: "1" }, + { + sandboxes: [box("ws-b", drive("fra1", sha("2"))), box("ws-c")], + next: null, + }, + ), + ); + expect(calls).toEqual([`remove ${drive("sfo1", sha("3"))}`]); + expect(result).toMatchObject({ + outcome: "pruned", + deleted: [drive("sfo1", sha("3"))], + }); + expect(lines).toContain( + `agent-tools: keep ${drive("iad1", sha("1"))}: mounted by 1 sandbox(es), e.g. ws-a`, + ); + expect(lines).toContain(`agent-tools: deleted ${drive("sfo1", sha("3"))}`); + }); + + test.each([ + ["a page fails", pages({ sandboxes: [box("ws-a")], next: "1" }), []], + [ + "a cursor repeats", + pages( + { sandboxes: [box("ws-a")], next: "1" }, + { sandboxes: [], next: "1" }, + ), + [], + ], + ["the listing is empty", pages({ sandboxes: [], next: null }), []], + [ + "a sandbox known to exist is missing", + pages({ sandboxes: [box("ws-a")], next: null }), + ["env-internal-x"], + ], + ])("deletes nothing when %s", async (_, sandboxes, expected) => { + const { host, calls } = fakeHost({ [drive("sfo1", sha("3"))]: {} }); + const { result } = await prune(host, sandboxes, { expected }); + expect(result.outcome).toBe("skipped"); + expect(calls).toEqual([]); + }); + + test("skips a drive Vercel refuses to delete and goes on", async () => { const { host } = fakeHost({ - "agent-tools-sfo1-aaaaaaaaaaaa": {}, - "agent-tools-sfo1-bbbbbbbbbbbb": { attachedTo: "box" }, + [drive("sfo1", sha("3"))]: {}, + [drive("sfo1", sha("4"))]: {}, }); - expect( - await pruneAgentToolsDrives({ host, referenced: new Set() }), - ).toEqual({ - deleted: ["agent-tools-sfo1-aaaaaaaaaaaa"], - kept: ["agent-tools-sfo1-bbbbbbbbbbbb"], + const refusing: DriveHost = { + ...host, + remove: async (name) => { + if (name === drive("sfo1", sha("3"))) + throw new Error("409 drive attached"); + await host.remove(name); + }, + }; + const { result } = await prune( + refusing, + pages({ sandboxes: [box("ws-a")], next: null }), + ); + expect(result).toMatchObject({ + deleted: [drive("sfo1", sha("4"))], + refused: [drive("sfo1", sha("3"))], + }); + }); + + test("does not try to delete a drive a sandbox holds attached", async () => { + const { host, calls } = fakeHost({ + [drive("sfo1", sha("3"))]: { attachedTo: "builder" }, + }); + const { result } = await prune( + host, + pages({ sandboxes: [box("ws-a")], next: null }), + ); + expect(calls).toEqual([]); + expect(result).toMatchObject({ deleted: [], refused: [] }); + }); + + test("a dry run reports what it would delete and deletes nothing", async () => { + const { host, calls } = fakeHost({ [drive("sfo1", sha("3"))]: {} }); + const { result, lines } = await prune( + host, + pages({ sandboxes: [box("ws-a")], next: null }), + { dryRun: true }, + ); + expect(calls).toEqual([]); + expect(result).toMatchObject({ + outcome: "dry-run", + deleted: [drive("sfo1", sha("3"))], + }); + expect(lines).toContain( + `agent-tools: would delete ${drive("sfo1", sha("3"))}`, + ); + }); + + test("the Vercel lister pages by cursor and throws on a refused page", async () => { + const seen: string[] = []; + const lister = vercelSandboxLister( + { token: "t", teamId: "team", projectId: "prj" }, + (async (url: string) => { + seen.push(new URL(url).searchParams.get("cursor") ?? ""); + return seen.length === 1 + ? Response.json({ + sandboxes: [ + { name: "ws-a", mounts: { "/opt/agents": { drive: "d" } } }, + ], + pagination: { next: "n1" }, + }) + : new Response("rate limited", { status: 429 }); + }) as typeof fetch, + ); + expect(await lister(null)).toEqual({ + sandboxes: [{ name: "ws-a", mounts: { "/opt/agents": { drive: "d" } } }], + next: "n1", }); + await expect(lister("n1")).rejects.toThrow("429"); + expect(seen).toEqual(["", "n1"]); }); }); diff --git a/packages/sandbox/src/agent-tools.ts b/packages/sandbox/src/agent-tools.ts index f289e3143a0..a56084a925e 100644 --- a/packages/sandbox/src/agent-tools.ts +++ b/packages/sandbox/src/agent-tools.ts @@ -17,6 +17,7 @@ * image's copy. * * bun run agent-tools --bundle [--region sfo1 ...] + * bun run agent-tools --bundle --prune-dry-run * * Needs VERCEL_SANDBOX_TOKEN, VERCEL_SANDBOX_TEAM_ID, VERCEL_SANDBOX_PROJECT_ID. * The agent rows come from bundle/assets.json, so they must be that bundle's. @@ -452,56 +453,175 @@ export function agentToolsGaps( }); } +/** A sandbox as the project listing shows it; `mounts` is what its next session mounts. */ +export interface ListedSandbox { + name: string; + mounts?: Record; +} + +/** One page of the project's sandbox listing; `next` is null on the last page. */ +export interface SandboxPage { + sandboxes: ListedSandbox[]; + next: string | null; +} + +export type SandboxLister = (cursor: string | null) => Promise; + +export const MAX_SANDBOX_LISTING_PAGES = 10_000; + /** - * Every drive a box may still mount: each environment row's bundle in its - * region, and the newest released bundles in every region, so a rollback and - * an environment created mid-release still find theirs. + * Every drive any sandbox in the project mounts, whatever its status or + * database, with the sandboxes that mount it. Throws unless the listing is + * whole: every page answered, each cursor new, and the sandboxes the caller + * knows exist are in it. */ -export function referencedAgentToolsDrives(args: { - rows: readonly { region: string; bundleSha: string | null }[]; - recentBundles: readonly string[]; - regions: readonly string[]; -}): Set { - return new Set([ - ...args.rows.flatMap((row) => - row.bundleSha ? [sandboxAgentToolsDrive(row.region, row.bundleSha)] : [], - ), - ...args.recentBundles.flatMap((sha) => - args.regions.map((region) => sandboxAgentToolsDrive(region, sha)), - ), - ]); +export async function listMountedDrives( + lister: SandboxLister, + expected: readonly string[] = [], +): Promise<{ sandboxes: number; mounted: Map }> { + const mounted = new Map(); + const seenCursors = new Set(); + const names = new Set(); + let cursor: string | null = null; + for (let pages = 0; ; pages++) { + if (pages >= MAX_SANDBOX_LISTING_PAGES) { + throw new Error(`listing has more than ${pages} pages`); + } + const page = await lister(cursor); + for (const sandbox of page.sandboxes) { + names.add(sandbox.name); + for (const { drive } of Object.values(sandbox.mounts ?? {})) { + mounted.set(drive, [...(mounted.get(drive) ?? []), sandbox.name]); + } + } + if (page.next === null) break; + if (seenCursors.has(page.next)) { + throw new Error(`listing repeated cursor ${page.next.slice(0, 16)}`); + } + seenCursors.add(page.next); + cursor = page.next; + } + if (names.size === 0) throw new Error("listing has no sandboxes"); + const missing = expected.filter((name) => !names.has(name)); + if (missing.length) { + throw new Error(`listing lacks ${missing.join(", ")}`); + } + return { sandboxes: names.size, mounted }; } -/** Agent-tools drives no environment row's bundle and region name any more. */ -export function agentToolsDrivesToPrune( - drives: readonly string[], - referenced: ReadonlySet, -): string[] { - return drives.filter( - (name) => - SANDBOX_AGENT_TOOLS_DRIVE_PATTERN.test(name) && !referenced.has(name), - ); +export type PruneDecision = + | { drive: string; action: "keep"; reason: string } + | { drive: string; action: "delete" }; + +/** + * A drive lives as long as a sandbox that mounts it: it is deleted once it + * is not the current release's bundle and no sandbox lists it under mounts. + */ +export function agentToolsPrunePlan(args: { + drives: readonly DriveEntry[]; + mounted: ReadonlyMap; + currentBundle: string; +}): PruneDecision[] { + const current = `-${args.currentBundle.slice(0, 12)}`; + return args.drives + .filter(({ name }) => SANDBOX_AGENT_TOOLS_DRIVE_PATTERN.test(name)) + .map(({ name, attachedTo }): PruneDecision => { + if (name.endsWith(current)) { + return { drive: name, action: "keep", reason: "current release" }; + } + const boxes = args.mounted.get(name) ?? []; + if (boxes.length) { + return { + drive: name, + action: "keep", + reason: `mounted by ${boxes.length} sandbox(es), e.g. ${boxes[0]}`, + }; + } + if (attachedTo) { + return { + drive: name, + action: "keep", + reason: `attached to ${attachedTo}; next release retries`, + }; + } + return { drive: name, action: "delete" }; + }) + .sort((a, b) => a.drive.localeCompare(b.drive)); } -/** Returns the drives it could not delete; a drive a sandbox holds is retried next release. */ +export type PruneOutcome = + | { outcome: "skipped"; reason: string } + | { + outcome: "pruned" | "dry-run"; + kept: { drive: string; reason: string }[]; + deleted: string[]; + refused: string[]; + }; + +/** + * Deletes nothing unless both the drive listing and the whole sandbox + * listing were read. A delete Vercel refuses (a drive still attached) is + * skipped and retried by the next release; nothing is forced. + */ export async function pruneAgentToolsDrives(args: { host: DriveHost; - referenced: ReadonlySet; -}): Promise<{ deleted: string[]; kept: string[] }> { + sandboxes: SandboxLister; + currentBundle: string; + expectedSandboxes?: readonly string[]; + dryRun: boolean; + log: (line: string) => void; +}): Promise { + let drives: DriveEntry[]; + let listing: Awaited>; + try { + drives = await args.host.list("agent-tools-"); + listing = await listMountedDrives(args.sandboxes, args.expectedSandboxes); + } catch (error) { + const reason = `listing incomplete, deleted nothing: ${String(error).slice(0, 300)}`; + args.log(`agent-tools: prune skipped; ${reason}`); + return { outcome: "skipped", reason }; + } + args.log( + `agent-tools: ${drives.length} drives, ${listing.sandboxes} sandboxes listed, ${listing.mounted.size} drives mounted`, + ); + const kept: { drive: string; reason: string }[] = []; const deleted: string[] = []; - const kept: string[] = []; - for (const name of agentToolsDrivesToPrune( - (await args.host.list("agent-tools-")).map((drive) => drive.name), - args.referenced, - )) { + const refused: string[] = []; + for (const decision of agentToolsPrunePlan({ + drives, + mounted: listing.mounted, + currentBundle: args.currentBundle, + })) { + if (decision.action === "keep") { + kept.push({ drive: decision.drive, reason: decision.reason }); + args.log(`agent-tools: keep ${decision.drive}: ${decision.reason}`); + continue; + } + if (args.dryRun) { + deleted.push(decision.drive); + args.log(`agent-tools: would delete ${decision.drive}`); + continue; + } try { - await args.host.remove(name); - deleted.push(name); - } catch { - kept.push(name); + await args.host.remove(decision.drive); + deleted.push(decision.drive); + args.log(`agent-tools: deleted ${decision.drive}`); + } catch (error) { + refused.push(decision.drive); + args.log( + `agent-tools: keep ${decision.drive}: Vercel refused the delete (${String(error).slice(0, 200)}); next release retries`, + ); } } - return { deleted, kept }; + args.log( + `agent-tools: ${args.dryRun ? "dry run: would delete" : "deleted"} ${deleted.length}, kept ${kept.length + refused.length}`, + ); + return { + outcome: args.dryRun ? "dry-run" : "pruned", + kept, + deleted, + refused, + }; } interface VercelCredentials { @@ -510,6 +630,42 @@ interface VercelCredentials { projectId: string; } +const SandboxListingPage = z.object({ + sandboxes: z.array( + z.object({ + name: z.string(), + mounts: z.record(z.string(), z.object({ drive: z.string() })).optional(), + }), + ), + pagination: z.object({ next: z.string().nullable() }), +}); + +/** Every sandbox in the project, of any status; the SDK's own listing hides the pages. */ +export function vercelSandboxLister( + credentials: VercelCredentials, + fetchPage: typeof fetch = fetch, +): SandboxLister { + return async (cursor) => { + const query = new URLSearchParams({ + teamId: credentials.teamId, + project: credentials.projectId, + limit: "50", + ...(cursor ? { cursor } : {}), + }); + const response = await fetchPage( + `https://api.vercel.com/v2/sandboxes?${query}`, + { headers: { authorization: `Bearer ${credentials.token}` } }, + ); + if (!response.ok) { + throw new Error( + `sandbox listing answered ${response.status}: ${(await response.text()).slice(0, 200)}`, + ); + } + const page = SandboxListingPage.parse(await response.json()); + return { sandboxes: page.sandboxes, next: page.pagination.next }; + }; +} + export function vercelDriveHost(credentials: VercelCredentials): DriveHost { const sdk = () => import("@vercel/sandbox"); const isNotFound = async (error: unknown) => { @@ -658,6 +814,7 @@ if (import.meta.main) { projectId: process.env.VERCEL_SANDBOX_PROJECT_ID ?? "", }; const [bundleSha] = argValues(argv, "--bundle"); + const pruneDryRun = argv.includes("--prune-dry-run"); if (!credentials.token || !credentials.teamId || !credentials.projectId) { console.error( "VERCEL_SANDBOX_TOKEN, VERCEL_SANDBOX_TEAM_ID and VERCEL_SANDBOX_PROJECT_ID are required", @@ -668,6 +825,16 @@ if (import.meta.main) { console.error("--bundle is required"); process.exit(1); } + if (pruneDryRun) { + const result = await pruneAgentToolsDrives({ + host: vercelDriveHost(credentials), + sandboxes: vercelSandboxLister(credentials), + currentBundle: bundleSha, + dryRun: true, + log: console.log, + }); + process.exit(result.outcome === "skipped" ? 1 : 0); + } const regions = argValues(argv, "--region"); const unknown = regions.filter((region) => !isSandboxRegionId(region)); if (unknown.length) { diff --git a/packages/sandbox/src/environments/release.ts b/packages/sandbox/src/environments/release.ts index 95dd3faf6c6..e661b7c683b 100644 --- a/packages/sandbox/src/environments/release.ts +++ b/packages/sandbox/src/environments/release.ts @@ -59,8 +59,6 @@ process.env.SKIP_ENV_VALIDATION ??= "1"; const SKIP_IMAGE = process.argv.includes("--skip-image"); const KEEP_OLD = process.argv.includes("--keep-old"); const PRODUCTION = process.argv.includes("--production"); -/** Released bundles whose drives stay in every region, so a rollback to one of them has its drives. */ -const KEPT_RELEASED_BUNDLES = 3; const FOLLOWED_UPSTREAM = ["claude", "codex", "chrome", "opencode", "pi"]; /** The environment the release rebuilds; without it, the release creates one named INTERNAL_NAME. */ const ENVIRONMENT_ID = process.env.SUPERSET_INTERNAL_ENVIRONMENT_ID; @@ -521,7 +519,7 @@ const { db, dbWs } = await import("@superset/db/client"); const { environments, environmentRepositories } = await import( "@superset/db/schema" ); -const { desc, eq } = await import("drizzle-orm"); +const { eq } = await import("drizzle-orm"); const { releaseRepositoryWhere } = await import( "./utils/releaseRepositoryWhere" ); @@ -605,38 +603,19 @@ if (!previous) `rows: created; set SUPERSET_INTERNAL_ENVIRONMENT_ID=${internal.id} so the next release updates it`, ); -// One Vercel project holds every database's drives (staging shares -// production's rows; preview branches copy them), so only the production -// release, whose rows the others mirror, may delete one. -if (!PRODUCTION) { - log("agent-tools: prune skipped; only --production prunes drives"); -} else { - try { - const { sandboxBundles } = await import("@superset/db/schema"); - const rows = await db.query.environments.findMany({ - columns: { region: true, bundleSha: true }, - }); - const recent = await db - .select({ sha: sandboxBundles.sha }) - .from(sandboxBundles) - .orderBy(desc(sandboxBundles.releasedAt)) - .limit(KEPT_RELEASED_BUNDLES); - const { deleted, kept } = await agentTools.pruneAgentToolsDrives({ - host: driveHost, - referenced: agentTools.referencedAgentToolsDrives({ - rows, - recentBundles: [bundle.sha256, ...recent.map((row) => row.sha)], - regions: SANDBOX_REGION_IDS, - }), - }); - log( - `agent-tools: pruned ${deleted.length}${kept.length ? `; ${kept.join(", ")} still attached, next release retries` : ""}`, - ); - } catch (error) { - console.log( - `::warning::agent-tools: prune failed: ${String(error).slice(0, 300)}`, - ); - } +// One Vercel project holds every database's drives, so a drive is kept by +// any sandbox in the project that mounts it, and only the production +// release deletes; any other release only reports what it would delete. +const pruned = await agentTools.pruneAgentToolsDrives({ + host: driveHost, + sandboxes: agentTools.vercelSandboxLister(credentials), + currentBundle: bundle.sha256, + expectedSandboxes: [golden], + dryRun: !PRODUCTION, + log, +}); +if (pruned.outcome === "skipped") { + console.log(`::warning::agent-tools: prune skipped: ${pruned.reason}`); } if (