From fb4ed8493ebf36ba1eed9b8e6d0c2fc9a1c4ba98 Mon Sep 17 00:00:00 2001 From: Satya Patel <14907857+saddlepaddle@users.noreply.github.com> Date: Wed, 18 Nov 2026 02:08:08 +0000 Subject: [PATCH] feat(sandbox): release agent-tools drives every box can rely on The release publishes the bundle's agents drive in every region and fails before the image, the golden or any row when a region with an environment has no drive, or a drive without claude, codex or pdev. A kept-previous agent warns with its region and the version that failed. Every environment row moves to the new bundle, archived ones too, so their workspaces' drives are not pruned. Only a --production release prunes, and it keeps each row's drive and the 3 newest released bundles in every region. The drive a box mounts is named by the box's own region on a wake. The golden, the probe and real-sandbox mount the drive, and the probe checks each cloud preset resolves into /opt/agents and that the wake relinks. --- packages/sandbox/README.md | 4 +- packages/sandbox/src/agent-tools.test.ts | 70 ++++++++++++- packages/sandbox/src/agent-tools.ts | 57 ++++++++++- packages/sandbox/src/environments/probe.ts | 12 +++ packages/sandbox/src/environments/release.ts | 97 +++++++++++-------- packages/sandbox/src/real-sandbox.ts | 26 ++++- .../trpc/src/lib/sandbox/agent-tools.test.ts | 23 +---- packages/trpc/src/lib/sandbox/agent-tools.ts | 11 --- packages/trpc/src/lib/sandbox/claim.ts | 8 +- packages/trpc/src/lib/sandbox/vercel.test.ts | 53 +++++++++- packages/trpc/src/lib/sandbox/vercel.ts | 23 ++++- 11 files changed, 298 insertions(+), 86 deletions(-) diff --git a/packages/sandbox/README.md b/packages/sandbox/README.md index ff1afd9e01f..664f3bd39a7 100644 --- a/packages/sandbox/README.md +++ b/packages/sandbox/README.md @@ -55,8 +55,8 @@ The rule of thumb: if you can answer "where is it on the box", put it there. | `bun run image` | Builds the image: OS, apt lists, vendor tools, the `ubuntu` user, then installs the bundle at its sha and runs every pass once, so a fresh box finds every hash sidecar and step marker matching. A push logs Docker into the registry itself with a credential minted from `VERCEL_SANDBOX_TOKEN`, so there is no login to renew. `--local` builds `superset-sandbox:local` for the boot-twice test; `--dry` prints the Dockerfile. | | boot (`superset-boot`) | On the box, as root, run by the control plane on every create and wake, after it has written `sandbox.conf`, with the host secret in its env: clears `/run/superset`, installs the pinned bundle if the hash differs, runs `apply-rootfs` and the steps marked `beforeHost` in `steps.json` (each a hash compare), starts host-service as `ubuntu` with the secret from the boot env, writes `resources.env`, starts `superset-memguard` (ranks every process for the OOM killer and reaps the largest before the kernel has to). Then, detached: `sync-assets` and the other steps, then the desktop, with the checkout beside them. `checkout.ready` waits for both, and the stamp `steps.end` says the steps are done. Once host-service answers, the environment has been pushed and the checkout is ready, the runner asks host-service to run the repo's `start` hook with that environment. | | `bun run release` | Rewrites the host-service and CLI assets from this checkout and the claude, codex, Chrome, opencode and pi rows to their latest upstream release, publishes the bundle, pushes the image under a candidate tag `release-` (unless `--skip-image`), builds the internal golden from it with the environment's `setup` hook, probes a fork of it (`src/environments/probe.ts`) including a stop and wake, and only after the probe passes moves `latest` to the candidate and writes the environment rows. Every image environment resolves `superset-hostsvc` to `latest`, so a failed release changes no box. | -| `bun run agent-tools --bundle ` | Writes that bundle's agent CLIs to `agent-tools--`, a drive boxes mount read-only at `/opt/agents` when their organization has the `sandbox-agent-tools-drive` flag. A drive is written once and never again; it starts as a fork of the region's newest earlier drive, so only changed agents are fetched, and an agent that fails its download or `--version` check keeps the earlier drive's version or is left out. The drive carries claude, codex and pdev (also in the image) and opencode and pi (only on the drive). `bun run release` does this for every region with an environment, six at a time, before it records the bundle, and afterwards deletes the drives no environment names. | -| `bun run src/real-sandbox.ts` | The Sandbox workflow dispatched with `real-sandbox`: one real sandbox from the registry image on this checkout's bundle, the same probe, a stop and a wake. | +| `bun run agent-tools --bundle ` | Writes that bundle's agent CLIs to `agent-tools--`, a drive boxes mount read-only at `/opt/agents` when their organization has the `sandbox-agent-tools-drive` flag. A drive is written once and never again; it starts as a fork of the region's newest earlier drive, so only changed agents are fetched, and an agent that fails its download or `--version` check keeps the earlier drive's version or is left out. The drive carries claude, codex and pdev (also in the image) and opencode and pi (only on the drive). `bun run release` does this for every region, six at a time, before it records the bundle. It fails when a region with an environment has no drive, or a drive without claude, codex or pdev. Afterwards a `--production` release deletes the drives that neither an environment row (archived ones too) nor one of the 3 newest released bundles names; one Vercel project holds every database's drives, so no other release deletes any. | +| `bun run src/real-sandbox.ts` | The Sandbox workflow dispatched with `real-sandbox`: one real sandbox from the registry image on this checkout's bundle and its agents drive in the default region, the same probe, a stop and a wake. | | Sandbox workflow, `production-release` | `bun run release --production` from `main` on a CI runner: the image, the internal golden and production's environment rows. Runs on a merge that touches the bundle and every night; a failed merge or nightly run opens an issue, or comments on the open one. | An asset URL is content-addressed, so the CDN caches it for hours — including a 404. An image diff --git a/packages/sandbox/src/agent-tools.test.ts b/packages/sandbox/src/agent-tools.test.ts index c8db13b598d..cb8744da80d 100644 --- a/packages/sandbox/src/agent-tools.test.ts +++ b/packages/sandbox/src/agent-tools.test.ts @@ -11,6 +11,7 @@ import { join } from "node:path"; import { type AgentToolsManifest, agentToolsDrivesToPrune, + agentToolsGaps, agentToolsManifest, type DriveHost, describeAgentToolsPublish, @@ -19,6 +20,8 @@ import { pruneAgentToolsDrives, publishAgentToolsDrive, publishAgentToolsDrives, + type RegionPublish, + referencedAgentToolsDrives, renderBuilderScript, } from "./agent-tools"; import { type Assets, loadAssets, sha256 } from "./manifest"; @@ -284,12 +287,20 @@ describe("publishAgentToolsDrive", () => { expect(agentLine(scripts[drive], "chrome")).toEndWith( "'--headless' '--version'", ); - const lines = describeAgentToolsPublish([{ region: "sfo1", result }]); + expect(result.available.map(({ name }) => name)).toEqual([ + "claude", + "codex", + "pdev", + ]); + const lines = describeAgentToolsPublish([{ region: "sfo1", result }], { + ...agentAssets, + pdev: { ...(agentAssets.pdev as Assets[string]), version: "0.2.6" }, + }); expect(lines[0]).toBe( `agent-tools: ${drive} published: claude written, codex reused, pdev kept-previous, chrome dropped`, ); expect(lines.slice(1)).toEqual([ - expect.stringMatching(/^::warning::.*pdev kept 999999999999/), + `::warning::agent-tools: sfo1: pdev 0.2.6 failed; ${drive} keeps 999999999999 from the earlier drive`, expect.stringMatching(/^::warning::.*chrome left out/), ]); }); @@ -341,6 +352,61 @@ test("builder results ignore lines that are not results", () => { ).toEqual([{ name: "claude", outcome: "reused", sha: sha("a") }]); }); +describe("agentToolsGaps", () => { + const published = ( + region: "sfo1" | "iad1" | "fra1", + names: string[], + ): RegionPublish => ({ + region, + result: { + drive: `agent-tools-${region}-0123456789ab`, + outcome: "exists", + agents: [], + available: names.map((name) => ({ name, sha: sha("a") })), + }, + }); + + test("names each needed region whose drive is missing or lacks a required agent", () => { + const results: RegionPublish[] = [ + published("sfo1", ["claude", "codex", "pdev", "pi"]), + published("iad1", ["claude", "pi"]), + { region: "fra1", error: new Error("builder exited 1") }, + ]; + expect(agentToolsGaps(results, new Set(["sfo1", "iad1", "fra1"]))).toEqual([ + "iad1: agent-tools-iad1-0123456789ab has no codex, pdev", + "fra1: no drive", + ]); + }); + + test("ignores regions no box needs, and an optional agent left out", () => { + const results: RegionPublish[] = [ + published("sfo1", ["claude", "codex", "pdev"]), + { region: "fra1", error: new Error("builder exited 1") }, + ]; + expect(agentToolsGaps(results, new Set(["sfo1"]))).toEqual([]); + }); +}); + +test("referenced drives: every row's bundle in its region, and the recent bundles in every region", () => { + const older = "1".repeat(64); + expect( + [ + ...referencedAgentToolsDrives({ + rows: [ + { region: "iad1", bundleSha: older }, + { region: "fra1", bundleSha: null }, + ], + recentBundles: [BUNDLE], + regions: ["sfo1", "iad1"], + }), + ].sort(), + ).toEqual([ + "agent-tools-iad1-0123456789ab", + "agent-tools-iad1-111111111111", + "agent-tools-sfo1-0123456789ab", + ]); +}); + describe("pruning", () => { test("deletes only agent-tools drives no environment references", () => { expect( diff --git a/packages/sandbox/src/agent-tools.ts b/packages/sandbox/src/agent-tools.ts index 2d94e54cdc6..f289e3143a0 100644 --- a/packages/sandbox/src/agent-tools.ts +++ b/packages/sandbox/src/agent-tools.ts @@ -57,6 +57,9 @@ const CHECK_TIMEOUT_SECONDS = 30; export const AGENT_TOOLS_MANIFEST = "manifest.json"; +/** Agents a box used to carry on its own disk; a drive without one of them must not be released. */ +export const REQUIRED_AGENT_TOOLS = ["claude", "codex", "pdev"] as const; + const agentToolSchema = z.object({ name: z.string().regex(/^[a-z0-9-]+$/), sha: z.string().regex(SHA256), @@ -270,6 +273,8 @@ export interface AgentToolsPublish { outcome: "exists" | "published"; forkedFrom?: string; agents: AgentResult[]; + /** What the drive's manifest names once published. */ + available: { name: string; sha: string }[]; } /** The region's newest other agent-tools drive that no sandbox holds read-write. */ @@ -316,7 +321,13 @@ export async function publishAgentToolsDrive(args: { const found = parseAgentToolsManifest( await args.host.readManifest(drive, args.region), ); - if (found.length > 0) return { drive, outcome: "exists", agents: [] }; + if (found.length > 0) + return { + drive, + outcome: "exists", + agents: [], + available: found.map(({ name, sha }) => ({ name, sha })), + }; await args.host.remove(drive); } let forkedFrom: string | undefined; @@ -348,11 +359,13 @@ export async function publishAgentToolsDrive(args: { tools: args.tools, }), ); + const agents = parseBuilderResults(stdout); return { drive, outcome: "published", ...(forkedFrom ? { forkedFrom } : {}), - agents: parseBuilderResults(stdout), + agents, + available: agents.flatMap(({ name, sha }) => (sha ? [{ name, sha }] : [])), }; } @@ -393,6 +406,7 @@ export async function publishAgentToolsDrives( /** One line per region and per agent that did not get the bundle's version; warnings start `::warning::`. */ export function describeAgentToolsPublish( results: readonly RegionPublish[], + assets: Assets = {}, ): string[] { return results.flatMap((entry) => { if ("error" in entry) { @@ -409,7 +423,7 @@ export function describeAgentToolsPublish( ...agents.flatMap((agent) => agent.outcome === "kept-previous" ? [ - `::warning::agent-tools: ${drive}: ${agent.name} kept ${agent.sha?.slice(0, 12)} from the earlier drive; the bundle's version failed`, + `::warning::agent-tools: ${entry.region}: ${agent.name} ${assets[agent.name]?.version ?? "of this bundle"} failed; ${drive} keeps ${agent.sha?.slice(0, 12)} from ${forkedFrom ?? "the earlier drive"}`, ] : agent.outcome === "dropped" ? [ @@ -421,6 +435,43 @@ export function describeAgentToolsPublish( }); } +/** One line per region a box needs whose drive is missing or lacks a required agent; empty means release. */ +export function agentToolsGaps( + results: readonly RegionPublish[], + needed: ReadonlySet, + required: readonly string[] = REQUIRED_AGENT_TOOLS, +): string[] { + return results.flatMap((entry) => { + if (!needed.has(entry.region)) return []; + if ("error" in entry) return [`${entry.region}: no drive`]; + const names = new Set(entry.result.available.map(({ name }) => name)); + const missing = required.filter((name) => !names.has(name)); + return missing.length + ? [`${entry.region}: ${entry.result.drive} has no ${missing.join(", ")}`] + : []; + }); +} + +/** + * Every drive a box may still mount: each environment row's bundle in its + * region, and the newest released bundles in every region, so a rollback and + * an environment created mid-release still find theirs. + */ +export function referencedAgentToolsDrives(args: { + rows: readonly { region: string; bundleSha: string | null }[]; + recentBundles: readonly string[]; + regions: readonly string[]; +}): Set { + return new Set([ + ...args.rows.flatMap((row) => + row.bundleSha ? [sandboxAgentToolsDrive(row.region, row.bundleSha)] : [], + ), + ...args.recentBundles.flatMap((sha) => + args.regions.map((region) => sandboxAgentToolsDrive(region, sha)), + ), + ]); +} + /** Agent-tools drives no environment row's bundle and region name any more. */ export function agentToolsDrivesToPrune( drives: readonly string[], diff --git a/packages/sandbox/src/environments/probe.ts b/packages/sandbox/src/environments/probe.ts index abeaae27ddb..e1b198bd79d 100644 --- a/packages/sandbox/src/environments/probe.ts +++ b/packages/sandbox/src/environments/probe.ts @@ -131,6 +131,14 @@ export async function probeBox(args: ProbeArgs): Promise { ); for (const preset of HOST_AGENT_PRESETS) { if (!isCloudAgentId(preset.presetId)) continue; + const resolved = ( + await run(`readlink -f "$(command -v ${preset.command})"`) + ).trim(); + check( + `${preset.presetId} runs from the agents drive`, + resolved.startsWith(`${SANDBOX_PATHS.agents}/`), + resolved, + ); const out = await run( `cd /tmp && timeout 15 ${[preset.command, ...preset.args].join(" ")} /tmp/preset.out 2>&1; echo "exit=$?"; head -5 /tmp/preset.out`, ); @@ -244,6 +252,10 @@ export async function checkWakeLog(args: { check("wake: every step skipped", !/step \S+ running/.test(last)); check("wake: checkout kept", /checkout\.skipped/.test(last)); check("wake: host-service ready", /host\.ready/.test(last)); + check( + "wake: agents linked from the remounted drive", + /agents\.linked [1-9]/.test(last) && !/agents\.unavailable/.test(last), + ); return failed; } diff --git a/packages/sandbox/src/environments/release.ts b/packages/sandbox/src/environments/release.ts index 23c65bef6db..95dd3faf6c6 100644 --- a/packages/sandbox/src/environments/release.ts +++ b/packages/sandbox/src/environments/release.ts @@ -43,13 +43,12 @@ import { SANDBOX_PATHS, SANDBOX_ROOT_CHECKOUT, type SandboxIdentity, - sandboxAgentToolsDrive, sandboxCheckoutDir, } from "@superset/shared/sandbox/contract"; import { DEFAULT_SANDBOX_REGION, isSandboxRegionId, - type SandboxRegionId, + SANDBOX_REGION_IDS, } from "@superset/shared/sandbox/regions"; import { Sandbox } from "@vercel/sandbox"; @@ -60,6 +59,8 @@ process.env.SKIP_ENV_VALIDATION ??= "1"; const SKIP_IMAGE = process.argv.includes("--skip-image"); const KEEP_OLD = process.argv.includes("--keep-old"); const PRODUCTION = process.argv.includes("--production"); +/** Released bundles whose drives stay in every region, so a rollback to one of them has its drives. */ +const KEPT_RELEASED_BUNDLES = 3; const FOLLOWED_UPSTREAM = ["claude", "codex", "chrome", "opencode", "pi"]; /** The environment the release rebuilds; without it, the release creates one named INTERNAL_NAME. */ const ENVIRONMENT_ID = process.env.SUPERSET_INTERNAL_ENVIRONMENT_ID; @@ -152,34 +153,38 @@ log( await publishBundle(bundle, { dry: false }); log(`bundle: ${bundle.sha256} published`); -// 1b. the bundle's agent-tools drive in every region with an environment, -// before any row points at the bundle so no box mounts one mid-write. Boxes -// fail open to the image's agents, so a failed publish only warns. +// 1b. the bundle's agent-tools drive in every region, before any row points +// at the bundle so no box mounts one mid-write. Every region, so an +// environment created there before the next release finds its drive. A +// region a box is in must have a drive with every required agent. const agentTools = await import("../agent-tools"); const driveHost = agentTools.vercelDriveHost(credentials); -async function environmentRegions(): Promise { +async function environmentRegions(): Promise> { const { db } = await import("@superset/db/client"); const rows = await db.query.environments.findMany({ columns: { region: true }, - where: (row, { isNull: unset }) => unset(row.archivedAt), }); - return [ - ...new Set( - [REGION, ...rows.map((row) => row.region)].filter(isSandboxRegionId), - ), - ]; + return new Set( + [REGION, ...rows.map((row) => row.region)].filter(isSandboxRegionId), + ); } +const neededRegions = await environmentRegions(); +const published = await agentTools.publishAgentToolsDrives({ + host: driveHost, + regions: SANDBOX_REGION_IDS, + bundleSha: bundle.sha256, + assets: bundle.assets, +}); for (const line of agentTools.describeAgentToolsPublish( - await agentTools.publishAgentToolsDrives({ - host: driveHost, - regions: await environmentRegions(), - bundleSha: bundle.sha256, - assets: bundle.assets, - }), + published, + bundle.assets, )) { if (line.startsWith("::warning::")) console.log(line); else log(line); } +const gaps = agentTools.agentToolsGaps(published, neededRegions); +if (gaps.length) + fail(`agent-tools: ${gaps.join("; ")}; no image, golden or row changed`); // 2. image const candidateTag = `release-${bundle.sha256.slice(0, 12)}`; @@ -256,6 +261,7 @@ for (;;) { managedEnv: {}, networkPolicy: "allow-all", vcpus: GOLDEN_BOX_VCPUS, + agentTools: true, }, }); break; @@ -279,6 +285,7 @@ await wakeSandbox({ managedEnv: {}, networkPolicy: "allow-all", vcpus: GOLDEN_BOX_VCPUS, + agentTools: true, }, }); log("golden: host-service up; waiting for the checkout"); @@ -440,6 +447,7 @@ const probeClaim = { managedEnv, networkPolicy, vcpus: GOLDEN_BOX_VCPUS, + agentTools: true, }; log(`probe: provisioning ${probe} as a fork of ${golden}`); await provisionSandbox({ @@ -513,7 +521,7 @@ const { db, dbWs } = await import("@superset/db/client"); const { environments, environmentRepositories } = await import( "@superset/db/schema" ); -const { eq, isNull } = await import("drizzle-orm"); +const { desc, eq } = await import("drizzle-orm"); const { releaseRepositoryWhere } = await import( "./utils/releaseRepositoryWhere" ); @@ -560,7 +568,6 @@ const { internal, bundled } = await dbWs.transaction(async (tx) => { const bundled = await tx .update(environments) .set({ bundleSha: bundle.sha256 }) - .where(isNull(environments.archivedAt)) .returning({ id: environments.id }); const [internal] = previous ? await tx @@ -598,24 +605,38 @@ if (!previous) `rows: created; set SUPERSET_INTERNAL_ENVIRONMENT_ID=${internal.id} so the next release updates it`, ); -try { - const live = await db.query.environments.findMany({ - columns: { region: true, bundleSha: true }, - where: (row, { isNull: unset }) => unset(row.archivedAt), - }); - const { deleted, kept } = await agentTools.pruneAgentToolsDrives({ - host: driveHost, - referenced: new Set( - live.map((row) => sandboxAgentToolsDrive(row.region, row.bundleSha)), - ), - }); - log( - `agent-tools: pruned ${deleted.length}${kept.length ? `; ${kept.join(", ")} still attached, next release retries` : ""}`, - ); -} catch (error) { - console.log( - `::warning::agent-tools: prune failed: ${String(error).slice(0, 300)}`, - ); +// One Vercel project holds every database's drives (staging shares +// production's rows; preview branches copy them), so only the production +// release, whose rows the others mirror, may delete one. +if (!PRODUCTION) { + log("agent-tools: prune skipped; only --production prunes drives"); +} else { + try { + const { sandboxBundles } = await import("@superset/db/schema"); + const rows = await db.query.environments.findMany({ + columns: { region: true, bundleSha: true }, + }); + const recent = await db + .select({ sha: sandboxBundles.sha }) + .from(sandboxBundles) + .orderBy(desc(sandboxBundles.releasedAt)) + .limit(KEPT_RELEASED_BUNDLES); + const { deleted, kept } = await agentTools.pruneAgentToolsDrives({ + host: driveHost, + referenced: agentTools.referencedAgentToolsDrives({ + rows, + recentBundles: [bundle.sha256, ...recent.map((row) => row.sha)], + regions: SANDBOX_REGION_IDS, + }), + }); + log( + `agent-tools: pruned ${deleted.length}${kept.length ? `; ${kept.join(", ")} still attached, next release retries` : ""}`, + ); + } catch (error) { + console.log( + `::warning::agent-tools: prune failed: ${String(error).slice(0, 300)}`, + ); + } } if ( diff --git a/packages/sandbox/src/real-sandbox.ts b/packages/sandbox/src/real-sandbox.ts index b339741cfbf..b04880cffec 100644 --- a/packages/sandbox/src/real-sandbox.ts +++ b/packages/sandbox/src/real-sandbox.ts @@ -6,7 +6,8 @@ * * bun run src/real-sandbox.ts (VERCEL_SANDBOX_*, SANDBOX_GATE_SECRET, CDN_R2_*) * - * Publishes the bundle first so the box can fetch it if the image is older. + * Publishes the bundle first so the box can fetch it if the image is older, + * and its agent-tools drive, which the box mounts. * Leaves the sandbox up for inspection on failure; deletes it on success. */ @@ -47,6 +48,28 @@ if ( const { buildBundle, publishBundle } = await import("./build"); const bundle = buildBundle(); await publishBundle(bundle, { dry: false }); +const agentTools = await import("./agent-tools"); +const drives = [ + { + region: DEFAULT_SANDBOX_REGION, + result: await agentTools.publishAgentToolsDrive({ + host: agentTools.vercelDriveHost(credentials), + region: DEFAULT_SANDBOX_REGION, + bundleSha: bundle.sha256, + assets: bundle.assets, + }), + }, +]; +for (const line of agentTools.describeAgentToolsPublish(drives, bundle.assets)) + log(line); +const gaps = agentTools.agentToolsGaps( + drives, + new Set([DEFAULT_SANDBOX_REGION]), +); +if (gaps.length) { + console.error(`agent-tools: ${gaps.join("; ")}`); + process.exit(1); +} const { deleteSandbox, provisionSandbox, wakeSandbox } = await import( "@superset/trpc/lib/sandbox/vercel" @@ -85,6 +108,7 @@ const claim = { managedEnv: { SUPERSET_REAL_CHECK: "1" }, networkPolicy: "allow-all" as const, vcpus: GOLDEN_BOX_VCPUS, + agentTools: true, }; log( diff --git a/packages/trpc/src/lib/sandbox/agent-tools.test.ts b/packages/trpc/src/lib/sandbox/agent-tools.test.ts index 00fe6117571..44ece4b0191 100644 --- a/packages/trpc/src/lib/sandbox/agent-tools.test.ts +++ b/packages/trpc/src/lib/sandbox/agent-tools.test.ts @@ -1,7 +1,7 @@ import { describe, expect, test } from "bun:test"; import { stub } from "../../../test/stub"; import { posthog } from "../analytics"; -import { agentToolsDriveFor } from "./agent-tools"; +import { agentToolsDriveEnabled } from "./agent-tools"; let answer: () => Promise = () => Promise.resolve(true); const asked: unknown[][] = []; @@ -12,19 +12,12 @@ stub(posthog, { }, }); -const BUNDLE = "0123456789abcdef".repeat(4); const ORG = "6f1c2b8e-1d2a-4c55-9a3e-0b7f5d2c9e11"; -describe("agentToolsDriveFor", () => { - test("names the bundle's drive in the box's region when the org's flag is on", async () => { +describe("agentToolsDriveEnabled", () => { + test("asks the org's flag", async () => { answer = () => Promise.resolve(true); - expect( - await agentToolsDriveFor({ - organizationId: ORG, - region: "sfo1", - bundleSha: BUNDLE, - }), - ).toBe("agent-tools-sfo1-0123456789ab"); + expect(await agentToolsDriveEnabled(ORG)).toBe(true); expect(asked.at(-1)?.[0]).toBe("sandbox-agent-tools-drive"); expect(asked.at(-1)?.[2]).toMatchObject({ groups: { organization: ORG }, @@ -38,13 +31,7 @@ describe("agentToolsDriveFor", () => { () => Promise.reject(new Error("offline")), ]) { answer = next; - expect( - await agentToolsDriveFor({ - organizationId: ORG, - region: "sfo1", - bundleSha: BUNDLE, - }), - ).toBeNull(); + expect(await agentToolsDriveEnabled(ORG)).toBe(false); } }); }); diff --git a/packages/trpc/src/lib/sandbox/agent-tools.ts b/packages/trpc/src/lib/sandbox/agent-tools.ts index 92d069b9d8e..bb14bb7bc0a 100644 --- a/packages/trpc/src/lib/sandbox/agent-tools.ts +++ b/packages/trpc/src/lib/sandbox/agent-tools.ts @@ -1,5 +1,4 @@ import { FEATURE_FLAGS } from "@superset/shared/constants"; -import { sandboxAgentToolsDrive } from "@superset/shared/sandbox/contract"; import { posthog } from "../analytics"; /** Fails closed: PostHog unreachable means no drive. */ @@ -14,13 +13,3 @@ export async function agentToolsDriveEnabled( .catch(() => undefined); return enabled === true; } - -/** The drive of the bundle the box boots, or null while the organization's flag is off. */ -export async function agentToolsDriveFor(args: { - organizationId: string; - region: string; - bundleSha: string; -}): Promise { - if (!(await agentToolsDriveEnabled(args.organizationId))) return null; - return sandboxAgentToolsDrive(args.region, args.bundleSha); -} diff --git a/packages/trpc/src/lib/sandbox/claim.ts b/packages/trpc/src/lib/sandbox/claim.ts index fc496d4d084..8e51b6e5e59 100644 --- a/packages/trpc/src/lib/sandbox/claim.ts +++ b/packages/trpc/src/lib/sandbox/claim.ts @@ -28,7 +28,7 @@ import { resolveEnvironment } from "../../router/environment/resolve-environment import { resolveMachineSize } from "../billing/plan"; import { githubUserConnectionFor, githubUserTokenFor } from "../github-user"; import { sandboxHostSecretFor } from "./access"; -import { agentToolsDriveFor } from "./agent-tools"; +import { agentToolsDriveEnabled } from "./agent-tools"; import { deriveSandboxCredentials, gitAuthorFor } from "./credentials"; import { creatorPlugins } from "./plugins"; import { readRepoHooks } from "./repo-hooks"; @@ -175,11 +175,7 @@ export async function buildSandboxClaim(args: { files, ports: repoHooks?.ports, vcpus: await boxVcpus(args.row), - agentToolsDrive: await agentToolsDriveFor({ - organizationId: args.row.organizationId, - region: environment.region, - bundleSha: environment.bundleSha, - }), + agentTools: await agentToolsDriveEnabled(args.row.organizationId), }, environment: { sourceKind: environment.sourceKind, diff --git a/packages/trpc/src/lib/sandbox/vercel.test.ts b/packages/trpc/src/lib/sandbox/vercel.test.ts index 4a24e20353a..272482b60dd 100644 --- a/packages/trpc/src/lib/sandbox/vercel.test.ts +++ b/packages/trpc/src/lib/sandbox/vercel.test.ts @@ -6,14 +6,16 @@ import { setSystemTime, test, } from "bun:test"; -import { Sandbox } from "@vercel/sandbox"; +import { Drive, Sandbox } from "@vercel/sandbox"; import { stub } from "../../../test/stub"; import { agentToolsMounts, findSandboxSession, listActiveWorkspaceSandboxes, + type SandboxClaim, SESSION_TIMEOUT_MS, topUpSandboxSession, + wakeSandbox, } from "./vercel"; type ListOptions = NonNullable[0]>; @@ -216,3 +218,52 @@ describe("agentToolsMounts", () => { expect(asked).toBe(2); }); }); + +describe("wakeSandbox", () => { + const asked: string[] = []; + const updates: unknown[] = []; + stub(Sandbox, { + get: async () => ({ + status: "stopped", + region: "iad1", + vcpus: 2, + mounts: {}, + domain: () => "https://host.example", + update: async (patch: unknown) => { + updates.push(patch); + }, + }), + }); + stub(Drive, { + get: async ({ name }: { name: string }) => { + asked.push(name); + return { currentSandboxName: undefined }; + }, + }); + + test("mounts the drive of the box's own region, not its environment's", async () => { + const stop = new Error("stop before resume"); + await expect( + wakeSandbox({ + providerSandboxId: "sbx", + claim: { + vcpus: 2, + agentTools: true, + identity: { SUPERSET_BUNDLE_SHA: "abcdef012345".repeat(5) }, + } as unknown as SandboxClaim, + beforeResume: () => Promise.reject(stop), + }), + ).rejects.toBe(stop); + expect(asked).toEqual(["agent-tools-iad1-abcdef012345"]); + expect(updates).toEqual([ + { + mounts: { + "/opt/agents": { + drive: "agent-tools-iad1-abcdef012345", + mode: "snapshot", + }, + }, + }, + ]); + }); +}); diff --git a/packages/trpc/src/lib/sandbox/vercel.ts b/packages/trpc/src/lib/sandbox/vercel.ts index 786002a5c37..911d4b286f7 100644 --- a/packages/trpc/src/lib/sandbox/vercel.ts +++ b/packages/trpc/src/lib/sandbox/vercel.ts @@ -17,6 +17,7 @@ import { SANDBOX_PORTS, SANDBOX_PUBLISHED_PORTS, type SandboxIdentity, + sandboxAgentToolsDrive, } from "@superset/shared/sandbox/contract"; import { APIError, @@ -121,8 +122,8 @@ export interface SandboxClaim { /** Ports the workspace's repository asks to publish, beside the platform's. */ ports?: readonly number[]; vcpus: number; - /** The bundle's agent-tools drive to mount read-only; absent keeps the image's agents. */ - agentToolsDrive?: string | null; + /** Mount the bundle's agent-tools drive read-only; absent keeps the image's agents. */ + agentTools?: boolean; } /** Vercel allows 5 tags per sandbox. */ @@ -176,6 +177,16 @@ export async function agentToolsMounts( return { [SANDBOX_PATHS.agents]: { drive: driveName, mode: "snapshot" } }; } +/** The box's own region names the drive: a workspace stays where it was created when its environment moves. */ +export function agentToolsDriveName( + claim: Pick, + region: string, +): string | null { + return claim.agentTools + ? sandboxAgentToolsDrive(region, claim.identity.SUPERSET_BUNDLE_SHA) + : null; +} + function publishedPorts(extra: readonly number[] = []): number[] { return [...new Set([...SANDBOX_PUBLISHED_PORTS, ...extra])]; } @@ -261,7 +272,9 @@ export async function provisionSandbox(args: { keepLastSnapshots: { count: 1 }, tags: sandboxTags(kind, args.claim.identity), }; - const mounts = await agentToolsMounts(args.claim.agentToolsDrive); + const mounts = await agentToolsMounts( + agentToolsDriveName(args.claim, args.environment.region), + ); const sandbox = (await getSandbox(args.name)) ?? (await Sentry.startSpan( @@ -522,7 +535,9 @@ export async function wakeSandbox(args: { if (!wasRunning) { const { [SANDBOX_PATHS.agents]: previous, ...others } = sandbox.mounts ?? {}; - const agents = await agentToolsMounts(args.claim.agentToolsDrive); + const agents = await agentToolsMounts( + agentToolsDriveName(args.claim, sandbox.region), + ); if (agents || previous) { await sandbox .update({ mounts: { ...others, ...agents } })