From 363c5befe41905b7c613673893e49613ffffda9e Mon Sep 17 00:00:00 2001 From: Satya Patel <14907857+saddlepaddle@users.noreply.github.com> Date: Wed, 18 Nov 2026 05:29:24 +0000 Subject: [PATCH 1/3] fix(cloud): only a wake or a create gets a handle that resumes a box (M9) Most calls on a @vercel/sandbox handle resume a stopped session and retry. getSandbox now returns a type with only the members that cannot resume; getSandboxToWake is the full handle, used by wakeSandbox and provisionSandbox. - promote snapshots a running source through its session and builds a stopped source's golden from its stop snapshot, instead of resuming it - applySandboxPolicy and topUpSandboxSession treat a session that stopped under the call as not running - docs: measured that HTTP to a stopped box's port answers 410 and resumes nothing --- docs/cloud-sandbox-mismatches.md | 17 +- packages/trpc/src/lib/sandbox/vercel.test.ts | 184 ++++++++++++++++--- packages/trpc/src/lib/sandbox/vercel.ts | 142 +++++++++----- 3 files changed, 267 insertions(+), 76 deletions(-) diff --git a/docs/cloud-sandbox-mismatches.md b/docs/cloud-sandbox-mismatches.md index 40cd448fefe..803c699ddca 100644 --- a/docs/cloud-sandbox-mismatches.md +++ b/docs/cloud-sandbox-mismatches.md @@ -790,12 +790,17 @@ timeout a moment later is resumed by the next call on that object, and no code chose to start it. Measured: 9 sessions in a week started 0.8–1.5 s after a timeout stop. `wakeSandbox` passes `onResume` to `Sandbox.get` and runs `beforeResume` (which authorizes the boot) for such a session; a refused one is -stopped at once. A top-up goes through the session, which never resumes. -Other callers on a box read `running` are covered only by the reconcile, which -adopts any session that has no boot row within one run: promote's -`source.snapshot()` (wrapped in `withResume`), and `applySandboxPolicy`, whose -`update` does not resume in SDK 4.0.0 but would if a later SDK wrapped it. -`provisionSandbox` on a box it already made is covered by its own boot. +stopped at once. **What we did:** only `wakeSandbox` and `provisionSandbox` +get a full handle (`getSandboxToWake` in `vercel.ts`). Every other caller gets +`getSandbox`, typed to the members that cannot resume (`stop`, `update`, +`delete`, `listSessions`, `currentSession()` and reads), and acts on a running +box through the session, whose calls answer 410 or 422 on a stopped one +instead. Top-up and the credential refresh treat that answer as "not +running"; promote builds a stopped source's golden from the snapshot its +stop took (measured: a create from that automatic snapshot has the stopped +box's files). An HTTP request to a stopped box's port does not resume it +either: measured, it answers `410 This sandbox was stopped` and no session +starts, so the archive hook and other host-service calls cannot wake a box. **`stop()` returns before the stop's snapshot is current.** The sandbox is still `stopping` when the call resolves, and a fork taken then boots from diff --git a/packages/trpc/src/lib/sandbox/vercel.test.ts b/packages/trpc/src/lib/sandbox/vercel.test.ts index c884fdeb54e..d527716874b 100644 --- a/packages/trpc/src/lib/sandbox/vercel.test.ts +++ b/packages/trpc/src/lib/sandbox/vercel.test.ts @@ -6,16 +6,21 @@ import { setSystemTime, test, } from "bun:test"; -import { Drive, Sandbox } from "@vercel/sandbox"; +import { APIError, Drive, Sandbox } from "@vercel/sandbox"; import { stub } from "../../../test/stub"; import { agentToolsMounts, + applySandboxPolicy, + deleteSandbox, + describeSandbox, findSandboxSession, listActiveWorkspaceSandboxes, listSandboxSessions, promoteSandboxToEnvironment, type SandboxClaim, SESSION_TIMEOUT_MS, + sandboxExists, + stopSandbox, topUpSandboxSession, wakeSandbox, } from "./vercel"; @@ -153,58 +158,181 @@ describe("listSandboxSessions", () => { }); }); -describe("promoteSandboxToEnvironment", () => { - const sourceCalls: string[] = []; - const source = { - status: "running", - region: "iad1", +/** + * A box the way the SDK presents it: the handle keeps the status it was read + * with, a session call on a stopped session fails, and a handle call resumes + * it first, which `resumes` records. + */ +function fakeBox(read: "running" | "stopped") { + const box = { + live: read, + snapshotId: "snap_stop_0", + resumes: [] as string[], + }; + const stoppedError = () => new APIError(new Response(null, { status: 410 })); + const session = { + sessionId: "sess_A", vcpus: 4, - currentSession: () => ({ sessionId: "sess_A" }), - snapshot: async () => { - sourceCalls.push("snapshot"); - return { snapshotId: "snap_1" }; + update: async () => { + if (box.live !== "running") throw stoppedError(); }, - writeFiles: async () => { - sourceCalls.push("writeFiles"); + extendTimeout: async () => { + if (box.live !== "running") throw stoppedError(); }, - runCommand: async () => { - sourceCalls.push("runCommand"); + snapshot: async () => { + if (box.live !== "running") throw stoppedError(); + box.live = "stopped"; + return { snapshotId: "snap_now" }; }, - mkDir: async () => { - sourceCalls.push("mkDir"); + }; + const resumingCall = (method: string) => async () => { + if (box.live !== "running") { + box.resumes.push(method); + box.live = "running"; + } + return method === "snapshot" ? session.snapshot() : undefined; + }; + const handle = () => ({ + name: "ws-box", + status: box.live, + region: "iad1", + vcpus: 4, + currentSnapshotId: box.snapshotId, + expiresAt: new Date(NOW + HOUR / 2), + domain: () => "https://box.test", + currentSession: () => session, + update: async (params: { networkPolicy?: unknown }) => { + if (params.networkPolicy && handleStatus === "running") { + await session.update(); + } + }, + delete: async () => {}, + listSessions: async () => (async function* () {})(), + snapshot: resumingCall("snapshot"), + runCommand: resumingCall("runCommand"), + writeFiles: resumingCall("writeFiles"), + extendTimeout: resumingCall("extendTimeout"), + }); + let handleStatus: string = read; + const stopNow = () => { + box.live = "stopped"; + box.snapshotId = "snap_stop_1"; + }; + return { + box, + stopNow, + get: async () => { + const read = handle(); + handleStatus = read.status; + return read; }, }; +} + +/** The idle timeout ends the session after the first read, before any call. */ +function stopsAfterRead(fake: ReturnType) { + const read = fake.get; + let reads = 0; + fake.get = async () => { + const handle = await read(); + if (++reads === 1) fake.stopNow(); + return handle; + }; + return fake; +} + +describe("a caller that is not a wake leaves a stopped box stopped", () => { + let fake = fakeBox("stopped"); + stub(Sandbox, { get: async () => fake.get() }); + + test("reads, stops, deletes, session lists, policy and top-up", async () => { + fake = fakeBox("stopped"); + await describeSandbox("ws-box"); + await sandboxExists("ws-box"); + await stopSandbox("ws-box"); + await listSandboxSessions("ws-box", NOW - HOUR); + await findSandboxSession("ws-box", null); + expect( + await applySandboxPolicy({ + providerSandboxId: "ws-box", + networkPolicy: {}, + }), + ).toBe("not-running"); + expect(await topUpSandboxSession("ws-box")).toBeNull(); + await deleteSandbox("ws-box"); + expect(fake.box).toMatchObject({ live: "stopped", resumes: [] }); + }); + + test("policy and top-up find the box stopped since they read it running", async () => { + fake = stopsAfterRead(fakeBox("running")); + expect( + await applySandboxPolicy({ + providerSandboxId: "ws-box", + networkPolicy: {}, + }), + ).toBe("not-running"); + expect(fake.box).toMatchObject({ live: "stopped", resumes: [] }); + + fake = stopsAfterRead(fakeBox("running")); + expect(await topUpSandboxSession("ws-box")).toBeNull(); + expect(fake.box).toMatchObject({ live: "stopped", resumes: [] }); + }); +}); + +describe("promoteSandboxToEnvironment", () => { + let fake = fakeBox("running"); + let sourceSnapshot: string | undefined; + let createTags: unknown; const golden = { currentSnapshotId: "snap_golden_0", runCommand: async () => {}, stop: async () => {}, }; - let createTags: unknown; stub(Sandbox, { get: async ({ name }: { name: string }) => name === "ws-box" - ? source + ? fake.get() : { status: "stopped", currentSnapshotId: "snap_golden_1" }, - create: async (params: { tags: unknown }) => { + create: async (params: { + tags: unknown; + source: { snapshotId: string }; + }) => { createTags = params.tags; + sourceSnapshot = params.source.snapshotId; return golden; }, }); + const promote = () => + promoteSandboxToEnvironment({ + sourceSandbox: "ws-box", + goldenName: "env-1", + organizationId: "org_1", + }); - test("ends the source's session for the snapshot and leaves its restart to the caller", async () => { - expect( - await promoteSandboxToEnvironment({ - sourceSandbox: "ws-box", - goldenName: "env-1", - organizationId: "org_1", - }), - ).toEqual({ + test("snapshots a running source through its session and leaves its restart to the caller", async () => { + fake = fakeBox("running"); + expect(await promote()).toEqual({ goldenName: "env-1", region: "iad1", stopped: { sessionId: "sess_A" }, }); - expect(sourceCalls).toEqual(["snapshot"]); + expect(sourceSnapshot).toBe("snap_now"); expect(createTags).toEqual({ kind: "environment", org: "org_1" }); + expect(fake.box.resumes).toEqual([]); + }); + + test("builds from a stopped source's last stop without resuming it", async () => { + fake = fakeBox("stopped"); + expect((await promote()).stopped).toBeNull(); + expect(sourceSnapshot).toBe("snap_stop_0"); + expect(fake.box).toMatchObject({ live: "stopped", resumes: [] }); + }); + + test("a source that stops before its snapshot is built from that stop's snapshot", async () => { + fake = stopsAfterRead(fakeBox("running")); + await promote(); + expect(sourceSnapshot).toBe("snap_stop_1"); + expect(fake.box).toMatchObject({ live: "stopped", resumes: [] }); }); }); diff --git a/packages/trpc/src/lib/sandbox/vercel.ts b/packages/trpc/src/lib/sandbox/vercel.ts index 812681d0020..7cae6c7982b 100644 --- a/packages/trpc/src/lib/sandbox/vercel.ts +++ b/packages/trpc/src/lib/sandbox/vercel.ts @@ -73,7 +73,7 @@ function isNotFound(error: unknown): boolean { export class SandboxUnavailableError extends Error { constructor( readonly providerSandboxId: string, - cause: unknown, + cause?: unknown, ) { super(`Sandbox ${providerSandboxId} is unavailable`, { cause }); } @@ -87,7 +87,7 @@ function isUnavailable(error: unknown): boolean { } /** Vercel starts a new session on every resume, so the id tells a stale record from a fresh one. */ -function sessionIdOf(sandbox: Sandbox): string | null { +function sessionIdOf(sandbox: Pick): string | null { try { return sandbox.currentSession().sessionId; } catch { @@ -95,15 +95,61 @@ function sessionIdOf(sandbox: Sandbox): string | null { } } -async function getSandbox(name: string): Promise { +/** The session ended under the call: the SDK's own resume-and-retry case. */ +function isSessionStopped(error: unknown): boolean { + if (!(error instanceof APIError)) return false; + if (error.response.status === 410) return true; + const code = (error.json as { error?: { code?: string } } | undefined)?.error + ?.code; + return ( + error.response.status === 422 && + (code === "sandbox_stopping" || code === "sandbox_snapshotting") + ); +} + +/** + * Most of a handle's calls resume a stopped session and retry. Only a wake + * or a create may do that, so every other caller gets the members that + * cannot, and acts on a running session through `currentSession()`, whose + * calls fail on a stopped one instead. + */ +type NonResumingSandbox = Pick< + Sandbox, + | "name" + | "status" + | "region" + | "vcpus" + | "expiresAt" + | "currentSnapshotId" + | "domain" + | "currentSession" + | "update" + | "stop" + | "delete" + | "listSessions" +>; + +/** A handle whose calls resume a stopped session: for a wake or a create only. */ +async function getSandboxToWake( + name: string, + onResume?: (sandbox: Sandbox) => Promise, +): Promise { try { - return await Sandbox.get({ ...credentials(), name, resume: false }); + return await Sandbox.get({ + ...credentials(), + name, + resume: false, + onResume, + }); } catch (error) { if (isNotFound(error)) return null; throw error; } } +const getSandbox: (name: string) => Promise = + getSandboxToWake; + export interface SandboxEnvironment { sourceKind: "image" | "fork"; sourceRef: string; @@ -281,7 +327,7 @@ export async function provisionSandbox(args: { agentToolsDriveName(args.claim, args.environment.region), ); const sandbox = - (await getSandbox(args.name)) ?? + (await getSandboxToWake(args.name)) ?? (await Sentry.startSpan( { name: @@ -412,16 +458,14 @@ export async function applySandboxPolicy(args: { providerSandboxId: string; networkPolicy: NetworkPolicy; }): Promise<"applied" | "not-running"> { - const sandbox = await Sandbox.get({ - ...credentials(), - name: args.providerSandboxId, - resume: false, - }).catch((error: unknown) => { - if (isNotFound(error)) return null; + const sandbox = await getSandbox(args.providerSandboxId); + if (sandbox?.status !== "running") return "not-running"; + try { + await sandbox.update({ networkPolicy: args.networkPolicy }); + } catch (error) { + if (isSessionStopped(error)) return "not-running"; throw error; - }); - if (!sandbox || sandbox.status !== "running") return "not-running"; - await sandbox.update({ networkPolicy: args.networkPolicy }); + } return "applied"; } @@ -435,11 +479,8 @@ export async function describeSandbox(providerSandboxId: string): Promise<{ sessionId: string | null; }> { try { - const sandbox = await Sandbox.get({ - ...credentials(), - name: providerSandboxId, - resume: false, - }); + const sandbox = await getSandbox(providerSandboxId); + if (!sandbox) throw new SandboxUnavailableError(providerSandboxId); const running = sandbox.status === "running"; return { hostTarget: sandbox.domain(HOST_SERVICE_PORT), @@ -470,7 +511,12 @@ export async function topUpSandboxSession( const extension = Date.now() + SESSION_TIMEOUT_MS - sandbox.expiresAt.getTime(); if (extension > MIN_TOP_UP_MS) { - await sandbox.currentSession().extendTimeout(extension); + try { + await sandbox.currentSession().extendTimeout(extension); + } catch (error) { + if (isSessionStopped(error)) return null; + throw error; + } } return sandbox.expiresAt ? { expiresAt: sandbox.expiresAt, sessionId: sessionIdOf(sandbox) } @@ -524,17 +570,15 @@ export async function wakeSandbox(args: { try { let wasRunning = false; let resumed = null as { vcpus: number; sessionId: string | null } | null; - const sandbox = await Sandbox.get({ - ...credentials(), - name: args.providerSandboxId, - resume: false, - // Any call on a Sandbox object resumes a stopped session and retries. - onResume: async (box) => { + const sandbox = await getSandboxToWake( + args.providerSandboxId, + async (box) => { if (!wasRunning) return; const session = box.currentSession(); resumed = { vcpus: session.vcpus, sessionId: session.sessionId }; }, - }); + ); + if (!sandbox) throw new SandboxUnavailableError(args.providerSandboxId); const hostTarget = sandbox.domain(HOST_SERVICE_PORT); wasRunning = sandbox.status === "running"; let vcpus = sandbox.vcpus ?? args.claim.vcpus; @@ -679,7 +723,8 @@ export async function stripWorkspaceIdentity(sandbox: Sandbox): Promise { export async function stopAndSnapshot( name: string, ): Promise<{ sessionId: string | null }> { - const sandbox = await Sandbox.get({ ...credentials(), name, resume: false }); + const sandbox = await getSandbox(name); + if (!sandbox) throw new SandboxUnavailableError(name); const before = sandbox.currentSnapshotId; const sessionId = sessionIdOf(sandbox); await sandbox.stop(); @@ -704,18 +749,14 @@ export async function promoteSandboxToEnvironment(args: { /** The session the snapshot ended; null when the source was not running. */ stopped: { sessionId: string | null } | null; }> { - const source = await Sandbox.get({ - ...credentials(), - name: args.sourceSandbox, - resume: false, - }); + const source = await getSandbox(args.sourceSandbox); + if (!source) throw new SandboxUnavailableError(args.sourceSandbox); const stopped = source.status === "running" ? { sessionId: sessionIdOf(source) } : null; - const snapshot = await source.snapshot(); const golden = await Sandbox.create({ ...credentials(), name: args.goldenName, - source: { type: "snapshot", snapshotId: snapshot.snapshotId }, + source: { type: "snapshot", snapshotId: await snapshotNow(source) }, ports: publishedPorts(), timeout: GOLDEN_SESSION_TIMEOUT_MS, region: source.region as SandboxRegion, @@ -733,6 +774,24 @@ export async function promoteSandboxToEnvironment(args: { return { goldenName: args.goldenName, region: source.region, stopped }; } +/** + * A running box's disk, snapshotted now; a stopped box's disk is the snapshot + * its stop took, so it is not resumed to take another. + */ +async function snapshotNow(source: NonResumingSandbox): Promise { + if (source.status === "running") { + try { + return (await source.currentSession().snapshot()).snapshotId; + } catch (error) { + if (!isSessionStopped(error)) throw error; + } + } + return waitForStopSnapshot( + source.name, + source.status === "stopped" ? undefined : source.currentSnapshotId, + ); +} + /** * stop() returns while the sandbox is still `stopping`; a fork taken before * the stop's snapshot is current boots from whatever was current before. @@ -740,16 +799,15 @@ export async function promoteSandboxToEnvironment(args: { export async function waitForStopSnapshot( name: string, previous: string | undefined, -): Promise { +): Promise { const deadline = Date.now() + 3 * 60 * 1000; while (Date.now() < deadline) { - const sandbox = await Sandbox.get({ - ...credentials(), - name, - resume: false, - }); + const sandbox = await getSandbox(name); + if (!sandbox) throw new SandboxUnavailableError(name); const current = sandbox.currentSnapshotId; - if (sandbox.status === "stopped" && current && current !== previous) return; + if (sandbox.status === "stopped" && current && current !== previous) { + return current; + } await new Promise((resolve) => setTimeout(resolve, 2000)); } throw new Error(`${name} did not snapshot in time`); From 3a1f6bb25aa0dd094471597251708d7951b34854 Mon Sep 17 00:00:00 2001 From: Satya Patel <14907857+saddlepaddle@users.noreply.github.com> Date: Wed, 18 Nov 2026 05:29:24 +0000 Subject: [PATCH 2/3] fix(cloud): applySignIns leaves a stopped box for its next wake; staff paths cannot start a box (M9) The next wake writes the sign-in files, re-applies the credential rules and records the credential digest, so a stopped box needs no wake to get them. A test walks the imports of router/admin, router/support and every file with adminProcedure and fails if any reaches the wake path or calls a function that starts a box. --- packages/trpc/src/router/admin/admin.test.ts | 62 +++++++++++++++++++ .../cloud-workspace/cloud-workspace.test.ts | 24 +++++++ .../router/cloud-workspace/cloud-workspace.ts | 4 ++ 3 files changed, 90 insertions(+) create mode 100644 packages/trpc/src/router/admin/admin.test.ts diff --git a/packages/trpc/src/router/admin/admin.test.ts b/packages/trpc/src/router/admin/admin.test.ts new file mode 100644 index 00000000000..5134a49bf24 --- /dev/null +++ b/packages/trpc/src/router/admin/admin.test.ts @@ -0,0 +1,62 @@ +import { describe, expect, test } from "bun:test"; +import { existsSync, readFileSync } from "node:fs"; +import { dirname, relative, resolve } from "node:path"; + +const SRC = resolve(import.meta.dir, "../.."); +const WAKE_PATH = resolve(SRC, "router/cloud-workspace/wake.ts"); +const SANDBOX_PROVIDER = resolve(SRC, "lib/sandbox/vercel.ts"); +const STARTS_A_BOX = + /\b(wakeCloudWorkspace|restartCloudWorkspace|wakeAfterStop|wakeSandbox|provisionSandbox|topUpSandboxSession|getSandboxToWake)\(/; + +const transpilers = { + ts: new Bun.Transpiler({ loader: "ts" }), + tsx: new Bun.Transpiler({ loader: "tsx" }), +}; + +function resolveImport(from: string, specifier: string): string | null { + if (!specifier.startsWith(".")) return null; + const base = resolve(dirname(from), specifier); + return ( + [`${base}.ts`, `${base}.tsx`, `${base}/index.ts`, base].find( + (path) => /\.tsx?$/.test(path) && existsSync(path), + ) ?? null + ); +} + +function reachableFrom(entries: string[]): Set { + const reached = new Set(); + const pending = [...entries]; + for (let file = pending.pop(); file; file = pending.pop()) { + if (reached.has(file)) continue; + reached.add(file); + const transpiler = file.endsWith(".tsx") ? transpilers.tsx : transpilers.ts; + for (const { path } of transpiler.scanImports(readFileSync(file, "utf8"))) { + const target = resolveImport(file, path); + if (target) pending.push(target); + } + } + return reached; +} + +const staffEntries = [ + ...new Bun.Glob("router/**/*.ts").scanSync({ cwd: SRC, absolute: true }), +] + .filter((file) => !file.endsWith(".test.ts")) + .filter( + (file) => + /\/router\/(admin|support)\//.test(file) || + readFileSync(file, "utf8").includes("adminProcedure"), + ); + +describe("staff tooling", () => { + test("can never start a customer's box", () => { + expect(staffEntries.length).toBeGreaterThan(1); + const reached = reachableFrom(staffEntries); + expect(reached.has(WAKE_PATH)).toBe(false); + const starters = [...reached] + .filter((file) => file !== SANDBOX_PROVIDER) + .filter((file) => STARTS_A_BOX.test(readFileSync(file, "utf8"))) + .map((file) => relative(SRC, file)); + expect(starters).toEqual([]); + }); +}); diff --git a/packages/trpc/src/router/cloud-workspace/cloud-workspace.test.ts b/packages/trpc/src/router/cloud-workspace/cloud-workspace.test.ts index ea6331a1a1b..50c1f03506f 100644 --- a/packages/trpc/src/router/cloud-workspace/cloud-workspace.test.ts +++ b/packages/trpc/src/router/cloud-workspace/cloud-workspace.test.ts @@ -29,6 +29,7 @@ const ROW = { let running = false; let boxVcpus = 8; let restartedWith: string[] = []; +let woken: string[] = []; const ENVIRONMENT_ID = "5e6f7a8b-9c0d-4e1f-8a2b-3c4d5e6f7a8b"; let inserted: unknown[] = []; @@ -128,6 +129,13 @@ stub(sandbox, { workspaceBranchName: () => "paradev/ws-1", }); stub(wake, { + wakeCloudWorkspace: async (row: { id: string }) => { + woken.push(row.id); + return { + hostTarget: "https://box-1.example", + agentCredentialsChanged: false, + }; + }, restartCloudWorkspace: async (row: { machineSize: string }) => { restartedWith.push(row.machineSize); boxVcpus = 16; @@ -160,6 +168,7 @@ beforeEach(() => { running = false; boxVcpus = 8; restartedWith = []; + woken = []; recordedAsleep = []; inserted = []; queuedJobs = []; @@ -253,6 +262,21 @@ describe("cloudWorkspace.setMachineSize", () => { }); }); +describe("cloudWorkspace.applySignIns", () => { + test("hands sign-ins to a running box, and leaves a stopped one for its next wake", async () => { + expect( + await caller.cloudWorkspace.applySignIns({ id: WORKSPACE_ID }), + ).toEqual({ applied: false }); + expect(woken).toEqual([]); + + running = true; + expect( + await caller.cloudWorkspace.applySignIns({ id: WORKSPACE_ID }), + ).toEqual({ applied: true }); + expect(woken).toEqual([WORKSPACE_ID]); + }); +}); + describe("cloudWorkspace.sleep", () => { test("records asleep only for the session it stopped", async () => { running = true; diff --git a/packages/trpc/src/router/cloud-workspace/cloud-workspace.ts b/packages/trpc/src/router/cloud-workspace/cloud-workspace.ts index 934f0a4820d..852906ff626 100644 --- a/packages/trpc/src/router/cloud-workspace/cloud-workspace.ts +++ b/packages/trpc/src/router/cloud-workspace/cloud-workspace.ts @@ -664,6 +664,10 @@ export const cloudWorkspaceRouter = { .input(z.object({ id: z.string().uuid() })) .mutation(async ({ ctx, input }) => { const row = await loadReadyWorkspace(ctx, input.id); + // A stopped box takes the current sign-ins when it next wakes. + if (!(await describeSandbox(row.providerSandboxId)).running) { + return { applied: false }; + } await wakeCloudWorkspace(row, { adoptSignIns: true }); return { applied: true }; }), From 84db333de0e460c4e61cb893b871d3c9d2576a9f Mon Sep 17 00:00:00 2001 From: Satya Patel <14907857+saddlepaddle@users.noreply.github.com> Date: Wed, 18 Nov 2026 05:43:57 +0000 Subject: [PATCH 3/3] fix(cloud): review fixes for M9 - applySignIns: the wake itself leaves a stopped box (onlyIfRunning on wakeSandbox, through adoptSignIns), so a box that stops while the claim is built is not booted - promote: a source that is stopped with no snapshot, or in a state that will not produce one, fails at once instead of polling - lint: @vercel/sandbox may be imported only in packages/trpc/src/lib/sandbox and packages/sandbox - the staff test is router/staff-cannot-wake.test.ts and no longer lists topUpSandboxSession --- biome.jsonc | 15 +- packages/sandbox/src/real-sandbox.ts | 1 + packages/trpc/src/lib/sandbox/vercel.test.ts | 36 ++++- packages/trpc/src/lib/sandbox/vercel.ts | 15 +- .../cloud-workspace/cloud-workspace.test.ts | 8 +- .../router/cloud-workspace/cloud-workspace.ts | 8 +- .../trpc/src/router/cloud-workspace/wake.ts | 133 ++++++++++-------- ...dmin.test.ts => staff-cannot-wake.test.ts} | 4 +- 8 files changed, 139 insertions(+), 81 deletions(-) rename packages/trpc/src/router/{admin/admin.test.ts => staff-cannot-wake.test.ts} (94%) diff --git a/biome.jsonc b/biome.jsonc index 66a352785b7..6150ed587b2 100644 --- a/biome.jsonc +++ b/biome.jsonc @@ -52,7 +52,8 @@ "child_process": { "importNames": ["execSync", "spawnSync", "execFileSync"], "message": "Sync subprocesses freeze the serving event loop (Electron main / host-service / pty-daemon) until the child exits — every request queues behind them. Prefer async spawn/execFile: the caller awaits the same result, but the loop keeps serving while the child runs. Git reads belong in the worker pools." - } + }, + "@vercel/sandbox": "Most calls on a raw sandbox handle resume a stopped box. Use the functions in packages/trpc/src/lib/sandbox, where only a wake or a create may resume one." } } } @@ -92,6 +93,18 @@ } } }, + { + // The one place that holds sandbox handles, and the release tooling that + // builds its own boxes. Neither runs on a desktop or host-service loop. + "includes": ["packages/trpc/src/lib/sandbox/**", "packages/sandbox/**"], + "linter": { + "rules": { + "style": { + "noRestrictedImports": "off" + } + } + } + }, { // Legacy on-loop sync-subprocess call sites, frozen by the // no-*-blocking ratchet tests — delete entries here as they're fixed. diff --git a/packages/sandbox/src/real-sandbox.ts b/packages/sandbox/src/real-sandbox.ts index b04880cffec..1ce5208b71a 100644 --- a/packages/sandbox/src/real-sandbox.ts +++ b/packages/sandbox/src/real-sandbox.ts @@ -139,6 +139,7 @@ for (;;) { } } const woken = await wakeSandbox({ providerSandboxId: name, claim }); +if (!woken) throw new Error(`${name} did not wake`); log( `host-service answered at ${woken.hostTarget} (${Date.now() - started} ms since start)`, ); diff --git a/packages/trpc/src/lib/sandbox/vercel.test.ts b/packages/trpc/src/lib/sandbox/vercel.test.ts index d527716874b..d2f832f0742 100644 --- a/packages/trpc/src/lib/sandbox/vercel.test.ts +++ b/packages/trpc/src/lib/sandbox/vercel.test.ts @@ -18,6 +18,7 @@ import { listSandboxSessions, promoteSandboxToEnvironment, type SandboxClaim, + SandboxUnavailableError, SESSION_TIMEOUT_MS, sandboxExists, stopSandbox, @@ -328,6 +329,15 @@ describe("promoteSandboxToEnvironment", () => { expect(fake.box).toMatchObject({ live: "stopped", resumes: [] }); }); + test("a stopped source with no snapshot to build from fails at once", async () => { + fake = fakeBox("stopped"); + fake.box.snapshotId = undefined; + sourceSnapshot = undefined; + await expect(promote()).rejects.toBeInstanceOf(SandboxUnavailableError); + expect(sourceSnapshot).toBeUndefined(); + expect(fake.box.resumes).toEqual([]); + }); + test("a source that stops before its snapshot is built from that stop's snapshot", async () => { fake = stopsAfterRead(fakeBox("running")); await promote(); @@ -340,6 +350,7 @@ describe("wakeSandbox", () => { type Box = { memoryMb: number; vcpus: number }; let running: boolean; let onResume: (sandbox: unknown) => Promise; + const calls: string[] = []; const box = { get status() { return running ? "running" : "stopped"; @@ -348,13 +359,19 @@ describe("wakeSandbox", () => { vcpus: 4, domain: () => "https://box.test", currentSession: () => ({ vcpus: 8, sessionId: "sess_new" }), - update: async () => {}, - writeFiles: async () => {}, + update: async () => { + calls.push("update"); + }, + writeFiles: async () => { + calls.push("writeFiles"); + }, // The SDK's withResume: a call on a stopped session resumes it, then retries. extendTimeout: async () => { + calls.push("extendTimeout"); await onResume(box); }, runCommand: async () => { + calls.push("runCommand"); await onResume(box); throw new Error("stop before settle"); }, @@ -406,6 +423,21 @@ describe("wakeSandbox", () => { running = false; expect(await wake()).toEqual([{ memoryMb: 8192, vcpus: 4 }]); }); + + test("a wake only for a running box leaves a box that stopped since the caller looked", async () => { + running = false; + calls.length = 0; + const woken = await wakeSandbox({ + providerSandboxId: "ws-box", + claim: { vcpus: 4, identity, networkPolicy: {} } as never, + onlyIfRunning: true, + beforeResume: async () => { + calls.push("beforeResume"); + }, + }); + expect(woken).toBeNull(); + expect(calls).toEqual([]); + }); }); describe("listActiveWorkspaceSandboxes", () => { diff --git a/packages/trpc/src/lib/sandbox/vercel.ts b/packages/trpc/src/lib/sandbox/vercel.ts index 7cae6c7982b..359bfc7007e 100644 --- a/packages/trpc/src/lib/sandbox/vercel.ts +++ b/packages/trpc/src/lib/sandbox/vercel.ts @@ -547,6 +547,8 @@ async function hostServiceAnswers(hostTarget: string): Promise { export async function wakeSandbox(args: { providerSandboxId: string; claim: SandboxClaim; + /** A stopped box is left stopped, and the wake answers null. */ + onlyIfRunning?: boolean; /** * Runs once the box is known to be stopped, before anything resumes it; or, * for a box read running that a call here resumed (`resumedSession`), once @@ -566,7 +568,7 @@ export async function wakeSandbox(args: { answer: HostAnswer; expiresAt: Date | null; sessionId: string | null; -}> { +} | null> { try { let wasRunning = false; let resumed = null as { vcpus: number; sessionId: string | null } | null; @@ -581,6 +583,7 @@ export async function wakeSandbox(args: { if (!sandbox) throw new SandboxUnavailableError(args.providerSandboxId); const hostTarget = sandbox.domain(HOST_SERVICE_PORT); wasRunning = sandbox.status === "running"; + if (args.onlyIfRunning && !wasRunning) return null; let vcpus = sandbox.vcpus ?? args.claim.vcpus; if (!wasRunning && vcpus !== args.claim.vcpus) { await Sentry.startSpan({ name: "resize", op: "sandbox.provider" }, () => @@ -785,11 +788,13 @@ async function snapshotNow(source: NonResumingSandbox): Promise { } catch (error) { if (!isSessionStopped(error)) throw error; } + } else if (source.status === "stopped") { + if (source.currentSnapshotId) return source.currentSnapshotId; + throw new SandboxUnavailableError(source.name); + } else if (source.status !== "stopping" && source.status !== "snapshotting") { + throw new SandboxUnavailableError(source.name); } - return waitForStopSnapshot( - source.name, - source.status === "stopped" ? undefined : source.currentSnapshotId, - ); + return waitForStopSnapshot(source.name, source.currentSnapshotId); } /** diff --git a/packages/trpc/src/router/cloud-workspace/cloud-workspace.test.ts b/packages/trpc/src/router/cloud-workspace/cloud-workspace.test.ts index 50c1f03506f..317ee8f885b 100644 --- a/packages/trpc/src/router/cloud-workspace/cloud-workspace.test.ts +++ b/packages/trpc/src/router/cloud-workspace/cloud-workspace.test.ts @@ -29,7 +29,6 @@ const ROW = { let running = false; let boxVcpus = 8; let restartedWith: string[] = []; -let woken: string[] = []; const ENVIRONMENT_ID = "5e6f7a8b-9c0d-4e1f-8a2b-3c4d5e6f7a8b"; let inserted: unknown[] = []; @@ -129,8 +128,8 @@ stub(sandbox, { workspaceBranchName: () => "paradev/ws-1", }); stub(wake, { - wakeCloudWorkspace: async (row: { id: string }) => { - woken.push(row.id); + wakeCloudWorkspace: async () => { + if (!running) return null; return { hostTarget: "https://box-1.example", agentCredentialsChanged: false, @@ -168,7 +167,6 @@ beforeEach(() => { running = false; boxVcpus = 8; restartedWith = []; - woken = []; recordedAsleep = []; inserted = []; queuedJobs = []; @@ -267,13 +265,11 @@ describe("cloudWorkspace.applySignIns", () => { expect( await caller.cloudWorkspace.applySignIns({ id: WORKSPACE_ID }), ).toEqual({ applied: false }); - expect(woken).toEqual([]); running = true; expect( await caller.cloudWorkspace.applySignIns({ id: WORKSPACE_ID }), ).toEqual({ applied: true }); - expect(woken).toEqual([WORKSPACE_ID]); }); }); diff --git a/packages/trpc/src/router/cloud-workspace/cloud-workspace.ts b/packages/trpc/src/router/cloud-workspace/cloud-workspace.ts index 852906ff626..64decf6a80f 100644 --- a/packages/trpc/src/router/cloud-workspace/cloud-workspace.ts +++ b/packages/trpc/src/router/cloud-workspace/cloud-workspace.ts @@ -664,12 +664,8 @@ export const cloudWorkspaceRouter = { .input(z.object({ id: z.string().uuid() })) .mutation(async ({ ctx, input }) => { const row = await loadReadyWorkspace(ctx, input.id); - // A stopped box takes the current sign-ins when it next wakes. - if (!(await describeSandbox(row.providerSandboxId)).running) { - return { applied: false }; - } - await wakeCloudWorkspace(row, { adoptSignIns: true }); - return { applied: true }; + const woken = await wakeCloudWorkspace(row, { adoptSignIns: true }); + return { applied: woken !== null }; }), /** Who forwarded which of the workspace's ports to which local port, for `superset ports`. */ diff --git a/packages/trpc/src/router/cloud-workspace/wake.ts b/packages/trpc/src/router/cloud-workspace/wake.ts index 08c75096525..16479baa745 100644 --- a/packages/trpc/src/router/cloud-workspace/wake.ts +++ b/packages/trpc/src/router/cloud-workspace/wake.ts @@ -41,10 +41,18 @@ export interface WokenCloudWorkspace { } /** Resumes or extends the sandbox and records its current address. */ +export async function wakeCloudWorkspace( + row: CloudWorkspaceRow, +): Promise; +/** Hands the current sign-ins to a running box; null when the box is stopped. */ +export async function wakeCloudWorkspace( + row: CloudWorkspaceRow, + options: { adoptSignIns: true }, +): Promise; export async function wakeCloudWorkspace( row: CloudWorkspaceRow, options: { adoptSignIns?: boolean } = {}, -): Promise { +): Promise { try { return await Sentry.startSpan( { @@ -54,7 +62,9 @@ export async function wakeCloudWorkspace( attributes: { "cloud_workspace.id": row.id }, }, async (span) => { - const { booted, rebooted, ...woken } = await wake(row, options, span); + const result = await wake(row, options, span); + if (!result) return null; + const { booted, rebooted, ...woken } = result; span.setAttributes({ "cloud_workspace.resumed": booted, [CLOUD_WORKSPACE_BOOTED_ATTRIBUTE]: rebooted, @@ -68,14 +78,17 @@ export async function wakeCloudWorkspace( } /** - * `adoptSignIns` marks the box's sign-ins current without a boot: the caller - * restarts the one agent that needs them, and the others keep theirs. + * `adoptSignIns` marks a running box's sign-ins current without a boot: the + * caller restarts the one agent that needs them, and the others keep theirs. + * A stopped box is left for its next wake, which hands them over anyway. */ async function wake( row: CloudWorkspaceRow, { adoptSignIns = false }: { adoptSignIns?: boolean }, span: Sentry.Span, -): Promise { +): Promise< + (WokenCloudWorkspace & { booted: boolean; rebooted: boolean }) | null +> { const { claim, agentCredentialDigest } = await Sentry.startSpan( { name: "claim", op: "sandbox" }, () => buildSandboxClaim({ row }), @@ -87,65 +100,67 @@ async function wake( let bootId: string | null = null; let resumed: { sessionId: string | null; expiresAt: Date | null } | undefined; const wakeStartedAt = Date.now(); - const { hostTarget, booted, rebooted, answer, expiresAt, sessionId } = - await wakeSandbox({ - providerSandboxId: row.providerSandboxId, - claim, - beforeResume: async ({ resumedSession, ...box }) => { - resumed = resumedSession; - ({ bootId } = await Sentry.startSpan( - { name: "authorize boot", op: "billing" }, - () => - authorizeBoot({ - organizationId: row.organizationId, - cloudWorkspaceId: row.id, - createdByUserId: row.createdByUserId, - ...box, - }), - )); - if (bootId) { - scope.bootId = bootId; - span.setAttribute("boot.id", bootId); - } - }, - }).catch(async (error) => { - if (!isCloudUsageRefusal(error)) { - logControl( - "wake.failed", - scope, - { ms: Date.now() - wakeStartedAt }, - "error", - ); - } + const woken = await wakeSandbox({ + providerSandboxId: row.providerSandboxId, + claim, + onlyIfRunning: adoptSignIns, + beforeResume: async ({ resumedSession, ...box }) => { + resumed = resumedSession; + ({ bootId } = await Sentry.startSpan( + { name: "authorize boot", op: "billing" }, + () => + authorizeBoot({ + organizationId: row.organizationId, + cloudWorkspaceId: row.id, + createdByUserId: row.createdByUserId, + ...box, + }), + )); if (bootId) { - await attachBootSession({ - bootId, - providerSandboxId: row.providerSandboxId, - }).catch(() => {}); + scope.bootId = bootId; + span.setAttribute("boot.id", bootId); } - // A call here resumed the box, and no boot meters that session. - if (resumed && isCloudUsageRefusal(error)) { - console.error( - `[cloud-workspace] ${row.id} was resumed past its limit; stopping it`, - ); - try { - if (resumed.expiresAt) { - await recordSandboxAwake(row.id, { - expiresAt: resumed.expiresAt, - sessionId: resumed.sessionId, - }); - } - const stopped = await stopSandbox(row.providerSandboxId); - await recordSandboxAsleep(row.id, { - sessionId: stopped.sessionId ?? resumed.sessionId, + }, + }).catch(async (error) => { + if (!isCloudUsageRefusal(error)) { + logControl( + "wake.failed", + scope, + { ms: Date.now() - wakeStartedAt }, + "error", + ); + } + if (bootId) { + await attachBootSession({ + bootId, + providerSandboxId: row.providerSandboxId, + }).catch(() => {}); + } + // A call here resumed the box, and no boot meters that session. + if (resumed && isCloudUsageRefusal(error)) { + console.error( + `[cloud-workspace] ${row.id} was resumed past its limit; stopping it`, + ); + try { + if (resumed.expiresAt) { + await recordSandboxAwake(row.id, { + expiresAt: resumed.expiresAt, + sessionId: resumed.sessionId, }); - logControl("meter.cap_stopped", scope, { source: "wake" }, "warn"); - } catch (stopError) { - Sentry.captureException(stopError, { tags: controlTags(scope) }); } + const stopped = await stopSandbox(row.providerSandboxId); + await recordSandboxAsleep(row.id, { + sessionId: stopped.sessionId ?? resumed.sessionId, + }); + logControl("meter.cap_stopped", scope, { source: "wake" }, "warn"); + } catch (stopError) { + Sentry.captureException(stopError, { tags: controlTags(scope) }); } - throw error; - }); + } + throw error; + }); + if (!woken) return null; + const { hostTarget, booted, rebooted, answer, expiresAt, sessionId } = woken; if (bootId) { await attachBootSession({ bootId, diff --git a/packages/trpc/src/router/admin/admin.test.ts b/packages/trpc/src/router/staff-cannot-wake.test.ts similarity index 94% rename from packages/trpc/src/router/admin/admin.test.ts rename to packages/trpc/src/router/staff-cannot-wake.test.ts index 5134a49bf24..13fd36e28a6 100644 --- a/packages/trpc/src/router/admin/admin.test.ts +++ b/packages/trpc/src/router/staff-cannot-wake.test.ts @@ -2,11 +2,11 @@ import { describe, expect, test } from "bun:test"; import { existsSync, readFileSync } from "node:fs"; import { dirname, relative, resolve } from "node:path"; -const SRC = resolve(import.meta.dir, "../.."); +const SRC = resolve(import.meta.dir, ".."); const WAKE_PATH = resolve(SRC, "router/cloud-workspace/wake.ts"); const SANDBOX_PROVIDER = resolve(SRC, "lib/sandbox/vercel.ts"); const STARTS_A_BOX = - /\b(wakeCloudWorkspace|restartCloudWorkspace|wakeAfterStop|wakeSandbox|provisionSandbox|topUpSandboxSession|getSandboxToWake)\(/; + /\b(wakeCloudWorkspace|restartCloudWorkspace|wakeAfterStop|wakeSandbox|provisionSandbox)\(/; const transpilers = { ts: new Bun.Transpiler({ loader: "ts" }),