From 36626c312ee9d09ed47b303aec8d5d109920649c Mon Sep 17 00:00:00 2001 From: Satya Patel <14907857+saddlepaddle@users.noreply.github.com> Date: Wed, 18 Nov 2026 03:26:51 +0000 Subject: [PATCH 1/6] refactor(cloud): one reserveLock for the meter's Autumn locks Preparation, no behaviour change: authorizeBoot and the settle renewal take a boot's lock through one function instead of two copies of the same check call. The adopt in the next commit uses it too. --- .../trpc/src/router/cloud-workspace/meter.ts | 56 +++++++++++-------- 1 file changed, 32 insertions(+), 24 deletions(-) diff --git a/packages/trpc/src/router/cloud-workspace/meter.ts b/packages/trpc/src/router/cloud-workspace/meter.ts index cfedc78260b..f7352679d50 100644 --- a/packages/trpc/src/router/cloud-workspace/meter.ts +++ b/packages/trpc/src/router/cloud-workspace/meter.ts @@ -84,6 +84,24 @@ function reservation(memoryMb: number): number { return gbMinutes({ ms: LOCK_WINDOW_MS, memoryMb }); } +function reserveLock( + boot: Pick, + memoryMb: number, + user: UsageUser | null, +) { + return autumnClient.check({ + customerId: boot.organizationId, + featureId: BOX_GB_MINUTES_FEATURE_ID, + requiredBalance: reservation(memoryMb), + properties: usageProperties(boot, user), + lock: { + enabled: true, + lockId: lockIdFor(boot.id, boot.lockSeq), + expiresAt: Date.now() + LOCK_TTL_MS, + }, + }); +} + function usageProperties( boot: Pick, user: UsageUser | null, @@ -166,20 +184,16 @@ export async function authorizeBoot( const bootId = `boot_${randomUUID()}`; const user = await usageUser(args.createdByUserId); const result = await checkOrFailOpen(args.organizationId, () => - autumnClient.check({ - customerId: args.organizationId, - featureId: BOX_GB_MINUTES_FEATURE_ID, - requiredBalance: reservation(args.memoryMb), - properties: usageProperties( - { id: bootId, lockSeq: 0, cloudWorkspaceId: args.cloudWorkspaceId }, - user, - ), - lock: { - enabled: true, - lockId: lockIdFor(bootId, 0), - expiresAt: Date.now() + LOCK_TTL_MS, + reserveLock( + { + id: bootId, + lockSeq: 0, + organizationId: args.organizationId, + cloudWorkspaceId: args.cloudWorkspaceId, }, - }), + args.memoryMb, + user, + ), ); const scope = { cloudWorkspaceId: args.cloudWorkspaceId, @@ -486,17 +500,11 @@ export async function settleBoot( } const nextSeq = boot.lockSeq + 1; - const renewal = await autumnClient.check({ - customerId: boot.organizationId, - featureId: BOX_GB_MINUTES_FEATURE_ID, - requiredBalance: reservation(session.memoryMb), - properties: usageProperties({ ...boot, lockSeq: nextSeq }, user), - lock: { - enabled: true, - lockId: lockIdFor(bootId, nextSeq), - expiresAt: Date.now() + LOCK_TTL_MS, - }, - }); + const renewal = await reserveLock( + { ...boot, lockSeq: nextSeq }, + session.memoryMb, + user, + ); await db .update(cloudWorkspaceBoots) .set({ From a2c4629d0c8b0af839f08e56f41db48b9d58cb34 Mon Sep 17 00:00:00 2001 From: Satya Patel <14907857+saddlepaddle@users.noreply.github.com> Date: Wed, 18 Nov 2026 03:28:48 +0000 Subject: [PATCH 2/6] fix(cloud): meter every box session (M8) The promote restart and a resume right after a timeout stop started box sessions with no boot row, so they were never billed: 79 sessions and about 6,256 GB-minutes in 7 days (W0 measurements, item 7). - promote: promoteSandboxToEnvironment no longer boots the source itself. It returns the session its snapshot ended, and environment.promote starts the box again through wakeAfterStop, the same authorize and attach path as restartCloudWorkspace. A refused or failed restart leaves the box asleep and still saves the environment; it now waits for host-service. - wake: wakeSandbox passes onResume to Sandbox.get. When a call resumed a box read running, it runs the wake's own beforeResume for the new session. A refused wake that finds the box running records that session awake, stops it, and records it asleep, so no minutes run past the cap. - reconcile: adoptRunningBoxes skipped a workspace while the boot of the session that just ended was open, looked only at the newest running session, and billed an adopted session from the adopt. Now the run queues the settles first, then adopts (RECONCILE_CONCURRENCY at a time) every recent session that no boot row names, billed from its start: lock first, lockOpen in the insert, the lock released on a conflict or a failed insert, and a settle queued for the new boot. The newest session is left to an open boot that has not attached yet. - The unused adopt_failed release reason is gone. --- .agents/skills/verify-sandbox-asleep/SKILL.md | 6 +- docs/cloud-sandbox-considerations.md | 2 + docs/cloud-sandbox-mismatches.md | 23 ++- packages/trpc/src/lib/sandbox/AGENTS.md | 7 +- packages/trpc/src/lib/sandbox/index.ts | 1 + packages/trpc/src/lib/sandbox/vercel.test.ts | 159 ++++++++++++++++++ packages/trpc/src/lib/sandbox/vercel.ts | 68 ++++++-- .../src/router/cloud-workspace/meter.test.ts | 139 ++++++++++++++- .../trpc/src/router/cloud-workspace/meter.ts | 159 ++++++++++++------ .../src/router/cloud-workspace/wake.test.ts | 101 +++++++++++ .../trpc/src/router/cloud-workspace/wake.ts | 39 ++++- .../router/environment/environment.test.ts | 59 ++++++- .../src/router/environment/environment.ts | 23 ++- 13 files changed, 696 insertions(+), 90 deletions(-) diff --git a/.agents/skills/verify-sandbox-asleep/SKILL.md b/.agents/skills/verify-sandbox-asleep/SKILL.md index e77f22508ec..3146b0a085c 100644 --- a/.agents/skills/verify-sandbox-asleep/SKILL.md +++ b/.agents/skills/verify-sandbox-asleep/SKILL.md @@ -31,14 +31,14 @@ one TLC worker, so a failure always prints the shortest trace. A pass takes abou ## Coverage record -Synced with commit `623b52bf4`. Paths are under `packages/trpc/src/`. +Synced with commit `cf52bc20`. Paths are under `packages/trpc/src/`. | Action | Code | | --- | --- | | `ProvisionDone` | `provisionCloudWorkspace` in `router/cloud-workspace/provision.ts`: `provisionSandbox`, the transition to `awake`, then `recordSandboxAwake` | -| `WakeResume` / `WakeRecord` | `wakeCloudWorkspace` in `router/cloud-workspace/wake.ts`: `wakeSandbox` in `lib/sandbox/vercel.ts`, then `recordSandboxAwake` in `router/cloud-workspace/sandbox-expiry.ts` | +| `WakeResume` / `WakeRecord` | `wakeCloudWorkspace` in `router/cloud-workspace/wake.ts`: `wakeSandbox` in `lib/sandbox/vercel.ts`, then `recordSandboxAwake` in `router/cloud-workspace/sandbox-expiry.ts`. A wake that read the box running and resumed it after a timeout stop is the same `WakeResume`: the box was stopped when the call resumed it | | `TopUpExtend` / `TopUpRecord` | `sandbox.keepalive` in `router/sandbox/sandbox.ts`: `topUpSandboxSession`, then `recordSandboxAwake` | -| `StopBox` / `StopRecord` | `stopAndSnapshot` or `stopSandbox`, then `recordSandboxAsleep` in `router/cloud-workspace/sandbox-sleep.ts`, from `cloudWorkspace.sleep` (`cloud-workspace.ts`), `restartCloudWorkspace` (`wake.ts`), and the spend-cap stops in `meter.ts` (`settleBoot`, `settleOrganizationNow`, the refused adopt in `adoptRunningBoxes`) | +| `StopBox` / `StopRecord` | `stopAndSnapshot` or `stopSandbox`, then `recordSandboxAsleep` in `router/cloud-workspace/sandbox-sleep.ts`, from `cloudWorkspace.sleep` (`cloud-workspace.ts`), `restartCloudWorkspace` and `wakeAfterStop` (`wake.ts`; `environment.promote` calls `wakeAfterStop` after the snapshot in `promoteSandboxToEnvironment` stopped the source, so `StopBox` there is the snapshot), a refused `wakeCloudWorkspace` that finds the box running (it records that session awake first, a `WakeRecord`, then stops it), and the spend-cap stops in `meter.ts` (`settleBoot`, `settleOrganizationNow`, the refused adopt in `adoptRunningBoxes`) | | `WatchFire` | `watchSandboxExpiry` in `sandbox-expiry.ts`, queued by `queueWatch` through `publishCloudWorkspaceJob` (`jobs.ts`), served by `apps/api/src/app/api/cloud-workspaces/sandbox-expiry/route.ts` | | `ReconcileList` / `ReconcileApply` | `reconcileCloudWorkspaces` in `sandbox-expiry.ts`: `listActiveWorkspaceSandboxes`, then `syncSandboxExpiries` against each listing's `currentSessionId` | | `Archive` | `cloudWorkspace.delete` in `cloud-workspace.ts`: the transition to `archived`, then `queueReap` | diff --git a/docs/cloud-sandbox-considerations.md b/docs/cloud-sandbox-considerations.md index 170cb936a90..e840f0dfcae 100644 --- a/docs/cloud-sandbox-considerations.md +++ b/docs/cloud-sandbox-considerations.md @@ -38,6 +38,8 @@ schedule (`/api/cloud-workspaces/jobs/reconcile-usage`, every 30 minutes, set in the Upstash console and watched by a Sentry cron monitor) finalizes each running box's memory GB-minutes and takes the next lock; a box whose next lock is refused is stopped. Stopping, archiving or deleting a box settles the rest. +The same run opens a boot for any session from the last hour that has none +(`adoptRunningBoxes` in `meter.ts`), billed from the session's start. Environment builds are not billed. Prices and limits live in Autumn, not here. Each paid seat adds its plan's GB-minutes to one team pool through Autumn seat licenses; see [Seats and cloud hours in Autumn](autumn-seat-licenses.md). diff --git a/docs/cloud-sandbox-mismatches.md b/docs/cloud-sandbox-mismatches.md index ffc8144c02c..40cd448fefe 100644 --- a/docs/cloud-sandbox-mismatches.md +++ b/docs/cloud-sandbox-mismatches.md @@ -776,9 +776,26 @@ obvious tidy-up once the golden exists — leaves the workspace unable to ever resume (`410 Cannot resume sandbox: no snapshot available`), which is how one e2e workspace died. So the snapshot stays (`keepLastSnapshots` evicts it on the source's next stop), and a source that was running is started again -before promote returns. A row whose sandbox is gone or unresumable is marked -`failed` by the next `access`, so it gets the failed screen and a Remove -button rather than a sidebar entry that never opens. +before promote returns, through `wakeAfterStop`: the new session is a wake, +authorized and metered like any other. Until 2026-11 the restart ran the boot +command directly, and 69 sessions in a week ran with no boot row. A row whose +sandbox is gone or unresumable is marked `failed` by the next `access`, so it +gets the failed screen and a Remove button rather than a sidebar entry that +never opens. + +**Any call on a `Sandbox` object resumes a stopped session.** The SDK wraps +`runCommand`, `writeFiles`, `extendTimeout` and the rest in `withResume`: on a +"stopped" answer it resumes and retries. A box read `running` that hits its +timeout a moment later is resumed by the next call on that object, and no +code chose to start it. Measured: 9 sessions in a week started 0.8–1.5 s after +a timeout stop. `wakeSandbox` passes `onResume` to `Sandbox.get` and runs +`beforeResume` (which authorizes the boot) for such a session; a refused one is +stopped at once. A top-up goes through the session, which never resumes. +Other callers on a box read `running` are covered only by the reconcile, which +adopts any session that has no boot row within one run: promote's +`source.snapshot()` (wrapped in `withResume`), and `applySandboxPolicy`, whose +`update` does not resume in SDK 4.0.0 but would if a later SDK wrapped it. +`provisionSandbox` on a box it already made is covered by its own boot. **`stop()` returns before the stop's snapshot is current.** The sandbox is still `stopping` when the call resolves, and a fork taken then boots from diff --git a/packages/trpc/src/lib/sandbox/AGENTS.md b/packages/trpc/src/lib/sandbox/AGENTS.md index 9b2a192f262..5487d379e0d 100644 --- a/packages/trpc/src/lib/sandbox/AGENTS.md +++ b/packages/trpc/src/lib/sandbox/AGENTS.md @@ -36,9 +36,10 @@ Two habits that pay for themselves here: a fix that only exists on your machine. - **The claim is the whole handoff.** `buildSandboxClaim` assembles what a box needs to be one workspace — the identity file, the firewall rules, the - managed environment, the host secret — and `provisionSandbox`, `wakeSandbox` - and `promoteSandboxToEnvironment` all take it, so a create, a wake and a - restart after promote hand the box the same thing. + managed environment, the host secret — and `provisionSandbox` and + `wakeSandbox` take it. A restart after promote is a wake (`wakeAfterStop`), + so a create, a wake and that restart hand the box the same thing, and each + new session gets a boot row. `docs/cloud-sandbox-considerations.md` is the companion list: what we still owe before a non-internal user can create a sandbox. Several entries there are only diff --git a/packages/trpc/src/lib/sandbox/index.ts b/packages/trpc/src/lib/sandbox/index.ts index 7038875b600..df06a04faf2 100644 --- a/packages/trpc/src/lib/sandbox/index.ts +++ b/packages/trpc/src/lib/sandbox/index.ts @@ -55,6 +55,7 @@ export { findSandboxSession, HOST_SERVICE_PORT, listActiveWorkspaceSandboxes, + listSandboxSessions, MEMORY_MB_PER_VCPU, promoteSandboxToEnvironment, provisionSandbox, diff --git a/packages/trpc/src/lib/sandbox/vercel.test.ts b/packages/trpc/src/lib/sandbox/vercel.test.ts index 4a24e20353a..d5dc9e00f70 100644 --- a/packages/trpc/src/lib/sandbox/vercel.test.ts +++ b/packages/trpc/src/lib/sandbox/vercel.test.ts @@ -12,8 +12,11 @@ import { agentToolsMounts, findSandboxSession, listActiveWorkspaceSandboxes, + listSandboxSessions, + promoteSandboxToEnvironment, SESSION_TIMEOUT_MS, topUpSandboxSession, + wakeSandbox, } from "./vercel"; type ListOptions = NonNullable[0]>; @@ -114,6 +117,162 @@ describe("findSandboxSession", () => { }); }); +describe("listSandboxSessions", () => { + const raw = (id: string, startedAt: number, stoppedAt?: number) => ({ + id, + memory: 8192, + vcpus: 4, + status: stoppedAt === undefined ? "running" : "stopped", + startedAt, + ...(stoppedAt === undefined ? {} : { stoppedAt }), + updatedAt: stoppedAt ?? startedAt, + }); + let read = 0; + stub(Sandbox, { + get: async () => ({ + listSessions: async () => + (async function* () { + for (const session of [ + raw("live", NOW - 10 * 60_000), + raw("recent", NOW - 50 * 60_000, NOW - 20 * 60_000), + raw("old", NOW - 3 * HOUR, NOW - 2 * HOUR), + raw("older", NOW - 5 * HOUR, NOW - 4 * HOUR), + ]) { + read += 1; + yield session; + } + })(), + }), + }); + + test("lists the live session and those ended since, and reads no further back", async () => { + const sessions = await listSandboxSessions("ws-box", NOW - HOUR); + expect(sessions?.map((session) => session.id)).toEqual(["live", "recent"]); + expect(read).toBe(3); + }); +}); + +describe("promoteSandboxToEnvironment", () => { + const sourceCalls: string[] = []; + const source = { + status: "running", + region: "iad1", + vcpus: 4, + currentSession: () => ({ sessionId: "sess_A" }), + snapshot: async () => { + sourceCalls.push("snapshot"); + return { snapshotId: "snap_1" }; + }, + writeFiles: async () => { + sourceCalls.push("writeFiles"); + }, + runCommand: async () => { + sourceCalls.push("runCommand"); + }, + mkDir: async () => { + sourceCalls.push("mkDir"); + }, + }; + const golden = { + currentSnapshotId: "snap_golden_0", + runCommand: async () => {}, + stop: async () => {}, + }; + let createTags: unknown; + stub(Sandbox, { + get: async ({ name }: { name: string }) => + name === "ws-box" + ? source + : { status: "stopped", currentSnapshotId: "snap_golden_1" }, + create: async (params: { tags: unknown }) => { + createTags = params.tags; + return golden; + }, + }); + + test("ends the source's session for the snapshot and leaves its restart to the caller", async () => { + expect( + await promoteSandboxToEnvironment({ + sourceSandbox: "ws-box", + goldenName: "env-1", + organizationId: "org_1", + }), + ).toEqual({ + goldenName: "env-1", + region: "iad1", + stopped: { sessionId: "sess_A" }, + }); + expect(sourceCalls).toEqual(["snapshot"]); + expect(createTags).toEqual({ kind: "environment", org: "org_1" }); + }); +}); + +describe("wakeSandbox", () => { + type Box = { memoryMb: number; vcpus: number }; + let running: boolean; + let onResume: (sandbox: unknown) => Promise; + const box = { + get status() { + return running ? "running" : "stopped"; + }, + expiresAt: new Date(NOW + 60_000), + vcpus: 4, + domain: () => "https://box.test", + currentSession: () => ({ vcpus: 8 }), + update: async () => {}, + writeFiles: async () => {}, + // The SDK's withResume: a call on a stopped session resumes it, then retries. + extendTimeout: async () => { + await onResume(box); + }, + runCommand: async () => { + await onResume(box); + throw new Error("stop before settle"); + }, + }; + stub(Sandbox, { + get: async (params: { onResume: typeof onResume }) => { + onResume = params.onResume; + return box; + }, + }); + stub(globalThis, { fetch: async () => new Response("{}") }); + const identity = { + SUPERSET_SANDBOX_CONTRACT: "1", + SUPERSET_API_URL: "https://api.test", + SUPERSET_SANDBOX_WORKSPACE_ID: "2b3c4d5e-6f7a-4b8c-9d0e-1f2a3b4c5d6e", + SUPERSET_SANDBOX_ORGANIZATION_ID: "1a2b3c4d-5e6f-4a7b-8c9d-0e1f2a3b4c5d", + SUPERSET_SANDBOX_REPOSITORIES: "[]", + SUPERSET_SANDBOX_IMAGE_TAG: "image", + SUPERSET_SANDBOX_PROVIDER: "vercel", + SUPERSET_BUNDLE_SHA: "a".repeat(64), + }; + async function wake(): Promise { + const authorized: Box[] = []; + await wakeSandbox({ + providerSandboxId: "ws-box", + claim: { vcpus: 4, identity, networkPolicy: {} } as never, + beforeResume: async (resumed) => { + authorized.push(resumed); + if (running) throw new Error("stop before settle"); + }, + }).catch((error: Error) => { + if (error.message !== "stop before settle") throw error; + }); + return authorized; + } + + test("a box read running that a call resumed runs beforeResume for the new session", async () => { + running = true; + expect(await wake()).toEqual([{ memoryMb: 16_384, vcpus: 8 }]); + }); + + test("a box read stopped runs beforeResume once, before its own resume", async () => { + running = false; + expect(await wake()).toEqual([{ memoryMb: 8192, vcpus: 4 }]); + }); +}); + describe("listActiveWorkspaceSandboxes", () => { test("finds older resumed workspaces when tag-filtered lists use creation order", async () => { expect(await listActiveWorkspaceSandboxes(NOW - 4 * HOUR)).toEqual([ diff --git a/packages/trpc/src/lib/sandbox/vercel.ts b/packages/trpc/src/lib/sandbox/vercel.ts index 786002a5c37..953fd3670fd 100644 --- a/packages/trpc/src/lib/sandbox/vercel.ts +++ b/packages/trpc/src/lib/sandbox/vercel.ts @@ -483,7 +483,10 @@ async function hostServiceAnswers(hostTarget: string): Promise { export async function wakeSandbox(args: { providerSandboxId: string; claim: SandboxClaim; - /** Runs once the box is known to be stopped, before anything resumes it. */ + /** + * Runs once the box is known to be stopped, before anything resumes it; or, + * for a box read running that a call here resumed, once those calls end. + */ beforeResume?: (box: { memoryMb: number; vcpus: number }) => Promise; }): Promise<{ hostTarget: string; @@ -496,13 +499,19 @@ export async function wakeSandbox(args: { sessionId: string | null; }> { try { + let wasRunning = false; + let resumedVcpus: number | null = null; const sandbox = await Sandbox.get({ ...credentials(), name: args.providerSandboxId, resume: false, + // Any call on a Sandbox object resumes a stopped session and retries. + onResume: async (resumed) => { + if (wasRunning) resumedVcpus = resumed.currentSession().vcpus; + }, }); const hostTarget = sandbox.domain(HOST_SERVICE_PORT); - const wasRunning = sandbox.status === "running"; + wasRunning = sandbox.status === "running"; let vcpus = sandbox.vcpus ?? args.claim.vcpus; if (!wasRunning && vcpus !== args.claim.vcpus) { await Sentry.startSpan({ name: "resize", op: "sandbox.provider" }, () => @@ -569,6 +578,12 @@ export async function wakeSandbox(args: { runBoot(sandbox, args.claim.hostSecret), ); } + if (resumedVcpus && args.beforeResume) { + await args.beforeResume({ + memoryMb: resumedVcpus * MEMORY_MB_PER_VCPU, + vcpus: resumedVcpus, + }); + } const answer = await Sentry.startSpan( { name: "settle", op: "sandbox" }, () => @@ -645,21 +660,26 @@ export async function stopAndSnapshot( * A golden is a stopped sandbox whose current snapshot is what forks start * from. It is built from a snapshot of the source taken now (a fork alone * would start from the source's last stop) and created with an empty env. - * Taking that snapshot ends the source's session, so a running source is - * started again before this returns. + * Taking that snapshot ends the source's session. The source stays stopped: + * the caller starts it again through a wake, which meters the new session. */ export async function promoteSandboxToEnvironment(args: { sourceSandbox: string; goldenName: string; - /** What restarts the source: it boots the same way a wake does. */ - claim: SandboxClaim; -}): Promise<{ goldenName: string; region: string }> { + organizationId: string; +}): Promise<{ + goldenName: string; + region: string; + /** The session the snapshot ended; null when the source was not running. */ + stopped: { sessionId: string | null } | null; +}> { const source = await Sandbox.get({ ...credentials(), name: args.sourceSandbox, resume: false, }); - const wasRunning = source.status === "running"; + const stopped = + source.status === "running" ? { sessionId: sessionIdOf(source) } : null; const snapshot = await source.snapshot(); const golden = await Sandbox.create({ ...credentials(), @@ -673,18 +693,13 @@ export async function promoteSandboxToEnvironment(args: { persistent: true, snapshotExpiration: 0, keepLastSnapshots: { count: 1 }, - tags: sandboxTags("environment", args.claim.identity), + tags: { kind: "environment", org: args.organizationId }, }); await stripWorkspaceIdentity(golden); const created = golden.currentSnapshotId; await golden.stop(); await waitForStopSnapshot(args.goldenName, created); - if (wasRunning) { - await writeIdentity(source, args.claim.identity); - await writeSignInFiles(source, args.claim.files); - await runBoot(source, args.claim.hostSecret); - } - return { goldenName: args.goldenName, region: source.region }; + return { goldenName: args.goldenName, region: source.region, stopped }; } /** @@ -784,6 +799,29 @@ function toBoxSession(session: RawSession): BoxSession | null { }; } +/** + * The box's sessions that are live or ended at or after `since`, newest + * first. Null when the sandbox no longer exists. + */ +export async function listSandboxSessions( + providerSandboxId: string, + since: number, +): Promise { + const sandbox = await getSandbox(providerSandboxId); + if (!sandbox) return null; + const sessions: BoxSession[] = []; + for await (const raw of await sandbox.listSessions({ + sortOrder: "desc", + limit: SESSION_PAGE_SIZE, + })) { + const session = toBoxSession(raw); + if (!session) continue; + if (session.endedAt !== null && session.endedAt < since) break; + sessions.push(session); + } + return sessions; +} + /** Null when the sandbox no longer exists. */ export async function findSandboxSession( providerSandboxId: string, diff --git a/packages/trpc/src/router/cloud-workspace/meter.test.ts b/packages/trpc/src/router/cloud-workspace/meter.test.ts index 9b3a49656d3..c7c925b82b5 100644 --- a/packages/trpc/src/router/cloud-workspace/meter.test.ts +++ b/packages/trpc/src/router/cloud-workspace/meter.test.ts @@ -38,6 +38,10 @@ let stopped: string[]; let usageStopAt: Date | null; let delayedSettles: number[]; let updateMatches: boolean; +let insertConflicts: boolean; +let insertError: Error | null; +let queuedSettles: string[]; +let listed: BoxSession[]; type UsageEvent = { featureId: string; value: number; @@ -69,8 +73,16 @@ function chain(rows: () => unknown[], onFrom?: (table: unknown) => void) { stub(db, { update: () => chain(() => (boot && updateMatches ? [boot] : [])), insert: () => ({ - values: async (values: Record) => { + values: (values: Record) => { writes.push(values); + return Object.assign(Promise.resolve(), { + onConflictDoNothing: () => ({ + returning: async () => { + if (insertError) throw insertError; + return insertConflicts ? [] : [{ id: values.id }]; + }, + }), + }); }, }), select: () => { @@ -83,6 +95,8 @@ stub(db, { } return [ { + id: "ws_1", + organizationId: "org_1", providerSandboxId: "ws-box", createdByUserId: "user_1", usageStopAt, @@ -99,6 +113,7 @@ stub(db, { }); stub(sandbox, { findSandboxSession: async () => ({ session }), + listSandboxSessions: async () => listed, stopSandbox: async (id: string) => { stopped.push(id); return { sessionId: "session_1" }; @@ -115,8 +130,12 @@ stub(sandboxSleep, { }, }); stub(jobs, { - publishCloudWorkspaceJob: async (job: { delaySeconds?: number }) => { + publishCloudWorkspaceJob: async (job: { + delaySeconds?: number; + body: { bootId: string }; + }) => { delayedSettles.push(job.delaySeconds ?? 0); + queuedSettles.push(job.body.bootId); }, }); stub(autumnClient, { @@ -211,6 +230,10 @@ beforeEach(() => { usageStopAt = null; delayedSettles = []; updateMatches = true; + insertConflicts = false; + insertError = null; + queuedSettles = []; + listed = []; usageEvents = null; reservations = new Map(); }); @@ -418,6 +441,118 @@ describe("settleBoot", () => { }); }); +describe("the reconcile adopts sessions no boot meters", () => { + stub(billing, { ensureAutumnCustomer: async () => ({ id: "org_1" }) }); + const box = { name: "ws-box", expiresAt: null, sessionId: "sess_C" }; + const resumed = (endedAt: number | null): BoxSession => ({ + id: "sess_C", + memoryMb: 16_384, + vcpus: 8, + startedAt: NOW - 12 * MINUTE, + endedAt, + activeCpuMs: null, + }); + const timedOut: BoxSession = { + id: "sess_1", + memoryMb: 16_384, + vcpus: 8, + startedAt: NOW - 72 * MINUTE, + endedAt: NOW - 12 * MINUTE - 1_000, + activeCpuMs: null, + }; + const reconcile = () => + reconcileCloudWorkspaceUsage({ active: [box], billingEnabled: true }); + const adoptedBoot = () => + writes.find((write) => write.vercelSessionId === "sess_C"); + + test("a session resumed after a timeout stop is adopted in the same run, though the stopped session's boot is still open", async () => { + listed = [resumed(NOW - 2 * MINUTE), timedOut]; + expect((await reconcile()).adopted).toBe(1); + expect(adoptedBoot()).toMatchObject({ + cloudWorkspaceId: "ws_1", + startedAt: new Date(NOW - 12 * MINUTE), + memoryMb: 16_384, + lockOpen: false, + }); + expect(adoptedBoot()).not.toHaveProperty("finalizedThrough"); + expect(autumnCalls.some((call) => call.method === "check")).toBe(false); + expect(stopped).toEqual([]); + }); + + test("a running unmetered session takes a lock and is billed from its start, not from the adopt", async () => { + listed = [resumed(null), timedOut]; + expect((await reconcile()).adopted).toBe(1); + expect(adoptedBoot()).toMatchObject({ + startedAt: new Date(NOW - 12 * MINUTE), + }); + expect(adoptedBoot()).not.toHaveProperty("finalizedThrough"); + expect( + autumnCalls.find((call) => call.method === "check")?.params, + ).toMatchObject({ + lock: { lockId: `${adoptedBoot()?.id}:0` }, + }); + expect(adoptedBoot()).toMatchObject({ lockOpen: true }); + }); + + test("a session that already has a boot is not adopted again", async () => { + boot = { ...boot, vercelSessionId: "sess_C" }; + listed = [resumed(null)]; + expect((await reconcile()).adopted).toBe(0); + expect(adoptedBoot()).toBeUndefined(); + }); + + test("a session another caller attached meanwhile is left to that boot, and the lock taken for it is released", async () => { + insertConflicts = true; + listed = [resumed(null), timedOut]; + expect((await reconcile()).adopted).toBe(0); + const lockId = ( + autumnCalls.find((call) => call.method === "check")?.params as { + lock: { lockId: string }; + } + ).lock.lockId; + expect( + autumnCalls.find((call) => call.method === "finalize")?.params, + ).toEqual({ lockId, action: "release" }); + }); + + test("a lock taken for an adopt whose insert fails is released", async () => { + insertError = new Error("db down"); + listed = [resumed(null), timedOut]; + expect((await reconcile()).adopted).toBe(0); + expect( + autumnCalls.find((call) => call.method === "finalize")?.params, + ).toMatchObject({ action: "release" }); + }); + + test("the run's settles are queued before the adopt, and an adopted boot queues its own", async () => { + listed = [resumed(NOW - 2 * MINUTE), timedOut]; + await reconcile(); + const adoptedId = adoptedBoot()?.id; + expect(adoptedId).toEqual(expect.any(String)); + expect(queuedSettles).toEqual(["boot_1", adoptedId]); + }); + + test("a refused running session is stopped and recorded asleep, and its minutes stay on the boot", async () => { + renewalAllowed = false; + listed = [resumed(null), timedOut]; + expect((await reconcile()).adopted).toBe(1); + expect(adoptedBoot()).toBeDefined(); + expect(stopped).toEqual(["ws-box"]); + expect(recordedAsleep).toEqual([["ws_1", { sessionId: "session_1" }]]); + expect(adoptedBoot()).toMatchObject({ lockOpen: false }); + }); + + test("the newest session is left to an open boot that has not attached yet", async () => { + boot = { ...boot, vercelSessionId: null }; + listed = [resumed(null), timedOut]; + expect((await reconcile()).adopted).toBe(1); + expect(adoptedBoot()).toBeUndefined(); + expect(writes).toContainEqual( + expect.objectContaining({ vercelSessionId: "sess_1" }), + ); + }); +}); + describe("Autumn attribution and Vercel runtime parity", () => { stub(billing, { ensureAutumnCustomer: async () => ({ id: "org_1" }) }); diff --git a/packages/trpc/src/router/cloud-workspace/meter.ts b/packages/trpc/src/router/cloud-workspace/meter.ts index f7352679d50..60a10516554 100644 --- a/packages/trpc/src/router/cloud-workspace/meter.ts +++ b/packages/trpc/src/router/cloud-workspace/meter.ts @@ -12,15 +12,15 @@ import { cloudWorkspaces, users, } from "@superset/db/schema"; -import { isCloudUsageRefusal } from "@superset/shared/billing"; import { AutumnError } from "autumn-js"; -import { and, eq, inArray, isNull, lt, ne, or, sql } from "drizzle-orm"; +import { and, eq, gt, inArray, isNull, lt, ne, or, sql } from "drizzle-orm"; import { userError } from "../../i18n-error"; import { controlTags, logControl } from "../../lib/control-log"; import { nudge } from "../../lib/realtime"; import { type ActiveWorkspaceSandbox, findSandboxSession, + listSandboxSessions, MEMORY_MB_PER_VCPU, stopSandbox, } from "../../lib/sandbox"; @@ -230,8 +230,7 @@ type BootReleaseReason = | "provision_abandoned" | "session_taken" | "never_attached" - | "sandbox_lost" - | "adopt_failed"; + | "sandbox_lost"; export async function releaseBoot( bootId: string, @@ -696,50 +695,86 @@ async function adoptRunningBoxes( eq(cloudWorkspaces.provider, "vercel"), ne(cloudWorkspaces.status, "failed"), inArray(cloudWorkspaces.providerSandboxId, names), - sql`not exists (select 1 from ${cloudWorkspaceBoots} boot where boot.cloud_workspace_id = ${cloudWorkspaces.id} and boot.stopped_at is null)`, ), ); - let adopted = 0; - for (const workspace of unmetered) { - const found = await findSandboxSession(workspace.providerSandboxId, null); - const session = found?.session; - if (!session || session.endedAt !== null) continue; - const claimed = await db - .select({ id: cloudWorkspaceBoots.id }) + const adoptSessions = async (workspace: (typeof unmetered)[number]) => { + const sessions = await listSandboxSessions( + workspace.providerSandboxId, + Date.now() - RECONCILE_LOOKBACK_MS, + ); + if (!sessions?.length) return 0; + const boots = await db + .select({ vercelSessionId: cloudWorkspaceBoots.vercelSessionId }) .from(cloudWorkspaceBoots) - .where(eq(cloudWorkspaceBoots.vercelSessionId, session.id)); - if (claimed.length > 0) continue; - let bootId: string | null = null; - try { - ({ bootId } = await authorizeBoot({ - organizationId: workspace.organizationId, - cloudWorkspaceId: workspace.id, - createdByUserId: workspace.createdByUserId, - memoryMb: session.memoryMb, - vcpus: session.vcpus, - })); - if (!bootId) continue; - await db - .update(cloudWorkspaceBoots) - .set({ - vercelSessionId: session.id, - startedAt: new Date(session.startedAt), - finalizedThrough: new Date(), - }) - .where(eq(cloudWorkspaceBoots.id, bootId)); - adopted += 1; - } catch (error) { - if (bootId) await releaseBoot(bootId, "adopt_failed").catch(() => {}); - if (!isCloudUsageRefusal(error)) { - Sentry.captureException(error, { - tags: controlTags({ - cloudWorkspaceId: workspace.id, - organizationId: workspace.organizationId, - bootId, - }), - }); + .where( + or( + inArray( + cloudWorkspaceBoots.vercelSessionId, + sessions.map((session) => session.id), + ), + and( + eq(cloudWorkspaceBoots.cloudWorkspaceId, workspace.id), + isNull(cloudWorkspaceBoots.vercelSessionId), + isNull(cloudWorkspaceBoots.stoppedAt), + gt( + cloudWorkspaceBoots.createdAt, + new Date(Date.now() - UNATTACHED_BOOT_GRACE_MS), + ), + ), + ), + ); + const metered = new Set(boots.map((boot) => boot.vercelSessionId)); + let adopted = 0; + for (const [index, session] of sessions.entries()) { + // An open boot with no session yet attaches the newest one itself. + if (metered.has(session.id) || (index === 0 && metered.has(null))) { continue; } + const bootId = `boot_${randomUUID()}`; + const lock = + session.endedAt === null + ? await checkOrFailOpen(workspace.organizationId, async () => + reserveLock( + { + id: bootId, + lockSeq: 0, + organizationId: workspace.organizationId, + cloudWorkspaceId: workspace.id, + }, + session.memoryMb, + await usageUser(workspace.createdByUserId), + ), + ) + : null; + const lockOpen = lock?.allowed === true; + let inserted: { id: string }[] = []; + try { + inserted = await db + .insert(cloudWorkspaceBoots) + .values({ + id: bootId, + organizationId: workspace.organizationId, + cloudWorkspaceId: workspace.id, + vercelSessionId: session.id, + memoryMb: session.memoryMb, + vcpus: session.vcpus, + startedAt: new Date(session.startedAt), + lockOpen, + }) + .onConflictDoNothing({ target: cloudWorkspaceBoots.vercelSessionId }) + .returning({ id: cloudWorkspaceBoots.id }); + } finally { + if (inserted.length === 0 && lockOpen) { + await autumnClient.balances.finalize({ + lockId: lockIdFor(bootId, 0), + action: "release", + }); + } + } + if (inserted.length === 0) continue; + adopted += 1; + await queueSettle(bootId); + if (!lock || lockOpen) continue; console.error( `[cloud-workspace/meter] ${workspace.id} runs past its limit; stopping it`, ); @@ -757,31 +792,45 @@ async function adoptRunningBoxes( "warn", ); } + return adopted; + }; + let adopted = 0; + for (let i = 0; i < unmetered.length; i += RECONCILE_CONCURRENCY) { + const batch = unmetered.slice(i, i + RECONCILE_CONCURRENCY); + const results = await Promise.allSettled(batch.map(adoptSessions)); + for (const [index, result] of results.entries()) { + if (result.status === "fulfilled") adopted += result.value; + else { + const workspace = batch[index] as (typeof unmetered)[number]; + Sentry.captureException(result.reason, { + tags: controlTags({ + cloudWorkspaceId: workspace.id, + organizationId: workspace.organizationId, + }), + }); + } + } } return adopted; } /** - * Queues a settle for every open boot, and starts metering boxes running - * without one (boxes already up when metering shipped). + * Queues a settle for every open boot, then opens a boot for every recent + * session that has none (a box resumed by a call no one authorized). */ export async function reconcileCloudWorkspaceUsage({ active = null, + billingEnabled = billingUsesAutumn, }: { /** Workspace boxes whose state changed lately; null when the listing failed. */ active?: ActiveWorkspaceSandbox[] | null; + billingEnabled?: boolean; } = {}): Promise<{ adopted: number; queued: number; failed: number; }> { - if (!billingUsesAutumn) return { adopted: 0, queued: 0, failed: 0 }; - const adopted = active - ? await adoptRunningBoxes(active).catch((error) => { - Sentry.captureException(error); - return 0; - }) - : 0; + if (!billingEnabled) return { adopted: 0, queued: 0, failed: 0 }; const ids = await openBootIds(); let failed = 0; for (let i = 0; i < ids.length; i += RECONCILE_CONCURRENCY) { @@ -795,5 +844,11 @@ export async function reconcileCloudWorkspaceUsage({ } } } + const adopted = active + ? await adoptRunningBoxes(active).catch((error) => { + Sentry.captureException(error); + return 0; + }) + : 0; return { adopted, queued: ids.length - failed, failed }; } diff --git a/packages/trpc/src/router/cloud-workspace/wake.test.ts b/packages/trpc/src/router/cloud-workspace/wake.test.ts index 42f9ae83fd3..c86dfb023f1 100644 --- a/packages/trpc/src/router/cloud-workspace/wake.test.ts +++ b/packages/trpc/src/router/cloud-workspace/wake.test.ts @@ -1,6 +1,8 @@ import { describe, expect, test } from "bun:test"; import * as Sentry from "@sentry/core"; +import { db } from "@superset/db/client"; import { stub } from "../../../test/stub"; +import { userError } from "../../i18n-error"; import * as realtime from "../../lib/realtime"; import * as sandbox from "../../lib/sandbox"; import * as meter from "./meter"; @@ -10,6 +12,7 @@ import * as transition from "./transition"; import { markSandboxUnavailable, restartCloudWorkspace, + wakeAfterStop, wakeCloudWorkspace, } from "./wake"; @@ -81,6 +84,104 @@ describe("a keepalive wake of a serving box", () => { }); }); +describe("wakeAfterStop", () => { + const calls: string[] = []; + const recordedAsleep: Array<[string, { sessionId: string | null }]> = []; + stub(sandbox, { + buildSandboxClaim: async () => ({ + claim: {}, + agentCredentialDigest: "digest", + }), + wakeSandbox: async (args: { + beforeResume: (box: { memoryMb: number; vcpus: number }) => Promise; + }) => { + await args.beforeResume({ memoryMb: 8192, vcpus: 4 }); + return { hostTarget: "https://box.test", answer: { stamps: [] } }; + }, + }); + stub(meter, { + queueSettleWorkspace: async () => { + calls.push("settle"); + }, + authorizeBoot: async () => { + calls.push("authorize"); + return { bootId: "boot_2" }; + }, + attachBootSession: async ({ bootId }: { bootId: string }) => { + calls.push(`attach ${bootId}`); + }, + }); + stub(sandboxSleep, { + recordSandboxAsleep: async ( + id: string, + stopped: { sessionId: string | null }, + ) => { + recordedAsleep.push([id, stopped]); + }, + }); + stub(sandboxExpiry, { recordSandboxAwake: async () => {} }); + stub(db, { update: () => ({ set: () => ({ where: async () => [] }) }) }); + + test("the box a promote snapshot stopped is recorded asleep, then woken through authorizeBoot and attached", async () => { + await wakeAfterStop(ROW, { sessionId: "sess_A" }); + expect(recordedAsleep).toEqual([["ws_1", { sessionId: "sess_A" }]]); + expect(calls).toEqual(["settle", "authorize", "attach boot_2"]); + }); +}); + +describe("a refused wake", () => { + const calls: string[] = []; + const recordedAsleep: Array<[string, { sessionId: string | null }]> = []; + stub(sandbox, { + buildSandboxClaim: async () => ({ + claim: {}, + agentCredentialDigest: "digest", + }), + wakeSandbox: async (args: { + beforeResume: (box: { memoryMb: number; vcpus: number }) => Promise; + }) => { + await args.beforeResume({ memoryMb: 16_384, vcpus: 8 }); + }, + describeSandbox: async () => ({ + running: true, + expiresAt: new Date(Date.now() + 60_000), + sessionId: "sess_C", + }), + stopSandbox: async () => { + calls.push("stop"); + return { sessionId: "sess_C" }; + }, + }); + stub(meter, { + authorizeBoot: async () => { + throw userError({ + code: "FORBIDDEN", + message: "limit reached", + i18nKey: "serverError.cloudWorkspace.usageLimitReached", + }); + }, + }); + stub(sandboxExpiry, { + recordSandboxAwake: async (_: string, session: { sessionId: string }) => { + calls.push(`awake ${session.sessionId}`); + }, + }); + stub(sandboxSleep, { + recordSandboxAsleep: async ( + id: string, + stopped: { sessionId: string | null }, + ) => { + recordedAsleep.push([id, stopped]); + }, + }); + + test("stops a box a call resumed before the refusal, and records it asleep for that session", async () => { + await expect(wakeCloudWorkspace(ROW)).rejects.toThrow("limit reached"); + expect(calls).toEqual(["awake sess_C", "stop"]); + expect(recordedAsleep).toEqual([["ws_1", { sessionId: "sess_C" }]]); + }); +}); + describe("markSandboxUnavailable", () => { stub(realtime, { nudge: () => {} }); diff --git a/packages/trpc/src/router/cloud-workspace/wake.ts b/packages/trpc/src/router/cloud-workspace/wake.ts index 33116da469c..4e8e617d467 100644 --- a/packages/trpc/src/router/cloud-workspace/wake.ts +++ b/packages/trpc/src/router/cloud-workspace/wake.ts @@ -20,6 +20,7 @@ import { describeSandbox, recordBootSpans, stopAndSnapshot, + stopSandbox, wakeSandbox, } from "../../lib/sandbox"; import { @@ -120,6 +121,24 @@ async function wake( providerSandboxId: row.providerSandboxId, }).catch(() => {}); } + // A call here may have resumed the box before the refusal. + if (isCloudUsageRefusal(error)) { + try { + const live = await describeSandbox(row.providerSandboxId); + if (live.running && live.expiresAt) { + await recordSandboxAwake(row.id, { + expiresAt: live.expiresAt, + sessionId: live.sessionId, + }); + const stopped = await stopSandbox(row.providerSandboxId); + await recordSandboxAsleep(row.id, { + sessionId: stopped.sessionId ?? live.sessionId, + }); + } + } catch (stopError) { + Sentry.captureException(stopError, { tags: controlTags(scope) }); + } + } throw error; }); if (bootId) { @@ -176,13 +195,19 @@ export async function restartCloudWorkspace( row: CloudWorkspaceRow, ): Promise { const { running } = await describeSandbox(row.providerSandboxId); - if (running) { - const stopped = await stopAndSnapshot(row.providerSandboxId); - await recordSandboxAsleep(row.id, { - sessionId: stopped.sessionId ?? row.sandboxSessionId, - }); - await queueSettleWorkspace(row.id); - } + if (!running) return wakeCloudWorkspace(row); + return wakeAfterStop(row, await stopAndSnapshot(row.providerSandboxId)); +} + +/** Starts a box again after the caller ended its session, as a metered wake. */ +export async function wakeAfterStop( + row: CloudWorkspaceRow, + stopped: { sessionId: string | null }, +): Promise { + await recordSandboxAsleep(row.id, { + sessionId: stopped.sessionId ?? row.sandboxSessionId, + }); + await queueSettleWorkspace(row.id); return wakeCloudWorkspace(row); } diff --git a/packages/trpc/src/router/environment/environment.test.ts b/packages/trpc/src/router/environment/environment.test.ts index e8d4a3a343a..08187e737c2 100644 --- a/packages/trpc/src/router/environment/environment.test.ts +++ b/packages/trpc/src/router/environment/environment.test.ts @@ -1,12 +1,15 @@ import { beforeEach, describe, expect, test } from "bun:test"; -import { db } from "@superset/db/client"; +import { db, dbWs } from "@superset/db/client"; import { stub } from "../../../test/stub"; import * as plan from "../../lib/billing/plan"; import * as guards from "../../lib/cloud-guards"; import * as sandbox from "../../lib/sandbox"; import { userError } from "../../trpc"; import * as access from "../cloud-workspace/access"; +import * as wake from "../cloud-workspace/wake"; import * as integration from "../integration/utils"; +import * as currentBundle from "./current-bundle"; +import * as limit from "./limit"; const ENVIRONMENT_ID = "6f7a8b9c-0d1e-4f2a-8b3c-4d5e6f7a8b9c"; const ORGANIZATION_ID = "1a2b3c4d-5e6f-4a7b-8c9d-0e1f2a3b4c5d"; @@ -128,3 +131,57 @@ describe("environment.promote onto an environment", () => { expect(written).toEqual([]); }); }); + +describe("environment.promote of a running workspace", () => { + const restarted: Array<[string, { sessionId: string | null }]> = []; + let saved = 0; + stub(sandbox, { + promoteSandboxToEnvironment: async () => ({ + goldenName: "env-new", + region: "iad1", + stopped: { sessionId: "sess_A" }, + }), + deleteSandbox: async () => {}, + }); + stub(wake, { + wakeAfterStop: async ( + row: { id: string }, + stopped: { sessionId: string | null }, + ) => { + restarted.push([row.id, stopped]); + throw userError({ + code: "FORBIDDEN", + message: "limit reached", + i18nKey: "serverError.cloudWorkspace.usageLimitReached", + }); + }, + }); + stub(db, { + select: () => ({ from: () => ({ where: async () => [] }) }), + }); + stub(limit, { assertEnvironmentCapacity: async () => null }); + stub(currentBundle, { currentBundleSha: async () => "sha" }); + stub(dbWs, { + transaction: async (run: (tx: unknown) => Promise) => { + const rows = () => [{ id: ENVIRONMENT_ID }]; + const tx = { + insert: () => ({ + values: () => + Object.assign(Promise.resolve(), { returning: async () => rows() }), + }), + delete: () => ({ where: async () => {} }), + }; + saved += 1; + return run(tx); + }, + }); + + test("starts the box again through the wake path, and a refused restart still saves the environment", async () => { + await caller.environment.promote({ + cloudWorkspaceId: WORKSPACE_ID, + name: "Saved", + }); + expect(restarted).toEqual([[WORKSPACE_ID, { sessionId: "sess_A" }]]); + expect(saved).toBe(1); + }); +}); diff --git a/packages/trpc/src/router/environment/environment.ts b/packages/trpc/src/router/environment/environment.ts index d8ce3b4e274..8bcd5f6370b 100644 --- a/packages/trpc/src/router/environment/environment.ts +++ b/packages/trpc/src/router/environment/environment.ts @@ -1,3 +1,4 @@ +import * as Sentry from "@sentry/core"; import { db, dbWs } from "@superset/db/client"; import { environmentProjects, @@ -7,7 +8,7 @@ import { environments, githubRepositories, } from "@superset/db/schema"; -import { MACHINE_SIZES } from "@superset/shared/billing"; +import { isCloudUsageRefusal, MACHINE_SIZES } from "@superset/shared/billing"; import { SANDBOX_IMAGE_NAME } from "@superset/shared/constants"; import { SANDBOX_REGION_IDS } from "@superset/shared/sandbox/regions"; import type { TRPCRouterRecord } from "@trpc/server"; @@ -20,7 +21,6 @@ import { type MemberContext, } from "../../lib/cloud-guards"; import { - buildSandboxClaim, deleteSandbox, loadRepositories, primaryRepository, @@ -33,6 +33,7 @@ import { personProcedure, userError } from "../../trpc"; import { loadVisibleWorkspace } from "../cloud-workspace/access"; import { isLiveStatus } from "../cloud-workspace/live"; import { assertMachineSizeOffered } from "../cloud-workspace/start"; +import { wakeAfterStop } from "../cloud-workspace/wake"; import { verifyOrgAdmin } from "../integration/utils"; import { currentBundleSha } from "./current-bundle"; import { @@ -498,12 +499,26 @@ export const environmentRouter = { const limit = target ? null : await assertEnvironmentCapacity(workspace.organizationId); - const { claim } = await buildSandboxClaim({ row: workspace }); const golden = await promoteSandboxToEnvironment({ sourceSandbox: workspace.providerSandboxId, goldenName, - claim, + organizationId: workspace.organizationId, }); + if (golden.stopped) { + await wakeAfterStop(workspace, golden.stopped).catch((error) => { + console.error( + `[environment/promote] could not restart ${workspace.id} after its snapshot`, + error, + ); + if (isCloudUsageRefusal(error)) return; + Sentry.captureException(error, { + tags: { + "cloud_workspace.id": workspace.id, + "organization.id": workspace.organizationId, + }, + }); + }); + } const primaryRepositoryId = checkouts[0]?.repository.id ?? null; const fromGolden = { provider: workspace.provider, From 78864be9a5352ecbcf12dd96317373012acfa8c9 Mon Sep 17 00:00:00 2001 From: Satya Patel <14907857+saddlepaddle@users.noreply.github.com> Date: Wed, 18 Nov 2026 03:28:48 +0000 Subject: [PATCH 3/6] docs(skill): sync the asleep coverage record to a2c4629d --- .agents/skills/verify-sandbox-asleep/SKILL.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.agents/skills/verify-sandbox-asleep/SKILL.md b/.agents/skills/verify-sandbox-asleep/SKILL.md index 3146b0a085c..cedac7447f7 100644 --- a/.agents/skills/verify-sandbox-asleep/SKILL.md +++ b/.agents/skills/verify-sandbox-asleep/SKILL.md @@ -31,7 +31,7 @@ one TLC worker, so a failure always prints the shortest trace. A pass takes abou ## Coverage record -Synced with commit `cf52bc20`. Paths are under `packages/trpc/src/`. +Synced with commit `a2c4629d`. Paths are under `packages/trpc/src/`. | Action | Code | | --- | --- | From 183e91e8774460336adfc3ca8d3e92dcc06a3b5e Mon Sep 17 00:00:00 2001 From: Satya Patel <14907857+saddlepaddle@users.noreply.github.com> Date: Wed, 18 Nov 2026 03:53:24 +0000 Subject: [PATCH 4/6] fix(cloud): second review fixes for M8 - A refused wake stops the box only when its own call resumed it: wakeSandbox hands that session to beforeResume as resumedSession. A refusal for a box another wake authorized and resumed stops nothing. - environment.promote saves the environment row first, then restarts the box; a failed save also restarts it before it rethrows. - Sentry tags through controlTags; the wake's cap stop writes meter.cap_stopped with source "wake". - RECONCILE_LOOKBACK_MS is 2 h, the attach grace plus one reconcile period, so one missed run cannot leave a short session unbilled. - The golden's tags come from environmentTags, which sandboxTags also uses. --- .agents/skills/verify-sandbox-asleep/SKILL.md | 2 +- packages/trpc/src/lib/sandbox/vercel.test.ts | 10 ++++- packages/trpc/src/lib/sandbox/vercel.ts | 40 +++++++++++++------ .../trpc/src/router/cloud-workspace/meter.ts | 2 +- .../src/router/cloud-workspace/wake.test.ts | 30 +++++++++++--- .../trpc/src/router/cloud-workspace/wake.ts | 27 ++++++++----- .../router/environment/environment.test.ts | 12 +++--- .../src/router/environment/environment.ts | 19 +++++---- 8 files changed, 98 insertions(+), 44 deletions(-) diff --git a/.agents/skills/verify-sandbox-asleep/SKILL.md b/.agents/skills/verify-sandbox-asleep/SKILL.md index cedac7447f7..d528d08c448 100644 --- a/.agents/skills/verify-sandbox-asleep/SKILL.md +++ b/.agents/skills/verify-sandbox-asleep/SKILL.md @@ -38,7 +38,7 @@ Synced with commit `a2c4629d`. Paths are under `packages/trpc/src/`. | `ProvisionDone` | `provisionCloudWorkspace` in `router/cloud-workspace/provision.ts`: `provisionSandbox`, the transition to `awake`, then `recordSandboxAwake` | | `WakeResume` / `WakeRecord` | `wakeCloudWorkspace` in `router/cloud-workspace/wake.ts`: `wakeSandbox` in `lib/sandbox/vercel.ts`, then `recordSandboxAwake` in `router/cloud-workspace/sandbox-expiry.ts`. A wake that read the box running and resumed it after a timeout stop is the same `WakeResume`: the box was stopped when the call resumed it | | `TopUpExtend` / `TopUpRecord` | `sandbox.keepalive` in `router/sandbox/sandbox.ts`: `topUpSandboxSession`, then `recordSandboxAwake` | -| `StopBox` / `StopRecord` | `stopAndSnapshot` or `stopSandbox`, then `recordSandboxAsleep` in `router/cloud-workspace/sandbox-sleep.ts`, from `cloudWorkspace.sleep` (`cloud-workspace.ts`), `restartCloudWorkspace` and `wakeAfterStop` (`wake.ts`; `environment.promote` calls `wakeAfterStop` after the snapshot in `promoteSandboxToEnvironment` stopped the source, so `StopBox` there is the snapshot), a refused `wakeCloudWorkspace` that finds the box running (it records that session awake first, a `WakeRecord`, then stops it), and the spend-cap stops in `meter.ts` (`settleBoot`, `settleOrganizationNow`, the refused adopt in `adoptRunningBoxes`) | +| `StopBox` / `StopRecord` | `stopAndSnapshot` or `stopSandbox`, then `recordSandboxAsleep` in `router/cloud-workspace/sandbox-sleep.ts`, from `cloudWorkspace.sleep` (`cloud-workspace.ts`), `restartCloudWorkspace` and `wakeAfterStop` (`wake.ts`; `environment.promote` calls `wakeAfterStop` after the snapshot in `promoteSandboxToEnvironment` stopped the source, so `StopBox` there is the snapshot), a refused `wakeCloudWorkspace` whose own call resumed the box (it records that session awake first, a `WakeRecord`, then stops it; a refusal for a box another wake resumed stops nothing), and the spend-cap stops in `meter.ts` (`settleBoot`, `settleOrganizationNow`, the refused adopt in `adoptRunningBoxes`) | | `WatchFire` | `watchSandboxExpiry` in `sandbox-expiry.ts`, queued by `queueWatch` through `publishCloudWorkspaceJob` (`jobs.ts`), served by `apps/api/src/app/api/cloud-workspaces/sandbox-expiry/route.ts` | | `ReconcileList` / `ReconcileApply` | `reconcileCloudWorkspaces` in `sandbox-expiry.ts`: `listActiveWorkspaceSandboxes`, then `syncSandboxExpiries` against each listing's `currentSessionId` | | `Archive` | `cloudWorkspace.delete` in `cloud-workspace.ts`: the transition to `archived`, then `queueReap` | diff --git a/packages/trpc/src/lib/sandbox/vercel.test.ts b/packages/trpc/src/lib/sandbox/vercel.test.ts index d5dc9e00f70..78fd25f4461 100644 --- a/packages/trpc/src/lib/sandbox/vercel.test.ts +++ b/packages/trpc/src/lib/sandbox/vercel.test.ts @@ -218,7 +218,7 @@ describe("wakeSandbox", () => { expiresAt: new Date(NOW + 60_000), vcpus: 4, domain: () => "https://box.test", - currentSession: () => ({ vcpus: 8 }), + currentSession: () => ({ vcpus: 8, sessionId: "sess_new" }), update: async () => {}, writeFiles: async () => {}, // The SDK's withResume: a call on a stopped session resumes it, then retries. @@ -264,7 +264,13 @@ describe("wakeSandbox", () => { test("a box read running that a call resumed runs beforeResume for the new session", async () => { running = true; - expect(await wake()).toEqual([{ memoryMb: 16_384, vcpus: 8 }]); + expect(await wake()).toEqual([ + { + memoryMb: 16_384, + vcpus: 8, + resumedSession: { sessionId: "sess_new", expiresAt: box.expiresAt }, + }, + ]); }); test("a box read stopped runs beforeResume once, before its own resume", async () => { diff --git a/packages/trpc/src/lib/sandbox/vercel.ts b/packages/trpc/src/lib/sandbox/vercel.ts index 953fd3670fd..aa5ed67e57e 100644 --- a/packages/trpc/src/lib/sandbox/vercel.ts +++ b/packages/trpc/src/lib/sandbox/vercel.ts @@ -126,16 +126,21 @@ export interface SandboxClaim { } /** Vercel allows 5 tags per sandbox. */ +function environmentTags(organizationId: string): Record { + return { kind: "environment", org: organizationId }; +} + function sandboxTags( kind: "workspace" | "environment", identity: SandboxIdentity, ): Record { - const tags = { kind, org: identity.SUPERSET_SANDBOX_ORGANIZATION_ID }; + const org = identity.SUPERSET_SANDBOX_ORGANIZATION_ID; // A golden's identity is the workspace it was built from, not its own. - if (kind === "environment") return tags; + if (kind === "environment") return environmentTags(org); const user = identity.SUPERSET_SANDBOX_CREATOR_USER_ID; return { - ...tags, + kind, + org, workspace: identity.SUPERSET_SANDBOX_WORKSPACE_ID, ...(user ? { user } : {}), }; @@ -485,9 +490,14 @@ export async function wakeSandbox(args: { claim: SandboxClaim; /** * Runs once the box is known to be stopped, before anything resumes it; or, - * for a box read running that a call here resumed, once those calls end. + * for a box read running that a call here resumed (`resumedSession`), once + * those calls end. */ - beforeResume?: (box: { memoryMb: number; vcpus: number }) => Promise; + beforeResume?: (box: { + memoryMb: number; + vcpus: number; + resumedSession?: { sessionId: string | null; expiresAt: Date | null }; + }) => Promise; }): Promise<{ hostTarget: string; /** The session was stopped, so every process on the box started from this claim. */ @@ -500,14 +510,16 @@ export async function wakeSandbox(args: { }> { try { let wasRunning = false; - let resumedVcpus: number | null = null; + let resumed = null as { vcpus: number; sessionId: string | null } | null; const sandbox = await Sandbox.get({ ...credentials(), name: args.providerSandboxId, resume: false, // Any call on a Sandbox object resumes a stopped session and retries. - onResume: async (resumed) => { - if (wasRunning) resumedVcpus = resumed.currentSession().vcpus; + onResume: async (box) => { + if (!wasRunning) return; + const session = box.currentSession(); + resumed = { vcpus: session.vcpus, sessionId: session.sessionId }; }, }); const hostTarget = sandbox.domain(HOST_SERVICE_PORT); @@ -578,10 +590,14 @@ export async function wakeSandbox(args: { runBoot(sandbox, args.claim.hostSecret), ); } - if (resumedVcpus && args.beforeResume) { + if (resumed && args.beforeResume) { await args.beforeResume({ - memoryMb: resumedVcpus * MEMORY_MB_PER_VCPU, - vcpus: resumedVcpus, + memoryMb: resumed.vcpus * MEMORY_MB_PER_VCPU, + vcpus: resumed.vcpus, + resumedSession: { + sessionId: resumed.sessionId, + expiresAt: sandbox.expiresAt ?? null, + }, }); } const answer = await Sentry.startSpan( @@ -693,7 +709,7 @@ export async function promoteSandboxToEnvironment(args: { persistent: true, snapshotExpiration: 0, keepLastSnapshots: { count: 1 }, - tags: { kind: "environment", org: args.organizationId }, + tags: environmentTags(args.organizationId), }); await stripWorkspaceIdentity(golden); const created = golden.currentSnapshotId; diff --git a/packages/trpc/src/router/cloud-workspace/meter.ts b/packages/trpc/src/router/cloud-workspace/meter.ts index 60a10516554..9a06bc84626 100644 --- a/packages/trpc/src/router/cloud-workspace/meter.ts +++ b/packages/trpc/src/router/cloud-workspace/meter.ts @@ -32,7 +32,7 @@ const LOCK_TTL_MS = 70 * 60 * 1000; const CAP_GRACE_MS = 10 * 60 * 1000; const SETTLE_LEASE_MS = 2 * 60 * 1000; const UNATTACHED_BOOT_GRACE_MS = 60 * 60 * 1000; -export const RECONCILE_LOOKBACK_MS = 60 * 60 * 1000; +export const RECONCILE_LOOKBACK_MS = 2 * 60 * 60 * 1000; const RECONCILE_CONCURRENCY = 25; export function boxFor(vcpus: number): { memoryMb: number; vcpus: number } { diff --git a/packages/trpc/src/router/cloud-workspace/wake.test.ts b/packages/trpc/src/router/cloud-workspace/wake.test.ts index c86dfb023f1..6ebecc8a2b9 100644 --- a/packages/trpc/src/router/cloud-workspace/wake.test.ts +++ b/packages/trpc/src/router/cloud-workspace/wake.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, test } from "bun:test"; +import { beforeEach, describe, expect, test } from "bun:test"; import * as Sentry from "@sentry/core"; import { db } from "@superset/db/client"; import { stub } from "../../../test/stub"; @@ -132,20 +132,25 @@ describe("wakeAfterStop", () => { describe("a refused wake", () => { const calls: string[] = []; const recordedAsleep: Array<[string, { sessionId: string | null }]> = []; + let resumedSession: { sessionId: string; expiresAt: Date } | undefined; stub(sandbox, { buildSandboxClaim: async () => ({ claim: {}, agentCredentialDigest: "digest", }), wakeSandbox: async (args: { - beforeResume: (box: { memoryMb: number; vcpus: number }) => Promise; + beforeResume: (box: { + memoryMb: number; + vcpus: number; + resumedSession?: { sessionId: string; expiresAt: Date }; + }) => Promise; }) => { - await args.beforeResume({ memoryMb: 16_384, vcpus: 8 }); + await args.beforeResume({ memoryMb: 16_384, vcpus: 8, resumedSession }); }, describeSandbox: async () => ({ running: true, expiresAt: new Date(Date.now() + 60_000), - sessionId: "sess_C", + sessionId: "sess_B", }), stopSandbox: async () => { calls.push("stop"); @@ -174,12 +179,27 @@ describe("a refused wake", () => { recordedAsleep.push([id, stopped]); }, }); + beforeEach(() => { + calls.length = 0; + recordedAsleep.length = 0; + }); - test("stops a box a call resumed before the refusal, and records it asleep for that session", async () => { + test("stops the box a call here resumed, and records it asleep for that session", async () => { + resumedSession = { + sessionId: "sess_C", + expiresAt: new Date(Date.now() + 60_000), + }; await expect(wakeCloudWorkspace(ROW)).rejects.toThrow("limit reached"); expect(calls).toEqual(["awake sess_C", "stop"]); expect(recordedAsleep).toEqual([["ws_1", { sessionId: "sess_C" }]]); }); + + test("leaves alone a box another wake authorized and resumed meanwhile", async () => { + resumedSession = undefined; + await expect(wakeCloudWorkspace(ROW)).rejects.toThrow("limit reached"); + expect(calls).toEqual([]); + expect(recordedAsleep).toEqual([]); + }); }); describe("markSandboxUnavailable", () => { diff --git a/packages/trpc/src/router/cloud-workspace/wake.ts b/packages/trpc/src/router/cloud-workspace/wake.ts index 4e8e617d467..08c75096525 100644 --- a/packages/trpc/src/router/cloud-workspace/wake.ts +++ b/packages/trpc/src/router/cloud-workspace/wake.ts @@ -85,12 +85,14 @@ async function wake( organizationId: row.organizationId, }; let bootId: string | null = null; + let resumed: { sessionId: string | null; expiresAt: Date | null } | undefined; const wakeStartedAt = Date.now(); const { hostTarget, booted, rebooted, answer, expiresAt, sessionId } = await wakeSandbox({ providerSandboxId: row.providerSandboxId, claim, - beforeResume: async (box) => { + beforeResume: async ({ resumedSession, ...box }) => { + resumed = resumedSession; ({ bootId } = await Sentry.startSpan( { name: "authorize boot", op: "billing" }, () => @@ -121,20 +123,23 @@ async function wake( providerSandboxId: row.providerSandboxId, }).catch(() => {}); } - // A call here may have resumed the box before the refusal. - if (isCloudUsageRefusal(error)) { + // A call here resumed the box, and no boot meters that session. + if (resumed && isCloudUsageRefusal(error)) { + console.error( + `[cloud-workspace] ${row.id} was resumed past its limit; stopping it`, + ); try { - const live = await describeSandbox(row.providerSandboxId); - if (live.running && live.expiresAt) { + if (resumed.expiresAt) { await recordSandboxAwake(row.id, { - expiresAt: live.expiresAt, - sessionId: live.sessionId, - }); - const stopped = await stopSandbox(row.providerSandboxId); - await recordSandboxAsleep(row.id, { - sessionId: stopped.sessionId ?? live.sessionId, + expiresAt: resumed.expiresAt, + sessionId: resumed.sessionId, }); } + const stopped = await stopSandbox(row.providerSandboxId); + await recordSandboxAsleep(row.id, { + sessionId: stopped.sessionId ?? resumed.sessionId, + }); + logControl("meter.cap_stopped", scope, { source: "wake" }, "warn"); } catch (stopError) { Sentry.captureException(stopError, { tags: controlTags(scope) }); } diff --git a/packages/trpc/src/router/environment/environment.test.ts b/packages/trpc/src/router/environment/environment.test.ts index 08187e737c2..76404214510 100644 --- a/packages/trpc/src/router/environment/environment.test.ts +++ b/packages/trpc/src/router/environment/environment.test.ts @@ -134,7 +134,7 @@ describe("environment.promote onto an environment", () => { describe("environment.promote of a running workspace", () => { const restarted: Array<[string, { sessionId: string | null }]> = []; - let saved = 0; + const order: string[] = []; stub(sandbox, { promoteSandboxToEnvironment: async () => ({ goldenName: "env-new", @@ -149,6 +149,7 @@ describe("environment.promote of a running workspace", () => { stopped: { sessionId: string | null }, ) => { restarted.push([row.id, stopped]); + order.push("restart"); throw userError({ code: "FORBIDDEN", message: "limit reached", @@ -171,17 +172,18 @@ describe("environment.promote of a running workspace", () => { }), delete: () => ({ where: async () => {} }), }; - saved += 1; - return run(tx); + const saved = await run(tx); + order.push("save"); + return saved; }, }); - test("starts the box again through the wake path, and a refused restart still saves the environment", async () => { + test("saves the environment, then starts the box again through the wake path; a refused restart does not fail the promote", async () => { await caller.environment.promote({ cloudWorkspaceId: WORKSPACE_ID, name: "Saved", }); expect(restarted).toEqual([[WORKSPACE_ID, { sessionId: "sess_A" }]]); - expect(saved).toBe(1); + expect(order).toEqual(["save", "restart"]); }); }); diff --git a/packages/trpc/src/router/environment/environment.ts b/packages/trpc/src/router/environment/environment.ts index 8bcd5f6370b..060826df00e 100644 --- a/packages/trpc/src/router/environment/environment.ts +++ b/packages/trpc/src/router/environment/environment.ts @@ -20,6 +20,7 @@ import { isMember, type MemberContext, } from "../../lib/cloud-guards"; +import { controlTags } from "../../lib/control-log"; import { deleteSandbox, loadRepositories, @@ -504,21 +505,23 @@ export const environmentRouter = { goldenName, organizationId: workspace.organizationId, }); - if (golden.stopped) { - await wakeAfterStop(workspace, golden.stopped).catch((error) => { + const { stopped } = golden; + const restart = async () => { + if (!stopped) return; + await wakeAfterStop(workspace, stopped).catch((error) => { console.error( `[environment/promote] could not restart ${workspace.id} after its snapshot`, error, ); if (isCloudUsageRefusal(error)) return; Sentry.captureException(error, { - tags: { - "cloud_workspace.id": workspace.id, - "organization.id": workspace.organizationId, - }, + tags: controlTags({ + cloudWorkspaceId: workspace.id, + organizationId: workspace.organizationId, + }), }); }); - } + }; const primaryRepositoryId = checkouts[0]?.repository.id ?? null; const fromGolden = { provider: workspace.provider, @@ -619,6 +622,7 @@ export const environmentRouter = { cleanup, ), ); + await restart(); throw error; }); if (target?.sourceKind === "fork") { @@ -629,6 +633,7 @@ export const environmentRouter = { ), ); } + await restart(); return row; }), From 9b1420760dd3950e43f3006e4f3ff7118f4d041b Mon Sep 17 00:00:00 2001 From: Satya Patel <14907857+saddlepaddle@users.noreply.github.com> Date: Wed, 18 Nov 2026 03:53:24 +0000 Subject: [PATCH 5/6] docs(skill): sync the asleep coverage record to 183e91e8 --- .agents/skills/verify-sandbox-asleep/SKILL.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.agents/skills/verify-sandbox-asleep/SKILL.md b/.agents/skills/verify-sandbox-asleep/SKILL.md index d528d08c448..99cd618c1b2 100644 --- a/.agents/skills/verify-sandbox-asleep/SKILL.md +++ b/.agents/skills/verify-sandbox-asleep/SKILL.md @@ -31,7 +31,7 @@ one TLC worker, so a failure always prints the shortest trace. A pass takes abou ## Coverage record -Synced with commit `a2c4629d`. Paths are under `packages/trpc/src/`. +Synced with commit `183e91e8`. Paths are under `packages/trpc/src/`. | Action | Code | | --- | --- | From 98f61b95634e27f6384f3c41a96e2a2c22bef7a6 Mon Sep 17 00:00:00 2001 From: Satya Patel <14907857+saddlepaddle@users.noreply.github.com> Date: Wed, 18 Nov 2026 04:50:19 +0000 Subject: [PATCH 6/6] refactor(cloud): separate adopt and listing lookbacks; tag-limit note on sandboxTags Same values (2 h), no behaviour change: the meter's adopt window and the reconcile's box listing were one shared constant, so a meter change moved the expiry sync. The listing must stay at least as long as the adopt window. --- packages/trpc/src/lib/sandbox/vercel.ts | 2 +- packages/trpc/src/router/cloud-workspace/meter.ts | 4 ++-- .../trpc/src/router/cloud-workspace/sandbox-expiry.ts | 10 ++++------ 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/packages/trpc/src/lib/sandbox/vercel.ts b/packages/trpc/src/lib/sandbox/vercel.ts index f98b33a4c31..812681d0020 100644 --- a/packages/trpc/src/lib/sandbox/vercel.ts +++ b/packages/trpc/src/lib/sandbox/vercel.ts @@ -126,11 +126,11 @@ export interface SandboxClaim { agentTools?: boolean; } -/** Vercel allows 5 tags per sandbox. */ function environmentTags(organizationId: string): Record { return { kind: "environment", org: organizationId }; } +/** Vercel allows 5 tags per sandbox. */ function sandboxTags( kind: "workspace" | "environment", identity: SandboxIdentity, diff --git a/packages/trpc/src/router/cloud-workspace/meter.ts b/packages/trpc/src/router/cloud-workspace/meter.ts index 9a06bc84626..57f51e9e328 100644 --- a/packages/trpc/src/router/cloud-workspace/meter.ts +++ b/packages/trpc/src/router/cloud-workspace/meter.ts @@ -32,7 +32,7 @@ const LOCK_TTL_MS = 70 * 60 * 1000; const CAP_GRACE_MS = 10 * 60 * 1000; const SETTLE_LEASE_MS = 2 * 60 * 1000; const UNATTACHED_BOOT_GRACE_MS = 60 * 60 * 1000; -export const RECONCILE_LOOKBACK_MS = 2 * 60 * 60 * 1000; +const ADOPT_LOOKBACK_MS = 2 * 60 * 60 * 1000; const RECONCILE_CONCURRENCY = 25; export function boxFor(vcpus: number): { memoryMb: number; vcpus: number } { @@ -700,7 +700,7 @@ async function adoptRunningBoxes( const adoptSessions = async (workspace: (typeof unmetered)[number]) => { const sessions = await listSandboxSessions( workspace.providerSandboxId, - Date.now() - RECONCILE_LOOKBACK_MS, + Date.now() - ADOPT_LOOKBACK_MS, ); if (!sessions?.length) return 0; const boots = await db diff --git a/packages/trpc/src/router/cloud-workspace/sandbox-expiry.ts b/packages/trpc/src/router/cloud-workspace/sandbox-expiry.ts index d5ca4661598..36710a1953f 100644 --- a/packages/trpc/src/router/cloud-workspace/sandbox-expiry.ts +++ b/packages/trpc/src/router/cloud-workspace/sandbox-expiry.ts @@ -11,11 +11,7 @@ import { SandboxUnavailableError, } from "../../lib/sandbox"; import { publishCloudWorkspaceJob } from "./jobs"; -import { - queueSettleWorkspace, - RECONCILE_LOOKBACK_MS, - reconcileCloudWorkspaceUsage, -} from "./meter"; +import { queueSettleWorkspace, reconcileCloudWorkspaceUsage } from "./meter"; import { keepUpdatedAt, recordSandboxAsleep, @@ -23,6 +19,8 @@ import { } from "./sandbox-sleep"; const WATCH_AFTER_EXPIRY_MS = 5_000; +// The meter adopts only boxes this listing returns: keep it at least its ADOPT_LOOKBACK_MS. +const LISTING_LOOKBACK_MS = 2 * 60 * 60 * 1000; export interface WatchSandboxExpiryInput { cloudWorkspaceId: string; @@ -191,7 +189,7 @@ export async function syncSandboxExpiries( /** The 30-minute reconcile: sleep state on every stack, then the meter where billing runs. */ export async function reconcileCloudWorkspaces() { const active = await listActiveWorkspaceSandboxes( - Date.now() - RECONCILE_LOOKBACK_MS, + Date.now() - LISTING_LOOKBACK_MS, ).catch((error) => { Sentry.captureException(error); return null;