diff --git a/.agents/skills/verify-sandbox-asleep/SKILL.md b/.agents/skills/verify-sandbox-asleep/SKILL.md index e77f22508ec..99cd618c1b2 100644 --- a/.agents/skills/verify-sandbox-asleep/SKILL.md +++ b/.agents/skills/verify-sandbox-asleep/SKILL.md @@ -31,14 +31,14 @@ one TLC worker, so a failure always prints the shortest trace. A pass takes abou ## Coverage record -Synced with commit `623b52bf4`. Paths are under `packages/trpc/src/`. +Synced with commit `183e91e8`. Paths are under `packages/trpc/src/`. | Action | Code | | --- | --- | | `ProvisionDone` | `provisionCloudWorkspace` in `router/cloud-workspace/provision.ts`: `provisionSandbox`, the transition to `awake`, then `recordSandboxAwake` | -| `WakeResume` / `WakeRecord` | `wakeCloudWorkspace` in `router/cloud-workspace/wake.ts`: `wakeSandbox` in `lib/sandbox/vercel.ts`, then `recordSandboxAwake` in `router/cloud-workspace/sandbox-expiry.ts` | +| `WakeResume` / `WakeRecord` | `wakeCloudWorkspace` in `router/cloud-workspace/wake.ts`: `wakeSandbox` in `lib/sandbox/vercel.ts`, then `recordSandboxAwake` in `router/cloud-workspace/sandbox-expiry.ts`. A wake that read the box running and resumed it after a timeout stop is the same `WakeResume`: the box was stopped when the call resumed it | | `TopUpExtend` / `TopUpRecord` | `sandbox.keepalive` in `router/sandbox/sandbox.ts`: `topUpSandboxSession`, then `recordSandboxAwake` | -| `StopBox` / `StopRecord` | `stopAndSnapshot` or `stopSandbox`, then `recordSandboxAsleep` in `router/cloud-workspace/sandbox-sleep.ts`, from `cloudWorkspace.sleep` (`cloud-workspace.ts`), `restartCloudWorkspace` (`wake.ts`), and the spend-cap stops in `meter.ts` (`settleBoot`, `settleOrganizationNow`, the refused adopt in `adoptRunningBoxes`) | +| `StopBox` / `StopRecord` | `stopAndSnapshot` or `stopSandbox`, then `recordSandboxAsleep` in `router/cloud-workspace/sandbox-sleep.ts`, from `cloudWorkspace.sleep` (`cloud-workspace.ts`), `restartCloudWorkspace` and `wakeAfterStop` (`wake.ts`; `environment.promote` calls `wakeAfterStop` after the snapshot in `promoteSandboxToEnvironment` stopped the source, so `StopBox` there is the snapshot), a refused `wakeCloudWorkspace` whose own call resumed the box (it records that session awake first, a `WakeRecord`, then stops it; a refusal for a box another wake resumed stops nothing), and the spend-cap stops in `meter.ts` (`settleBoot`, `settleOrganizationNow`, the refused adopt in `adoptRunningBoxes`) | | `WatchFire` | `watchSandboxExpiry` in `sandbox-expiry.ts`, queued by `queueWatch` through `publishCloudWorkspaceJob` (`jobs.ts`), served by `apps/api/src/app/api/cloud-workspaces/sandbox-expiry/route.ts` | | `ReconcileList` / `ReconcileApply` | `reconcileCloudWorkspaces` in `sandbox-expiry.ts`: `listActiveWorkspaceSandboxes`, then `syncSandboxExpiries` against each listing's `currentSessionId` | | `Archive` | `cloudWorkspace.delete` in `cloud-workspace.ts`: the transition to `archived`, then `queueReap` | diff --git a/docs/cloud-sandbox-considerations.md b/docs/cloud-sandbox-considerations.md index 7c1364c4275..60031668f38 100644 --- a/docs/cloud-sandbox-considerations.md +++ b/docs/cloud-sandbox-considerations.md @@ -38,6 +38,8 @@ schedule (`/api/cloud-workspaces/jobs/reconcile-usage`, every 30 minutes, set in the Upstash console and watched by a Sentry cron monitor) finalizes each running box's memory GB-minutes and takes the next lock; a box whose next lock is refused is stopped. Stopping, archiving or deleting a box settles the rest. +The same run opens a boot for any session from the last hour that has none +(`adoptRunningBoxes` in `meter.ts`), billed from the session's start. Environment builds are not billed. Prices and limits live in Autumn, not here. Each paid seat adds its plan's GB-minutes to one team pool through Autumn seat licenses; see [Seats and cloud hours in Autumn](autumn-seat-licenses.md). diff --git a/docs/cloud-sandbox-mismatches.md b/docs/cloud-sandbox-mismatches.md index ffc8144c02c..40cd448fefe 100644 --- a/docs/cloud-sandbox-mismatches.md +++ b/docs/cloud-sandbox-mismatches.md @@ -776,9 +776,26 @@ obvious tidy-up once the golden exists — leaves the workspace unable to ever resume (`410 Cannot resume sandbox: no snapshot available`), which is how one e2e workspace died. So the snapshot stays (`keepLastSnapshots` evicts it on the source's next stop), and a source that was running is started again -before promote returns. A row whose sandbox is gone or unresumable is marked -`failed` by the next `access`, so it gets the failed screen and a Remove -button rather than a sidebar entry that never opens. +before promote returns, through `wakeAfterStop`: the new session is a wake, +authorized and metered like any other. Until 2026-11 the restart ran the boot +command directly, and 69 sessions in a week ran with no boot row. A row whose +sandbox is gone or unresumable is marked `failed` by the next `access`, so it +gets the failed screen and a Remove button rather than a sidebar entry that +never opens. + +**Any call on a `Sandbox` object resumes a stopped session.** The SDK wraps +`runCommand`, `writeFiles`, `extendTimeout` and the rest in `withResume`: on a +"stopped" answer it resumes and retries. A box read `running` that hits its +timeout a moment later is resumed by the next call on that object, and no +code chose to start it. Measured: 9 sessions in a week started 0.8–1.5 s after +a timeout stop. `wakeSandbox` passes `onResume` to `Sandbox.get` and runs +`beforeResume` (which authorizes the boot) for such a session; a refused one is +stopped at once. A top-up goes through the session, which never resumes. +Other callers on a box read `running` are covered only by the reconcile, which +adopts any session that has no boot row within one run: promote's +`source.snapshot()` (wrapped in `withResume`), and `applySandboxPolicy`, whose +`update` does not resume in SDK 4.0.0 but would if a later SDK wrapped it. +`provisionSandbox` on a box it already made is covered by its own boot. **`stop()` returns before the stop's snapshot is current.** The sandbox is still `stopping` when the call resolves, and a fork taken then boots from diff --git a/packages/trpc/src/lib/sandbox/AGENTS.md b/packages/trpc/src/lib/sandbox/AGENTS.md index 9b2a192f262..5487d379e0d 100644 --- a/packages/trpc/src/lib/sandbox/AGENTS.md +++ b/packages/trpc/src/lib/sandbox/AGENTS.md @@ -36,9 +36,10 @@ Two habits that pay for themselves here: a fix that only exists on your machine. - **The claim is the whole handoff.** `buildSandboxClaim` assembles what a box needs to be one workspace — the identity file, the firewall rules, the - managed environment, the host secret — and `provisionSandbox`, `wakeSandbox` - and `promoteSandboxToEnvironment` all take it, so a create, a wake and a - restart after promote hand the box the same thing. + managed environment, the host secret — and `provisionSandbox` and + `wakeSandbox` take it. A restart after promote is a wake (`wakeAfterStop`), + so a create, a wake and that restart hand the box the same thing, and each + new session gets a boot row. `docs/cloud-sandbox-considerations.md` is the companion list: what we still owe before a non-internal user can create a sandbox. Several entries there are only diff --git a/packages/trpc/src/lib/sandbox/index.ts b/packages/trpc/src/lib/sandbox/index.ts index 7038875b600..df06a04faf2 100644 --- a/packages/trpc/src/lib/sandbox/index.ts +++ b/packages/trpc/src/lib/sandbox/index.ts @@ -55,6 +55,7 @@ export { findSandboxSession, HOST_SERVICE_PORT, listActiveWorkspaceSandboxes, + listSandboxSessions, MEMORY_MB_PER_VCPU, promoteSandboxToEnvironment, provisionSandbox, diff --git a/packages/trpc/src/lib/sandbox/vercel.test.ts b/packages/trpc/src/lib/sandbox/vercel.test.ts index 272482b60dd..c884fdeb54e 100644 --- a/packages/trpc/src/lib/sandbox/vercel.test.ts +++ b/packages/trpc/src/lib/sandbox/vercel.test.ts @@ -12,6 +12,8 @@ import { agentToolsMounts, findSandboxSession, listActiveWorkspaceSandboxes, + listSandboxSessions, + promoteSandboxToEnvironment, type SandboxClaim, SESSION_TIMEOUT_MS, topUpSandboxSession, @@ -116,6 +118,168 @@ describe("findSandboxSession", () => { }); }); +describe("listSandboxSessions", () => { + const raw = (id: string, startedAt: number, stoppedAt?: number) => ({ + id, + memory: 8192, + vcpus: 4, + status: stoppedAt === undefined ? "running" : "stopped", + startedAt, + ...(stoppedAt === undefined ? {} : { stoppedAt }), + updatedAt: stoppedAt ?? startedAt, + }); + let read = 0; + stub(Sandbox, { + get: async () => ({ + listSessions: async () => + (async function* () { + for (const session of [ + raw("live", NOW - 10 * 60_000), + raw("recent", NOW - 50 * 60_000, NOW - 20 * 60_000), + raw("old", NOW - 3 * HOUR, NOW - 2 * HOUR), + raw("older", NOW - 5 * HOUR, NOW - 4 * HOUR), + ]) { + read += 1; + yield session; + } + })(), + }), + }); + + test("lists the live session and those ended since, and reads no further back", async () => { + const sessions = await listSandboxSessions("ws-box", NOW - HOUR); + expect(sessions?.map((session) => session.id)).toEqual(["live", "recent"]); + expect(read).toBe(3); + }); +}); + +describe("promoteSandboxToEnvironment", () => { + const sourceCalls: string[] = []; + const source = { + status: "running", + region: "iad1", + vcpus: 4, + currentSession: () => ({ sessionId: "sess_A" }), + snapshot: async () => { + sourceCalls.push("snapshot"); + return { snapshotId: "snap_1" }; + }, + writeFiles: async () => { + sourceCalls.push("writeFiles"); + }, + runCommand: async () => { + sourceCalls.push("runCommand"); + }, + mkDir: async () => { + sourceCalls.push("mkDir"); + }, + }; + const golden = { + currentSnapshotId: "snap_golden_0", + runCommand: async () => {}, + stop: async () => {}, + }; + let createTags: unknown; + stub(Sandbox, { + get: async ({ name }: { name: string }) => + name === "ws-box" + ? source + : { status: "stopped", currentSnapshotId: "snap_golden_1" }, + create: async (params: { tags: unknown }) => { + createTags = params.tags; + return golden; + }, + }); + + test("ends the source's session for the snapshot and leaves its restart to the caller", async () => { + expect( + await promoteSandboxToEnvironment({ + sourceSandbox: "ws-box", + goldenName: "env-1", + organizationId: "org_1", + }), + ).toEqual({ + goldenName: "env-1", + region: "iad1", + stopped: { sessionId: "sess_A" }, + }); + expect(sourceCalls).toEqual(["snapshot"]); + expect(createTags).toEqual({ kind: "environment", org: "org_1" }); + }); +}); + +describe("wakeSandbox", () => { + type Box = { memoryMb: number; vcpus: number }; + let running: boolean; + let onResume: (sandbox: unknown) => Promise; + const box = { + get status() { + return running ? "running" : "stopped"; + }, + expiresAt: new Date(NOW + 60_000), + vcpus: 4, + domain: () => "https://box.test", + currentSession: () => ({ vcpus: 8, sessionId: "sess_new" }), + update: async () => {}, + writeFiles: async () => {}, + // The SDK's withResume: a call on a stopped session resumes it, then retries. + extendTimeout: async () => { + await onResume(box); + }, + runCommand: async () => { + await onResume(box); + throw new Error("stop before settle"); + }, + }; + stub(Sandbox, { + get: async (params: { onResume: typeof onResume }) => { + onResume = params.onResume; + return box; + }, + }); + stub(globalThis, { fetch: async () => new Response("{}") }); + const identity = { + SUPERSET_SANDBOX_CONTRACT: "1", + SUPERSET_API_URL: "https://api.test", + SUPERSET_SANDBOX_WORKSPACE_ID: "2b3c4d5e-6f7a-4b8c-9d0e-1f2a3b4c5d6e", + SUPERSET_SANDBOX_ORGANIZATION_ID: "1a2b3c4d-5e6f-4a7b-8c9d-0e1f2a3b4c5d", + SUPERSET_SANDBOX_REPOSITORIES: "[]", + SUPERSET_SANDBOX_IMAGE_TAG: "image", + SUPERSET_SANDBOX_PROVIDER: "vercel", + SUPERSET_BUNDLE_SHA: "a".repeat(64), + }; + async function wake(): Promise { + const authorized: Box[] = []; + await wakeSandbox({ + providerSandboxId: "ws-box", + claim: { vcpus: 4, identity, networkPolicy: {} } as never, + beforeResume: async (resumed) => { + authorized.push(resumed); + if (running) throw new Error("stop before settle"); + }, + }).catch((error: Error) => { + if (error.message !== "stop before settle") throw error; + }); + return authorized; + } + + test("a box read running that a call resumed runs beforeResume for the new session", async () => { + running = true; + expect(await wake()).toEqual([ + { + memoryMb: 16_384, + vcpus: 8, + resumedSession: { sessionId: "sess_new", expiresAt: box.expiresAt }, + }, + ]); + }); + + test("a box read stopped runs beforeResume once, before its own resume", async () => { + running = false; + expect(await wake()).toEqual([{ memoryMb: 8192, vcpus: 4 }]); + }); +}); + describe("listActiveWorkspaceSandboxes", () => { test("finds older resumed workspaces when tag-filtered lists use creation order", async () => { expect(await listActiveWorkspaceSandboxes(NOW - 4 * HOUR)).toEqual([ diff --git a/packages/trpc/src/lib/sandbox/vercel.ts b/packages/trpc/src/lib/sandbox/vercel.ts index 911d4b286f7..812681d0020 100644 --- a/packages/trpc/src/lib/sandbox/vercel.ts +++ b/packages/trpc/src/lib/sandbox/vercel.ts @@ -126,17 +126,22 @@ export interface SandboxClaim { agentTools?: boolean; } +function environmentTags(organizationId: string): Record { + return { kind: "environment", org: organizationId }; +} + /** Vercel allows 5 tags per sandbox. */ function sandboxTags( kind: "workspace" | "environment", identity: SandboxIdentity, ): Record { - const tags = { kind, org: identity.SUPERSET_SANDBOX_ORGANIZATION_ID }; + const org = identity.SUPERSET_SANDBOX_ORGANIZATION_ID; // A golden's identity is the workspace it was built from, not its own. - if (kind === "environment") return tags; + if (kind === "environment") return environmentTags(org); const user = identity.SUPERSET_SANDBOX_CREATOR_USER_ID; return { - ...tags, + kind, + org, workspace: identity.SUPERSET_SANDBOX_WORKSPACE_ID, ...(user ? { user } : {}), }; @@ -496,8 +501,16 @@ async function hostServiceAnswers(hostTarget: string): Promise { export async function wakeSandbox(args: { providerSandboxId: string; claim: SandboxClaim; - /** Runs once the box is known to be stopped, before anything resumes it. */ - beforeResume?: (box: { memoryMb: number; vcpus: number }) => Promise; + /** + * Runs once the box is known to be stopped, before anything resumes it; or, + * for a box read running that a call here resumed (`resumedSession`), once + * those calls end. + */ + beforeResume?: (box: { + memoryMb: number; + vcpus: number; + resumedSession?: { sessionId: string | null; expiresAt: Date | null }; + }) => Promise; }): Promise<{ hostTarget: string; /** The session was stopped, so every process on the box started from this claim. */ @@ -509,13 +522,21 @@ export async function wakeSandbox(args: { sessionId: string | null; }> { try { + let wasRunning = false; + let resumed = null as { vcpus: number; sessionId: string | null } | null; const sandbox = await Sandbox.get({ ...credentials(), name: args.providerSandboxId, resume: false, + // Any call on a Sandbox object resumes a stopped session and retries. + onResume: async (box) => { + if (!wasRunning) return; + const session = box.currentSession(); + resumed = { vcpus: session.vcpus, sessionId: session.sessionId }; + }, }); const hostTarget = sandbox.domain(HOST_SERVICE_PORT); - const wasRunning = sandbox.status === "running"; + wasRunning = sandbox.status === "running"; let vcpus = sandbox.vcpus ?? args.claim.vcpus; if (!wasRunning && vcpus !== args.claim.vcpus) { await Sentry.startSpan({ name: "resize", op: "sandbox.provider" }, () => @@ -584,6 +605,16 @@ export async function wakeSandbox(args: { runBoot(sandbox, args.claim.hostSecret), ); } + if (resumed && args.beforeResume) { + await args.beforeResume({ + memoryMb: resumed.vcpus * MEMORY_MB_PER_VCPU, + vcpus: resumed.vcpus, + resumedSession: { + sessionId: resumed.sessionId, + expiresAt: sandbox.expiresAt ?? null, + }, + }); + } const answer = await Sentry.startSpan( { name: "settle", op: "sandbox" }, () => @@ -660,21 +691,26 @@ export async function stopAndSnapshot( * A golden is a stopped sandbox whose current snapshot is what forks start * from. It is built from a snapshot of the source taken now (a fork alone * would start from the source's last stop) and created with an empty env. - * Taking that snapshot ends the source's session, so a running source is - * started again before this returns. + * Taking that snapshot ends the source's session. The source stays stopped: + * the caller starts it again through a wake, which meters the new session. */ export async function promoteSandboxToEnvironment(args: { sourceSandbox: string; goldenName: string; - /** What restarts the source: it boots the same way a wake does. */ - claim: SandboxClaim; -}): Promise<{ goldenName: string; region: string }> { + organizationId: string; +}): Promise<{ + goldenName: string; + region: string; + /** The session the snapshot ended; null when the source was not running. */ + stopped: { sessionId: string | null } | null; +}> { const source = await Sandbox.get({ ...credentials(), name: args.sourceSandbox, resume: false, }); - const wasRunning = source.status === "running"; + const stopped = + source.status === "running" ? { sessionId: sessionIdOf(source) } : null; const snapshot = await source.snapshot(); const golden = await Sandbox.create({ ...credentials(), @@ -688,18 +724,13 @@ export async function promoteSandboxToEnvironment(args: { persistent: true, snapshotExpiration: 0, keepLastSnapshots: { count: 1 }, - tags: sandboxTags("environment", args.claim.identity), + tags: environmentTags(args.organizationId), }); await stripWorkspaceIdentity(golden); const created = golden.currentSnapshotId; await golden.stop(); await waitForStopSnapshot(args.goldenName, created); - if (wasRunning) { - await writeIdentity(source, args.claim.identity); - await writeSignInFiles(source, args.claim.files); - await runBoot(source, args.claim.hostSecret); - } - return { goldenName: args.goldenName, region: source.region }; + return { goldenName: args.goldenName, region: source.region, stopped }; } /** @@ -799,6 +830,29 @@ function toBoxSession(session: RawSession): BoxSession | null { }; } +/** + * The box's sessions that are live or ended at or after `since`, newest + * first. Null when the sandbox no longer exists. + */ +export async function listSandboxSessions( + providerSandboxId: string, + since: number, +): Promise { + const sandbox = await getSandbox(providerSandboxId); + if (!sandbox) return null; + const sessions: BoxSession[] = []; + for await (const raw of await sandbox.listSessions({ + sortOrder: "desc", + limit: SESSION_PAGE_SIZE, + })) { + const session = toBoxSession(raw); + if (!session) continue; + if (session.endedAt !== null && session.endedAt < since) break; + sessions.push(session); + } + return sessions; +} + /** Null when the sandbox no longer exists. */ export async function findSandboxSession( providerSandboxId: string, diff --git a/packages/trpc/src/router/cloud-workspace/meter.test.ts b/packages/trpc/src/router/cloud-workspace/meter.test.ts index 9b3a49656d3..c7c925b82b5 100644 --- a/packages/trpc/src/router/cloud-workspace/meter.test.ts +++ b/packages/trpc/src/router/cloud-workspace/meter.test.ts @@ -38,6 +38,10 @@ let stopped: string[]; let usageStopAt: Date | null; let delayedSettles: number[]; let updateMatches: boolean; +let insertConflicts: boolean; +let insertError: Error | null; +let queuedSettles: string[]; +let listed: BoxSession[]; type UsageEvent = { featureId: string; value: number; @@ -69,8 +73,16 @@ function chain(rows: () => unknown[], onFrom?: (table: unknown) => void) { stub(db, { update: () => chain(() => (boot && updateMatches ? [boot] : [])), insert: () => ({ - values: async (values: Record) => { + values: (values: Record) => { writes.push(values); + return Object.assign(Promise.resolve(), { + onConflictDoNothing: () => ({ + returning: async () => { + if (insertError) throw insertError; + return insertConflicts ? [] : [{ id: values.id }]; + }, + }), + }); }, }), select: () => { @@ -83,6 +95,8 @@ stub(db, { } return [ { + id: "ws_1", + organizationId: "org_1", providerSandboxId: "ws-box", createdByUserId: "user_1", usageStopAt, @@ -99,6 +113,7 @@ stub(db, { }); stub(sandbox, { findSandboxSession: async () => ({ session }), + listSandboxSessions: async () => listed, stopSandbox: async (id: string) => { stopped.push(id); return { sessionId: "session_1" }; @@ -115,8 +130,12 @@ stub(sandboxSleep, { }, }); stub(jobs, { - publishCloudWorkspaceJob: async (job: { delaySeconds?: number }) => { + publishCloudWorkspaceJob: async (job: { + delaySeconds?: number; + body: { bootId: string }; + }) => { delayedSettles.push(job.delaySeconds ?? 0); + queuedSettles.push(job.body.bootId); }, }); stub(autumnClient, { @@ -211,6 +230,10 @@ beforeEach(() => { usageStopAt = null; delayedSettles = []; updateMatches = true; + insertConflicts = false; + insertError = null; + queuedSettles = []; + listed = []; usageEvents = null; reservations = new Map(); }); @@ -418,6 +441,118 @@ describe("settleBoot", () => { }); }); +describe("the reconcile adopts sessions no boot meters", () => { + stub(billing, { ensureAutumnCustomer: async () => ({ id: "org_1" }) }); + const box = { name: "ws-box", expiresAt: null, sessionId: "sess_C" }; + const resumed = (endedAt: number | null): BoxSession => ({ + id: "sess_C", + memoryMb: 16_384, + vcpus: 8, + startedAt: NOW - 12 * MINUTE, + endedAt, + activeCpuMs: null, + }); + const timedOut: BoxSession = { + id: "sess_1", + memoryMb: 16_384, + vcpus: 8, + startedAt: NOW - 72 * MINUTE, + endedAt: NOW - 12 * MINUTE - 1_000, + activeCpuMs: null, + }; + const reconcile = () => + reconcileCloudWorkspaceUsage({ active: [box], billingEnabled: true }); + const adoptedBoot = () => + writes.find((write) => write.vercelSessionId === "sess_C"); + + test("a session resumed after a timeout stop is adopted in the same run, though the stopped session's boot is still open", async () => { + listed = [resumed(NOW - 2 * MINUTE), timedOut]; + expect((await reconcile()).adopted).toBe(1); + expect(adoptedBoot()).toMatchObject({ + cloudWorkspaceId: "ws_1", + startedAt: new Date(NOW - 12 * MINUTE), + memoryMb: 16_384, + lockOpen: false, + }); + expect(adoptedBoot()).not.toHaveProperty("finalizedThrough"); + expect(autumnCalls.some((call) => call.method === "check")).toBe(false); + expect(stopped).toEqual([]); + }); + + test("a running unmetered session takes a lock and is billed from its start, not from the adopt", async () => { + listed = [resumed(null), timedOut]; + expect((await reconcile()).adopted).toBe(1); + expect(adoptedBoot()).toMatchObject({ + startedAt: new Date(NOW - 12 * MINUTE), + }); + expect(adoptedBoot()).not.toHaveProperty("finalizedThrough"); + expect( + autumnCalls.find((call) => call.method === "check")?.params, + ).toMatchObject({ + lock: { lockId: `${adoptedBoot()?.id}:0` }, + }); + expect(adoptedBoot()).toMatchObject({ lockOpen: true }); + }); + + test("a session that already has a boot is not adopted again", async () => { + boot = { ...boot, vercelSessionId: "sess_C" }; + listed = [resumed(null)]; + expect((await reconcile()).adopted).toBe(0); + expect(adoptedBoot()).toBeUndefined(); + }); + + test("a session another caller attached meanwhile is left to that boot, and the lock taken for it is released", async () => { + insertConflicts = true; + listed = [resumed(null), timedOut]; + expect((await reconcile()).adopted).toBe(0); + const lockId = ( + autumnCalls.find((call) => call.method === "check")?.params as { + lock: { lockId: string }; + } + ).lock.lockId; + expect( + autumnCalls.find((call) => call.method === "finalize")?.params, + ).toEqual({ lockId, action: "release" }); + }); + + test("a lock taken for an adopt whose insert fails is released", async () => { + insertError = new Error("db down"); + listed = [resumed(null), timedOut]; + expect((await reconcile()).adopted).toBe(0); + expect( + autumnCalls.find((call) => call.method === "finalize")?.params, + ).toMatchObject({ action: "release" }); + }); + + test("the run's settles are queued before the adopt, and an adopted boot queues its own", async () => { + listed = [resumed(NOW - 2 * MINUTE), timedOut]; + await reconcile(); + const adoptedId = adoptedBoot()?.id; + expect(adoptedId).toEqual(expect.any(String)); + expect(queuedSettles).toEqual(["boot_1", adoptedId]); + }); + + test("a refused running session is stopped and recorded asleep, and its minutes stay on the boot", async () => { + renewalAllowed = false; + listed = [resumed(null), timedOut]; + expect((await reconcile()).adopted).toBe(1); + expect(adoptedBoot()).toBeDefined(); + expect(stopped).toEqual(["ws-box"]); + expect(recordedAsleep).toEqual([["ws_1", { sessionId: "session_1" }]]); + expect(adoptedBoot()).toMatchObject({ lockOpen: false }); + }); + + test("the newest session is left to an open boot that has not attached yet", async () => { + boot = { ...boot, vercelSessionId: null }; + listed = [resumed(null), timedOut]; + expect((await reconcile()).adopted).toBe(1); + expect(adoptedBoot()).toBeUndefined(); + expect(writes).toContainEqual( + expect.objectContaining({ vercelSessionId: "sess_1" }), + ); + }); +}); + describe("Autumn attribution and Vercel runtime parity", () => { stub(billing, { ensureAutumnCustomer: async () => ({ id: "org_1" }) }); diff --git a/packages/trpc/src/router/cloud-workspace/meter.ts b/packages/trpc/src/router/cloud-workspace/meter.ts index cfedc78260b..57f51e9e328 100644 --- a/packages/trpc/src/router/cloud-workspace/meter.ts +++ b/packages/trpc/src/router/cloud-workspace/meter.ts @@ -12,15 +12,15 @@ import { cloudWorkspaces, users, } from "@superset/db/schema"; -import { isCloudUsageRefusal } from "@superset/shared/billing"; import { AutumnError } from "autumn-js"; -import { and, eq, inArray, isNull, lt, ne, or, sql } from "drizzle-orm"; +import { and, eq, gt, inArray, isNull, lt, ne, or, sql } from "drizzle-orm"; import { userError } from "../../i18n-error"; import { controlTags, logControl } from "../../lib/control-log"; import { nudge } from "../../lib/realtime"; import { type ActiveWorkspaceSandbox, findSandboxSession, + listSandboxSessions, MEMORY_MB_PER_VCPU, stopSandbox, } from "../../lib/sandbox"; @@ -32,7 +32,7 @@ const LOCK_TTL_MS = 70 * 60 * 1000; const CAP_GRACE_MS = 10 * 60 * 1000; const SETTLE_LEASE_MS = 2 * 60 * 1000; const UNATTACHED_BOOT_GRACE_MS = 60 * 60 * 1000; -export const RECONCILE_LOOKBACK_MS = 60 * 60 * 1000; +const ADOPT_LOOKBACK_MS = 2 * 60 * 60 * 1000; const RECONCILE_CONCURRENCY = 25; export function boxFor(vcpus: number): { memoryMb: number; vcpus: number } { @@ -84,6 +84,24 @@ function reservation(memoryMb: number): number { return gbMinutes({ ms: LOCK_WINDOW_MS, memoryMb }); } +function reserveLock( + boot: Pick, + memoryMb: number, + user: UsageUser | null, +) { + return autumnClient.check({ + customerId: boot.organizationId, + featureId: BOX_GB_MINUTES_FEATURE_ID, + requiredBalance: reservation(memoryMb), + properties: usageProperties(boot, user), + lock: { + enabled: true, + lockId: lockIdFor(boot.id, boot.lockSeq), + expiresAt: Date.now() + LOCK_TTL_MS, + }, + }); +} + function usageProperties( boot: Pick, user: UsageUser | null, @@ -166,20 +184,16 @@ export async function authorizeBoot( const bootId = `boot_${randomUUID()}`; const user = await usageUser(args.createdByUserId); const result = await checkOrFailOpen(args.organizationId, () => - autumnClient.check({ - customerId: args.organizationId, - featureId: BOX_GB_MINUTES_FEATURE_ID, - requiredBalance: reservation(args.memoryMb), - properties: usageProperties( - { id: bootId, lockSeq: 0, cloudWorkspaceId: args.cloudWorkspaceId }, - user, - ), - lock: { - enabled: true, - lockId: lockIdFor(bootId, 0), - expiresAt: Date.now() + LOCK_TTL_MS, + reserveLock( + { + id: bootId, + lockSeq: 0, + organizationId: args.organizationId, + cloudWorkspaceId: args.cloudWorkspaceId, }, - }), + args.memoryMb, + user, + ), ); const scope = { cloudWorkspaceId: args.cloudWorkspaceId, @@ -216,8 +230,7 @@ type BootReleaseReason = | "provision_abandoned" | "session_taken" | "never_attached" - | "sandbox_lost" - | "adopt_failed"; + | "sandbox_lost"; export async function releaseBoot( bootId: string, @@ -486,17 +499,11 @@ export async function settleBoot( } const nextSeq = boot.lockSeq + 1; - const renewal = await autumnClient.check({ - customerId: boot.organizationId, - featureId: BOX_GB_MINUTES_FEATURE_ID, - requiredBalance: reservation(session.memoryMb), - properties: usageProperties({ ...boot, lockSeq: nextSeq }, user), - lock: { - enabled: true, - lockId: lockIdFor(bootId, nextSeq), - expiresAt: Date.now() + LOCK_TTL_MS, - }, - }); + const renewal = await reserveLock( + { ...boot, lockSeq: nextSeq }, + session.memoryMb, + user, + ); await db .update(cloudWorkspaceBoots) .set({ @@ -688,50 +695,86 @@ async function adoptRunningBoxes( eq(cloudWorkspaces.provider, "vercel"), ne(cloudWorkspaces.status, "failed"), inArray(cloudWorkspaces.providerSandboxId, names), - sql`not exists (select 1 from ${cloudWorkspaceBoots} boot where boot.cloud_workspace_id = ${cloudWorkspaces.id} and boot.stopped_at is null)`, ), ); - let adopted = 0; - for (const workspace of unmetered) { - const found = await findSandboxSession(workspace.providerSandboxId, null); - const session = found?.session; - if (!session || session.endedAt !== null) continue; - const claimed = await db - .select({ id: cloudWorkspaceBoots.id }) + const adoptSessions = async (workspace: (typeof unmetered)[number]) => { + const sessions = await listSandboxSessions( + workspace.providerSandboxId, + Date.now() - ADOPT_LOOKBACK_MS, + ); + if (!sessions?.length) return 0; + const boots = await db + .select({ vercelSessionId: cloudWorkspaceBoots.vercelSessionId }) .from(cloudWorkspaceBoots) - .where(eq(cloudWorkspaceBoots.vercelSessionId, session.id)); - if (claimed.length > 0) continue; - let bootId: string | null = null; - try { - ({ bootId } = await authorizeBoot({ - organizationId: workspace.organizationId, - cloudWorkspaceId: workspace.id, - createdByUserId: workspace.createdByUserId, - memoryMb: session.memoryMb, - vcpus: session.vcpus, - })); - if (!bootId) continue; - await db - .update(cloudWorkspaceBoots) - .set({ - vercelSessionId: session.id, - startedAt: new Date(session.startedAt), - finalizedThrough: new Date(), - }) - .where(eq(cloudWorkspaceBoots.id, bootId)); - adopted += 1; - } catch (error) { - if (bootId) await releaseBoot(bootId, "adopt_failed").catch(() => {}); - if (!isCloudUsageRefusal(error)) { - Sentry.captureException(error, { - tags: controlTags({ - cloudWorkspaceId: workspace.id, - organizationId: workspace.organizationId, - bootId, - }), - }); + .where( + or( + inArray( + cloudWorkspaceBoots.vercelSessionId, + sessions.map((session) => session.id), + ), + and( + eq(cloudWorkspaceBoots.cloudWorkspaceId, workspace.id), + isNull(cloudWorkspaceBoots.vercelSessionId), + isNull(cloudWorkspaceBoots.stoppedAt), + gt( + cloudWorkspaceBoots.createdAt, + new Date(Date.now() - UNATTACHED_BOOT_GRACE_MS), + ), + ), + ), + ); + const metered = new Set(boots.map((boot) => boot.vercelSessionId)); + let adopted = 0; + for (const [index, session] of sessions.entries()) { + // An open boot with no session yet attaches the newest one itself. + if (metered.has(session.id) || (index === 0 && metered.has(null))) { continue; } + const bootId = `boot_${randomUUID()}`; + const lock = + session.endedAt === null + ? await checkOrFailOpen(workspace.organizationId, async () => + reserveLock( + { + id: bootId, + lockSeq: 0, + organizationId: workspace.organizationId, + cloudWorkspaceId: workspace.id, + }, + session.memoryMb, + await usageUser(workspace.createdByUserId), + ), + ) + : null; + const lockOpen = lock?.allowed === true; + let inserted: { id: string }[] = []; + try { + inserted = await db + .insert(cloudWorkspaceBoots) + .values({ + id: bootId, + organizationId: workspace.organizationId, + cloudWorkspaceId: workspace.id, + vercelSessionId: session.id, + memoryMb: session.memoryMb, + vcpus: session.vcpus, + startedAt: new Date(session.startedAt), + lockOpen, + }) + .onConflictDoNothing({ target: cloudWorkspaceBoots.vercelSessionId }) + .returning({ id: cloudWorkspaceBoots.id }); + } finally { + if (inserted.length === 0 && lockOpen) { + await autumnClient.balances.finalize({ + lockId: lockIdFor(bootId, 0), + action: "release", + }); + } + } + if (inserted.length === 0) continue; + adopted += 1; + await queueSettle(bootId); + if (!lock || lockOpen) continue; console.error( `[cloud-workspace/meter] ${workspace.id} runs past its limit; stopping it`, ); @@ -749,31 +792,45 @@ async function adoptRunningBoxes( "warn", ); } + return adopted; + }; + let adopted = 0; + for (let i = 0; i < unmetered.length; i += RECONCILE_CONCURRENCY) { + const batch = unmetered.slice(i, i + RECONCILE_CONCURRENCY); + const results = await Promise.allSettled(batch.map(adoptSessions)); + for (const [index, result] of results.entries()) { + if (result.status === "fulfilled") adopted += result.value; + else { + const workspace = batch[index] as (typeof unmetered)[number]; + Sentry.captureException(result.reason, { + tags: controlTags({ + cloudWorkspaceId: workspace.id, + organizationId: workspace.organizationId, + }), + }); + } + } } return adopted; } /** - * Queues a settle for every open boot, and starts metering boxes running - * without one (boxes already up when metering shipped). + * Queues a settle for every open boot, then opens a boot for every recent + * session that has none (a box resumed by a call no one authorized). */ export async function reconcileCloudWorkspaceUsage({ active = null, + billingEnabled = billingUsesAutumn, }: { /** Workspace boxes whose state changed lately; null when the listing failed. */ active?: ActiveWorkspaceSandbox[] | null; + billingEnabled?: boolean; } = {}): Promise<{ adopted: number; queued: number; failed: number; }> { - if (!billingUsesAutumn) return { adopted: 0, queued: 0, failed: 0 }; - const adopted = active - ? await adoptRunningBoxes(active).catch((error) => { - Sentry.captureException(error); - return 0; - }) - : 0; + if (!billingEnabled) return { adopted: 0, queued: 0, failed: 0 }; const ids = await openBootIds(); let failed = 0; for (let i = 0; i < ids.length; i += RECONCILE_CONCURRENCY) { @@ -787,5 +844,11 @@ export async function reconcileCloudWorkspaceUsage({ } } } + const adopted = active + ? await adoptRunningBoxes(active).catch((error) => { + Sentry.captureException(error); + return 0; + }) + : 0; return { adopted, queued: ids.length - failed, failed }; } diff --git a/packages/trpc/src/router/cloud-workspace/sandbox-expiry.ts b/packages/trpc/src/router/cloud-workspace/sandbox-expiry.ts index d5ca4661598..36710a1953f 100644 --- a/packages/trpc/src/router/cloud-workspace/sandbox-expiry.ts +++ b/packages/trpc/src/router/cloud-workspace/sandbox-expiry.ts @@ -11,11 +11,7 @@ import { SandboxUnavailableError, } from "../../lib/sandbox"; import { publishCloudWorkspaceJob } from "./jobs"; -import { - queueSettleWorkspace, - RECONCILE_LOOKBACK_MS, - reconcileCloudWorkspaceUsage, -} from "./meter"; +import { queueSettleWorkspace, reconcileCloudWorkspaceUsage } from "./meter"; import { keepUpdatedAt, recordSandboxAsleep, @@ -23,6 +19,8 @@ import { } from "./sandbox-sleep"; const WATCH_AFTER_EXPIRY_MS = 5_000; +// The meter adopts only boxes this listing returns: keep it at least its ADOPT_LOOKBACK_MS. +const LISTING_LOOKBACK_MS = 2 * 60 * 60 * 1000; export interface WatchSandboxExpiryInput { cloudWorkspaceId: string; @@ -191,7 +189,7 @@ export async function syncSandboxExpiries( /** The 30-minute reconcile: sleep state on every stack, then the meter where billing runs. */ export async function reconcileCloudWorkspaces() { const active = await listActiveWorkspaceSandboxes( - Date.now() - RECONCILE_LOOKBACK_MS, + Date.now() - LISTING_LOOKBACK_MS, ).catch((error) => { Sentry.captureException(error); return null; diff --git a/packages/trpc/src/router/cloud-workspace/wake.test.ts b/packages/trpc/src/router/cloud-workspace/wake.test.ts index 42f9ae83fd3..6ebecc8a2b9 100644 --- a/packages/trpc/src/router/cloud-workspace/wake.test.ts +++ b/packages/trpc/src/router/cloud-workspace/wake.test.ts @@ -1,6 +1,8 @@ -import { describe, expect, test } from "bun:test"; +import { beforeEach, describe, expect, test } from "bun:test"; import * as Sentry from "@sentry/core"; +import { db } from "@superset/db/client"; import { stub } from "../../../test/stub"; +import { userError } from "../../i18n-error"; import * as realtime from "../../lib/realtime"; import * as sandbox from "../../lib/sandbox"; import * as meter from "./meter"; @@ -10,6 +12,7 @@ import * as transition from "./transition"; import { markSandboxUnavailable, restartCloudWorkspace, + wakeAfterStop, wakeCloudWorkspace, } from "./wake"; @@ -81,6 +84,124 @@ describe("a keepalive wake of a serving box", () => { }); }); +describe("wakeAfterStop", () => { + const calls: string[] = []; + const recordedAsleep: Array<[string, { sessionId: string | null }]> = []; + stub(sandbox, { + buildSandboxClaim: async () => ({ + claim: {}, + agentCredentialDigest: "digest", + }), + wakeSandbox: async (args: { + beforeResume: (box: { memoryMb: number; vcpus: number }) => Promise; + }) => { + await args.beforeResume({ memoryMb: 8192, vcpus: 4 }); + return { hostTarget: "https://box.test", answer: { stamps: [] } }; + }, + }); + stub(meter, { + queueSettleWorkspace: async () => { + calls.push("settle"); + }, + authorizeBoot: async () => { + calls.push("authorize"); + return { bootId: "boot_2" }; + }, + attachBootSession: async ({ bootId }: { bootId: string }) => { + calls.push(`attach ${bootId}`); + }, + }); + stub(sandboxSleep, { + recordSandboxAsleep: async ( + id: string, + stopped: { sessionId: string | null }, + ) => { + recordedAsleep.push([id, stopped]); + }, + }); + stub(sandboxExpiry, { recordSandboxAwake: async () => {} }); + stub(db, { update: () => ({ set: () => ({ where: async () => [] }) }) }); + + test("the box a promote snapshot stopped is recorded asleep, then woken through authorizeBoot and attached", async () => { + await wakeAfterStop(ROW, { sessionId: "sess_A" }); + expect(recordedAsleep).toEqual([["ws_1", { sessionId: "sess_A" }]]); + expect(calls).toEqual(["settle", "authorize", "attach boot_2"]); + }); +}); + +describe("a refused wake", () => { + const calls: string[] = []; + const recordedAsleep: Array<[string, { sessionId: string | null }]> = []; + let resumedSession: { sessionId: string; expiresAt: Date } | undefined; + stub(sandbox, { + buildSandboxClaim: async () => ({ + claim: {}, + agentCredentialDigest: "digest", + }), + wakeSandbox: async (args: { + beforeResume: (box: { + memoryMb: number; + vcpus: number; + resumedSession?: { sessionId: string; expiresAt: Date }; + }) => Promise; + }) => { + await args.beforeResume({ memoryMb: 16_384, vcpus: 8, resumedSession }); + }, + describeSandbox: async () => ({ + running: true, + expiresAt: new Date(Date.now() + 60_000), + sessionId: "sess_B", + }), + stopSandbox: async () => { + calls.push("stop"); + return { sessionId: "sess_C" }; + }, + }); + stub(meter, { + authorizeBoot: async () => { + throw userError({ + code: "FORBIDDEN", + message: "limit reached", + i18nKey: "serverError.cloudWorkspace.usageLimitReached", + }); + }, + }); + stub(sandboxExpiry, { + recordSandboxAwake: async (_: string, session: { sessionId: string }) => { + calls.push(`awake ${session.sessionId}`); + }, + }); + stub(sandboxSleep, { + recordSandboxAsleep: async ( + id: string, + stopped: { sessionId: string | null }, + ) => { + recordedAsleep.push([id, stopped]); + }, + }); + beforeEach(() => { + calls.length = 0; + recordedAsleep.length = 0; + }); + + test("stops the box a call here resumed, and records it asleep for that session", async () => { + resumedSession = { + sessionId: "sess_C", + expiresAt: new Date(Date.now() + 60_000), + }; + await expect(wakeCloudWorkspace(ROW)).rejects.toThrow("limit reached"); + expect(calls).toEqual(["awake sess_C", "stop"]); + expect(recordedAsleep).toEqual([["ws_1", { sessionId: "sess_C" }]]); + }); + + test("leaves alone a box another wake authorized and resumed meanwhile", async () => { + resumedSession = undefined; + await expect(wakeCloudWorkspace(ROW)).rejects.toThrow("limit reached"); + expect(calls).toEqual([]); + expect(recordedAsleep).toEqual([]); + }); +}); + describe("markSandboxUnavailable", () => { stub(realtime, { nudge: () => {} }); diff --git a/packages/trpc/src/router/cloud-workspace/wake.ts b/packages/trpc/src/router/cloud-workspace/wake.ts index 33116da469c..08c75096525 100644 --- a/packages/trpc/src/router/cloud-workspace/wake.ts +++ b/packages/trpc/src/router/cloud-workspace/wake.ts @@ -20,6 +20,7 @@ import { describeSandbox, recordBootSpans, stopAndSnapshot, + stopSandbox, wakeSandbox, } from "../../lib/sandbox"; import { @@ -84,12 +85,14 @@ async function wake( organizationId: row.organizationId, }; let bootId: string | null = null; + let resumed: { sessionId: string | null; expiresAt: Date | null } | undefined; const wakeStartedAt = Date.now(); const { hostTarget, booted, rebooted, answer, expiresAt, sessionId } = await wakeSandbox({ providerSandboxId: row.providerSandboxId, claim, - beforeResume: async (box) => { + beforeResume: async ({ resumedSession, ...box }) => { + resumed = resumedSession; ({ bootId } = await Sentry.startSpan( { name: "authorize boot", op: "billing" }, () => @@ -120,6 +123,27 @@ async function wake( providerSandboxId: row.providerSandboxId, }).catch(() => {}); } + // A call here resumed the box, and no boot meters that session. + if (resumed && isCloudUsageRefusal(error)) { + console.error( + `[cloud-workspace] ${row.id} was resumed past its limit; stopping it`, + ); + try { + if (resumed.expiresAt) { + await recordSandboxAwake(row.id, { + expiresAt: resumed.expiresAt, + sessionId: resumed.sessionId, + }); + } + const stopped = await stopSandbox(row.providerSandboxId); + await recordSandboxAsleep(row.id, { + sessionId: stopped.sessionId ?? resumed.sessionId, + }); + logControl("meter.cap_stopped", scope, { source: "wake" }, "warn"); + } catch (stopError) { + Sentry.captureException(stopError, { tags: controlTags(scope) }); + } + } throw error; }); if (bootId) { @@ -176,13 +200,19 @@ export async function restartCloudWorkspace( row: CloudWorkspaceRow, ): Promise { const { running } = await describeSandbox(row.providerSandboxId); - if (running) { - const stopped = await stopAndSnapshot(row.providerSandboxId); - await recordSandboxAsleep(row.id, { - sessionId: stopped.sessionId ?? row.sandboxSessionId, - }); - await queueSettleWorkspace(row.id); - } + if (!running) return wakeCloudWorkspace(row); + return wakeAfterStop(row, await stopAndSnapshot(row.providerSandboxId)); +} + +/** Starts a box again after the caller ended its session, as a metered wake. */ +export async function wakeAfterStop( + row: CloudWorkspaceRow, + stopped: { sessionId: string | null }, +): Promise { + await recordSandboxAsleep(row.id, { + sessionId: stopped.sessionId ?? row.sandboxSessionId, + }); + await queueSettleWorkspace(row.id); return wakeCloudWorkspace(row); } diff --git a/packages/trpc/src/router/environment/environment.test.ts b/packages/trpc/src/router/environment/environment.test.ts index e8d4a3a343a..76404214510 100644 --- a/packages/trpc/src/router/environment/environment.test.ts +++ b/packages/trpc/src/router/environment/environment.test.ts @@ -1,12 +1,15 @@ import { beforeEach, describe, expect, test } from "bun:test"; -import { db } from "@superset/db/client"; +import { db, dbWs } from "@superset/db/client"; import { stub } from "../../../test/stub"; import * as plan from "../../lib/billing/plan"; import * as guards from "../../lib/cloud-guards"; import * as sandbox from "../../lib/sandbox"; import { userError } from "../../trpc"; import * as access from "../cloud-workspace/access"; +import * as wake from "../cloud-workspace/wake"; import * as integration from "../integration/utils"; +import * as currentBundle from "./current-bundle"; +import * as limit from "./limit"; const ENVIRONMENT_ID = "6f7a8b9c-0d1e-4f2a-8b3c-4d5e6f7a8b9c"; const ORGANIZATION_ID = "1a2b3c4d-5e6f-4a7b-8c9d-0e1f2a3b4c5d"; @@ -128,3 +131,59 @@ describe("environment.promote onto an environment", () => { expect(written).toEqual([]); }); }); + +describe("environment.promote of a running workspace", () => { + const restarted: Array<[string, { sessionId: string | null }]> = []; + const order: string[] = []; + stub(sandbox, { + promoteSandboxToEnvironment: async () => ({ + goldenName: "env-new", + region: "iad1", + stopped: { sessionId: "sess_A" }, + }), + deleteSandbox: async () => {}, + }); + stub(wake, { + wakeAfterStop: async ( + row: { id: string }, + stopped: { sessionId: string | null }, + ) => { + restarted.push([row.id, stopped]); + order.push("restart"); + throw userError({ + code: "FORBIDDEN", + message: "limit reached", + i18nKey: "serverError.cloudWorkspace.usageLimitReached", + }); + }, + }); + stub(db, { + select: () => ({ from: () => ({ where: async () => [] }) }), + }); + stub(limit, { assertEnvironmentCapacity: async () => null }); + stub(currentBundle, { currentBundleSha: async () => "sha" }); + stub(dbWs, { + transaction: async (run: (tx: unknown) => Promise) => { + const rows = () => [{ id: ENVIRONMENT_ID }]; + const tx = { + insert: () => ({ + values: () => + Object.assign(Promise.resolve(), { returning: async () => rows() }), + }), + delete: () => ({ where: async () => {} }), + }; + const saved = await run(tx); + order.push("save"); + return saved; + }, + }); + + test("saves the environment, then starts the box again through the wake path; a refused restart does not fail the promote", async () => { + await caller.environment.promote({ + cloudWorkspaceId: WORKSPACE_ID, + name: "Saved", + }); + expect(restarted).toEqual([[WORKSPACE_ID, { sessionId: "sess_A" }]]); + expect(order).toEqual(["save", "restart"]); + }); +}); diff --git a/packages/trpc/src/router/environment/environment.ts b/packages/trpc/src/router/environment/environment.ts index d8ce3b4e274..060826df00e 100644 --- a/packages/trpc/src/router/environment/environment.ts +++ b/packages/trpc/src/router/environment/environment.ts @@ -1,3 +1,4 @@ +import * as Sentry from "@sentry/core"; import { db, dbWs } from "@superset/db/client"; import { environmentProjects, @@ -7,7 +8,7 @@ import { environments, githubRepositories, } from "@superset/db/schema"; -import { MACHINE_SIZES } from "@superset/shared/billing"; +import { isCloudUsageRefusal, MACHINE_SIZES } from "@superset/shared/billing"; import { SANDBOX_IMAGE_NAME } from "@superset/shared/constants"; import { SANDBOX_REGION_IDS } from "@superset/shared/sandbox/regions"; import type { TRPCRouterRecord } from "@trpc/server"; @@ -19,8 +20,8 @@ import { isMember, type MemberContext, } from "../../lib/cloud-guards"; +import { controlTags } from "../../lib/control-log"; import { - buildSandboxClaim, deleteSandbox, loadRepositories, primaryRepository, @@ -33,6 +34,7 @@ import { personProcedure, userError } from "../../trpc"; import { loadVisibleWorkspace } from "../cloud-workspace/access"; import { isLiveStatus } from "../cloud-workspace/live"; import { assertMachineSizeOffered } from "../cloud-workspace/start"; +import { wakeAfterStop } from "../cloud-workspace/wake"; import { verifyOrgAdmin } from "../integration/utils"; import { currentBundleSha } from "./current-bundle"; import { @@ -498,12 +500,28 @@ export const environmentRouter = { const limit = target ? null : await assertEnvironmentCapacity(workspace.organizationId); - const { claim } = await buildSandboxClaim({ row: workspace }); const golden = await promoteSandboxToEnvironment({ sourceSandbox: workspace.providerSandboxId, goldenName, - claim, + organizationId: workspace.organizationId, }); + const { stopped } = golden; + const restart = async () => { + if (!stopped) return; + await wakeAfterStop(workspace, stopped).catch((error) => { + console.error( + `[environment/promote] could not restart ${workspace.id} after its snapshot`, + error, + ); + if (isCloudUsageRefusal(error)) return; + Sentry.captureException(error, { + tags: controlTags({ + cloudWorkspaceId: workspace.id, + organizationId: workspace.organizationId, + }), + }); + }); + }; const primaryRepositoryId = checkouts[0]?.repository.id ?? null; const fromGolden = { provider: workspace.provider, @@ -604,6 +622,7 @@ export const environmentRouter = { cleanup, ), ); + await restart(); throw error; }); if (target?.sourceKind === "fork") { @@ -614,6 +633,7 @@ export const environmentRouter = { ), ); } + await restart(); return row; }),