diff --git a/.github/workflows/publish-cloud-workspace-job.yml b/.github/workflows/publish-cloud-workspace-job.yml index 51f02cea3eb..20dacb277e2 100644 --- a/.github/workflows/publish-cloud-workspace-job.yml +++ b/.github/workflows/publish-cloud-workspace-job.yml @@ -9,6 +9,7 @@ on: options: - reconcile-usage - renew-chatgpt-sign-ins + - reap-retention default: reconcile-usage jobs: diff --git a/apps/api/src/app/api/cloud-workspaces/jobs/reap-retention/route.ts b/apps/api/src/app/api/cloud-workspaces/jobs/reap-retention/route.ts new file mode 100644 index 00000000000..84bc28ef098 --- /dev/null +++ b/apps/api/src/app/api/cloud-workspaces/jobs/reap-retention/route.ts @@ -0,0 +1,31 @@ +import * as Sentry from "@sentry/nextjs"; +import { + RETENTION_JOB_PATH, + reapCloudWorkspaceRetention, +} from "@superset/trpc/cloud-workspace-retention"; +import { verifyQstashRequest } from "@/lib/verifyQstash"; + +export const dynamic = "force-dynamic"; +export const maxDuration = 300; + +/** + * Run by a QStash schedule every 6 hours. That schedule lives in the + * Upstash console, not in this repo. + */ +export async function POST(request: Request): Promise { + const body = await request.text(); + const rejected = await verifyQstashRequest(request, body, RETENTION_JOB_PATH); + if (rejected) return rejected; + + const outcome = await Sentry.withMonitor( + "reap-retention", + () => + reapCloudWorkspaceRetention({ + production: process.env.VERCEL_ENV === "production", + }), + { schedule: { type: "crontab", value: "0 */6 * * *" } }, + ); + // Turbopack builds do not wrap route handlers, so nothing else flushes the closing check-in. + await Sentry.flush(2_000).catch(() => false); + return Response.json({ ok: true, ...outcome }); +} diff --git a/apps/api/src/app/api/cloud-workspaces/teardown/route.ts b/apps/api/src/app/api/cloud-workspaces/teardown/route.ts new file mode 100644 index 00000000000..f446716d52c --- /dev/null +++ b/apps/api/src/app/api/cloud-workspaces/teardown/route.ts @@ -0,0 +1,33 @@ +import { teardownSandbox } from "@superset/trpc/cloud-workspace-teardown"; +import { z } from "zod"; +import { verifyQstashRequest } from "@/lib/verifyQstash"; + +export const dynamic = "force-dynamic"; +export const maxDuration = 60; + +const payloadSchema = z + .object({ + providerSandboxId: z.string().min(1), + driveName: z.null(), + attempt: z.number().int().min(0), + }) + .strict(); + +/** Stops a box, then deletes it with its snapshots; requeues itself on failure. */ +export async function POST(request: Request): Promise { + const body = await request.text(); + const rejected = await verifyQstashRequest( + request, + body, + "/api/cloud-workspaces/teardown", + ); + if (rejected) return rejected; + + const parsed = payloadSchema.safeParse(JSON.parse(body)); + if (!parsed.success) { + console.error("[cloud-workspaces/teardown] invalid payload", parsed.error); + return Response.json({ error: "Invalid payload" }, { status: 400 }); + } + const outcome = await teardownSandbox(parsed.data); + return Response.json({ ok: true, outcome }); +} diff --git a/docs/cloud-sandbox-mismatches.md b/docs/cloud-sandbox-mismatches.md index 803c699ddca..52e8d922dd3 100644 --- a/docs/cloud-sandbox-mismatches.md +++ b/docs/cloud-sandbox-mismatches.md @@ -783,6 +783,42 @@ sandbox is gone or unresumable is marked `failed` by the next `access`, so it gets the failed screen and a Remove button rather than a sidebar entry that never opens. +**Vercel's snapshot expiry was silent, so we own retention now.** The app +assumed a stopped box keeps its disk until someone deletes it. In fact Vercel +expired a workspace box's snapshot 30 days after its last use, and nothing told +us: the row learned only from the 410 its next wake got. New workspace boxes +are now created with `snapshotExpiration: 0`, and one cleanup job +(`router/cloud-workspace/retention.ts`, every 6 hours, production only) is the +only thing that deletes for age. It applies three rules from one table: + +1. A workspace not in use is retired after its wait: asleep 30 days after last + use, archived 7 days, failed 30 days, stuck creating with no boot 1 day. + Retiring is one conditional write (still in that status, still past the + wait) that takes the box name off the row, then the retried teardown job + deletes the box. A wake that lands first wins. An unarchive after the 7 days + is refused even before the job runs. +2. An environment keeps its active golden and its newest others, 3 in all; an + archived environment keeps none. On main an environment records only its + active golden, so the job deletes archived environments' goldens. +3. A box, drive or golden no production row names goes 7 days after it was + made; an agents drive goes once it is not the current release's and no box, + stopped ones included, mounts it. + +Everything else only sets a status: archive (its 60 s stop stays), a failed +create and a 410 on wake. Only production deletes: `sandboxDeletesAllowed` +(`lib/sandbox/deletes.ts`), checked by the teardown job, the drive delete and a +replaced golden's delete, logs "would delete" unless `VERCEL_ENV` is +`production`. `deleteSandbox` itself stays unchecked so a job still deletes the +temporary boxes it made (the release probe, real-sandbox). This is because +every deployment shares one Vercel project and a preview database copied from +production names production's boxes. Staging deploys as production on the +production database, so it counts. The job reads the whole box and drive +listing first and deletes nothing if it cannot. Boxes created before this +change keep a 30-day expiry until `bun run backfill-snapshot-expiration +--apply` (`packages/sandbox`) is run by hand. **Open:** no row names a +workspace drive on main; #9847's `drive_name` and `golden_drive_name` must +join rule 3's names when it lands. + **Any call on a `Sandbox` object resumes a stopped session.** The SDK wraps `runCommand`, `writeFiles`, `extendTimeout` and the rest in `withResume`: on a "stopped" answer it resumes and retries. A box read `running` that hits its diff --git a/packages/sandbox/package.json b/packages/sandbox/package.json index e00d676269f..f03e0f590c9 100644 --- a/packages/sandbox/package.json +++ b/packages/sandbox/package.json @@ -9,6 +9,7 @@ "pin": "bun run src/pin.ts", "assets": "bun run src/assets/produce.ts", "inventory": "bun run src/inventory.ts", + "backfill-snapshot-expiration": "bun run src/backfill-snapshot-expiration.ts", "release": "bun run src/environments/release.ts", "agent-tools": "bun run src/agent-tools.ts", "record-bundle": "bun run src/environments/record-bundle.ts", diff --git a/packages/sandbox/src/agent-tools.test.ts b/packages/sandbox/src/agent-tools.test.ts index cb8744da80d..09aae7263fd 100644 --- a/packages/sandbox/src/agent-tools.test.ts +++ b/packages/sandbox/src/agent-tools.test.ts @@ -10,18 +10,15 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { type AgentToolsManifest, - agentToolsDrivesToPrune, agentToolsGaps, agentToolsManifest, type DriveHost, describeAgentToolsPublish, parseAgentToolsManifest, parseBuilderResults, - pruneAgentToolsDrives, publishAgentToolsDrive, publishAgentToolsDrives, type RegionPublish, - referencedAgentToolsDrives, renderBuilderScript, } from "./agent-tools"; import { type Assets, loadAssets, sha256 } from "./manifest"; @@ -387,59 +384,6 @@ describe("agentToolsGaps", () => { }); }); -test("referenced drives: every row's bundle in its region, and the recent bundles in every region", () => { - const older = "1".repeat(64); - expect( - [ - ...referencedAgentToolsDrives({ - rows: [ - { region: "iad1", bundleSha: older }, - { region: "fra1", bundleSha: null }, - ], - recentBundles: [BUNDLE], - regions: ["sfo1", "iad1"], - }), - ].sort(), - ).toEqual([ - "agent-tools-iad1-0123456789ab", - "agent-tools-iad1-111111111111", - "agent-tools-sfo1-0123456789ab", - ]); -}); - -describe("pruning", () => { - test("deletes only agent-tools drives no environment references", () => { - expect( - agentToolsDrivesToPrune( - [ - "agent-tools-sfo1-0123456789ab", - "agent-tools-sfo1-aaaaaaaaaaaa", - "agent-tools-iad1-0123456789ab", - "agent-tools-sfo1", - "agent-tools-builder-sfo1-xyz", - ], - new Set(["agent-tools-sfo1-0123456789ab"]), - ), - ).toEqual([ - "agent-tools-sfo1-aaaaaaaaaaaa", - "agent-tools-iad1-0123456789ab", - ]); - }); - - test("keeps a drive a sandbox holds for the next release", async () => { - const { host } = fakeHost({ - "agent-tools-sfo1-aaaaaaaaaaaa": {}, - "agent-tools-sfo1-bbbbbbbbbbbb": { attachedTo: "box" }, - }); - expect( - await pruneAgentToolsDrives({ host, referenced: new Set() }), - ).toEqual({ - deleted: ["agent-tools-sfo1-aaaaaaaaaaaa"], - kept: ["agent-tools-sfo1-bbbbbbbbbbbb"], - }); - }); -}); - async function runScript(script: string) { const proc = Bun.spawn(["bash", "-c", script], { stdout: "pipe", diff --git a/packages/sandbox/src/agent-tools.ts b/packages/sandbox/src/agent-tools.ts index f289e3143a0..6d7a506208a 100644 --- a/packages/sandbox/src/agent-tools.ts +++ b/packages/sandbox/src/agent-tools.ts @@ -452,58 +452,6 @@ export function agentToolsGaps( }); } -/** - * Every drive a box may still mount: each environment row's bundle in its - * region, and the newest released bundles in every region, so a rollback and - * an environment created mid-release still find theirs. - */ -export function referencedAgentToolsDrives(args: { - rows: readonly { region: string; bundleSha: string | null }[]; - recentBundles: readonly string[]; - regions: readonly string[]; -}): Set { - return new Set([ - ...args.rows.flatMap((row) => - row.bundleSha ? [sandboxAgentToolsDrive(row.region, row.bundleSha)] : [], - ), - ...args.recentBundles.flatMap((sha) => - args.regions.map((region) => sandboxAgentToolsDrive(region, sha)), - ), - ]); -} - -/** Agent-tools drives no environment row's bundle and region name any more. */ -export function agentToolsDrivesToPrune( - drives: readonly string[], - referenced: ReadonlySet, -): string[] { - return drives.filter( - (name) => - SANDBOX_AGENT_TOOLS_DRIVE_PATTERN.test(name) && !referenced.has(name), - ); -} - -/** Returns the drives it could not delete; a drive a sandbox holds is retried next release. */ -export async function pruneAgentToolsDrives(args: { - host: DriveHost; - referenced: ReadonlySet; -}): Promise<{ deleted: string[]; kept: string[] }> { - const deleted: string[] = []; - const kept: string[] = []; - for (const name of agentToolsDrivesToPrune( - (await args.host.list("agent-tools-")).map((drive) => drive.name), - args.referenced, - )) { - try { - await args.host.remove(name); - deleted.push(name); - } catch { - kept.push(name); - } - } - return { deleted, kept }; -} - interface VercelCredentials { token: string; teamId: string; diff --git a/packages/sandbox/src/backfill-snapshot-expiration.test.ts b/packages/sandbox/src/backfill-snapshot-expiration.test.ts new file mode 100644 index 00000000000..f7c072aa666 --- /dev/null +++ b/packages/sandbox/src/backfill-snapshot-expiration.test.ts @@ -0,0 +1,39 @@ +import { expect, test } from "bun:test"; +import { backfillSnapshotExpiration } from "./backfill-snapshot-expiration"; + +const boxes = [ + { name: "ws-default", snapshotExpiration: undefined }, + { name: "ws-30d", snapshotExpiration: 30 * 24 * 60 * 60 * 1000 }, + { name: "ws-done", snapshotExpiration: 0 }, +]; + +test("changes only boxes that still expire, paced, and only with --apply", async () => { + const updated: string[] = []; + const slept: number[] = []; + const host = { + listWorkspaceBoxes: async () => boxes, + neverExpire: async (name: string) => { + updated.push(name); + }, + sleep: async (ms: number) => { + slept.push(ms); + }, + }; + const dry = await backfillSnapshotExpiration({ + host, + apply: false, + perSecond: 2, + log: () => {}, + }); + expect(dry).toMatchObject({ pending: 2, updated: 0 }); + expect(updated).toEqual([]); + + await backfillSnapshotExpiration({ + host, + apply: true, + perSecond: 2, + log: () => {}, + }); + expect(updated).toEqual(["ws-default", "ws-30d"]); + expect(slept).toEqual([500]); +}); diff --git a/packages/sandbox/src/backfill-snapshot-expiration.ts b/packages/sandbox/src/backfill-snapshot-expiration.ts new file mode 100644 index 00000000000..7e5449292d2 --- /dev/null +++ b/packages/sandbox/src/backfill-snapshot-expiration.ts @@ -0,0 +1,105 @@ +/** + * Sets `snapshotExpiration: 0` on workspace boxes made before new boxes got + * it, so Vercel stops expiring their snapshots and the retention reaper + * decides instead. It changes the default for snapshots taken from then on: + * a box's current snapshot keeps the expiry it was taken with, so a box only + * stops expiring after its next stop. Wakes nothing. Dry run by default. + * + * bun run backfill-snapshot-expiration lists what it would change + * bun run backfill-snapshot-expiration --apply changes them, 2 a second + * + * Needs VERCEL_SANDBOX_TOKEN, VERCEL_SANDBOX_TEAM_ID and VERCEL_SANDBOX_PROJECT_ID. + */ +import { Sandbox } from "@vercel/sandbox"; +import { missingCredentials } from "./inventory"; + +export interface BackfillBox { + name: string; + snapshotExpiration: number | undefined; +} + +export interface BackfillHost { + listWorkspaceBoxes(): Promise; + neverExpire(name: string): Promise; + sleep(ms: number): Promise; +} + +export async function backfillSnapshotExpiration(args: { + host: BackfillHost; + apply: boolean; + perSecond: number; + log: (line: string) => void; +}): Promise<{ + listed: number; + pending: number; + updated: number; + failed: number; +}> { + const boxes = await args.host.listWorkspaceBoxes(); + const pending = boxes.filter((box) => box.snapshotExpiration !== 0); + args.log(`${boxes.length} workspace boxes, ${pending.length} still expire`); + let updated = 0; + let failed = 0; + if (!args.apply) { + for (const box of pending) { + args.log( + `would set ${box.name} (now ${box.snapshotExpiration ?? "default"})`, + ); + } + if (pending.length) args.log("dry run: pass --apply to change them"); + return { listed: boxes.length, pending: pending.length, updated, failed }; + } + for (const [index, box] of pending.entries()) { + if (index > 0) await args.host.sleep(1000 / args.perSecond); + try { + await args.host.neverExpire(box.name); + updated++; + } catch (error) { + failed++; + args.log(`::warning::could not update ${box.name}: ${error}`); + } + } + args.log(`updated ${updated}, failed ${failed}`); + return { listed: boxes.length, pending: pending.length, updated, failed }; +} + +if (import.meta.main) { + const missing = missingCredentials(process.env); + if (missing.length > 0) { + console.error( + `backfill needs real Vercel sandbox credentials; missing or placeholder: ${missing.join(", ")}`, + ); + process.exit(64); + } + const credentials = { + token: process.env.VERCEL_SANDBOX_TOKEN ?? "", + teamId: process.env.VERCEL_SANDBOX_TEAM_ID ?? "", + projectId: process.env.VERCEL_SANDBOX_PROJECT_ID ?? "", + }; + const { failed } = await backfillSnapshotExpiration({ + host: { + async listWorkspaceBoxes() { + const boxes: BackfillBox[] = []; + for await (const box of await Sandbox.list({ + ...credentials, + tags: { kind: "workspace" }, + })) { + boxes.push({ + name: box.name, + snapshotExpiration: box.snapshotExpiration, + }); + } + return boxes; + }, + async neverExpire(name) { + const box = await Sandbox.get({ ...credentials, name, resume: false }); + await box.update({ snapshotExpiration: 0 }); + }, + sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms)), + }, + apply: process.argv.includes("--apply"), + perSecond: 2, + log: console.log, + }); + process.exit(failed ? 1 : 0); +} diff --git a/packages/sandbox/src/environments/release.ts b/packages/sandbox/src/environments/release.ts index 95dd3faf6c6..3f388d4ff27 100644 --- a/packages/sandbox/src/environments/release.ts +++ b/packages/sandbox/src/environments/release.ts @@ -59,8 +59,6 @@ process.env.SKIP_ENV_VALIDATION ??= "1"; const SKIP_IMAGE = process.argv.includes("--skip-image"); const KEEP_OLD = process.argv.includes("--keep-old"); const PRODUCTION = process.argv.includes("--production"); -/** Released bundles whose drives stay in every region, so a rollback to one of them has its drives. */ -const KEPT_RELEASED_BUNDLES = 3; const FOLLOWED_UPSTREAM = ["claude", "codex", "chrome", "opencode", "pi"]; /** The environment the release rebuilds; without it, the release creates one named INTERNAL_NAME. */ const ENVIRONMENT_ID = process.env.SUPERSET_INTERNAL_ENVIRONMENT_ID; @@ -521,7 +519,7 @@ const { db, dbWs } = await import("@superset/db/client"); const { environments, environmentRepositories } = await import( "@superset/db/schema" ); -const { desc, eq } = await import("drizzle-orm"); +const { eq } = await import("drizzle-orm"); const { releaseRepositoryWhere } = await import( "./utils/releaseRepositoryWhere" ); @@ -605,39 +603,7 @@ if (!previous) `rows: created; set SUPERSET_INTERNAL_ENVIRONMENT_ID=${internal.id} so the next release updates it`, ); -// One Vercel project holds every database's drives (staging shares -// production's rows; preview branches copy them), so only the production -// release, whose rows the others mirror, may delete one. -if (!PRODUCTION) { - log("agent-tools: prune skipped; only --production prunes drives"); -} else { - try { - const { sandboxBundles } = await import("@superset/db/schema"); - const rows = await db.query.environments.findMany({ - columns: { region: true, bundleSha: true }, - }); - const recent = await db - .select({ sha: sandboxBundles.sha }) - .from(sandboxBundles) - .orderBy(desc(sandboxBundles.releasedAt)) - .limit(KEPT_RELEASED_BUNDLES); - const { deleted, kept } = await agentTools.pruneAgentToolsDrives({ - host: driveHost, - referenced: agentTools.referencedAgentToolsDrives({ - rows, - recentBundles: [bundle.sha256, ...recent.map((row) => row.sha)], - regions: SANDBOX_REGION_IDS, - }), - }); - log( - `agent-tools: pruned ${deleted.length}${kept.length ? `; ${kept.join(", ")} still attached, next release retries` : ""}`, - ); - } catch (error) { - console.log( - `::warning::agent-tools: prune failed: ${String(error).slice(0, 300)}`, - ); - } -} +// Old agents drives are deleted by the API's cleanup job (rule 3), once no box mounts them. if ( previous && diff --git a/packages/trpc/package.json b/packages/trpc/package.json index c444ba70db4..a71a18c6c48 100644 --- a/packages/trpc/package.json +++ b/packages/trpc/package.json @@ -136,6 +136,14 @@ "types": "./src/router/cloud-workspace/reap.ts", "default": "./src/router/cloud-workspace/reap.ts" }, + "./cloud-workspace-retention": { + "types": "./src/router/cloud-workspace/retention.ts", + "default": "./src/router/cloud-workspace/retention.ts" + }, + "./cloud-workspace-teardown": { + "types": "./src/router/cloud-workspace/teardown.ts", + "default": "./src/router/cloud-workspace/teardown.ts" + }, "./automation-dispatch": { "types": "./src/router/automation/dispatch.ts", "default": "./src/router/automation/dispatch.ts" diff --git a/packages/trpc/scripts/cleanup-dry-run.ts b/packages/trpc/scripts/cleanup-dry-run.ts new file mode 100644 index 00000000000..ab913415981 --- /dev/null +++ b/packages/trpc/scripts/cleanup-dry-run.ts @@ -0,0 +1,289 @@ +/** + * What the cleanup job's next run would do, against a real database and the + * real Vercel listing, with nothing written or deleted. Runs the job itself + * (listing checks included) with `dryRun`, then reports each rule's + * decisions grouped by kind, action and reason. + * + * Four guards keep it from deleting anything: + * 1. it refuses unless DATABASE_URL logs in as the `readonly` role, so no row can change; + * 2. VERCEL_ENV is set to "development" before anything loads, so `sandboxDeletesAllowed` refuses; + * 3. every SDK call that stops, updates, creates or deletes a box or drive throws; + * 4. the job runs with `dryRun`, which replaces every writing or deleting dependency. + * + * Usage: DATABASE_URL= VERCEL_SANDBOX_TOKEN=… VERCEL_SANDBOX_TEAM_ID=… \ + * VERCEL_SANDBOX_PROJECT_ID=… bun run packages/trpc/scripts/cleanup-dry-run.ts [--json out.json] + */ + +const databaseUrl = process.env.DATABASE_URL ?? ""; +let role = ""; +try { + role = decodeURIComponent(new URL(databaseUrl).username); +} catch {} +if (role !== "readonly") { + console.error( + `DATABASE_URL must log in as the readonly role, not "${role || "none"}"`, + ); + process.exit(64); +} +for (const key of [ + "VERCEL_SANDBOX_TOKEN", + "VERCEL_SANDBOX_TEAM_ID", + "VERCEL_SANDBOX_PROJECT_ID", +]) { + if (!process.env[key]) { + console.error(`${key} is required`); + process.exit(64); + } +} +process.env.VERCEL_ENV = "development"; +process.env.SKIP_ENV_VALIDATION = "1"; +process.env.DATABASE_URL_UNPOOLED = databaseUrl; + +const { Drive, Sandbox } = await import("@vercel/sandbox"); +const trap = (what: string) => () => { + throw new Error(`dry run reached ${what}`); +}; +const trapped: Array<[object, string]> = [ + [Sandbox.prototype, "delete"], + [Sandbox.prototype, "stop"], + [Sandbox.prototype, "update"], + [Sandbox, "create"], + [Drive.prototype, "delete"], + [Drive, "getOrCreate"], +]; +for (const [target, key] of trapped) { + Object.defineProperty(target, key, { + value: trap(key), + configurable: false, + writable: false, + }); +} + +const { db } = await import("@superset/db/client"); +const { cloudWorkspaces, environments, organizations } = await import( + "@superset/db/schema" +); +const { count, inArray } = await import("drizzle-orm"); +const retention = await import("../src/router/cloud-workspace/retention"); +type Deps = import("../src/router/cloud-workspace/retention").RetentionDeps; +type ListedSandbox = import("../src/lib/sandbox").ListedSandbox; +type ListedDrive = import("../src/lib/sandbox").ListedDrive; + +const now = Date.now(); +let boxes: ListedSandbox[] = []; +let drives: ListedDrive[] = []; +const rows = new Map(); +const lines: string[] = []; + +const base = retention.defaultRetentionDeps; +const deps: Deps = { + ...base, + listSandboxes: async (expected) => { + boxes = await base.listSandboxes(expected); + return boxes; + }, + listDrives: async () => { + drives = await base.listDrives(); + return drives; + }, + findPastWait: async (args) => { + const found = await base.findPastWait(args); + for (const row of found) rows.set(row.id, row); + return found; + }, + log: (line) => lines.push(line), +}; + +const result = await retention.reapCloudWorkspaceRetention({ + now, + production: true, + deps, + dryRun: true, +}); + +type Decision = { + rule: number; + kind: string; + name: string; + action: string; + reason: string; +}; +const decisions: Decision[] = lines + .filter((line) => line.startsWith("[cleanup] {")) + .map((line) => JSON.parse(line.slice("[cleanup] ".length))); +const reached = decisions.filter((d) => d.action === "delete"); +if (reached.length) throw new Error("dry run logged a real delete"); + +const boxByName = new Map(boxes.map((box) => [box.name, box])); +const driveByName = new Map(drives.map((drive) => [drive.name, drive])); +const createdAt = (d: Decision): number | null => { + if (d.kind === "workspace") { + const id = d.name.split(" ")[0] ?? ""; + return rows.get(id)?.createdAt.getTime() ?? null; + } + if (d.kind === "drive" || d.kind === "agents drive") { + return driveByName.get(d.name)?.createdAt ?? null; + } + return boxByName.get(d.name)?.createdAt ?? null; +}; +const iso = (ms: number | null) => (ms ? new Date(ms).toISOString() : "?"); + +// Rule 1's select is capped at a batch; count everything past its wait too. +const pastWait = await db + .select({ status: cloudWorkspaces.status, n: count() }) + .from(cloudWorkspaces) + .where( + retention.waitOverWhere({ now, listedNames: boxes.map((b) => b.name) }), + ) + .groupBy(cloudWorkspaces.status); + +// Where an unnamed box came from: a production row, or a database that is not production. +const unnamed = decisions.filter( + (d) => d.rule === 3 && d.kind === "box" && d.action === "would delete", +); +const tagged = (key: string) => [ + ...new Set( + unnamed.flatMap((d) => { + const value = boxByName.get(d.name)?.tags[key]; + return value ? [value] : []; + }), + ), +]; +const knownWorkspaces = new Set(); +for (let i = 0; i < tagged("workspace").length; i += 500) { + for (const row of await db + .select({ id: cloudWorkspaces.id }) + .from(cloudWorkspaces) + .where( + inArray(cloudWorkspaces.id, tagged("workspace").slice(i, i + 500)), + )) { + knownWorkspaces.add(row.id); + } +} +const knownOrgs = new Set(); +for (let i = 0; i < tagged("org").length; i += 500) { + for (const row of await db + .select({ id: organizations.id }) + .from(organizations) + .where(inArray(organizations.id, tagged("org").slice(i, i + 500)))) { + knownOrgs.add(row.id); + } +} +const originOf = (name: string): string => { + const tags = boxByName.get(name)?.tags ?? {}; + if (!tags.kind) return "untagged (made before tags, or not by the API)"; + if (tags.workspace && knownWorkspaces.has(tags.workspace)) { + return "its production workspace row names another box now"; + } + if (tags.org && !knownOrgs.has(tags.org)) { + return "organization not in production: a preview's or dev's, or deleted"; + } + if (tags.workspace) { + return "workspace not in production: made by a preview or dev database, or its row was deleted"; + } + return `kind ${tags.kind}, no production row`; +}; + +const envRows = await db + .select({ + id: environments.id, + name: environments.name, + sourceRef: environments.sourceRef, + archivedAt: environments.archivedAt, + }) + .from(environments); +const envByRef = new Map(envRows.map((env) => [env.sourceRef, env])); + +const groups = new Map(); +for (const d of decisions) { + const key = `${d.rule}|${d.kind}|${d.action}|${d.reason}`; + groups.set(key, [...(groups.get(key) ?? []), d]); +} +const report = [...groups.entries()] + .map(([key, items]) => { + const [rule = "", kind = "", action = "", reason = ""] = key.split("|"); + const sorted = [...items].sort( + (a, b) => (createdAt(a) ?? Infinity) - (createdAt(b) ?? Infinity), + ); + const oldest = sorted[0]; + return { + rule: Number(rule), + kind, + action, + reason, + count: items.length, + examples: items.slice(0, 5).map((d) => d.name), + oldest: oldest + ? { name: oldest.name, createdAt: iso(createdAt(oldest)) } + : null, + }; + }) + .sort( + (a, b) => + a.rule - b.rule || + a.kind.localeCompare(b.kind) || + a.action.localeCompare(b.action) || + b.count - a.count, + ); + +const wouldDelete = decisions.filter((d) => d.action === "would delete"); +const running = new Set(["pending", "running"]); +const flags = { + runningNowAmongDeletes: wouldDelete + .filter((d) => { + const name = + d.kind === "workspace" ? d.name.match(/\((.*)\)$/)?.[1] : d.name; + return name && running.has(boxByName.get(name)?.status ?? ""); + }) + .map((d) => `${d.rule}:${d.kind}:${d.name}`), + runningNowListed: boxes.filter((box) => running.has(box.status)).length, + environmentGoldensAmongDeletes: wouldDelete + .filter((d) => envByRef.has(d.name)) + .map((d) => { + const env = envByRef.get(d.name); + return `${d.name} (${env?.name}${env?.archivedAt ? ", archived" : ""})`; + }), + internalGoldens: envRows + .filter((env) => /superset/i.test(env.name)) + .map((env) => ({ + environment: env.name, + golden: env.sourceRef, + decision: + decisions.find((d) => d.name === env.sourceRef)?.action ?? "not listed", + })), + workspaceDrivesLike9847: drives + .filter((d) => /^(env-)?drive-/.test(d.name)) + .map((d) => ({ + name: d.name, + createdAt: iso(d.createdAt), + decision: decisions.find((x) => x.name === d.name)?.action ?? "none", + })), + unnamedBoxOrigins: Object.entries( + unnamed.reduce>((acc, d) => { + const origin = originOf(d.name); + acc[origin] = (acc[origin] ?? 0) + 1; + return acc; + }, {}), + ), +}; + +const summary = { + ranAt: new Date(now).toISOString(), + listed: { boxes: boxes.length, drives: drives.length }, + result, + rule1PastWaitInTotal: Object.fromEntries( + pastWait.map((row) => [row.status, row.n]), + ), + report, + flags, +}; +const jsonAt = process.argv.indexOf("--json"); +if (jsonAt > 0 && process.argv[jsonAt + 1]) { + const { writeFile } = await import("node:fs/promises"); + await writeFile( + process.argv[jsonAt + 1] as string, + JSON.stringify(summary, null, 2), + ); +} +console.log(JSON.stringify(summary, null, 2)); +process.exit(0); diff --git a/packages/trpc/src/lib/sandbox/deletes.ts b/packages/trpc/src/lib/sandbox/deletes.ts new file mode 100644 index 00000000000..43ff545dfdd --- /dev/null +++ b/packages/trpc/src/lib/sandbox/deletes.ts @@ -0,0 +1,11 @@ +/** Previews and local dev use production's Vercel project with rows copied from its database: only production deletes. */ +export function sandboxDeletesAllowed( + what: string, + vercelEnv: string | undefined = process.env.VERCEL_ENV, +): boolean { + if (vercelEnv === "production") return true; + console.log( + `[sandbox] would delete ${what}; VERCEL_ENV is ${vercelEnv ?? "unset"}, not production`, + ); + return false; +} diff --git a/packages/trpc/src/lib/sandbox/index.ts b/packages/trpc/src/lib/sandbox/index.ts index ec376ff4943..45808addf3b 100644 --- a/packages/trpc/src/lib/sandbox/index.ts +++ b/packages/trpc/src/lib/sandbox/index.ts @@ -26,6 +26,7 @@ export { } from "./box-pressure"; export { buildSandboxClaim } from "./claim"; export { deriveSandboxCredentials } from "./credentials"; +export { sandboxDeletesAllowed } from "./deletes"; export { listRemoteBranches, type RemoteBranch, @@ -55,12 +56,19 @@ export { export { type ActiveWorkspaceSandbox, type BoxSession, + deleteDrive, deleteSandbox, describeSandbox, findSandboxSession, HOST_SERVICE_PORT, + type ListedDrive, + type ListedSandbox, + type ListingPage, listActiveWorkspaceSandboxes, + listProjectDrives, + listProjectSandboxes, listSandboxSessions, + listWhole, MEMORY_MB_PER_VCPU, promoteSandboxToEnvironment, provisionSandbox, diff --git a/packages/trpc/src/lib/sandbox/vercel.test.ts b/packages/trpc/src/lib/sandbox/vercel.test.ts index 2141b68005c..e8bd65409f9 100644 --- a/packages/trpc/src/lib/sandbox/vercel.test.ts +++ b/packages/trpc/src/lib/sandbox/vercel.test.ts @@ -8,15 +8,19 @@ import { } from "bun:test"; import { APIError, Drive, Sandbox } from "@vercel/sandbox"; import { stub } from "../../../test/stub"; +import { sandboxDeletesAllowed } from "./deletes"; import { agentToolsMounts, applySandboxPolicy, + deleteDrive, deleteSandbox, describeSandbox, findSandboxSession, listActiveWorkspaceSandboxes, listSandboxSessions, + listWhole, promoteSandboxToEnvironment, + provisionSandbox, type SandboxClaim, SandboxUnavailableError, SESSION_TIMEOUT_MS, @@ -307,6 +311,100 @@ describe("a caller that is not a wake leaves a stopped box stopped", () => { }); }); +describe("only production deletes", () => { + stub(Drive, { + get: async () => { + throw new Error("reached Vercel"); + }, + }); + + test("outside production a drive delete logs and returns without reaching Vercel", async () => { + process.env.VERCEL_ENV = "preview"; + try { + await deleteDrive("drive-1"); + } finally { + process.env.VERCEL_ENV = "production"; + } + expect(sandboxDeletesAllowed("box b", "production")).toBe(true); + expect(sandboxDeletesAllowed("box b", "preview")).toBe(false); + expect(sandboxDeletesAllowed("box b", "development")).toBe(false); + }); +}); + +describe("listWhole", () => { + const pages = (...list: Array<{ items: string[]; next: string | null }>) => { + let index = 0; + return async () => { + const page = list[index++]; + if (!page) throw new Error("no such page"); + return { items: page.items.map((name) => ({ name })), next: page.next }; + }; + }; + + test("returns every page, and throws on a repeated cursor, a missing known name or an empty listing", async () => { + expect( + await listWhole({ + fetchPage: pages( + { items: ["a"], next: "c1" }, + { items: ["b"], next: null }, + ), + expected: ["b"], + nonEmpty: true, + }), + ).toEqual([{ name: "a" }, { name: "b" }]); + await expect( + listWhole({ + fetchPage: pages( + { items: ["a"], next: "c1" }, + { items: ["b"], next: "c1" }, + ), + nonEmpty: true, + }), + ).rejects.toThrow("repeated cursor"); + await expect( + listWhole({ + fetchPage: pages({ items: ["a"], next: null }), + expected: ["ws-running"], + nonEmpty: true, + }), + ).rejects.toThrow("lacks ws-running"); + await expect( + listWhole({ + fetchPage: pages({ items: [], next: null }), + nonEmpty: true, + }), + ).rejects.toThrow("empty"); + }); +}); + +describe("provisionSandbox", () => { + let created: { snapshotExpiration?: number } | undefined; + stub(Sandbox, { + get: async () => { + throw new APIError(new Response(null, { status: 404 })); + }, + create: async (params: { snapshotExpiration?: number }) => { + created = params; + throw new Error("created"); + }, + }); + + test("a workspace box's snapshots never expire on Vercel's clock", async () => { + const provision = provisionSandbox({ + name: "ws-new", + environment: { sourceKind: "image", sourceRef: "img", region: "iad1" }, + claim: { + identity: {}, + hostSecret: "secret", + networkPolicy: "allow-all", + vcpus: 2, + } as unknown as SandboxClaim, + }); + await expect(provision).rejects.toThrow("created"); + expect(created?.snapshotExpiration).toBe(0); + }); +}); + describe("promoteSandboxToEnvironment", () => { let fake = fakeBox("running"); let sourceSnapshot: string | undefined; diff --git a/packages/trpc/src/lib/sandbox/vercel.ts b/packages/trpc/src/lib/sandbox/vercel.ts index b79ff3809e9..1313143fb46 100644 --- a/packages/trpc/src/lib/sandbox/vercel.ts +++ b/packages/trpc/src/lib/sandbox/vercel.ts @@ -34,6 +34,7 @@ import { CODEX_SYSTEM_CONFIG_PATH, type SandboxFile, } from "./credentials"; +import { sandboxDeletesAllowed } from "./deletes"; export const HOST_SERVICE_PORT = SANDBOX_PORTS.hostService; /** @@ -46,7 +47,6 @@ export const SESSION_TIMEOUT_MS = 60 * 60 * 1000; /** Above the 10-minute client keepalive, so an open workspace is always extended in time. */ const EXTEND_BELOW_MS = 30 * 60 * 1000; const MIN_TOP_UP_MS = 60 * 1000; -const WORKSPACE_SNAPSHOT_EXPIRATION_MS = 30 * 24 * 60 * 60 * 1000; /** 2 GB of memory per vCPU; disk is 64 GB regardless. */ export const MEMORY_MB_PER_VCPU = 2048; const GOLDEN_SESSION_TIMEOUT_MS = 30 * 60 * 1000; @@ -304,7 +304,6 @@ export async function provisionSandbox(args: { name: string; environment: SandboxEnvironment; claim: SandboxClaim; - /** A golden under construction keeps its snapshots until its row goes. */ kind?: "workspace" | "environment"; }): Promise<{ providerSandboxId: string; @@ -322,8 +321,8 @@ export async function provisionSandbox(args: { env: {}, networkPolicy: args.claim.networkPolicy, persistent: true, - snapshotExpiration: - kind === "environment" ? 0 : WORKSPACE_SNAPSHOT_EXPIRATION_MS, + // Vercel never expires a snapshot: the retention reaper decides when a box goes. + snapshotExpiration: 0, keepLastSnapshots: { count: 1 }, tags: sandboxTags(kind, args.claim.identity), }; @@ -989,6 +988,132 @@ export async function listActiveWorkspaceSandboxes( return found; } +export interface ListedSandbox { + name: string; + /** Our `kind` tag; absent on a box made before tags or not made by us. */ + kind: string | undefined; + createdAt: number; + status: string; + /** What it mounts, running or stopped: a stopped box mounts it again on its next boot. */ + mounts: string[]; + tags: Record; +} + +export interface ListedDrive { + name: string; + createdAt: number; + attachedTo: string | undefined; +} + +export interface ListingPage { + items: T[]; + next: string | null; +} + +export const MAX_LISTING_PAGES = 10_000; + +/** + * Every page of a listing, or a throw: each page answered, no cursor seen + * twice, and the names the caller knows exist all in it. A deleter that acts + * on a partial listing deletes what it did not see named. + */ +export async function listWhole({ + fetchPage, + expected = [], + nonEmpty, +}: { + fetchPage: (cursor: string | null) => Promise>; + expected?: readonly string[]; + nonEmpty: boolean; +}): Promise { + const items: T[] = []; + const seen = new Set(); + let cursor: string | null = null; + for (let pages = 0; ; pages++) { + if (pages >= MAX_LISTING_PAGES) { + throw new Error(`listing has more than ${pages} pages`); + } + const page = await fetchPage(cursor); + items.push(...page.items); + if (page.next === null) break; + if (seen.has(page.next)) { + throw new Error(`listing repeated cursor ${page.next.slice(0, 16)}`); + } + seen.add(page.next); + cursor = page.next; + } + if (nonEmpty && items.length === 0) throw new Error("listing is empty"); + const names = new Set(items.map((item) => item.name)); + const missing = expected.filter((name) => !names.has(name)); + if (missing.length) throw new Error(`listing lacks ${missing.join(", ")}`); + return items; +} + +/** The page size #9943 used against the same endpoint. */ +const LISTING_PAGE_SIZE = 50; + +/** Every box in the project, whatever its status, tags or database; throws unless whole. */ +export function listProjectSandboxes( + expected: readonly string[], +): Promise { + return listWhole({ + expected, + nonEmpty: true, + fetchPage: async (cursor) => { + const page = await Sandbox.list({ + ...credentials(), + limit: LISTING_PAGE_SIZE, + ...(cursor ? { cursor } : {}), + }); + return { + items: page.sandboxes.map((sandbox) => ({ + name: sandbox.name, + kind: sandbox.tags?.kind, + createdAt: sandbox.createdAt, + status: sandbox.status, + mounts: Object.values(sandbox.mounts ?? {}).map( + (mount) => mount.drive, + ), + tags: sandbox.tags ?? {}, + })), + next: page.pagination.next, + }; + }, + }); +} + +/** Every drive in the project; throws unless whole. */ +export function listProjectDrives(): Promise { + return listWhole({ + nonEmpty: false, + fetchPage: async (cursor) => { + const page = await Drive.list({ + ...credentials(), + limit: LISTING_PAGE_SIZE, + ...(cursor ? { cursor } : {}), + }); + return { + items: page.drives.map((drive) => ({ + name: drive.name, + createdAt: drive.createdAt.getTime(), + attachedTo: drive.currentSandboxName, + })), + next: page.pagination.next, + }; + }, + }); +} + +/** Throws while a box holds it; a drive already gone is the state we wanted. */ +export async function deleteDrive(name: string): Promise { + if (!sandboxDeletesAllowed(`drive ${name}`)) return; + const drive = await Drive.get({ ...credentials(), name }).catch((error) => { + if (isNotFound(error)) return null; + throw error; + }); + await drive?.delete(); +} + /** Best-effort: a sandbox already gone is the state we wanted. */ export async function deleteSandbox(providerSandboxId: string): Promise { const sandbox = await getSandbox(providerSandboxId); diff --git a/packages/trpc/src/router/cloud-workspace/cloud-workspace.test.ts b/packages/trpc/src/router/cloud-workspace/cloud-workspace.test.ts index 09f35d9d042..2115f531801 100644 --- a/packages/trpc/src/router/cloud-workspace/cloud-workspace.test.ts +++ b/packages/trpc/src/router/cloud-workspace/cloud-workspace.test.ts @@ -332,13 +332,33 @@ describe("an address read that finds the box stopped", () => { }); describe("cloudWorkspace.delete when the reap cannot be queued", () => { - test("ends every open boot as archived before the final settle", async () => { + test("archive only sets a status: it ends no boot and deletes nothing", async () => { + endedBoots = []; expect(await caller.cloudWorkspace.delete({ id: WORKSPACE_ID })).toEqual({ deleted: true, }); - expect(endedBoots).toEqual([ - [WORKSPACE_ID, { sessionId: null, reason: "archived" }], - ]); + expect(endedBoots).toEqual([]); + }); +}); + +describe("cloudWorkspace.unarchive", () => { + test("is refused once the 7 days are over, before the cleanup job has run", async () => { + const before = { ...ROW }; + Object.assign(ROW, { + status: "archived", + deletedAt: new Date(Date.now() - 8 * 24 * 60 * 60 * 1000), + updatedAt: new Date(), + }); + try { + expect( + await caller.cloudWorkspace.unarchive({ id: WORKSPACE_ID }), + ).toEqual({ unarchived: false }); + } finally { + Object.assign(ROW, before, { + deletedAt: undefined, + updatedAt: undefined, + }); + } }); }); diff --git a/packages/trpc/src/router/cloud-workspace/cloud-workspace.ts b/packages/trpc/src/router/cloud-workspace/cloud-workspace.ts index c9bd47c194d..647b2421a1f 100644 --- a/packages/trpc/src/router/cloud-workspace/cloud-workspace.ts +++ b/packages/trpc/src/router/cloud-workspace/cloud-workspace.ts @@ -37,7 +37,6 @@ import { logControl } from "../../lib/control-log"; import { callMoves } from "../../lib/moves"; import { nudge, readWorkspaceForwards } from "../../lib/realtime"; import { - deleteSandbox, describeSandbox, HOST_SERVICE_PORT, listRemoteBranches, @@ -69,21 +68,13 @@ import { import { recordCloudWorkspaceActivity } from "./activity"; import { withClientStatus } from "./client-status"; import { isLiveStatus } from "./live"; -import { - assertMayBoot, - queueSettleWorkspace, - settleWorkspaceNow, -} from "./meter"; +import { assertMayBoot, queueSettleWorkspace } from "./meter"; import { nextSandboxNameFor } from "./provision"; import { listPullRequests } from "./pull-requests"; -import { queueReap, reapWorkspaceFiles } from "./reap"; +import { archiveWaitIsOver, queueReap } from "./reap"; import { cloudWorkspaceRecordRouter } from "./record"; import { recordSandboxAwake } from "./sandbox-expiry"; -import { - recordBootEnd, - recordBoxStopped, - recordStopSeen, -} from "./sandbox-sleep"; +import { recordBoxStopped, recordStopSeen } from "./sandbox-sleep"; import { cloudWorkspaceSharingRouter } from "./sharing"; import { machineSizeNotAvailable, @@ -870,8 +861,9 @@ export const cloudWorkspaceRouter = { const archivedAt = new Date(); // From any state, provisioning included: the job checks the row - // before it marks it ready and tears its box down when this won. - // The box keeps running until the reap stops it, so an undo is instant. + // before it marks it ready and stops its box when this won. + // The box keeps running until the reap stops it, so an undo is instant; + // the cleanup job deletes it once the grace is over. const archived = await transitionCloudWorkspace({ id: row.id, from: ["provisioning", "awake", "asleep", "failed"], @@ -898,27 +890,12 @@ export const cloudWorkspaceRouter = { await queueReap( { cloudWorkspaceId: row.id, archivedAt: archivedAt.toISOString() }, row, - ).catch(async (error) => { + ).catch((error) => console.error( - `[cloud-workspace] could not queue the reap for ${row.id}`, + `[cloud-workspace] could not stop the box of ${row.id}; the cleanup job deletes it after the grace`, error, - ); - await recordBootEnd( - { cloudWorkspaceId: row.id, organizationId: row.organizationId }, - { - sessionId: null, - reason: "archived", - }, - ); - await settleWorkspaceNow(row.id).catch((meterError) => - console.error( - `[cloud-workspace] ${row.id} final meter failed`, - meterError, - ), - ); - await deleteSandbox(row.providerSandboxId); - await reapWorkspaceFiles(row.id); - }); + ), + ); } await recordCloudWorkspaceActivity( db, @@ -940,6 +917,9 @@ export const cloudWorkspaceRouter = { .input(z.object({ id: z.string().uuid() })) .mutation(async ({ ctx, input }) => { const row = await loadVisibleWorkspace(ctx, input.id); + if (row.status === "archived" && archiveWaitIsOver(row)) { + return { unarchived: false }; + } const resumable = row.status === "archived" && row.provider === "vercel" && diff --git a/packages/trpc/src/router/cloud-workspace/golden-rotation.ts b/packages/trpc/src/router/cloud-workspace/golden-rotation.ts new file mode 100644 index 00000000000..4b7f197a6a3 --- /dev/null +++ b/packages/trpc/src/router/cloud-workspace/golden-rotation.ts @@ -0,0 +1,16 @@ +/** Rule 2: an environment keeps its active golden and its newest others, this many in all. */ +export const KEPT_GOLDENS = 3; + +/** + * Goldens past the active one and the `KEPT_GOLDENS - 1` newest others. + * Takes undeleted goldens, newest first; never returns the active one. + * The same as `goldensToDelete` in #10000's environment-build policy. + */ +export function goldensToDelete( + goldens: readonly T[], + activeRef: string, +): T[] { + return goldens + .filter((golden) => golden.goldenRef !== activeRef) + .slice(KEPT_GOLDENS - 1); +} diff --git a/packages/trpc/src/router/cloud-workspace/provision.test.ts b/packages/trpc/src/router/cloud-workspace/provision.test.ts index 83d1dacd057..cfa68314784 100644 --- a/packages/trpc/src/router/cloud-workspace/provision.test.ts +++ b/packages/trpc/src/router/cloud-workspace/provision.test.ts @@ -10,6 +10,7 @@ import * as meter from "./meter"; import { provisionCloudWorkspace } from "./provision"; const logged: unknown[] = []; +const boxCalls: string[] = []; const captured: unknown[] = []; let claimError: unknown; stub(db.query.cloudWorkspaces, { @@ -24,8 +25,13 @@ stub(sandbox, { buildSandboxClaim: async () => { throw claimError; }, - deleteSandbox: async () => {}, - stopSandbox: async () => ({ sessionId: null }), + deleteSandbox: async (name: string) => { + boxCalls.push(`delete:${name}`); + }, + stopSandbox: async (name: string) => { + boxCalls.push(`stop:${name}`); + return { sessionId: null }; + }, }); stub(realtime, { nudge: () => {} }); stub(chatMoves, { failChatMovesTo: async () => {} }); @@ -56,6 +62,15 @@ function rowUpdate(moved: boolean, sets: unknown[] = []) { describe("a provision that Autumn refuses", () => { stub(db, rowUpdate(true)); + test("a failed create only stops its box and marks the row failed: the cleanup job deletes it later", async () => { + boxCalls.length = 0; + claimError = new Error("boom"); + expect(await provisionCloudWorkspace({ cloudWorkspaceId: "ws_1" })).toBe( + "failed", + ); + expect(boxCalls).toEqual(["stop:ws-box"]); + }); + test("fails the workspace with a warning and opens no Sentry issue", async () => { logged.length = 0; captured.length = 0; diff --git a/packages/trpc/src/router/cloud-workspace/provision.ts b/packages/trpc/src/router/cloud-workspace/provision.ts index 8bc3b4c365c..4716d4a6446 100644 --- a/packages/trpc/src/router/cloud-workspace/provision.ts +++ b/packages/trpc/src/router/cloud-workspace/provision.ts @@ -12,7 +12,6 @@ import { nudge } from "../../lib/realtime"; import { bootPhases, buildSandboxClaim, - deleteSandbox, provisionSandbox, SandboxNotReadyError, settleSandbox, @@ -161,10 +160,10 @@ async function provision( }, }); if (!ready) { - // Deleted while the box was being made: the delete won the row, so - // the box it never knew about goes with it. A duplicate delivery of - // this job loses the row to the first one and must leave the box be: - // the name is per workspace, so it is the same box. + // Archived while the box was being made: the archive won the row, so + // the box is stopped and left to the cleanup job's teardown. A + // duplicate delivery of this job loses the row to the first one and + // must leave the box be: the name is per workspace, so it is the same box. const current = await db.query.cloudWorkspaces.findFirst({ where: eq(cloudWorkspaces.id, row.id), columns: { status: true }, @@ -177,7 +176,7 @@ async function provision( error, ), ); - await deleteSandbox(providerSandboxId); + await stopSandbox(providerSandboxId); if (bootId) await releaseBoot(bootId, "provision_abandoned"); } await naming.catch(() => {}); @@ -238,18 +237,13 @@ async function provision( ), ); } - // A box that booted but whose host-service never answered is kept for - // diagnosis, stopped: its boot log, desktop and shell are a resume - // away, it costs storage rather than compute meanwhile, and a delete - // from the sidebar still removes it. Any other failure must not leak - // a box, since billing started at provision. - const keepForDiagnosis = error instanceof SandboxNotReadyError; - await (keepForDiagnosis ? stopSandbox : deleteSandbox)( - providerSandboxId, - ).catch((teardownError) => { + // A failed create only stops the box: it costs storage, not compute, + // its boot log is a resume away, and the cleanup job deletes it 30 + // days after the failure (rule 1). + await stopSandbox(providerSandboxId).catch((stopError) => { console.error( - `[cloud-workspace] ${keepForDiagnosis ? "could not stop" : "leaked"} sandbox ${providerSandboxId}`, - teardownError, + `[cloud-workspace] could not stop sandbox ${providerSandboxId}`, + stopError, ); }); await failCloudWorkspace({ @@ -259,7 +253,7 @@ async function provision( bootId, }, from: ["provisioning", "awake", "asleep"], - set: keepForDiagnosis ? { providerSandboxId } : {}, + set: { providerSandboxId }, reason: provisionFailureReason(error), error, }); diff --git a/packages/trpc/src/router/cloud-workspace/reap.test.ts b/packages/trpc/src/router/cloud-workspace/reap.test.ts index b200c936b57..f117c940422 100644 --- a/packages/trpc/src/router/cloud-workspace/reap.test.ts +++ b/packages/trpc/src/router/cloud-workspace/reap.test.ts @@ -13,6 +13,7 @@ type Row = { deletedAt: Date | null; provider: string; providerSandboxId: string; + updatedAt?: Date; }; type Stage = "stop" | "delete"; @@ -100,14 +101,15 @@ stub(meter, { }, }); -const { queueReap, reapArchivedCloudWorkspace } = await import("./reap"); +const { archiveWaitIsOver, queueReap, reapArchivedCloudWorkspace } = + await import("./reap"); const archivedAt = new Date("2026-10-01T00:00:00.000Z"); const input = { cloudWorkspaceId: "00000000-0000-0000-0000-000000000001", archivedAt: archivedAt.toISOString(), }; -const SEVEN_DAYS = 7 * 24 * 60 * 60; +const DAY_MS = 24 * 60 * 60 * 1000; const BOX = { organizationId: "org_1", @@ -123,14 +125,9 @@ describe("queueReap", () => { recordedAsleep = []; }); - test("queues the delete after seven days and the stop after a minute", async () => { + test("queues only the stop, after a minute: the cleanup job deletes the box", async () => { await queueReap(input, BOX); expect(queued).toEqual([ - { - path: "/api/cloud-workspaces/reap", - body: { ...input, stage: "delete" }, - delaySeconds: SEVEN_DAYS, - }, { path: "/api/cloud-workspaces/reap", body: { ...input, stage: "stop" }, @@ -147,9 +144,7 @@ describe("queueReap", () => { expect(recordedAsleep).toEqual([ [input.cloudWorkspaceId, { sessionId: "session_1" }], ]); - expect(queued.map((job) => (job.body as { stage: Stage }).stage)).toEqual([ - "delete", - ]); + expect(queued).toEqual([]); }); test("a stop that could not be queued ends the stopped session's boot as archived", async () => { @@ -160,12 +155,6 @@ describe("queueReap", () => { [input.cloudWorkspaceId, { sessionId: "session_1", reason: "archived" }], ]); }); - - test("a delete that could not be queued throws so the caller deletes the box", async () => { - failingStage = "delete"; - await expect(queueReap(input, BOX)).rejects.toThrow("qstash down"); - expect(queued).toEqual([]); - }); }); describe("reapArchivedCloudWorkspace", () => { @@ -196,13 +185,6 @@ describe("reapArchivedCloudWorkspace", () => { ]); }); - test("the delete stage ends every boot still open as archived", async () => { - await reapArchivedCloudWorkspace({ ...input, stage: "delete" }); - expect(endedBoots).toEqual([ - [input.cloudWorkspaceId, { sessionId: null, reason: "archived" }], - ]); - }); - test("the stop stage runs the archive hook, stops the box and queues its final settle", async () => { expect(await reapArchivedCloudWorkspace({ ...input, stage: "stop" })).toBe( "stopped", @@ -252,25 +234,14 @@ describe("reapArchivedCloudWorkspace", () => { expect(meterCalls).toEqual([]); }); - test("the delete stage settles the box's usage before deleting it", async () => { - expect( - await reapArchivedCloudWorkspace({ ...input, stage: "delete" }), - ).toBe("reaped"); - expect(meterCalls).toEqual(["settle"]); - expect(sandboxCalls).toEqual(["delete:ws-box"]); - }); - - test("the delete stage removes the files sent to the workspace", async () => { - await reapArchivedCloudWorkspace({ ...input, stage: "delete" }); - expect(reapedFiles).toEqual([`file-of-${input.cloudWorkspaceId}`]); - }); - - test("a delete still deletes when the final settle fails: at most one lock window goes unbilled", async () => { - meterFailure = new Error("autumn down"); + test("the delete stage of an archive queued before the cleanup job deletes nothing", async () => { expect( await reapArchivedCloudWorkspace({ ...input, stage: "delete" }), - ).toBe("reaped"); - expect(sandboxCalls).toEqual(["delete:ws-box"]); + ).toBe("skipped"); + expect(sandboxCalls).toEqual([]); + expect(meterCalls).toEqual([]); + expect(endedBoots).toEqual([]); + expect(reapedFiles).toEqual([]); }); for (const stage of ["stop", "delete"] as const) { @@ -313,3 +284,27 @@ describe("reapArchivedCloudWorkspace", () => { expect(sandboxCalls).toEqual([]); }); }); + +describe("archiveWaitIsOver", () => { + test("an unarchive is refused from day 7, counting an undated archive from its last write", () => { + const now = archivedAt.getTime(); + expect( + archiveWaitIsOver( + { deletedAt: new Date(now - 6 * DAY_MS), updatedAt: new Date(0) }, + now, + ), + ).toBe(false); + expect( + archiveWaitIsOver( + { deletedAt: new Date(now - 7 * DAY_MS), updatedAt: new Date(now) }, + now, + ), + ).toBe(true); + expect( + archiveWaitIsOver( + { deletedAt: null, updatedAt: new Date(now - 8 * DAY_MS) }, + now, + ), + ).toBe(true); + }); +}); diff --git a/packages/trpc/src/router/cloud-workspace/reap.ts b/packages/trpc/src/router/cloud-workspace/reap.ts index 1f815c4a577..5acbbba3791 100644 --- a/packages/trpc/src/router/cloud-workspace/reap.ts +++ b/packages/trpc/src/router/cloud-workspace/reap.ts @@ -4,7 +4,6 @@ import { eq } from "drizzle-orm"; import { logControl } from "../../lib/control-log"; import { detachAll, reapOrphanFiles } from "../../lib/files"; import { - deleteSandbox, describeSandbox, rerunBoxStartHook, runBoxArchiveHook, @@ -12,18 +11,31 @@ import { stopSandbox, } from "../../lib/sandbox"; import { publishCloudWorkspaceJob } from "./jobs"; -import { queueSettleWorkspace, settleWorkspaceNow } from "./meter"; -import { recordBootEnd, recordBoxStopped } from "./sandbox-sleep"; +import { queueSettleWorkspace } from "./meter"; +import { recordBoxStopped } from "./sandbox-sleep"; /** How long an archived workspace keeps its box running, so an undo finds it live. */ export const ARCHIVE_STOP_DELAY_SECONDS = 60; /** How long an archived workspace keeps its stopped box for an unarchive to resume. */ export const ARCHIVE_GRACE_SECONDS = 7 * 24 * 60 * 60; +/** + * Past it an unarchive is refused, even before the cleanup job deletes the + * box. Rows archived before `deleted_at` existed count from their last write. + */ +export function archiveWaitIsOver( + row: { deletedAt: Date | null; updatedAt: Date }, + now = Date.now(), +): boolean { + const archivedAt = row.deletedAt ?? row.updatedAt; + return now - archivedAt.getTime() >= ARCHIVE_GRACE_SECONDS * 1000; +} + export interface ReapArchivedCloudWorkspaceInput { cloudWorkspaceId: string; /** The archive this reap belongs to; a later archive queues its own. */ archivedAt: string; + /** `delete` only arrives from archives queued before the cleanup job took the delete over. */ stage: "stop" | "delete"; } @@ -83,26 +95,16 @@ async function runArchiveHook( export async function reapArchivedCloudWorkspace( input: ReapArchivedCloudWorkspaceInput, -): Promise<"stopped" | "reaped" | "skipped"> { +): Promise<"stopped" | "skipped"> { const row = await findRow(input.cloudWorkspaceId); if (!isStillArchived(row, input)) { return "skipped"; } if (input.stage === "delete") { - await recordBootEnd( - { cloudWorkspaceId: row.id, organizationId: row.organizationId }, - { sessionId: null, reason: "archived" }, - ); - await settleWorkspaceNow(row.id).catch((error) => - console.error(`[cloud-workspace] ${row.id} final settle failed`, error), + console.log( + `[cloud-workspace] ${row.id} reap delete left to the cleanup job (rule 1)`, ); - await deleteSandbox(row.providerSandboxId); - await reapWorkspaceFiles(row.id); - logControl("archive.deleted", { - cloudWorkspaceId: row.id, - organizationId: row.organizationId, - }); - return "reaped"; + return "skipped"; } if (!(await runArchiveHook(row, input))) return "skipped"; const stopped = await stopSandbox(row.providerSandboxId); @@ -149,9 +151,8 @@ function queueReapStage( } /** - * Stops the box after a short delay, and deletes it once the grace period is over. - * Throws only when the delete could not be queued; a stop that could not be - * queued happens now instead. + * Stops the box after a short delay; a stop that could not be queued happens + * now instead. The cleanup job deletes the box once the grace period is over. */ export async function queueReap( input: Omit, @@ -161,7 +162,6 @@ export async function queueReap( sandboxSessionId: string | null; }, ): Promise { - await queueReapStage({ ...input, stage: "delete" }, ARCHIVE_GRACE_SECONDS); await queueReapStage( { ...input, stage: "stop" }, ARCHIVE_STOP_DELAY_SECONDS, diff --git a/packages/trpc/src/router/cloud-workspace/retention.test.ts b/packages/trpc/src/router/cloud-workspace/retention.test.ts new file mode 100644 index 00000000000..92c0770554d --- /dev/null +++ b/packages/trpc/src/router/cloud-workspace/retention.test.ts @@ -0,0 +1,478 @@ +import { describe, expect, test } from "bun:test"; +import { PgDialect } from "drizzle-orm/pg-core"; +import { stub } from "../../../test/stub"; +import type { ListedDrive, ListedSandbox } from "../../lib/sandbox"; +import * as meter from "./meter"; +import * as reap from "./reap"; +import { + type EnvironmentGoldens, + LEFTOVER_AGE_MS, + MUTATING_DEPS, + RETIRE_BATCH, + type RetentionDeps, + RUN_BUDGET_MS, + reapCloudWorkspaceRetention, + retireWorkspace, + TEARDOWN_SPACING_MS, + WAIT_MS, + type WaitingWorkspace, + type WaitStatus, + waitOverWhere, +} from "./retention"; +import * as sandboxSleep from "./sandbox-sleep"; +import * as teardown from "./teardown"; + +const NOW = Date.parse("2026-11-19T06:00:00Z"); +const DAY = 24 * 60 * 60 * 1000; +const BUNDLE = "0123456789abcdef".repeat(4); + +const row = (n: number, status: WaitStatus = "asleep"): WaitingWorkspace => ({ + id: `ws_${n}`, + organizationId: "org_1", + providerSandboxId: `ws-box${n}`, + status, + createdAt: new Date(NOW - 90 * DAY), +}); + +const box = ( + name: string, + overrides: Partial = {}, +): ListedSandbox => ({ + name, + kind: "workspace", + createdAt: NOW - 60 * DAY, + status: "stopped", + mounts: [], + tags: {}, + ...overrides, +}); + +const drive = ( + name: string, + overrides: Partial = {}, +): ListedDrive => ({ + name, + createdAt: NOW - 60 * DAY, + attachedTo: undefined, + ...overrides, +}); + +const render = (where: Parameters[0]) => + new PgDialect().sqlToQuery(where); + +function fakeDeps(overrides: Partial = {}) { + const tornDown: string[] = []; + const deletedDrives: string[] = []; + const retired: string[] = []; + const paused: number[] = []; + const lines: string[] = []; + const deps: RetentionDeps = { + boxesRunningNow: async () => [], + listSandboxes: async () => [box("ws-live")], + listDrives: async () => [], + findPastWait: async () => [], + retireWorkspace: async (row) => { + retired.push(row.id); + return true; + }, + listEnvironmentGoldens: async () => [], + namedByLiveRow: async () => new Set(), + namedByAnyRow: async (names) => new Set(names), + namedDrives: async () => new Set(), + currentBundleSha: async () => BUNDLE, + tearDownBox: async (name) => { + tornDown.push(name); + }, + deleteDrive: async (name) => { + deletedDrives.push(name); + }, + pause: async (ms) => { + paused.push(ms); + }, + log: (line) => lines.push(line), + ...overrides, + }; + const decisions = () => + lines + .filter((line) => line.startsWith("[cleanup] {")) + .map((line) => JSON.parse(line.slice("[cleanup] ".length))); + return { deps, tornDown, deletedDrives, retired, paused, decisions }; +} + +const run = (deps: RetentionDeps, clock?: () => number) => + reapCloudWorkspaceRetention({ now: NOW, production: true, deps, clock }); + +describe("the cleanup job", () => { + test("a dry run reads everything, reaches no write or delete, and logs what it would delete", async () => { + const reached = () => { + throw new Error("a dry run reached a write or a delete"); + }; + const { deps, decisions } = fakeDeps({ + listSandboxes: async () => [box("ws-orphan"), box("env-old")], + namedByAnyRow: async () => new Set(), + findPastWait: async () => [row(1)], + listDrives: async () => [drive("drive-orphan")], + listEnvironmentGoldens: async () => [ + { id: "e", archived: true, activeRef: "env-old", goldens: ["env-old"] }, + ], + }); + const trapped = { + ...deps, + ...Object.fromEntries(MUTATING_DEPS.map((key) => [key, reached])), + } as RetentionDeps; + const result = await reapCloudWorkspaceRetention({ + now: NOW, + production: true, + deps: trapped, + dryRun: true, + }); + expect(result).toMatchObject({ + rule1: { failed: 0 }, + rule2: { deleted: 1, failed: 0 }, + rule3: { boxes: { deleted: 1, failed: 0 }, drives: { deleted: 1 } }, + }); + const actions = new Set(decisions().map((d) => d.action)); + expect(actions.has("delete")).toBe(false); + expect(actions.has("would delete")).toBe(true); + }); + + test("runs only in production", async () => { + let listed = false; + const { deps } = fakeDeps({ + listSandboxes: async () => { + listed = true; + return []; + }, + }); + expect( + await reapCloudWorkspaceRetention({ now: NOW, production: false, deps }), + ).toEqual({ skipped: "not production" }); + expect(listed).toBe(false); + }); + + test("fails closed: an incomplete listing deletes nothing", async () => { + let expected: readonly string[] = []; + const { deps, retired, tornDown, deletedDrives } = fakeDeps({ + boxesRunningNow: async () => ["ws-running"], + listSandboxes: async (names) => { + expected = names; + throw new Error("listing lacks ws-running"); + }, + findPastWait: async () => [row(1)], + }); + expect(await run(deps)).toMatchObject({ skipped: "listing incomplete" }); + expect(expected).toEqual(["ws-running"]); + expect(retired).toEqual([]); + expect(tornDown).toEqual([]); + expect(deletedDrives).toEqual([]); + }); + + test("rule 1: retires every row one select finds past its wait, a pause apart, and logs each", async () => { + let asked: Parameters[0] | undefined; + const { deps, retired, paused, tornDown, decisions } = fakeDeps({ + listSandboxes: async () => [box("ws-box2"), box("ws-box3")], + namedByAnyRow: async () => new Set(), + findPastWait: async (args) => { + asked = args; + return [row(1, "asleep"), row(2, "archived"), row(3, "failed")]; + }, + }); + const result = await run(deps); + expect(asked).toEqual({ + now: NOW, + listedNames: ["ws-box2", "ws-box3"], + limit: RETIRE_BATCH, + }); + expect(retired).toEqual(["ws_1", "ws_2", "ws_3"]); + expect(paused.slice(0, 3)).toEqual([ + TEARDOWN_SPACING_MS, + TEARDOWN_SPACING_MS, + TEARDOWN_SPACING_MS, + ]); + expect(tornDown).toEqual([]); + expect(decisions().filter((d) => d.rule === 1)).toEqual([ + expect.objectContaining({ + action: "delete", + reason: "asleep for more than 30 days", + }), + expect.objectContaining({ + action: "delete", + reason: "archived for more than 7 days", + }), + expect.objectContaining({ + action: "delete", + reason: "failed for more than 30 days", + }), + ]); + expect(result).toMatchObject({ + rule1: { + retired: { asleep: 1, archived: 1, failed: 1, provisioning: 0 }, + }, + }); + }); + + test("rule 1: an asleep row whose box runs now is kept", async () => { + const { deps, retired } = fakeDeps({ + listSandboxes: async () => [box("ws-box1", { status: "running" })], + findPastWait: async () => [row(1)], + }); + expect(await run(deps)).toMatchObject({ rule1: { skippedRunning: 1 } }); + expect(retired).toEqual([]); + }); + + test("rule 1: a wake that lands first wins, and the row keeps its box", async () => { + const { deps, decisions } = fakeDeps({ + findPastWait: async () => [row(1)], + retireWorkspace: async () => false, + }); + expect(await run(deps)).toMatchObject({ rule1: { skippedChanged: 1 } }); + expect(decisions()).toContainEqual( + expect.objectContaining({ rule: 1, action: "keep" }), + ); + }); + + test("stops starting work when its time is up and leaves the rest for the next run", async () => { + let time = 0; + const { deps, retired } = fakeDeps({ + findPastWait: async () => [row(1), row(2), row(3)], + retireWorkspace: async (row) => { + retired.push(row.id); + time += RUN_BUDGET_MS + 1; + return true; + }, + }); + expect(await run(deps, () => time)).toMatchObject({ + rule1: { deferred: 2 }, + }); + expect(retired).toEqual(["ws_1"]); + }); + + test("rule 2: an archived environment's golden goes; a live one keeps its active golden and newest others", async () => { + const envs: EnvironmentGoldens[] = [ + { id: "env_a", archived: true, activeRef: "env-a", goldens: ["env-a"] }, + { + id: "env_l", + archived: false, + activeRef: "env-l3", + goldens: ["env-l5", "env-l4", "env-l3", "env-l2", "env-l1"], + }, + ]; + const { deps, tornDown } = fakeDeps({ + listSandboxes: async () => + ["env-a", "env-l5", "env-l4", "env-l3", "env-l2", "env-l1"].map( + (name) => box(name, { kind: "environment" }), + ), + listEnvironmentGoldens: async () => envs, + }); + const result = await run(deps); + expect(tornDown).toEqual(["env-a", "env-l2", "env-l1"]); + expect(result).toMatchObject({ rule2: { deleted: 3, kept: 3 } }); + }); + + test("rule 2: a golden already gone from the listing is not deleted again, and one a live row names is kept", async () => { + const { deps, tornDown } = fakeDeps({ + listSandboxes: async () => [box("env-shared")], + listEnvironmentGoldens: async () => [ + { + id: "e1", + archived: true, + activeRef: "env-gone", + goldens: ["env-gone"], + }, + { + id: "e2", + archived: true, + activeRef: "env-shared", + goldens: ["env-shared"], + }, + ], + namedByLiveRow: async () => new Set(["env-shared"]), + }); + await run(deps); + expect(tornDown).toEqual([]); + }); + + test("rule 3: a box no production row names goes once it is 7 days old", async () => { + const { deps, tornDown, decisions } = fakeDeps({ + listSandboxes: async () => [ + box("ws-named"), + box("ws-orphan", { kind: undefined, status: "running" }), + box("ws-young", { createdAt: NOW - LEFTOVER_AGE_MS + DAY }), + ], + namedByAnyRow: async () => new Set(["ws-named"]), + }); + const result = await run(deps); + expect(tornDown).toEqual(["ws-orphan"]); + expect(decisions()).toContainEqual( + expect.objectContaining({ + rule: 3, + name: "ws-young", + action: "keep", + reason: "made less than 7 days ago", + }), + ); + expect(result).toMatchObject({ rule3: { boxes: { deleted: 1, kept: 2 } } }); + }); + + test("rule 3: a drive goes once no row names it, no box mounts it and it is 7 days old", async () => { + const { deps, deletedDrives } = fakeDeps({ + listSandboxes: async () => [ + box("ws-stopped", { mounts: ["drive-mounted"] }), + ], + listDrives: async () => [ + drive("drive-orphan"), + drive("drive-mounted"), + drive("drive-named"), + drive("drive-attached", { attachedTo: "ws-other" }), + drive("drive-young", { createdAt: NOW - DAY }), + ], + namedDrives: async () => new Set(["drive-named"]), + }); + const result = await run(deps); + expect(deletedDrives).toEqual(["drive-orphan"]); + expect(result).toMatchObject({ + rule3: { drives: { deleted: 1, kept: 4 } }, + }); + }); + + test("rule 3: an agents drive goes once it is not the current release's and no box, stopped ones included, mounts it", async () => { + const old = "aaaaaaaaaaaa"; + const { deps, deletedDrives } = fakeDeps({ + listSandboxes: async () => [ + box("ws-stopped", { mounts: [`agent-tools-iad1-${old}`] }), + ], + listDrives: async () => [ + drive(`agent-tools-sfo1-${BUNDLE.slice(0, 12)}`), + drive(`agent-tools-sfo1-${old}`), + drive(`agent-tools-iad1-${old}`), + drive("agent-tools-sfo1-bbbbbbbbbbbb", { createdAt: NOW - DAY / 2 }), + ], + }); + const result = await run(deps); + expect(deletedDrives).toEqual([`agent-tools-sfo1-${old}`]); + expect(result).toMatchObject({ + rule3: { agentsDrives: { deleted: 1, kept: 3 } }, + }); + }); + + test("rule 3: no current release recorded keeps every agents drive", async () => { + const { deps, deletedDrives } = fakeDeps({ + listDrives: async () => [drive("agent-tools-sfo1-aaaaaaaaaaaa")], + currentBundleSha: async () => null, + }); + await run(deps); + expect(deletedDrives).toEqual([]); + }); +}); + +describe("retireWorkspace", () => { + let retiring: Parameters | undefined; + let steps: string[] = []; + stub(teardown, { + retireAndTearDown: async ( + ...args: Parameters + ) => { + retiring = args; + steps.push("retire"); + return true; + }, + }); + stub(sandboxSleep, { + recordBootEnd: async () => { + steps.push("end boots"); + }, + }); + stub(meter, { + settleWorkspaceNow: async () => { + steps.push("settle"); + }, + }); + stub(reap, { + reapWorkspaceFiles: async () => { + steps.push("files"); + }, + }); + + async function retireAs(status: WaitStatus) { + retiring = undefined; + steps = []; + expect(await retireWorkspace(row(1, status), NOW)).toBe(true); + expect(retiring?.[0]).toEqual({ + id: "ws_1", + providerSandboxId: "ws-box1", + driveName: null, + }); + return { + condition: render( + retiring?.[1] as Parameters[0], + ), + set: retiring?.[2], + }; + } + + const cutoff = (status: WaitStatus) => + new Date(NOW - WAIT_MS[status]).toISOString(); + + test("asleep: 30 days after last use, the one write retires it as an expired snapshot", async () => { + const { condition, set } = await retireAs("asleep"); + expect(condition.params).toEqual( + expect.arrayContaining(["asleep", cutoff("asleep")]), + ); + expect(condition.sql).toContain('"sandbox_expires_at"'); + expect(set).toMatchObject({ + status: "failed", + failureReason: "snapshot_expired", + sandboxUrl: null, + }); + }); + + test("archived: 7 days after the archive, it ends its boots and settles, then retires and drops its files", async () => { + const { condition, set } = await retireAs("archived"); + expect(condition.params).toEqual( + expect.arrayContaining(["archived", cutoff("archived")]), + ); + expect(condition.sql).toContain('"deleted_at"'); + expect(set).toEqual({}); + expect(steps).toEqual(["end boots", "settle", "retire", "files"]); + }); + + test("failed: 30 days after it failed, it is retired and stays failed", async () => { + const { condition, set } = await retireAs("failed"); + expect(condition.params).toEqual( + expect.arrayContaining(["failed", cutoff("failed")]), + ); + expect(condition.sql).toContain('"failed_at"'); + expect(set).toEqual({}); + }); + + test("stuck creating: a day with no boot, it is retired as a failed create", async () => { + const { condition, set } = await retireAs("provisioning"); + expect(condition.params).toEqual( + expect.arrayContaining(["provisioning", cutoff("provisioning")]), + ); + expect(condition.sql).toContain('"started_at"'); + expect(set).toMatchObject({ + status: "failed", + failureReason: "create_failed", + }); + }); +}); + +describe("waitOverWhere", () => { + test("a retired row leaves the selection: by its new status, or, keeping its status, because its new name is not listed", () => { + const listed = render(waitOverWhere({ now: NOW, listedNames: ["ws-a"] })); + expect(listed.params).toEqual( + expect.arrayContaining([ + "asleep", + "archived", + "failed", + "provisioning", + "ws-a", + ]), + ); + expect(listed.sql.match(/string_to_array/g)).toHaveLength(2); + const none = render(waitOverWhere({ now: NOW, listedNames: [] })); + expect(none.sql).not.toContain("string_to_array"); + expect(none.sql.match(/false/g)).toHaveLength(2); + }); +}); diff --git a/packages/trpc/src/router/cloud-workspace/retention.ts b/packages/trpc/src/router/cloud-workspace/retention.ts new file mode 100644 index 00000000000..92511252541 --- /dev/null +++ b/packages/trpc/src/router/cloud-workspace/retention.ts @@ -0,0 +1,801 @@ +import * as Sentry from "@sentry/core"; +import { db } from "@superset/db/client"; +import type { CloudWorkspaceFailureReason } from "@superset/db/enums"; +import { + cloudWorkspaceBoots, + cloudWorkspaces, + environments, + sandboxBundles, +} from "@superset/db/schema"; +import { SANDBOX_AGENT_TOOLS_DRIVE_PATTERN } from "@superset/shared/sandbox/contract"; +import { + and, + asc, + desc, + eq, + gt, + isNull, + lt, + or, + type SQL, + sql, +} from "drizzle-orm"; +import { logControl } from "../../lib/control-log"; +import { nudge } from "../../lib/realtime"; +import { + deleteDrive, + type ListedDrive, + type ListedSandbox, + listProjectDrives, + listProjectSandboxes, +} from "../../lib/sandbox"; +import { goldensToDelete } from "./golden-rotation"; +import { settleWorkspaceNow } from "./meter"; +import { ARCHIVE_GRACE_SECONDS, reapWorkspaceFiles } from "./reap"; +import { recordBootEnd } from "./sandbox-sleep"; +import { deleteSandboxAndDriveLater, retireAndTearDown } from "./teardown"; +import { namedByAnyRow, namedSandboxes } from "./teardown-guard"; + +export const RETENTION_JOB_PATH = "/api/cloud-workspaces/jobs/reap-retention"; + +const DAY_MS = 24 * 60 * 60 * 1000; + +/** Rule 1: how long a workspace not in use keeps its box, snapshot and drive. */ +export const WAIT_MS = { + /** Since last use: what Vercel's snapshot expiry gave before we owned it. */ + asleep: 30 * DAY_MS, + /** Since the archive; an unarchive inside it keeps everything, after it is refused. */ + archived: ARCHIVE_GRACE_SECONDS * 1000, + /** Since it failed: long enough to look at what went wrong. */ + failed: 30 * DAY_MS, + /** Stuck creating with no boot since: every delivery of its job died. */ + provisioning: DAY_MS, +} as const; + +export type WaitStatus = keyof typeof WAIT_MS; +const WAIT_STATUSES = Object.keys(WAIT_MS) as WaitStatus[]; + +/** Rule 3: a box, drive or golden nothing in production names goes this long after it was made. */ +export const LEFTOVER_AGE_MS = 7 * DAY_MS; +/** A release publishes its agents drives before it records its bundle as current. */ +export const NEW_AGENTS_DRIVE_MS = DAY_MS; + +/** + * Every 6 hours, so 800 a day: production made a median 156 workspaces a day + * and at most 725 in the 30 days to 2026-11-18. A backlog takes more runs. + */ +export const RETIRE_BATCH = 200; +export const GOLDEN_BATCH = 20; +export const LEFTOVER_BOX_BATCH = 200; +export const DRIVE_BATCH = 50; +/** Stops starting new work here, well inside the route's 300 s; the rest waits for the next run. */ +export const RUN_BUDGET_MS = 200_000; +/** At most 4 teardowns queued a second: far under Vercel's 20 deletions a second per team. */ +export const TEARDOWN_SPACING_MS = 250; +const NAME_LOOKUP_CHUNK = 500; +/** Boxes running now, which the box listing must contain to count as whole. */ +const EXPECTED_IN_LISTING = 3; +const RUNNING_STATUSES = new Set(["pending", "running"]); + +const at = (cutoff: Date) => sql`${cutoff.toISOString()}::timestamptz`; + +/** + * Nothing used the workspace since `cutoff`. Boots count because a wake + * writes its boot before Vercel resumes the box, so one in flight keeps the + * row. `sandbox_expires_at` and `created_at` count too: when they disagree + * with the boots, the later one wins and the box is kept longer. + */ +export function idleSince(cutoff: Date): SQL { + return and( + lt(cloudWorkspaces.createdAt, cutoff), + or( + isNull(cloudWorkspaces.sandboxExpiresAt), + lt(cloudWorkspaces.sandboxExpiresAt, cutoff), + ), + sql`not exists (select 1 from ${cloudWorkspaceBoots} where ${cloudWorkspaceBoots.cloudWorkspaceId} = ${cloudWorkspaces.id} and (${cloudWorkspaceBoots.stoppedAt} is null or ${cloudWorkspaceBoots.stoppedAt} >= ${at(cutoff)} or ${cloudWorkspaceBoots.startedAt} >= ${at(cutoff)}))`, + ) as SQL; +} + +/** Rows archived before `deleted_at` existed count from their last write. */ +function archivedBefore(cutoff: Date): SQL { + return sql`coalesce(${cloudWorkspaces.deletedAt}, ${cloudWorkspaces.updatedAt}) < ${at(cutoff)}`; +} + +/** Rows failed before `failed_at` existed count from their last write. */ +function failedBefore(cutoff: Date): SQL { + return and( + isNull(cloudWorkspaces.deletedAt), + sql`coalesce(${cloudWorkspaces.failedAt}, ${cloudWorkspaces.updatedAt}) < ${at(cutoff)}`, + ) as SQL; +} + +/** An unarchive moves an old row to provisioning, so `updated_at` counts, not only `created_at`. */ +function noBootSince(cutoff: Date): SQL { + return and( + isNull(cloudWorkspaces.deletedAt), + lt(cloudWorkspaces.createdAt, cutoff), + lt(cloudWorkspaces.updatedAt, cutoff), + sql`not exists (select 1 from ${cloudWorkspaceBoots} where ${cloudWorkspaceBoots.cloudWorkspaceId} = ${cloudWorkspaces.id} and ${cloudWorkspaceBoots.startedAt} >= ${at(cutoff)})`, + ) as SQL; +} + +interface WaitRule { + since: (cutoff: Date) => SQL; + /** The status the retire write gives the row; null keeps its status. */ + failAs: CloudWorkspaceFailureReason | null; +} + +/** + * Rule 1's wait table. A retired row leaves the selection: an asleep or + * stuck row fails, and a row that keeps its status (archived, failed) is + * selected only while its box is listed, which a retired name never is. + */ +export const WAIT_RULES: Record = { + asleep: { + since: (cutoff) => + and(isNull(cloudWorkspaces.deletedAt), idleSince(cutoff)) as SQL, + failAs: "snapshot_expired", + }, + archived: { since: archivedBefore, failAs: null }, + failed: { since: failedBefore, failAs: null }, + provisioning: { since: noBootSince, failAs: "create_failed" }, +}; + +export function waitCutoff(status: WaitStatus, now: number): Date { + return new Date(now - WAIT_MS[status]); +} + +/** Still in `status` and still past its wait: the condition of the one retire write. */ +export function pastWait(status: WaitStatus, now: number): SQL { + return and( + eq(cloudWorkspaces.provider, "vercel"), + eq(cloudWorkspaces.status, status), + WAIT_RULES[status].since(waitCutoff(status, now)), + ) as SQL; +} + +function amongNames(names: readonly string[]): SQL { + const usable = names.filter((name) => !name.includes(",")); + if (usable.length === 0) return sql`false`; + return sql`${cloudWorkspaces.providerSandboxId} = any(string_to_array(${usable.join(",")}, ','))`; +} + +/** Rule 1 as one select. */ +export function waitOverWhere({ + now, + listedNames, +}: { + now: number; + listedNames: readonly string[]; +}): SQL { + return or( + ...WAIT_STATUSES.map((status) => + WAIT_RULES[status].failAs + ? pastWait(status, now) + : and(pastWait(status, now), amongNames(listedNames)), + ), + ) as SQL; +} + +export interface WaitingWorkspace { + id: string; + organizationId: string; + providerSandboxId: string; + status: WaitStatus; + createdAt: Date; +} + +async function findPastWait(args: { + now: number; + listedNames: readonly string[]; + limit: number; +}): Promise { + const rows = await db + .select({ + id: cloudWorkspaces.id, + organizationId: cloudWorkspaces.organizationId, + providerSandboxId: cloudWorkspaces.providerSandboxId, + status: cloudWorkspaces.status, + createdAt: cloudWorkspaces.createdAt, + }) + .from(cloudWorkspaces) + .where(waitOverWhere(args)) + .orderBy(asc(cloudWorkspaces.createdAt)) + .limit(args.limit); + return rows.filter((row): row is WaitingWorkspace => + (WAIT_STATUSES as string[]).includes(row.status), + ); +} + +/** + * teardown(row): one conditional write takes the box name off the row, and + * the status with it, while the row is still in its status and still past its + * wait; then the box is deleted through the retried teardown job. A wake that + * lands first wins, and this returns false. An archived row ends its boots + * and settles its usage first, and its files go with the box. + */ +export async function retireWorkspace( + row: WaitingWorkspace, + now: number, +): Promise { + const scope = { + cloudWorkspaceId: row.id, + organizationId: row.organizationId, + }; + if (row.status === "archived") { + await recordBootEnd(scope, { sessionId: null, reason: "archived" }); + await settleWorkspaceNow(row.id).catch((error) => + console.error(`[cloud-workspace] ${row.id} final settle failed`, error), + ); + } + const failAs = WAIT_RULES[row.status].failAs; + const retired = await retireAndTearDown( + { id: row.id, providerSandboxId: row.providerSandboxId, driveName: null }, + pastWait(row.status, now), + failAs + ? { + status: "failed", + failureReason: failAs, + failedAt: new Date(now), + sandboxUrl: null, + } + : {}, + ); + if (!retired) return false; + if (row.status === "archived") await reapWorkspaceFiles(row.id); + if (failAs) { + logControl("workspace.failed", scope, { "failure.reason": failAs }, "warn"); + nudge(row.organizationId, "cloud_workspaces"); + } + return true; +} + +/** A few boxes running now: a listing without them is not the whole project. */ +async function boxesRunningNow(now: number): Promise { + const rows = await db + .select({ name: cloudWorkspaces.providerSandboxId }) + .from(cloudWorkspaces) + .where( + and( + eq(cloudWorkspaces.provider, "vercel"), + eq(cloudWorkspaces.status, "awake"), + gt(cloudWorkspaces.sandboxExpiresAt, new Date(now + 60_000)), + ), + ) + .limit(EXPECTED_IN_LISTING); + return rows.map((row) => row.name); +} + +export interface EnvironmentGoldens { + id: string; + archived: boolean; + activeRef: string | null; + /** Undeleted goldens, newest first. */ + goldens: string[]; +} + +/** On main an environment records only its active golden; a fork's golden is a box. */ +async function listEnvironmentGoldens(): Promise { + const rows = await db + .select({ + id: environments.id, + archivedAt: environments.archivedAt, + sourceKind: environments.sourceKind, + sourceRef: environments.sourceRef, + }) + .from(environments) + .where(eq(environments.provider, "vercel")); + return rows.map((row) => ({ + id: row.id, + archived: row.archivedAt !== null, + activeRef: row.sourceKind === "fork" ? row.sourceRef : null, + goldens: row.sourceKind === "fork" ? [row.sourceRef] : [], + })); +} + +async function currentBundleSha(): Promise { + const [row] = await db + .select({ sha: sandboxBundles.sha }) + .from(sandboxBundles) + .orderBy(desc(sandboxBundles.releasedAt)) + .limit(1); + return row?.sha ?? null; +} + +async function inChunks( + names: readonly string[], + lookup: (chunk: string[]) => Promise>, +): Promise> { + const found = new Set(); + for (let i = 0; i < names.length; i += NAME_LOOKUP_CHUNK) { + for (const name of await lookup(names.slice(i, i + NAME_LOOKUP_CHUNK))) { + found.add(name); + } + } + return found; +} + +export interface RetentionDeps { + boxesRunningNow: typeof boxesRunningNow; + listSandboxes: typeof listProjectSandboxes; + listDrives: typeof listProjectDrives; + findPastWait: typeof findPastWait; + retireWorkspace: typeof retireWorkspace; + listEnvironmentGoldens: typeof listEnvironmentGoldens; + /** Named by a live workspace or an unarchived environment: what a golden delete must not take. */ + namedByLiveRow: (names: readonly string[]) => Promise>; + /** Named by any production row, archived ones included. */ + namedByAnyRow: (names: readonly string[]) => Promise>; + /** Workspace drives rows name: none on main; #9847's `drive_name` and `golden_drive_name`. */ + namedDrives: (names: readonly string[]) => Promise>; + currentBundleSha: typeof currentBundleSha; + tearDownBox: (providerSandboxId: string) => Promise; + deleteDrive: typeof deleteDrive; + pause: (ms: number) => Promise; + log: (line: string) => void; +} + +export const defaultRetentionDeps: RetentionDeps = { + boxesRunningNow, + listSandboxes: listProjectSandboxes, + listDrives: listProjectDrives, + findPastWait, + retireWorkspace, + listEnvironmentGoldens, + namedByLiveRow: (names) => inChunks(names, namedSandboxes), + namedByAnyRow: (names) => inChunks(names, namedByAnyRow), + namedDrives: async () => new Set(), + currentBundleSha, + tearDownBox: (providerSandboxId) => + deleteSandboxAndDriveLater({ providerSandboxId, driveName: null }), + deleteDrive, + pause: (ms) => new Promise((resolve) => setTimeout(resolve, ms)), + log: (line) => console.log(line), +}; + +/** Every dependency that writes a row or deletes in Vercel; a dry run replaces each one. */ +export const MUTATING_DEPS = [ + "retireWorkspace", + "tearDownBox", + "deleteDrive", + "pause", +] as const satisfies readonly (keyof RetentionDeps)[]; + +/** The job's reads, unchanged; its writes and deletes only recorded as if they had worked. */ +export function dryRunDeps(deps: RetentionDeps): RetentionDeps { + const replaced: Pick = { + retireWorkspace: async () => true, + tearDownBox: async () => {}, + deleteDrive: async () => {}, + pause: async () => {}, + }; + return { ...deps, ...replaced }; +} + +type Counts = { + kept: number; + deleted: number; + deferred: number; + failed: number; +}; + +const perStatus = (): Record => ({ + asleep: 0, + archived: 0, + failed: 0, + provisioning: 0, +}); + +export type RetentionRun = + | { skipped: "not production" } + | { skipped: "listing incomplete"; reason: string } + | { + rule1: { + found: Record; + retired: Record; + skippedRunning: number; + skippedChanged: number; + deferred: number; + failed: number; + }; + rule2: Counts & { environments: number }; + rule3: { boxes: Counts; drives: Counts; agentsDrives: Counts }; + }; + +type Decision = { + rule: 1 | 2 | 3; + kind: "workspace" | "box" | "golden" | "drive" | "agents drive"; + name: string; + action: "keep" | "delete" | "would delete"; + reason: string; +}; + +/** + * The cleanup job, every 6 hours, production only: the one place that + * deletes for age or for being unused. Rule 1 retires workspaces past their + * wait, rule 2 deletes goldens an environment no longer keeps, rule 3 + * deletes what nothing in production names. It reads the whole Vercel + * listing first and deletes nothing if it cannot. + */ +export async function reapCloudWorkspaceRetention({ + now = Date.now(), + production, + deps: given = defaultRetentionDeps, + clock = Date.now, + dryRun = false, +}: { + now?: number; + production: boolean; + deps?: RetentionDeps; + clock?: () => number; + /** Reads everything, writes and deletes nothing, and logs "would delete". */ + dryRun?: boolean; +}): Promise { + const deps = dryRun ? dryRunDeps(given) : given; + if (!production) { + deps.log("[cleanup] skipped: only production runs the cleanup job"); + return { skipped: "not production" }; + } + const startedAt = clock(); + const outOfTime = () => clock() - startedAt > RUN_BUDGET_MS; + const decide = (decision: Decision) => + deps.log( + `[cleanup] ${JSON.stringify( + dryRun && decision.action === "delete" + ? { ...decision, action: "would delete" } + : decision, + )}`, + ); + let boxes: ListedSandbox[]; + let drives: ListedDrive[]; + try { + boxes = await deps.listSandboxes(await deps.boxesRunningNow(now)); + drives = await deps.listDrives(); + } catch (error) { + const reason = String(error).slice(0, 300); + deps.log(`[cleanup] listing incomplete, deleted nothing: ${reason}`); + Sentry.captureMessage("cleanup job: listing incomplete, deleted nothing", { + level: "warning", + extra: { reason }, + }); + return { skipped: "listing incomplete", reason }; + } + const ctx = { now, deps, boxes, drives, outOfTime, decide }; + const retired = new Set(); + const rule1 = await retireWorkspaces({ ...ctx, retired }); + const rule2 = await pruneGoldens({ ...ctx, retired }); + const rule3 = await deleteLeftovers({ ...ctx, retired }); + const run = { rule1, rule2, rule3 }; + deps.log(`[cleanup] run ${JSON.stringify(run)}`); + Sentry.logger.info("retention.run", flatten(run)); + if ( + rule1.failed || + rule2.failed || + rule3.boxes.failed || + rule3.drives.failed || + rule3.agentsDrives.failed + ) { + Sentry.captureMessage("cleanup job run had failures", { + level: "warning", + extra: { ...run }, + }); + } + return run; +} + +type Ctx = { + now: number; + deps: RetentionDeps; + boxes: ListedSandbox[]; + drives: ListedDrive[]; + outOfTime: () => boolean; + decide: (decision: Decision) => void; + retired: Set; +}; + +async function retireWorkspaces({ + now, + deps, + boxes, + outOfTime, + decide, + retired, +}: Ctx) { + const status = new Map(boxes.map((box) => [box.name, box.status])); + const rows = await deps.findPastWait({ + now, + listedNames: boxes.map((box) => box.name), + limit: RETIRE_BATCH, + }); + const counts = { + found: perStatus(), + retired: perStatus(), + skippedRunning: 0, + skippedChanged: 0, + deferred: 0, + failed: 0, + }; + for (const row of rows) counts.found[row.status]++; + for (const [index, row] of rows.entries()) { + if (outOfTime()) { + counts.deferred = rows.length - index; + break; + } + const name = `${row.id} (${row.providerSandboxId})`; + try { + // With billing off no boot row guards a wake in flight; the listing does. + if ( + row.status === "asleep" && + RUNNING_STATUSES.has(status.get(row.providerSandboxId) ?? "") + ) { + counts.skippedRunning++; + decide({ + rule: 1, + kind: "workspace", + name, + action: "keep", + reason: "asleep past its wait, but its box runs a session now", + }); + continue; + } + if (!(await deps.retireWorkspace(row, now))) { + counts.skippedChanged++; + decide({ + rule: 1, + kind: "workspace", + name, + action: "keep", + reason: `no longer ${row.status} past its wait: changed since it was read`, + }); + continue; + } + counts.retired[row.status]++; + retired.add(row.providerSandboxId); + decide({ + rule: 1, + kind: "workspace", + name, + action: "delete", + reason: `${row.status} for more than ${WAIT_MS[row.status] / DAY_MS} days`, + }); + await deps.pause(TEARDOWN_SPACING_MS); + } catch (error) { + counts.failed++; + Sentry.captureException(error, { + tags: { "cloud_workspace.id": row.id }, + }); + } + } + return counts; +} + +async function pruneGoldens({ deps, boxes, outOfTime, decide, retired }: Ctx) { + const listed = new Set(boxes.map((box) => box.name)); + const envs = await deps.listEnvironmentGoldens(); + const counts = { + environments: envs.length, + kept: 0, + deleted: 0, + deferred: 0, + failed: 0, + }; + const due: string[] = []; + for (const env of envs) { + const drop = new Set( + env.archived + ? env.goldens + : goldensToDelete( + env.goldens.map((goldenRef) => ({ goldenRef })), + env.activeRef ?? "", + ).map((golden) => golden.goldenRef), + ); + for (const golden of env.goldens) { + if (drop.has(golden) && listed.has(golden)) { + due.push(golden); + continue; + } + if (drop.has(golden)) continue; + counts.kept++; + decide({ + rule: 2, + kind: "golden", + name: golden, + action: "keep", + reason: + golden === env.activeRef + ? `active golden of environment ${env.id}` + : `among the newest goldens of environment ${env.id}`, + }); + } + } + const shared = await deps.namedByLiveRow(due); + const batch = due.filter((name) => !shared.has(name) && !retired.has(name)); + for (const name of due.filter((name) => shared.has(name))) { + counts.kept++; + decide({ + rule: 2, + kind: "golden", + name, + action: "keep", + reason: "a live workspace or environment still names it", + }); + } + const work = batch.slice(0, GOLDEN_BATCH); + counts.deferred = batch.length - work.length; + for (const [index, name] of work.entries()) { + if (outOfTime()) { + counts.deferred += work.length - index; + break; + } + try { + await deps.tearDownBox(name); + retired.add(name); + counts.deleted++; + decide({ + rule: 2, + kind: "golden", + name, + action: "delete", + reason: "its environment is archived or keeps newer goldens", + }); + await deps.pause(TEARDOWN_SPACING_MS); + } catch (error) { + counts.failed++; + Sentry.captureException(error, { extra: { golden: name } }); + } + } + return counts; +} + +async function deleteLeftovers(ctx: Ctx) { + return { + boxes: await deleteLeftoverBoxes(ctx), + ...(await deleteLeftoverDrives(ctx)), + }; +} + +async function deleteLeftoverBoxes({ + now, + deps, + boxes, + outOfTime, + decide, + retired, +}: Ctx): Promise { + const counts = { kept: 0, deleted: 0, deferred: 0, failed: 0 }; + const fresh = boxes.filter((box) => !retired.has(box.name)); + const named = await deps.namedByAnyRow(fresh.map((box) => box.name)); + const due: string[] = []; + for (const box of fresh) { + const reason = named.has(box.name) + ? "a production row names it" + : now - box.createdAt < LEFTOVER_AGE_MS + ? "made less than 7 days ago" + : null; + if (reason) { + counts.kept++; + decide({ rule: 3, kind: "box", name: box.name, action: "keep", reason }); + } else { + due.push(box.name); + } + } + const work = due.slice(0, LEFTOVER_BOX_BATCH); + counts.deferred = due.length - work.length; + for (const [index, name] of work.entries()) { + if (outOfTime()) { + counts.deferred += work.length - index; + break; + } + try { + await deps.tearDownBox(name); + counts.deleted++; + decide({ + rule: 3, + kind: "box", + name, + action: "delete", + reason: "no production row names it, made more than 7 days ago", + }); + await deps.pause(TEARDOWN_SPACING_MS); + } catch (error) { + counts.failed++; + Sentry.captureException(error, { extra: { providerSandboxId: name } }); + } + } + return counts; +} + +async function deleteLeftoverDrives({ + now, + deps, + boxes, + drives, + outOfTime, + decide, +}: Ctx): Promise<{ drives: Counts; agentsDrives: Counts }> { + const mountedBy = new Map(); + for (const box of boxes) { + for (const drive of box.mounts) mountedBy.set(drive, box.name); + } + const current = await deps.currentBundleSha(); + const workspaceDrives = drives.filter( + (drive) => !SANDBOX_AGENT_TOOLS_DRIVE_PATTERN.test(drive.name), + ); + const named = await deps.namedDrives(workspaceDrives.map((d) => d.name)); + const keepReason = (drive: ListedDrive): string | null => { + const mounter = mountedBy.get(drive.name); + if (mounter) return `mounted by box ${mounter}`; + if (drive.attachedTo) return `attached to ${drive.attachedTo}`; + if (SANDBOX_AGENT_TOOLS_DRIVE_PATTERN.test(drive.name)) { + if (!current) return "no current release recorded"; + if (drive.name.endsWith(`-${current.slice(0, 12)}`)) { + return "the current release's"; + } + if (now - drive.createdAt < NEW_AGENTS_DRIVE_MS) { + return "published less than a day ago: its release may not be recorded yet"; + } + return null; + } + if (named.has(drive.name)) return "a production row names it"; + if (now - drive.createdAt < LEFTOVER_AGE_MS) { + return "made less than 7 days ago"; + } + return null; + }; + const results = { + drives: { kept: 0, deleted: 0, deferred: 0, failed: 0 }, + agentsDrives: { kept: 0, deleted: 0, deferred: 0, failed: 0 }, + }; + let started = 0; + for (const drive of drives) { + const agents = SANDBOX_AGENT_TOOLS_DRIVE_PATTERN.test(drive.name); + const counts = agents ? results.agentsDrives : results.drives; + const kind = agents ? "agents drive" : "drive"; + const reason = keepReason(drive); + if (reason) { + counts.kept++; + decide({ rule: 3, kind, name: drive.name, action: "keep", reason }); + continue; + } + if (started >= DRIVE_BATCH || outOfTime()) { + counts.deferred++; + continue; + } + started++; + try { + await deps.deleteDrive(drive.name); + counts.deleted++; + decide({ + rule: 3, + kind, + name: drive.name, + action: "delete", + reason: agents + ? "not the current release's, and no box mounts it" + : "no production row names it and no box mounts it, made more than 7 days ago", + }); + } catch (error) { + counts.failed++; + decide({ + rule: 3, + kind, + name: drive.name, + action: "keep", + reason: `Vercel refused the delete: ${String(error).slice(0, 200)}`, + }); + } + } + return results; +} + +function flatten(value: object, prefix = ""): Record { + const flat: Record = {}; + for (const [key, entry] of Object.entries(value)) { + if (entry && typeof entry === "object") { + Object.assign(flat, flatten(entry, `${prefix}${key}.`)); + } else { + flat[`${prefix}${key}`] = entry; + } + } + return flat; +} diff --git a/packages/trpc/src/router/cloud-workspace/teardown-guard.ts b/packages/trpc/src/router/cloud-workspace/teardown-guard.ts new file mode 100644 index 00000000000..fdd626bb8c1 --- /dev/null +++ b/packages/trpc/src/router/cloud-workspace/teardown-guard.ts @@ -0,0 +1,59 @@ +import { db } from "@superset/db/client"; +import { cloudWorkspaces, environments } from "@superset/db/schema"; +import { and, inArray, isNull } from "drizzle-orm"; + +/** + * The names among `names` that a workspace row, in any status, or an + * unarchived environment still names. A real teardown clears the row's name + * in the write that commits to it, so a name still on a row is that row's: + * an archived row may be undone onto it, a failed one keeps it for diagnosis. + */ +export async function namedSandboxes( + names: readonly string[], +): Promise> { + if (names.length === 0) return new Set(); + const [workspaces, goldens] = await Promise.all([ + db + .select({ name: cloudWorkspaces.providerSandboxId }) + .from(cloudWorkspaces) + .where(inArray(cloudWorkspaces.providerSandboxId, [...names])), + db + .select({ name: environments.sourceRef }) + .from(environments) + .where( + and( + inArray(environments.sourceRef, [...names]), + isNull(environments.archivedAt), + ), + ), + ]); + return new Set([...workspaces, ...goldens].map((row) => row.name)); +} + +/** + * The names among `names` that any production row names: a workspace in any + * status, or an environment, archived ones included (rule 2 deletes those). + */ +export async function namedByAnyRow( + names: readonly string[], +): Promise> { + if (names.length === 0) return new Set(); + const [workspaces, goldens] = await Promise.all([ + db + .select({ name: cloudWorkspaces.providerSandboxId }) + .from(cloudWorkspaces) + .where(inArray(cloudWorkspaces.providerSandboxId, [...names])), + db + .select({ name: environments.sourceRef }) + .from(environments) + .where(inArray(environments.sourceRef, [...names])), + ]); + return new Set([...workspaces, ...goldens].map((row) => row.name)); +} + +export async function liveReferences(args: { + providerSandboxId: string; +}): Promise<{ sandbox: boolean }> { + const named = await namedSandboxes([args.providerSandboxId]); + return { sandbox: named.has(args.providerSandboxId) }; +} diff --git a/packages/trpc/src/router/cloud-workspace/teardown.test.ts b/packages/trpc/src/router/cloud-workspace/teardown.test.ts new file mode 100644 index 00000000000..09051b05334 --- /dev/null +++ b/packages/trpc/src/router/cloud-workspace/teardown.test.ts @@ -0,0 +1,139 @@ +import { beforeEach, describe, expect, test } from "bun:test"; +import { db } from "@superset/db/client"; +import { stub } from "../../../test/stub"; +import * as sandbox from "../../lib/sandbox"; +import * as jobs from "./jobs"; +import { + deleteSandboxAndDriveLater, + retireAndTearDown, + TEARDOWN_ATTEMPTS, + teardownSandbox, +} from "./teardown"; +import * as guard from "./teardown-guard"; + +let calls: string[] = []; +let namedOnRead: boolean[] = []; +let deleteFails = false; +let queued: Array<{ path: string; body: unknown; delaySeconds?: number }> = []; +let updateMatches = true; +let renamedTo: unknown; +let written: unknown; + +stub(guard, { + liveReferences: async () => ({ sandbox: namedOnRead.shift() ?? false }), +}); +stub(sandbox, { + stopSandbox: async (name: string) => { + calls.push(`stop:${name}`); + return { sessionId: null }; + }, + deleteSandbox: async (name: string) => { + if (deleteFails) throw new Error("vercel down"); + calls.push(`delete:${name}`); + }, +}); +stub(db, { + update: () => ({ + set: (values: { providerSandboxId: string }) => { + renamedTo = values.providerSandboxId; + written = values; + return { + where: () => ({ + returning: async () => (updateMatches ? [{ id: "ws_1" }] : []), + }), + }; + }, + }), +}); +stub(jobs, { + publishCloudWorkspaceJob: async (job: { + path: string; + body: unknown; + delaySeconds?: number; + }) => { + queued.push(job); + }, +}); + +beforeEach(() => { + calls = []; + namedOnRead = []; + deleteFails = false; + queued = []; + updateMatches = true; + renamedTo = undefined; +}); + +const input = { providerSandboxId: "ws-old", driveName: null, attempt: 0 }; + +describe("teardownSandbox", () => { + test("outside production it stops and deletes nothing", async () => { + process.env.VERCEL_ENV = "preview"; + try { + expect(await teardownSandbox(input)).toBe("skipped"); + } finally { + process.env.VERCEL_ENV = "production"; + } + expect(calls).toEqual([]); + }); + + test("stops, then deletes a box no row names", async () => { + expect(await teardownSandbox(input)).toBe("deleted"); + expect(calls).toEqual(["stop:ws-old", "delete:ws-old"]); + }); + + test("keeps a box a row names by the time it would delete it", async () => { + namedOnRead = [false, true]; + expect(await teardownSandbox(input)).toBe("kept"); + expect(calls).toEqual(["stop:ws-old"]); + }); + + test("a failure requeues the next attempt, and the last attempt gives up", async () => { + deleteFails = true; + expect(await teardownSandbox(input)).toBe("retrying"); + expect(queued).toMatchObject([ + { + path: "/api/cloud-workspaces/teardown", + body: { ...input, attempt: 1 }, + }, + ]); + queued = []; + expect( + await teardownSandbox({ ...input, attempt: TEARDOWN_ATTEMPTS - 1 }), + ).toBe("abandoned"); + expect(queued).toEqual([]); + }); +}); + +describe("retireAndTearDown", () => { + const row = { id: "ws_1", providerSandboxId: "ws-old", driveName: null }; + + test("gives the row a new box name, and its status in the same write, and queues the old box's teardown only when the update matched", async () => { + updateMatches = false; + expect(await retireAndTearDown(row, undefined)).toBe(false); + expect(queued).toEqual([]); + updateMatches = true; + expect(await retireAndTearDown(row, undefined, { status: "failed" })).toBe( + true, + ); + expect(renamedTo).toMatch(/^ws-ws_1[0-9a-f]{8}$/); + expect(written).toMatchObject({ status: "failed" }); + expect(queued).toMatchObject([ + { path: "/api/cloud-workspaces/teardown", body: input }, + ]); + expect(calls).toEqual([]); + }); +}); + +describe("deleteSandboxAndDriveLater", () => { + test("a box without a drive goes through the retried job too, not an inline delete", async () => { + await deleteSandboxAndDriveLater({ + providerSandboxId: "ws-old", + driveName: null, + }); + expect(calls).toEqual([]); + expect(queued).toMatchObject([ + { path: "/api/cloud-workspaces/teardown", body: input }, + ]); + }); +}); diff --git a/packages/trpc/src/router/cloud-workspace/teardown.ts b/packages/trpc/src/router/cloud-workspace/teardown.ts new file mode 100644 index 00000000000..3ba7077351f --- /dev/null +++ b/packages/trpc/src/router/cloud-workspace/teardown.ts @@ -0,0 +1,145 @@ +import * as Sentry from "@sentry/core"; +import { db } from "@superset/db/client"; +import { cloudWorkspaces } from "@superset/db/schema"; +import { and, eq, type SQL } from "drizzle-orm"; +import { + deleteSandbox, + sandboxDeletesAllowed, + stopSandbox, +} from "../../lib/sandbox"; +import { publishCloudWorkspaceJob } from "./jobs"; +import { nextSandboxNameFor } from "./provision"; +import { liveReferences } from "./teardown-guard"; + +export interface TeardownSandboxInput { + providerSandboxId: string; + /** Always null until workspace drives exist; #9847 tears down the drive here. */ + driveName: null; + attempt: number; +} + +export const TEARDOWN_RETRY_SECONDS = 30; +export const TEARDOWN_ATTEMPTS = 20; + +export type TeardownOutcome = + | "deleted" + | "kept" + | "retrying" + | "abandoned" + | "skipped"; + +/** + * Stops the box, then deletes it with its snapshots. Before each destructive + * call it reads the rows again: a box a workspace or a live environment names + * is kept. Any failure requeues the job. + */ +export async function teardownSandbox( + input: TeardownSandboxInput, +): Promise { + if (!sandboxDeletesAllowed(`box ${input.providerSandboxId} and no drive`)) { + return "skipped"; + } + try { + return await teardownOnce(input); + } catch (error) { + Sentry.captureException(error, { extra: { ...input } }); + return retryOrAbandon(input, error); + } +} + +async function teardownOnce( + input: TeardownSandboxInput, +): Promise { + if ((await liveReferences(input)).sandbox) return kept(input); + await stopSandbox(input.providerSandboxId); + if ((await liveReferences(input)).sandbox) return kept(input); + await deleteSandbox(input.providerSandboxId); + console.log(`[cloud-workspace] teardown deleted ${input.providerSandboxId}`); + return "deleted"; +} + +function kept(input: TeardownSandboxInput): TeardownOutcome { + console.log( + `[cloud-workspace] teardown kept ${input.providerSandboxId}: a live row names it`, + ); + return "kept"; +} + +async function retryOrAbandon( + input: TeardownSandboxInput, + error: unknown, +): Promise { + if (input.attempt + 1 >= TEARDOWN_ATTEMPTS) { + Sentry.captureMessage( + `cloud workspace teardown gave up on ${input.providerSandboxId}`, + { level: "error", extra: { ...input, error: String(error) } }, + ); + return "abandoned"; + } + await queueTeardown( + { ...input, attempt: input.attempt + 1 }, + TEARDOWN_RETRY_SECONDS, + ); + return "retrying"; +} + +function queueTeardown( + input: TeardownSandboxInput, + delaySeconds?: number, +): Promise { + return publishCloudWorkspaceJob({ + path: "/api/cloud-workspaces/teardown", + body: input, + delaySeconds, + runLocally: teardownSandbox, + }); +} + +/** Every box goes through the retried job, which reads the rows again just before it deletes. */ +export async function deleteSandboxAndDriveLater(args: { + providerSandboxId: string; + driveName: string | null | undefined; + keepDrive?: boolean; +}): Promise { + if (args.driveName) { + throw new Error( + `no drive teardown yet: ${args.providerSandboxId} names drive ${args.driveName}`, + ); + } + await queueTeardown({ + providerSandboxId: args.providerSandboxId, + driveName: null, + attempt: 0, + }); +} + +/** + * Takes the row's box name out of use in the same write that commits to + * deleting the box, then queues its teardown. A wake after it finds a name + * that does not exist, never the box being deleted. `set` changes the row in + * that same write, such as the status a retired row ends in. Returns false + * when the row no longer matches `condition` or names another box. + */ +export async function retireAndTearDown( + row: { id: string; providerSandboxId: string; driveName: string | null }, + condition: SQL | undefined, + set: Partial = {}, +): Promise { + const [retired] = await db + .update(cloudWorkspaces) + .set({ ...set, providerSandboxId: nextSandboxNameFor(row.id) }) + .where( + and( + eq(cloudWorkspaces.id, row.id), + eq(cloudWorkspaces.providerSandboxId, row.providerSandboxId), + condition, + ), + ) + .returning({ id: cloudWorkspaces.id }); + if (!retired) return false; + await deleteSandboxAndDriveLater({ + providerSandboxId: row.providerSandboxId, + driveName: row.driveName, + }); + return true; +} diff --git a/packages/trpc/src/router/cloud-workspace/transition.ts b/packages/trpc/src/router/cloud-workspace/transition.ts index 7c26d5cfbe4..c222c050a02 100644 --- a/packages/trpc/src/router/cloud-workspace/transition.ts +++ b/packages/trpc/src/router/cloud-workspace/transition.ts @@ -4,7 +4,7 @@ import type { CloudWorkspaceFailureReason } from "@superset/db/enums"; import type { CloudWorkspaceStatus } from "@superset/db/schema"; import { cloudWorkspaces } from "@superset/db/schema"; import { isCloudUsageRefusal } from "@superset/shared/billing"; -import { and, eq, inArray } from "drizzle-orm"; +import { and, eq, inArray, type SQL } from "drizzle-orm"; import { type ControlScope, controlTags, @@ -16,6 +16,7 @@ type Move = { id: string; from: readonly CloudWorkspaceStatus[]; set?: Partial; + where?: SQL; }; async function moveCloudWorkspace( @@ -28,6 +29,7 @@ async function moveCloudWorkspace( and( eq(cloudWorkspaces.id, args.id), inArray(cloudWorkspaces.status, [...args.from]), + args.where, ), ) .returning({ id: cloudWorkspaces.id }); @@ -53,31 +55,39 @@ export function transitionCloudWorkspace( ); } -/** An Autumn refusal is an expected state: it is logged as a warning and opens no Sentry issue. */ +/** + * An expected failure (an Autumn refusal, or `expected`) is logged as a + * warning and opens no Sentry issue. Returns whether the row moved. + */ export async function failCloudWorkspace(args: { scope: ControlScope; from: readonly CloudWorkspaceStatus[]; set?: Partial; + /** Further conditions the same update must meet. */ + where?: SQL; reason: CloudWorkspaceFailureReason; error: unknown; -}): Promise { + expected?: boolean; +}): Promise { const { scope } = args; const changed = await moveCloudWorkspace({ id: scope.cloudWorkspaceId, from: args.from, to: "failed", set: { ...args.set, failureReason: args.reason, failedAt: new Date() }, + where: args.where, }); nudge(scope.organizationId, "cloud_workspaces"); - if (!changed) return; - const refused = isCloudUsageRefusal(args.error); + if (!changed) return false; + const expected = args.expected || isCloudUsageRefusal(args.error); logControl( "workspace.failed", scope, { "failure.reason": args.reason }, - refused ? "warn" : "error", + expected ? "warn" : "error", ); - if (!refused) { + if (!expected) { Sentry.captureException(args.error, { tags: controlTags(scope) }); } + return true; } diff --git a/packages/trpc/src/router/environment/environment.test.ts b/packages/trpc/src/router/environment/environment.test.ts index 76404214510..64deffa784b 100644 --- a/packages/trpc/src/router/environment/environment.test.ts +++ b/packages/trpc/src/router/environment/environment.test.ts @@ -58,7 +58,13 @@ stub(access, { branch: "main", }), }); -stub(sandbox, { workspaceRepositories: async () => [] }); +let deletedBoxes: string[] = []; +stub(sandbox, { + workspaceRepositories: async () => [], + deleteSandbox: async (name: string) => { + deletedBoxes.push(name); + }, +}); stub(plan, { resolveMachineSize: async (_: string, name: string | null | undefined) => name === "large" || name == null @@ -95,6 +101,22 @@ beforeEach(() => { }; isAdmin = false; written = []; + deletedBoxes = []; +}); + +describe("environment.archive", () => { + test("only sets a status: the cleanup job deletes its golden (rule 2)", async () => { + environment = { + ...environment, + sourceKind: "fork", + sourceRef: "env-golden", + }; + expect(await caller.environment.archive({ id: ENVIRONMENT_ID })).toEqual({ + archived: true, + }); + expect(written).toEqual([{ archivedAt: expect.any(Date) }]); + expect(deletedBoxes).toEqual([]); + }); }); describe("environment.update machineSize", () => { diff --git a/packages/trpc/src/router/environment/environment.ts b/packages/trpc/src/router/environment/environment.ts index 060826df00e..280e9779114 100644 --- a/packages/trpc/src/router/environment/environment.ts +++ b/packages/trpc/src/router/environment/environment.ts @@ -27,6 +27,7 @@ import { primaryRepository, promoteSandboxToEnvironment, RepositoryError, + sandboxDeletesAllowed, sortRepositories, workspaceRepositories, } from "../../lib/sandbox"; @@ -625,7 +626,10 @@ export const environmentRouter = { await restart(); throw error; }); - if (target?.sourceKind === "fork") { + if ( + target?.sourceKind === "fork" && + sandboxDeletesAllowed(`golden ${target.sourceRef}`) + ) { await deleteSandbox(target.sourceRef).catch((error: unknown) => console.error( `[environment/promote] ${target.id} replaced; could not delete its previous golden ${target.sourceRef}`, @@ -722,19 +726,11 @@ export const environmentRouter = { .mutation(async ({ ctx, input }) => { const environment = await loadEnvironment(input.id, ctx); await assertCloudAccess(ctx, environment.organizationId); - const [archived] = await db + await db .update(environments) .set({ archivedAt: new Date() }) - .where(eq(environments.id, environment.id)) - .returning({ - sourceKind: environments.sourceKind, - sourceRef: environments.sourceRef, - }); - // A golden is one environment's alone, and an archived environment - // never forks from it again; without this it bills storage forever. - if (archived?.sourceKind === "fork") { - await deleteSandbox(archived.sourceRef); - } + .where(eq(environments.id, environment.id)); + // Only a status: the cleanup job deletes an archived environment's goldens (rule 2). return { archived: true }; }), } satisfies TRPCRouterRecord; diff --git a/packages/trpc/test/setup-env.ts b/packages/trpc/test/setup-env.ts index 9d1b00eb4c0..246920a0b1c 100644 --- a/packages/trpc/test/setup-env.ts +++ b/packages/trpc/test/setup-env.ts @@ -2,6 +2,8 @@ import "../../../scripts/test-preload.ts"; process.env.SKIP_ENV_VALIDATION = "1"; process.env.BILLING_PROVIDER = "stripe"; +// `sandboxDeletesAllowed` deletes nothing outside production; the tests check the deletes. +process.env.VERCEL_ENV = "production"; // Importing a module that constructs a client throws without these. None // connects until it is used. process.env.STRIPE_SECRET_KEY ??= "sk_test_placeholder"; diff --git a/plans/done/20261118-1500-cloud-workspace-retention-reaper.md b/plans/done/20261118-1500-cloud-workspace-retention-reaper.md new file mode 100644 index 00000000000..cd807bb9142 --- /dev/null +++ b/plans/done/20261118-1500-cloud-workspace-retention-reaper.md @@ -0,0 +1,104 @@ +# Make cloud workspace retention ours: a daily reaper for idle boxes and orphans + +This ExecPlan is a living document. The sections `Progress`, `Surprises & Discoveries`, `Decision Log`, and `Outcomes & Retrospective` must be kept up to date as work proceeds. + +Reference: This plan follows conventions from AGENTS.md and the ExecPlan template. + +## Purpose / Big Picture + +A cloud workspace is a row in `cloud_workspaces` backed by a Vercel sandbox ("box"). When the box stops, Vercel keeps a snapshot of its disk, and the next wake resumes from it. Today `provisionSandbox` in `packages/trpc/src/lib/sandbox/vercel.ts` creates workspace boxes with `snapshotExpiration: 30 days`, so Vercel deletes the snapshot of a box nobody used for 30 days. The app learns this only when a wake, an agent turn or an automation dispatch gets a 410 and calls `markSandboxUnavailable` (`router/cloud-workspace/wake.ts`), which moves the row to `failed` with reason `snapshot_expired`. A workspace nobody opens again never learns. The box itself, and boxes that no row names (orphans), stay in Vercel and use storage. + +After this change Vercel never expires a new workspace's snapshot. A job that runs every 6 hours decides instead. It moves a workspace that has been asleep with no activity for 30 days to the same `failed`/`snapshot_expired` state the 410 path gives, takes the box name off the row first (through `retireAndTearDown`, the function #9847 owns), and queues a retried teardown that deletes the box. The same job queues teardown for orphan boxes. Each run logs and returns how many it retired, queued and skipped, and a Sentry cron monitor reports a run that did not happen. The customer sees the same 30-day retention as today. + +## Context and Orientation + +Apps and packages: `apps/api` (Next.js on Vercel, holds the QStash job routes) and `packages/trpc` (the cloud workspace logic). No tRPC procedure changes, no schema change, no migration. + +Terms. A "QStash job" is an HTTP POST that Upstash QStash delivers to `apps/api`, signed and retried. `publishCloudWorkspaceJob` (`packages/trpc/src/router/cloud-workspace/jobs.ts`) queues one, and runs it in-process when the API runs on localhost. A "scheduled job" in this repo is a route under `apps/api/src/app/api/cloud-workspaces/jobs//route.ts`. A QStash schedule in the Upstash console calls it, the route is wrapped in `Sentry.withMonitor` (a cron check-in), and `.github/workflows/publish-cloud-workspace-job.yml` can fire it once by hand. `reconcile-usage` is the model to copy. + +Names. A workspace box is named `sandboxNameFor(id)` or `nextSandboxNameFor(id)` (`router/cloud-workspace/provision.ts`) and tagged `{kind: "workspace", org, workspace, user}`. A golden (the box an environment forks from) is named `env-…`, is tagged `{kind: "environment", org}`, and is named by `environments.source_ref` of a fork environment. + +Last activity. `sandbox_expires_at` on an asleep row is when its last session ended. It is not reliable alone: in production on 2026-11-16 a one-off write moved it forward on 412 asleep rows past their last boot. Boot rows in `cloud_workspace_boots` (one per session: `started_at`, `stopped_at`) are written before Vercel resumes a box (`authorizeBoot` in `beforeResume`), so they are both the most accurate record and a guard against a wake in flight. The reaper therefore treats a row as idle since `cutoff` only when its `created_at` and `sandbox_expires_at` are older than `cutoff`, and it has no boot that is open or that started or stopped after `cutoff`. Taking the latest of these makes the reaper wait longer than Vercel when they disagree, never shorter. + +#9847 (draft, not merged) adds a `teardownSandbox` QStash job at `/api/cloud-workspaces/teardown` with payload `{providerSandboxId, driveName, attempt, keepDrive?}` and `retireAndTearDown(row, condition)`. Main has neither. This plan adds box-only versions with the same route, payload shape and function names, so #9847 replaces their bodies when it lands. That is the seam for drives. + +## Plan of Work + +1. `vercel.ts`: workspace boxes are created with `snapshotExpiration: 0` like goldens. Remove `WORKSPACE_SNAPSHOT_EXPIRATION_MS`. Add `listTaggedSandboxes(kind)`, which lists boxes of one tag kind with `createdAt`, `statusUpdatedAt`, `status` and `snapshotExpiration`. +2. `router/cloud-workspace/teardown.ts` (new): `teardownSandbox(input)` checks `liveReferences` (any workspace row in any status naming the box, or an unarchived environment's `source_ref`), stops the box, checks again, deletes it (`deleteSandbox`, which also deletes its snapshots). On error it requeues itself up to `TEARDOWN_ATTEMPTS` times. `deleteSandboxAndDriveLater({providerSandboxId, driveName})` queues that job for every box (box-only: a drive name throws). `retireAndTearDown(row, condition)` is one UPDATE WHERE id AND provider_sandbox_id AND condition that sets a new box name; only if it matched, it calls `deleteSandboxAndDriveLater`. Same names, signatures and module path as #9847, so #9847's file replaces this one. +3. `transition.ts`: `failCloudWorkspace` takes an optional `where` (extra conditions on the same update) and `expected` (log a warning, no Sentry exception), and returns whether it moved the row. Existing callers are unchanged. +4. `router/cloud-workspace/retention.ts` (new): `reapIdleCloudWorkspaces({now, production, deps})`. + - Idle stage: select up to `REAP_BATCH` asleep vercel rows idle since `now - 30d`. For each, a provider read skips a box that is running. Then `retireAndTearDown(row, status = asleep AND still idle AND not archived)` takes the name off the row and queues the old box's teardown. Only if it matched, `failCloudWorkspace` from `asleep` (still idle) sets `failure_reason = snapshot_expired` and `sandbox_url = null`. A wake between the two writes finds no box and fails the row itself, so the second write then moves nothing. + - Orphan stage (only when `production`): list `workspace` and `environment` tagged boxes. Keep those in a settled status whose `createdAt` and `statusUpdatedAt` are older than `ORPHAN_MIN_AGE` (7 days). Drop every name a workspace row or an unarchived environment names, and queue teardown for up to `ORPHAN_BATCH`. + - The run pauses `TEARDOWN_SPACING_MS` (250 ms) after each teardown it queues, so at most 4 a second, far below Vercel's 20 deletions/s per team. `retireAndTearDown` takes no delay, so the pacing is in the loop. + - It returns counts (`retired`, `orphansQueued`, `skipped{running,raced,orphanBatchFull,…}`) and logs them as one line. +5. `apps/api/.../cloud-workspaces/jobs/reap-retention/route.ts` (every 6 hours, `Sentry.withMonitor("reap-retention", …, "0 */6 * * *")`) and `.../cloud-workspaces/teardown/route.ts`. Add `reap-retention` to the publish workflow's choices. Add package exports. +6. `packages/sandbox/src/backfill-snapshot-expiration.ts` (new, not run): a dry run by default. `--apply` calls `sandbox.update({snapshotExpiration: 0})` on workspace boxes whose `snapshotExpiration` is not 0, at 2/s. +7. Docs: an entry in `docs/cloud-sandbox-mismatches.md`. The TLA+ model is not edited; the PR says which action this maps to. + +## Validation and Acceptance + + cd packages/trpc && bun test src/router/cloud-workspace/retention.test.ts src/router/cloud-workspace/teardown.test.ts src/lib/sandbox/vercel.test.ts + bun run lint && bunx turbo run typecheck --filter=@superset/trpc --filter=@superset/api --filter=@superset/sandbox + +Each behaviour has one test, with an injected clock and no real waiting. + +## Idempotence and Recovery + +A run that stops halfway is safe to repeat. The retire update matches only a row that still names the old box. A teardown that finds the box gone does nothing. A box retired but never torn down (its job abandoned) is an orphan, so a later run queues it again. To roll back, pause the QStash schedule. Rows already retired stay `failed`, as after a 410. + +## Assumptions + +All deployments probably share one Vercel sandbox project, so an orphan seen from a non-production database may be a production box. The orphan stage therefore runs only when `VERCEL_ENV === "production"`. + +## Open Questions + +None blocking. The QStash schedule must be created in the Upstash console by someone with access, as for the other jobs. + +## Progress + +- [x] (2026-11-18 15:00Z) Discovery, production counts, plan. +- [x] (2026-11-18 22:10Z) Implementation: `snapshotExpiration: 0`, `teardownSandbox` (box only), `failCloudWorkspace` `where`/`expected`, `retention.ts`, both routes, the backfill script, docs. +- [x] (2026-11-18 22:20Z) trpc 1104 pass, sandbox 85 pass; lint; typecheck of trpc, api and sandbox. Ran the reaper's `idleSince` SQL read-only on production: 0 reapable at 30 days, 0 at 7, 518 at 2. +- [ ] Create the QStash schedule `0 */6 * * *` for `/api/cloud-workspaces/jobs/reap-retention` in the Upstash console (needs console access; not done here). +- [ ] Run `bun run backfill-snapshot-expiration` dry, then `--apply` (not done here, on purpose). + +## Surprises & Discoveries + +- Observation: there is no `teardownSandbox` or box sweep on main; both are on #9847's branch only. +- Observation: `sandbox.update({snapshotExpiration})` exists in `@vercel/sandbox` 4.0.0 and does not resume a box. It sets the default for snapshots made from then on. The SDK has no call that changes an existing snapshot's expiry. +- Observation: production on 2026-11-18 had 857 asleep vercel rows, 0 idle for more than 30 days by either measure. By boot rows, 188 are idle for more than 7 days, and the oldest last boot is 2026-10-22. + +## Decision Log + +- Decision: reuse `failed` + `snapshot_expired`, no new enum value. + Rationale: the brief asks for the 410 path's terminal state, and a new value needs a migration and client copy. Date/Author: 2026-11-18, agent. +- Decision: last activity = latest of boot rows, `sandbox_expires_at` and `created_at`. + Rationale: deleting too late costs storage; deleting too early loses someone's work. Date/Author: 2026-11-18, agent. +- Decision: the orphan stage covers boxes only, not drives. + Rationale: on main no row names a workspace drive, so every `drive-` drive would look orphaned. Once #9847's columns exist, a drive stage must check them, and #9847's `teardownSandbox` deletes drives. Agent-tools drives are pruned by the release. Date/Author: 2026-11-18, agent. + +- Decision: run every 6 hours (`0 */6 * * *`) with batches of 200 idle rows and 200 orphans, and stop starting work after 200 s of a run (`RUN_BUDGET_MS`). + Rationale: Satya asked for 6-hourly runs sized to clear a normal day with room to spare. Production made a median 156 workspaces a day and at most 725 in the 30 days to 2026-11-18. Four runs of 200 is 800 a day, so even the peak day clears within one day, and a backlog takes several runs. The time budget keeps a large run inside the route's 300 s; the rest is reported as `deferred`. Date/Author: 2026-11-18, Satya via orchestrator. + +- Decision: add an archived-rows backstop (orchestrator, required). With `snapshotExpiration: 0` a lost 7-day archive delete would keep a box and its snapshot forever. Each production run lists every box once. Archived rows past grace + 1 day whose box is in that listing are deleted through `reapArchivedCloudWorkspace({stage: "delete", archivedAt: deleted_at})`, or through `reapUndatedArchivedCloudWorkspace` (status archived, `deleted_at` null, `updated_at` older than the cutoff) for the 71 rows with no archive date. At most 20 a run, inside the same time budget, counted as found / alreadyDeleted / deleted / skipped / deferred / failed. + Rationale: matching against the listing covers every row each run, with no per-row provider read and no marker column; checking rows one by one would keep re-reading the same oldest rows. It is production only for the same reason as the orphan stage: a dev database branched from production names production's boxes. On 2026-11-18, a sample of 20 such boxes were all already gone, and 3,553 rows matched the condition. Date/Author: 2026-11-18, orchestrator. + +- Decision: the idle stage retires names through `retireAndTearDown(row, condition)` with #9847's exact name, signature and module, and every box (with or without a drive) goes through the retried teardown job (orchestrator). + Rationale: #9847's file then replaces this one with no change to `retention.ts`. The cost: the name and the status are now two writes, not one. The name goes first, so the window is an asleep row that names a box that does not exist, which the 410 path already handles. The other order would leave a failed row naming a box being deleted. `deleteSandboxAndDriveLater` on #9847 deletes a box without a drive inline, with no retry and no re-read; this PR queues it, because the orphan stage depends on the re-read just before delete. Date/Author: 2026-11-18, orchestrator. + +- Decision: a stranded stage (production only, after the archived stage, batch 50) for the drive-liveness property that #9914's model found failing. It selects rows `provisioning` for `STRANDED_PROVISIONING_MS` (1 day, by `created_at`, `updated_at` and no boot since), and rows `failed` for `RETENTION_IDLE_MS` (by `coalesce(failed_at, updated_at)`) whose box is in this run's listing. It retires each through `retireAndTearDown` with the same status-and-age predicate as the condition; a stranded provisioning row then fails as `create_failed`. On main this covers boxes only; the drive half needs #9847's `drive_name` (see the rebase list in the PR). + Rationale: a stranded provisioning row leaves the selection by its status, a failed one because its retired name is in no listing, so no row can fill an oldest-first batch forever. Failed rows wait 30 days, because a failed row keeps its box for diagnosis. A call that does not complete is retried by a later run: a stranded row whose fail threw is selected again a day later (the rename moved `updated_at`); a box whose teardown was not queued is an orphan. On 2026-11-19 production had 2 failed rows (1 failed over 30 days ago) and no stranded ones. Date/Author: 2026-11-19, orchestrator. + +- Decision: rebuild #9987 to the agreed end state (page 20261119-sandbox-lifecycle-end-state, model #10006). The cleanup job applies three rules from one table and is the only thing that deletes for age: rule 1 (one select over the wait table: asleep 30 d since last use, archived 7 d, failed 30 d, stuck creating 1 d with no boot; each through one conditional retire write, then the retried teardown), rule 2 (archived environments' goldens; live environments through `goldensToDelete`, the same as #10000's), rule 3 (unnamed boxes, drives and goldens older than 7 days; agents drives that are not current and no box mounts). It runs only in production and deletes nothing unless the whole box and drive listing was read (#9943's checks). `deleteSandbox` and `deleteDrive` delete only when `VERCEL_ENV` is production; staging deploys with `--prod` on the production database, so it counts. Archive, a failed create, the provision race and environment archive only set a status; archive's delayed delete message is gone, its 60 s stop stays, and an unarchive after 7 days is refused. The release no longer prunes agents drives. The stranded and archived-backstop stages are dropped: rule 1 covers them; the orphan stage became rule 3. + Rationale: one deleter with one rule set, checked by a model, replaces five places that deleted on their own schedule. Stricter than the model in four places, all keeps: an asleep row whose box runs now; a workspace drive a box mounts; an agents drive younger than a day (a release publishes drives before it records its bundle); nothing at all on an incomplete listing. Date/Author: 2026-11-19, orchestrator. + +## Outcomes & Retrospective + +All seven parts of the brief are done except the drives. Workspace drives do not exist on main, so the drive half is a seam: `TeardownSandboxInput.driveName` is typed `null`, and #9847's version of `teardownSandbox` replaces this one at the same route with the same payload. The reaper's measure of last activity is deliberately later than Vercel's: the 412 rows whose `sandbox_expires_at` was moved forward on 2026-11-16 are reaped no earlier than 2026-12-16, and until then Vercel's own expiry and the 410 path still apply to them. + +Revision note (2026-11-18): aligned with #9847's `retireAndTearDown` and `deleteSandboxAndDriveLater`; the idle stage's single update became two writes, and teardowns are paced in the loop instead of by `delaySeconds`. + +Revision note (2026-11-19): added the stranded stage for failed and stuck-provisioning rows, for the drive-liveness property in #9914's model. + +Revision note (2026-11-19): rebuilt to the end-state rules table (rules 1 to 3) with the production-only delete check and status-only archive, failed create and environment archive.